From 6fbbd2e05d759b7e9d08e04093d0116197f5353d Mon Sep 17 00:00:00 2001 From: Alexey Vatolin Date: Fri, 20 Dec 2024 08:04:37 +0100 Subject: [PATCH] Add RuMTEB evaluation scores for most english top MTEB models (#76) --- .../CEDRClassification.json | 73 +++++ .../GeoreviewClassification.json | 73 +++++ .../GeoreviewClusteringP2P.json | 34 +++ .../HeadlineClassification.json | 73 +++++ .../InappropriatenessClassification.json | 95 ++++++ .../KinopoiskClassification.json | 73 +++++ .../MIRACLReranking.json | 130 ++++++++ .../MassiveIntentClassification.json | 137 +++++++++ .../MassiveScenarioClassification.json | 137 +++++++++ .../RUParaPhraserSTS.json | 26 ++ .../RiaNewsRetrieval.json | 158 ++++++++++ .../RuBQReranking.json | 26 ++ .../RuBQRetrieval.json | 158 ++++++++++ .../RuReviewsClassification.json | 73 +++++ .../RuSTSBenchmarkSTS.json | 26 ++ .../RuSciBenchGRNTIClassification.json | 73 +++++ .../RuSciBenchGRNTIClusteringP2P.json | 34 +++ .../RuSciBenchOECDClassification.json | 73 +++++ .../RuSciBenchOECDClusteringP2P.json | 34 +++ .../STS22.json | 26 ++ .../SensitiveTopicsClassification.json | 73 +++++ .../TERRa.json | 58 ++++ .../model_meta.json | 1 + .../CEDRClassification.json | 73 +++++ .../GeoreviewClassification.json | 73 +++++ .../GeoreviewClusteringP2P.json | 34 +++ .../HeadlineClassification.json | 73 +++++ .../InappropriatenessClassification.json | 95 ++++++ .../KinopoiskClassification.json | 73 +++++ .../MIRACLReranking.json | 130 ++++++++ .../MassiveIntentClassification.json | 137 +++++++++ .../MassiveScenarioClassification.json | 137 +++++++++ .../RUParaPhraserSTS.json | 26 ++ .../RiaNewsRetrieval.json | 158 ++++++++++ .../RuBQReranking.json | 26 ++ .../RuBQRetrieval.json | 158 ++++++++++ .../RuReviewsClassification.json | 73 +++++ .../RuSTSBenchmarkSTS.json | 26 ++ .../RuSciBenchGRNTIClassification.json | 73 +++++ .../RuSciBenchGRNTIClusteringP2P.json | 34 +++ .../RuSciBenchOECDClassification.json | 73 +++++ .../RuSciBenchOECDClusteringP2P.json | 34 +++ .../STS22.json | 26 ++ .../SensitiveTopicsClassification.json | 73 +++++ .../TERRa.json | 58 ++++ .../model_meta.json | 1 + .../CEDRClassification.json | 68 ++--- .../GeoreviewClassification.json | 73 +++++ .../GeoreviewClusteringP2P.json | 34 +++ .../HeadlineClassification.json | 73 +++++ .../InappropriatenessClassification.json | 95 ++++++ .../KinopoiskClassification.json | 73 +++++ .../MIRACLReranking.json | 130 ++++++++ .../MassiveScenarioClassification.json | 137 +++++++++ .../RUParaPhraserSTS.json | 26 ++ .../RiaNewsRetrieval.json | 158 ++++++++++ .../RuBQRetrieval.json | 158 ++++++++++ .../RuReviewsClassification.json | 73 +++++ .../RuSTSBenchmarkSTS.json | 26 ++ .../RuSciBenchGRNTIClassification.json | 73 +++++ .../RuSciBenchGRNTIClusteringP2P.json | 34 +++ .../RuSciBenchOECDClassification.json | 73 +++++ .../RuSciBenchOECDClusteringP2P.json | 34 +++ .../STS22.json | 26 ++ .../SensitiveTopicsClassification.json | 73 +++++ .../model_meta.json | 24 +- .../CEDRClassification.json | 73 +++++ .../GeoreviewClassification.json | 73 +++++ .../GeoreviewClusteringP2P.json | 34 +++ .../HeadlineClassification.json | 73 +++++ .../InappropriatenessClassification.json | 95 ++++++ .../KinopoiskClassification.json | 73 +++++ .../MIRACLReranking.json | 130 ++++++++ .../MIRACLRetrieval.json | 158 ++++++++++ .../MassiveScenarioClassification.json | 137 +++++++++ .../RUParaPhraserSTS.json | 26 ++ .../RiaNewsRetrieval.json | 158 ++++++++++ .../RuBQRetrieval.json | 158 ++++++++++ .../RuReviewsClassification.json | 73 +++++ .../RuSTSBenchmarkSTS.json | 26 ++ .../RuSciBenchGRNTIClassification.json | 73 +++++ .../RuSciBenchGRNTIClusteringP2P.json | 34 +++ .../RuSciBenchOECDClassification.json | 73 +++++ .../RuSciBenchOECDClusteringP2P.json | 34 +++ .../STS22.json | 26 ++ .../SensitiveTopicsClassification.json | 73 +++++ .../model_meta.json | 26 +- .../CEDRClassification.json | 68 ++--- .../GeoreviewClassification.json | 68 ++--- .../GeoreviewClusteringP2P.json | 26 +- .../HeadlineClassification.json | 68 ++--- .../InappropriatenessClassification.json | 112 +++---- .../KinopoiskClassification.json | 68 ++--- .../MIRACLReranking.json | 130 ++++++++ .../MIRACLRetrieval.json | 158 ++++++++++ .../RiaNewsRetrieval.json | 158 ++++++++++ .../RuBQReranking.json | 18 +- .../RuBQRetrieval.json | 280 ++++++++--------- .../RuReviewsClassification.json | 68 ++--- .../RuSciBenchGRNTIClassification.json | 68 ++--- .../RuSciBenchGRNTIClusteringP2P.json | 26 +- .../RuSciBenchOECDClassification.json | 68 ++--- .../RuSciBenchOECDClusteringP2P.json | 26 +- .../STS22.json | 26 ++ .../SensitiveTopicsClassification.json | 68 ++--- .../model_meta.json | 22 +- .../CEDRClassification.json | 73 +++++ .../GeoreviewClassification.json | 73 +++++ .../GeoreviewClusteringP2P.json | 34 +++ .../HeadlineClassification.json | 73 +++++ .../InappropriatenessClassification.json | 95 ++++++ .../KinopoiskClassification.json | 73 +++++ .../MIRACLReranking.json | 130 ++++++++ .../MIRACLRetrieval.json | 158 ++++++++++ .../MassiveIntentClassification.json | 137 +++++++++ .../MassiveScenarioClassification.json | 137 +++++++++ .../RUParaPhraserSTS.json | 26 ++ .../RiaNewsRetrieval.json | 158 ++++++++++ .../RuBQReranking.json | 26 ++ .../RuBQRetrieval.json | 158 ++++++++++ .../RuReviewsClassification.json | 73 +++++ .../RuSTSBenchmarkSTS.json | 26 ++ .../RuSciBenchGRNTIClassification.json | 73 +++++ .../RuSciBenchGRNTIClusteringP2P.json | 34 +++ .../RuSciBenchOECDClassification.json | 73 +++++ .../RuSciBenchOECDClusteringP2P.json | 34 +++ .../STS22.json | 26 ++ .../SensitiveTopicsClassification.json | 73 +++++ .../TERRa.json | 58 ++++ .../model_meta.json | 1 + .../model_meta.json | 1 + .../model_meta.json | 26 +- .../model_meta.json | 1 + .../GeoreviewClassification.json | 73 +++++ .../GeoreviewClusteringP2P.json | 34 +++ .../HeadlineClassification.json | 73 +++++ .../InappropriatenessClassification.json | 95 ++++++ .../KinopoiskClassification.json | 73 +++++ .../MIRACLReranking.json | 130 ++++++++ .../MassiveScenarioClassification.json | 137 +++++++++ .../RUParaPhraserSTS.json | 26 ++ .../RiaNewsRetrieval.json | 158 ++++++++++ .../RuBQRetrieval.json | 158 ++++++++++ .../RuReviewsClassification.json | 73 +++++ .../RuSTSBenchmarkSTS.json | 26 ++ .../RuSciBenchGRNTIClassification.json | 73 +++++ .../RuSciBenchGRNTIClusteringP2P.json | 34 +++ .../RuSciBenchOECDClassification.json | 73 +++++ .../RuSciBenchOECDClusteringP2P.json | 34 +++ .../STS22.json | 26 ++ .../SensitiveTopicsClassification.json | 73 +++++ .../model_meta.json | 26 +- .../CEDRClassification.json | 73 +++++ .../GeoreviewClassification.json | 73 +++++ .../GeoreviewClusteringP2P.json | 34 +++ .../HeadlineClassification.json | 73 +++++ .../InappropriatenessClassification.json | 95 ++++++ .../KinopoiskClassification.json | 73 +++++ .../MIRACLReranking.json | 130 ++++++++ .../MIRACLRetrieval.json | 158 ++++++++++ .../MassiveScenarioClassification.json | 137 +++++++++ .../RUParaPhraserSTS.json | 26 ++ .../RiaNewsRetrieval.json | 158 ++++++++++ .../RuBQRetrieval.json | 158 ++++++++++ .../RuReviewsClassification.json | 73 +++++ .../RuSTSBenchmarkSTS.json | 26 ++ .../RuSciBenchGRNTIClassification.json | 73 +++++ .../RuSciBenchGRNTIClusteringP2P.json | 34 +++ .../RuSciBenchOECDClassification.json | 73 +++++ .../RuSciBenchOECDClusteringP2P.json | 34 +++ .../STS22.json | 26 ++ .../SensitiveTopicsClassification.json | 73 +++++ .../model_meta.json | 26 +- .../CEDRClassification.json | 73 +++++ .../GeoreviewClassification.json | 73 +++++ .../GeoreviewClusteringP2P.json | 34 +++ .../HeadlineClassification.json | 73 +++++ .../InappropriatenessClassification.json | 95 ++++++ .../KinopoiskClassification.json | 73 +++++ .../MIRACLReranking.json | 130 ++++++++ .../MIRACLRetrieval.json | 158 ++++++++++ .../MassiveIntentClassification.json | 137 +++++++++ .../MassiveScenarioClassification.json | 137 +++++++++ .../RUParaPhraserSTS.json | 26 ++ .../RiaNewsRetrieval.json | 158 ++++++++++ .../RuBQReranking.json | 26 ++ .../RuBQRetrieval.json | 158 ++++++++++ .../RuReviewsClassification.json | 73 +++++ .../RuSTSBenchmarkSTS.json | 26 ++ .../RuSciBenchGRNTIClassification.json | 73 +++++ .../RuSciBenchGRNTIClusteringP2P.json | 34 +++ .../RuSciBenchOECDClassification.json | 73 +++++ .../RuSciBenchOECDClusteringP2P.json | 34 +++ .../STS22.json | 26 ++ .../SensitiveTopicsClassification.json | 73 +++++ .../TERRa.json | 58 ++++ .../model_meta.json | 22 ++ .../CEDRClassification.json | 73 +++++ .../GeoreviewClassification.json | 73 +++++ .../GeoreviewClusteringP2P.json | 34 +++ .../HeadlineClassification.json | 73 +++++ .../InappropriatenessClassification.json | 95 ++++++ .../KinopoiskClassification.json | 73 +++++ .../MIRACLReranking.json | 130 ++++++++ .../MIRACLRetrieval.json | 158 ++++++++++ .../MassiveIntentClassification.json | 137 +++++++++ .../MassiveScenarioClassification.json | 137 +++++++++ .../RUParaPhraserSTS.json | 26 ++ .../RiaNewsRetrieval.json | 158 ++++++++++ .../RuBQReranking.json | 26 ++ .../RuBQRetrieval.json | 158 ++++++++++ .../RuReviewsClassification.json | 73 +++++ .../RuSTSBenchmarkSTS.json | 26 ++ .../RuSciBenchGRNTIClassification.json | 73 +++++ .../RuSciBenchGRNTIClusteringP2P.json | 34 +++ .../RuSciBenchOECDClassification.json | 73 +++++ .../RuSciBenchOECDClusteringP2P.json | 34 +++ .../STS22.json | 26 ++ .../SensitiveTopicsClassification.json | 73 +++++ .../TERRa.json | 58 ++++ .../model_meta.json | 22 ++ .../STS22.json | 24 +- .../model_meta.json | 18 +- .../CEDRClassification.json | 64 ++-- .../GeoreviewClassification.json | 68 ++--- .../GeoreviewClusteringP2P.json | 26 +- .../HeadlineClassification.json | 64 ++-- .../InappropriatenessClassification.json | 112 +++---- .../KinopoiskClassification.json | 68 ++--- .../MIRACLRetrieval.json | 282 +++++++++--------- .../MassiveIntentClassification.json | 134 ++++----- .../MassiveScenarioClassification.json | 136 ++++----- .../RUParaPhraserSTS.json | 18 +- .../RiaNewsRetrieval.json | 278 ++++++++--------- .../RuReviewsClassification.json | 68 ++--- .../RuSTSBenchmarkSTS.json | 18 +- .../RuSciBenchGRNTIClusteringP2P.json | 26 +- .../RuSciBenchOECDClusteringP2P.json | 26 +- .../STS22.json | 18 +- .../SensitiveTopicsClassification.json | 68 ++--- .../TERRa.json | 53 ++-- .../model_meta.json | 18 +- .../GeoreviewClusteringP2P.json | 26 +- .../STS22.json | 24 +- .../SensitiveTopicsClassification.json | 68 ++--- .../model_meta.json | 18 +- .../CEDRClassification.json | 68 ++--- .../GeoreviewClassification.json | 73 +++++ .../GeoreviewClusteringP2P.json | 34 +++ .../HeadlineClassification.json | 73 +++++ .../InappropriatenessClassification.json | 95 ++++++ .../KinopoiskClassification.json | 73 +++++ .../MIRACLReranking.json | 130 ++++++++ .../MIRACLRetrieval.json | 158 ++++++++++ .../MassiveScenarioClassification.json | 137 +++++++++ .../RUParaPhraserSTS.json | 26 ++ .../RiaNewsRetrieval.json | 158 ++++++++++ .../RuBQRetrieval.json | 158 ++++++++++ .../RuReviewsClassification.json | 73 +++++ .../RuSTSBenchmarkSTS.json | 26 ++ .../RuSciBenchGRNTIClassification.json | 73 +++++ .../RuSciBenchGRNTIClusteringP2P.json | 34 +++ .../RuSciBenchOECDClassification.json | 73 +++++ .../RuSciBenchOECDClusteringP2P.json | 34 +++ .../STS22.json | 26 ++ .../SensitiveTopicsClassification.json | 73 +++++ .../model_meta.json | 27 +- .../CEDRClassification.json | 68 ++--- .../GeoreviewClassification.json | 73 +++++ .../GeoreviewClusteringP2P.json | 34 +++ .../HeadlineClassification.json | 73 +++++ .../InappropriatenessClassification.json | 95 ++++++ .../KinopoiskClassification.json | 73 +++++ .../MIRACLReranking.json | 130 ++++++++ .../MIRACLRetrieval.json | 158 ++++++++++ .../MassiveScenarioClassification.json | 137 +++++++++ .../RUParaPhraserSTS.json | 26 ++ .../RiaNewsRetrieval.json | 158 ++++++++++ .../RuBQRetrieval.json | 158 ++++++++++ .../RuReviewsClassification.json | 73 +++++ .../RuSTSBenchmarkSTS.json | 26 ++ .../RuSciBenchGRNTIClassification.json | 73 +++++ .../RuSciBenchGRNTIClusteringP2P.json | 34 +++ .../RuSciBenchOECDClassification.json | 73 +++++ .../RuSciBenchOECDClusteringP2P.json | 34 +++ .../STS22.json | 26 ++ .../SensitiveTopicsClassification.json | 73 +++++ .../model_meta.json | 27 +- .../MIRACLReranking.json | 218 +++++++------- .../MIRACLRetrieval.json | 282 +++++++++--------- .../RiaNewsRetrieval.json | 282 +++++++++--------- .../RuBQReranking.json | 18 +- .../RuBQRetrieval.json | 280 ++++++++--------- .../RuSciBenchOECDClusteringP2P.json | 26 +- .../STS22.json | 18 +- .../model_meta.json | 116 ++++++- .../CEDRClassification.json | 73 +++++ .../GeoreviewClassification.json | 73 +++++ .../GeoreviewClusteringP2P.json | 34 +++ .../HeadlineClassification.json | 73 +++++ .../InappropriatenessClassification.json | 95 ++++++ .../KinopoiskClassification.json | 73 +++++ .../MIRACLReranking.json | 130 ++++++++ .../MIRACLRetrieval.json | 158 ++++++++++ .../MassiveIntentClassification.json | 137 +++++++++ .../MassiveScenarioClassification.json | 137 +++++++++ .../RUParaPhraserSTS.json | 26 ++ .../RiaNewsRetrieval.json | 158 ++++++++++ .../RuBQReranking.json | 26 ++ .../RuBQRetrieval.json | 158 ++++++++++ .../RuReviewsClassification.json | 73 +++++ .../RuSTSBenchmarkSTS.json | 26 ++ .../RuSciBenchGRNTIClassification.json | 73 +++++ .../RuSciBenchGRNTIClusteringP2P.json | 34 +++ .../RuSciBenchOECDClassification.json | 73 +++++ .../RuSciBenchOECDClusteringP2P.json | 34 +++ .../STS22.json | 26 ++ .../SensitiveTopicsClassification.json | 73 +++++ .../TERRa.json | 58 ++++ .../model_meta.json | 124 +++++++- .../CEDRClassification.json | 68 ++--- .../GeoreviewClassification.json | 73 +++++ .../GeoreviewClusteringP2P.json | 34 +++ .../HeadlineClassification.json | 73 +++++ .../InappropriatenessClassification.json | 95 ++++++ .../KinopoiskClassification.json | 73 +++++ .../MIRACLReranking.json | 130 ++++++++ .../MassiveScenarioClassification.json | 137 +++++++++ .../RUParaPhraserSTS.json | 26 ++ .../RiaNewsRetrieval.json | 158 ++++++++++ .../RuBQRetrieval.json | 158 ++++++++++ .../RuReviewsClassification.json | 73 +++++ .../RuSTSBenchmarkSTS.json | 26 ++ .../RuSciBenchGRNTIClassification.json | 73 +++++ .../RuSciBenchGRNTIClusteringP2P.json | 34 +++ .../RuSciBenchOECDClassification.json | 73 +++++ .../RuSciBenchOECDClusteringP2P.json | 34 +++ .../STS22.json | 26 ++ .../SensitiveTopicsClassification.json | 73 +++++ .../model_meta.json | 124 +++++++- .../CEDRClassification.json | 73 +++++ .../GeoreviewClassification.json | 73 +++++ .../GeoreviewClusteringP2P.json | 34 +++ .../HeadlineClassification.json | 73 +++++ .../InappropriatenessClassification.json | 95 ++++++ .../KinopoiskClassification.json | 73 +++++ .../MIRACLReranking.json | 130 ++++++++ .../MIRACLRetrieval.json | 158 ++++++++++ .../MassiveScenarioClassification.json | 137 +++++++++ .../RUParaPhraserSTS.json | 26 ++ .../RiaNewsRetrieval.json | 158 ++++++++++ .../RuBQRetrieval.json | 158 ++++++++++ .../RuReviewsClassification.json | 73 +++++ .../RuSTSBenchmarkSTS.json | 26 ++ .../RuSciBenchGRNTIClassification.json | 73 +++++ .../RuSciBenchGRNTIClusteringP2P.json | 34 +++ .../RuSciBenchOECDClassification.json | 73 +++++ .../RuSciBenchOECDClusteringP2P.json | 34 +++ .../STS22.json | 26 ++ .../SensitiveTopicsClassification.json | 73 +++++ .../model_meta.json | 26 +- .../CEDRClassification.json | 73 +++++ .../GeoreviewClassification.json | 73 +++++ .../GeoreviewClusteringP2P.json | 34 +++ .../HeadlineClassification.json | 73 +++++ .../InappropriatenessClassification.json | 95 ++++++ .../KinopoiskClassification.json | 73 +++++ .../MIRACLReranking.json | 130 ++++++++ .../MassiveIntentClassification.json | 137 +++++++++ .../MassiveScenarioClassification.json | 137 +++++++++ .../RUParaPhraserSTS.json | 26 ++ .../RiaNewsRetrieval.json | 158 ++++++++++ .../RuBQReranking.json | 26 ++ .../RuBQRetrieval.json | 158 ++++++++++ .../RuReviewsClassification.json | 73 +++++ .../RuSTSBenchmarkSTS.json | 26 ++ .../RuSciBenchGRNTIClassification.json | 73 +++++ .../RuSciBenchGRNTIClusteringP2P.json | 34 +++ .../RuSciBenchOECDClassification.json | 73 +++++ .../RuSciBenchOECDClusteringP2P.json | 34 +++ .../STS22.json | 26 ++ .../SensitiveTopicsClassification.json | 73 +++++ .../TERRa.json | 58 ++++ .../model_meta.json | 1 + .../CEDRClassification.json | 73 +++++ .../GeoreviewClassification.json | 73 +++++ .../GeoreviewClusteringP2P.json | 34 +++ .../HeadlineClassification.json | 73 +++++ .../InappropriatenessClassification.json | 95 ++++++ .../KinopoiskClassification.json | 73 +++++ .../MIRACLReranking.json | 130 ++++++++ .../MassiveIntentClassification.json | 137 +++++++++ .../MassiveScenarioClassification.json | 137 +++++++++ .../RUParaPhraserSTS.json | 26 ++ .../RiaNewsRetrieval.json | 158 ++++++++++ .../RuBQReranking.json | 26 ++ .../RuBQRetrieval.json | 158 ++++++++++ .../RuReviewsClassification.json | 73 +++++ .../RuSTSBenchmarkSTS.json | 26 ++ .../RuSciBenchGRNTIClassification.json | 73 +++++ .../RuSciBenchGRNTIClusteringP2P.json | 34 +++ .../RuSciBenchOECDClassification.json | 73 +++++ .../RuSciBenchOECDClusteringP2P.json | 34 +++ .../STS22.json | 26 ++ .../SensitiveTopicsClassification.json | 73 +++++ .../TERRa.json | 58 ++++ .../model_meta.json | 26 +- .../RuSciBenchGRNTIClusteringP2P.json | 26 +- .../RuSciBenchOECDClusteringP2P.json | 26 +- .../STS22.json | 24 +- .../SensitiveTopicsClassification.json | 68 ++--- .../model_meta.json | 18 +- .../RuSciBenchGRNTIClusteringP2P.json | 26 +- .../RuSciBenchOECDClusteringP2P.json | 26 +- .../STS22.json | 24 +- .../SensitiveTopicsClassification.json | 66 ++-- .../model_meta.json | 18 +- 417 files changed, 29291 insertions(+), 2221 deletions(-) create mode 100644 results/Alibaba-NLP__gte-Qwen1.5-7B-instruct/07d27e5226328010336563bc1b564a5e3436a298/CEDRClassification.json create mode 100644 results/Alibaba-NLP__gte-Qwen1.5-7B-instruct/07d27e5226328010336563bc1b564a5e3436a298/GeoreviewClassification.json create mode 100644 results/Alibaba-NLP__gte-Qwen1.5-7B-instruct/07d27e5226328010336563bc1b564a5e3436a298/GeoreviewClusteringP2P.json create mode 100644 results/Alibaba-NLP__gte-Qwen1.5-7B-instruct/07d27e5226328010336563bc1b564a5e3436a298/HeadlineClassification.json create mode 100644 results/Alibaba-NLP__gte-Qwen1.5-7B-instruct/07d27e5226328010336563bc1b564a5e3436a298/InappropriatenessClassification.json create mode 100644 results/Alibaba-NLP__gte-Qwen1.5-7B-instruct/07d27e5226328010336563bc1b564a5e3436a298/KinopoiskClassification.json create mode 100644 results/Alibaba-NLP__gte-Qwen1.5-7B-instruct/07d27e5226328010336563bc1b564a5e3436a298/MIRACLReranking.json create mode 100644 results/Alibaba-NLP__gte-Qwen1.5-7B-instruct/07d27e5226328010336563bc1b564a5e3436a298/MassiveIntentClassification.json create mode 100644 results/Alibaba-NLP__gte-Qwen1.5-7B-instruct/07d27e5226328010336563bc1b564a5e3436a298/MassiveScenarioClassification.json create mode 100644 results/Alibaba-NLP__gte-Qwen1.5-7B-instruct/07d27e5226328010336563bc1b564a5e3436a298/RUParaPhraserSTS.json create mode 100644 results/Alibaba-NLP__gte-Qwen1.5-7B-instruct/07d27e5226328010336563bc1b564a5e3436a298/RiaNewsRetrieval.json create mode 100644 results/Alibaba-NLP__gte-Qwen1.5-7B-instruct/07d27e5226328010336563bc1b564a5e3436a298/RuBQReranking.json create mode 100644 results/Alibaba-NLP__gte-Qwen1.5-7B-instruct/07d27e5226328010336563bc1b564a5e3436a298/RuBQRetrieval.json create mode 100644 results/Alibaba-NLP__gte-Qwen1.5-7B-instruct/07d27e5226328010336563bc1b564a5e3436a298/RuReviewsClassification.json create mode 100644 results/Alibaba-NLP__gte-Qwen1.5-7B-instruct/07d27e5226328010336563bc1b564a5e3436a298/RuSTSBenchmarkSTS.json create mode 100644 results/Alibaba-NLP__gte-Qwen1.5-7B-instruct/07d27e5226328010336563bc1b564a5e3436a298/RuSciBenchGRNTIClassification.json create mode 100644 results/Alibaba-NLP__gte-Qwen1.5-7B-instruct/07d27e5226328010336563bc1b564a5e3436a298/RuSciBenchGRNTIClusteringP2P.json create mode 100644 results/Alibaba-NLP__gte-Qwen1.5-7B-instruct/07d27e5226328010336563bc1b564a5e3436a298/RuSciBenchOECDClassification.json create mode 100644 results/Alibaba-NLP__gte-Qwen1.5-7B-instruct/07d27e5226328010336563bc1b564a5e3436a298/RuSciBenchOECDClusteringP2P.json create mode 100644 results/Alibaba-NLP__gte-Qwen1.5-7B-instruct/07d27e5226328010336563bc1b564a5e3436a298/STS22.json create mode 100644 results/Alibaba-NLP__gte-Qwen1.5-7B-instruct/07d27e5226328010336563bc1b564a5e3436a298/SensitiveTopicsClassification.json create mode 100644 results/Alibaba-NLP__gte-Qwen1.5-7B-instruct/07d27e5226328010336563bc1b564a5e3436a298/TERRa.json create mode 100644 results/Alibaba-NLP__gte-Qwen1.5-7B-instruct/07d27e5226328010336563bc1b564a5e3436a298/model_meta.json create mode 100644 results/Alibaba-NLP__gte-Qwen2-1.5B-instruct/c6c1b92f4a3e1b92b326ad29dd3c8433457df8dd/CEDRClassification.json create mode 100644 results/Alibaba-NLP__gte-Qwen2-1.5B-instruct/c6c1b92f4a3e1b92b326ad29dd3c8433457df8dd/GeoreviewClassification.json create mode 100644 results/Alibaba-NLP__gte-Qwen2-1.5B-instruct/c6c1b92f4a3e1b92b326ad29dd3c8433457df8dd/GeoreviewClusteringP2P.json create mode 100644 results/Alibaba-NLP__gte-Qwen2-1.5B-instruct/c6c1b92f4a3e1b92b326ad29dd3c8433457df8dd/HeadlineClassification.json create mode 100644 results/Alibaba-NLP__gte-Qwen2-1.5B-instruct/c6c1b92f4a3e1b92b326ad29dd3c8433457df8dd/InappropriatenessClassification.json create mode 100644 results/Alibaba-NLP__gte-Qwen2-1.5B-instruct/c6c1b92f4a3e1b92b326ad29dd3c8433457df8dd/KinopoiskClassification.json create mode 100644 results/Alibaba-NLP__gte-Qwen2-1.5B-instruct/c6c1b92f4a3e1b92b326ad29dd3c8433457df8dd/MIRACLReranking.json create mode 100644 results/Alibaba-NLP__gte-Qwen2-1.5B-instruct/c6c1b92f4a3e1b92b326ad29dd3c8433457df8dd/MassiveIntentClassification.json create mode 100644 results/Alibaba-NLP__gte-Qwen2-1.5B-instruct/c6c1b92f4a3e1b92b326ad29dd3c8433457df8dd/MassiveScenarioClassification.json create mode 100644 results/Alibaba-NLP__gte-Qwen2-1.5B-instruct/c6c1b92f4a3e1b92b326ad29dd3c8433457df8dd/RUParaPhraserSTS.json create mode 100644 results/Alibaba-NLP__gte-Qwen2-1.5B-instruct/c6c1b92f4a3e1b92b326ad29dd3c8433457df8dd/RiaNewsRetrieval.json create mode 100644 results/Alibaba-NLP__gte-Qwen2-1.5B-instruct/c6c1b92f4a3e1b92b326ad29dd3c8433457df8dd/RuBQReranking.json create mode 100644 results/Alibaba-NLP__gte-Qwen2-1.5B-instruct/c6c1b92f4a3e1b92b326ad29dd3c8433457df8dd/RuBQRetrieval.json create mode 100644 results/Alibaba-NLP__gte-Qwen2-1.5B-instruct/c6c1b92f4a3e1b92b326ad29dd3c8433457df8dd/RuReviewsClassification.json create mode 100644 results/Alibaba-NLP__gte-Qwen2-1.5B-instruct/c6c1b92f4a3e1b92b326ad29dd3c8433457df8dd/RuSTSBenchmarkSTS.json create mode 100644 results/Alibaba-NLP__gte-Qwen2-1.5B-instruct/c6c1b92f4a3e1b92b326ad29dd3c8433457df8dd/RuSciBenchGRNTIClassification.json create mode 100644 results/Alibaba-NLP__gte-Qwen2-1.5B-instruct/c6c1b92f4a3e1b92b326ad29dd3c8433457df8dd/RuSciBenchGRNTIClusteringP2P.json create mode 100644 results/Alibaba-NLP__gte-Qwen2-1.5B-instruct/c6c1b92f4a3e1b92b326ad29dd3c8433457df8dd/RuSciBenchOECDClassification.json create mode 100644 results/Alibaba-NLP__gte-Qwen2-1.5B-instruct/c6c1b92f4a3e1b92b326ad29dd3c8433457df8dd/RuSciBenchOECDClusteringP2P.json create mode 100644 results/Alibaba-NLP__gte-Qwen2-1.5B-instruct/c6c1b92f4a3e1b92b326ad29dd3c8433457df8dd/STS22.json create mode 100644 results/Alibaba-NLP__gte-Qwen2-1.5B-instruct/c6c1b92f4a3e1b92b326ad29dd3c8433457df8dd/SensitiveTopicsClassification.json create mode 100644 results/Alibaba-NLP__gte-Qwen2-1.5B-instruct/c6c1b92f4a3e1b92b326ad29dd3c8433457df8dd/TERRa.json create mode 100644 results/Alibaba-NLP__gte-Qwen2-1.5B-instruct/c6c1b92f4a3e1b92b326ad29dd3c8433457df8dd/model_meta.json create mode 100644 results/Alibaba-NLP__gte-Qwen2-7B-instruct/e26182b2122f4435e8b3ebecbf363990f409b45b/GeoreviewClassification.json create mode 100644 results/Alibaba-NLP__gte-Qwen2-7B-instruct/e26182b2122f4435e8b3ebecbf363990f409b45b/GeoreviewClusteringP2P.json create mode 100644 results/Alibaba-NLP__gte-Qwen2-7B-instruct/e26182b2122f4435e8b3ebecbf363990f409b45b/HeadlineClassification.json create mode 100644 results/Alibaba-NLP__gte-Qwen2-7B-instruct/e26182b2122f4435e8b3ebecbf363990f409b45b/InappropriatenessClassification.json create mode 100644 results/Alibaba-NLP__gte-Qwen2-7B-instruct/e26182b2122f4435e8b3ebecbf363990f409b45b/KinopoiskClassification.json create mode 100644 results/Alibaba-NLP__gte-Qwen2-7B-instruct/e26182b2122f4435e8b3ebecbf363990f409b45b/MIRACLReranking.json create mode 100644 results/Alibaba-NLP__gte-Qwen2-7B-instruct/e26182b2122f4435e8b3ebecbf363990f409b45b/MassiveScenarioClassification.json create mode 100644 results/Alibaba-NLP__gte-Qwen2-7B-instruct/e26182b2122f4435e8b3ebecbf363990f409b45b/RUParaPhraserSTS.json create mode 100644 results/Alibaba-NLP__gte-Qwen2-7B-instruct/e26182b2122f4435e8b3ebecbf363990f409b45b/RiaNewsRetrieval.json create mode 100644 results/Alibaba-NLP__gte-Qwen2-7B-instruct/e26182b2122f4435e8b3ebecbf363990f409b45b/RuBQRetrieval.json create mode 100644 results/Alibaba-NLP__gte-Qwen2-7B-instruct/e26182b2122f4435e8b3ebecbf363990f409b45b/RuReviewsClassification.json create mode 100644 results/Alibaba-NLP__gte-Qwen2-7B-instruct/e26182b2122f4435e8b3ebecbf363990f409b45b/RuSTSBenchmarkSTS.json create mode 100644 results/Alibaba-NLP__gte-Qwen2-7B-instruct/e26182b2122f4435e8b3ebecbf363990f409b45b/RuSciBenchGRNTIClassification.json create mode 100644 results/Alibaba-NLP__gte-Qwen2-7B-instruct/e26182b2122f4435e8b3ebecbf363990f409b45b/RuSciBenchGRNTIClusteringP2P.json create mode 100644 results/Alibaba-NLP__gte-Qwen2-7B-instruct/e26182b2122f4435e8b3ebecbf363990f409b45b/RuSciBenchOECDClassification.json create mode 100644 results/Alibaba-NLP__gte-Qwen2-7B-instruct/e26182b2122f4435e8b3ebecbf363990f409b45b/RuSciBenchOECDClusteringP2P.json create mode 100644 results/Alibaba-NLP__gte-Qwen2-7B-instruct/e26182b2122f4435e8b3ebecbf363990f409b45b/STS22.json create mode 100644 results/Alibaba-NLP__gte-Qwen2-7B-instruct/e26182b2122f4435e8b3ebecbf363990f409b45b/SensitiveTopicsClassification.json create mode 100644 results/BAAI__bge-large-en-v1.5/d4aa6901d3a41ba39fb536a557fa166f842b0e09/CEDRClassification.json create mode 100644 results/BAAI__bge-large-en-v1.5/d4aa6901d3a41ba39fb536a557fa166f842b0e09/GeoreviewClassification.json create mode 100644 results/BAAI__bge-large-en-v1.5/d4aa6901d3a41ba39fb536a557fa166f842b0e09/GeoreviewClusteringP2P.json create mode 100644 results/BAAI__bge-large-en-v1.5/d4aa6901d3a41ba39fb536a557fa166f842b0e09/HeadlineClassification.json create mode 100644 results/BAAI__bge-large-en-v1.5/d4aa6901d3a41ba39fb536a557fa166f842b0e09/InappropriatenessClassification.json create mode 100644 results/BAAI__bge-large-en-v1.5/d4aa6901d3a41ba39fb536a557fa166f842b0e09/KinopoiskClassification.json create mode 100644 results/BAAI__bge-large-en-v1.5/d4aa6901d3a41ba39fb536a557fa166f842b0e09/MIRACLReranking.json create mode 100644 results/BAAI__bge-large-en-v1.5/d4aa6901d3a41ba39fb536a557fa166f842b0e09/MIRACLRetrieval.json create mode 100644 results/BAAI__bge-large-en-v1.5/d4aa6901d3a41ba39fb536a557fa166f842b0e09/MassiveScenarioClassification.json create mode 100644 results/BAAI__bge-large-en-v1.5/d4aa6901d3a41ba39fb536a557fa166f842b0e09/RUParaPhraserSTS.json create mode 100644 results/BAAI__bge-large-en-v1.5/d4aa6901d3a41ba39fb536a557fa166f842b0e09/RiaNewsRetrieval.json create mode 100644 results/BAAI__bge-large-en-v1.5/d4aa6901d3a41ba39fb536a557fa166f842b0e09/RuBQRetrieval.json create mode 100644 results/BAAI__bge-large-en-v1.5/d4aa6901d3a41ba39fb536a557fa166f842b0e09/RuReviewsClassification.json create mode 100644 results/BAAI__bge-large-en-v1.5/d4aa6901d3a41ba39fb536a557fa166f842b0e09/RuSTSBenchmarkSTS.json create mode 100644 results/BAAI__bge-large-en-v1.5/d4aa6901d3a41ba39fb536a557fa166f842b0e09/RuSciBenchGRNTIClassification.json create mode 100644 results/BAAI__bge-large-en-v1.5/d4aa6901d3a41ba39fb536a557fa166f842b0e09/RuSciBenchGRNTIClusteringP2P.json create mode 100644 results/BAAI__bge-large-en-v1.5/d4aa6901d3a41ba39fb536a557fa166f842b0e09/RuSciBenchOECDClassification.json create mode 100644 results/BAAI__bge-large-en-v1.5/d4aa6901d3a41ba39fb536a557fa166f842b0e09/RuSciBenchOECDClusteringP2P.json create mode 100644 results/BAAI__bge-large-en-v1.5/d4aa6901d3a41ba39fb536a557fa166f842b0e09/STS22.json create mode 100644 results/BAAI__bge-large-en-v1.5/d4aa6901d3a41ba39fb536a557fa166f842b0e09/SensitiveTopicsClassification.json create mode 100644 results/GritLM__GritLM-7B/13f00a0e36500c80ce12870ea513846a066004af/MIRACLReranking.json create mode 100644 results/GritLM__GritLM-7B/13f00a0e36500c80ce12870ea513846a066004af/MIRACLRetrieval.json create mode 100644 results/GritLM__GritLM-7B/13f00a0e36500c80ce12870ea513846a066004af/RiaNewsRetrieval.json create mode 100644 results/GritLM__GritLM-7B/13f00a0e36500c80ce12870ea513846a066004af/STS22.json create mode 100644 results/Linq-AI-Research__Linq-Embed-Mistral/0c1a0b0589177079acc552433cad51d7c9132379/CEDRClassification.json create mode 100644 results/Linq-AI-Research__Linq-Embed-Mistral/0c1a0b0589177079acc552433cad51d7c9132379/GeoreviewClassification.json create mode 100644 results/Linq-AI-Research__Linq-Embed-Mistral/0c1a0b0589177079acc552433cad51d7c9132379/GeoreviewClusteringP2P.json create mode 100644 results/Linq-AI-Research__Linq-Embed-Mistral/0c1a0b0589177079acc552433cad51d7c9132379/HeadlineClassification.json create mode 100644 results/Linq-AI-Research__Linq-Embed-Mistral/0c1a0b0589177079acc552433cad51d7c9132379/InappropriatenessClassification.json create mode 100644 results/Linq-AI-Research__Linq-Embed-Mistral/0c1a0b0589177079acc552433cad51d7c9132379/KinopoiskClassification.json create mode 100644 results/Linq-AI-Research__Linq-Embed-Mistral/0c1a0b0589177079acc552433cad51d7c9132379/MIRACLReranking.json create mode 100644 results/Linq-AI-Research__Linq-Embed-Mistral/0c1a0b0589177079acc552433cad51d7c9132379/MIRACLRetrieval.json create mode 100644 results/Linq-AI-Research__Linq-Embed-Mistral/0c1a0b0589177079acc552433cad51d7c9132379/MassiveIntentClassification.json create mode 100644 results/Linq-AI-Research__Linq-Embed-Mistral/0c1a0b0589177079acc552433cad51d7c9132379/MassiveScenarioClassification.json create mode 100644 results/Linq-AI-Research__Linq-Embed-Mistral/0c1a0b0589177079acc552433cad51d7c9132379/RUParaPhraserSTS.json create mode 100644 results/Linq-AI-Research__Linq-Embed-Mistral/0c1a0b0589177079acc552433cad51d7c9132379/RiaNewsRetrieval.json create mode 100644 results/Linq-AI-Research__Linq-Embed-Mistral/0c1a0b0589177079acc552433cad51d7c9132379/RuBQReranking.json create mode 100644 results/Linq-AI-Research__Linq-Embed-Mistral/0c1a0b0589177079acc552433cad51d7c9132379/RuBQRetrieval.json create mode 100644 results/Linq-AI-Research__Linq-Embed-Mistral/0c1a0b0589177079acc552433cad51d7c9132379/RuReviewsClassification.json create mode 100644 results/Linq-AI-Research__Linq-Embed-Mistral/0c1a0b0589177079acc552433cad51d7c9132379/RuSTSBenchmarkSTS.json create mode 100644 results/Linq-AI-Research__Linq-Embed-Mistral/0c1a0b0589177079acc552433cad51d7c9132379/RuSciBenchGRNTIClassification.json create mode 100644 results/Linq-AI-Research__Linq-Embed-Mistral/0c1a0b0589177079acc552433cad51d7c9132379/RuSciBenchGRNTIClusteringP2P.json create mode 100644 results/Linq-AI-Research__Linq-Embed-Mistral/0c1a0b0589177079acc552433cad51d7c9132379/RuSciBenchOECDClassification.json create mode 100644 results/Linq-AI-Research__Linq-Embed-Mistral/0c1a0b0589177079acc552433cad51d7c9132379/RuSciBenchOECDClusteringP2P.json create mode 100644 results/Linq-AI-Research__Linq-Embed-Mistral/0c1a0b0589177079acc552433cad51d7c9132379/STS22.json create mode 100644 results/Linq-AI-Research__Linq-Embed-Mistral/0c1a0b0589177079acc552433cad51d7c9132379/SensitiveTopicsClassification.json create mode 100644 results/Linq-AI-Research__Linq-Embed-Mistral/0c1a0b0589177079acc552433cad51d7c9132379/TERRa.json create mode 100644 results/Linq-AI-Research__Linq-Embed-Mistral/0c1a0b0589177079acc552433cad51d7c9132379/model_meta.json create mode 100644 results/McGill-NLP__LLM2Vec-Llama-2-7b-chat-hf-mntp-supervised/2c055a5d77126c0d3dc6cd8ffa30e2908f4f45f8/model_meta.json create mode 100644 results/McGill-NLP__LLM2Vec-Mistral-7B-Instruct-v2-mntp-supervised/0ae69bdd5816105778b971c3138e8f8a18eaa3ae/model_meta.json create mode 100644 results/Salesforce__SFR-Embedding-2_R/91762139d94ed4371a9fa31db5551272e0b83818/GeoreviewClassification.json create mode 100644 results/Salesforce__SFR-Embedding-2_R/91762139d94ed4371a9fa31db5551272e0b83818/GeoreviewClusteringP2P.json create mode 100644 results/Salesforce__SFR-Embedding-2_R/91762139d94ed4371a9fa31db5551272e0b83818/HeadlineClassification.json create mode 100644 results/Salesforce__SFR-Embedding-2_R/91762139d94ed4371a9fa31db5551272e0b83818/InappropriatenessClassification.json create mode 100644 results/Salesforce__SFR-Embedding-2_R/91762139d94ed4371a9fa31db5551272e0b83818/KinopoiskClassification.json create mode 100644 results/Salesforce__SFR-Embedding-2_R/91762139d94ed4371a9fa31db5551272e0b83818/MIRACLReranking.json create mode 100644 results/Salesforce__SFR-Embedding-2_R/91762139d94ed4371a9fa31db5551272e0b83818/MassiveScenarioClassification.json create mode 100644 results/Salesforce__SFR-Embedding-2_R/91762139d94ed4371a9fa31db5551272e0b83818/RUParaPhraserSTS.json create mode 100644 results/Salesforce__SFR-Embedding-2_R/91762139d94ed4371a9fa31db5551272e0b83818/RiaNewsRetrieval.json create mode 100644 results/Salesforce__SFR-Embedding-2_R/91762139d94ed4371a9fa31db5551272e0b83818/RuBQRetrieval.json create mode 100644 results/Salesforce__SFR-Embedding-2_R/91762139d94ed4371a9fa31db5551272e0b83818/RuReviewsClassification.json create mode 100644 results/Salesforce__SFR-Embedding-2_R/91762139d94ed4371a9fa31db5551272e0b83818/RuSTSBenchmarkSTS.json create mode 100644 results/Salesforce__SFR-Embedding-2_R/91762139d94ed4371a9fa31db5551272e0b83818/RuSciBenchGRNTIClassification.json create mode 100644 results/Salesforce__SFR-Embedding-2_R/91762139d94ed4371a9fa31db5551272e0b83818/RuSciBenchGRNTIClusteringP2P.json create mode 100644 results/Salesforce__SFR-Embedding-2_R/91762139d94ed4371a9fa31db5551272e0b83818/RuSciBenchOECDClassification.json create mode 100644 results/Salesforce__SFR-Embedding-2_R/91762139d94ed4371a9fa31db5551272e0b83818/RuSciBenchOECDClusteringP2P.json create mode 100644 results/Salesforce__SFR-Embedding-2_R/91762139d94ed4371a9fa31db5551272e0b83818/STS22.json create mode 100644 results/Salesforce__SFR-Embedding-2_R/91762139d94ed4371a9fa31db5551272e0b83818/SensitiveTopicsClassification.json create mode 100644 results/WhereIsAI__UAE-Large-V1/369c368f70f16a613f19f5598d4f12d9f44235d4/CEDRClassification.json create mode 100644 results/WhereIsAI__UAE-Large-V1/369c368f70f16a613f19f5598d4f12d9f44235d4/GeoreviewClassification.json create mode 100644 results/WhereIsAI__UAE-Large-V1/369c368f70f16a613f19f5598d4f12d9f44235d4/GeoreviewClusteringP2P.json create mode 100644 results/WhereIsAI__UAE-Large-V1/369c368f70f16a613f19f5598d4f12d9f44235d4/HeadlineClassification.json create mode 100644 results/WhereIsAI__UAE-Large-V1/369c368f70f16a613f19f5598d4f12d9f44235d4/InappropriatenessClassification.json create mode 100644 results/WhereIsAI__UAE-Large-V1/369c368f70f16a613f19f5598d4f12d9f44235d4/KinopoiskClassification.json create mode 100644 results/WhereIsAI__UAE-Large-V1/369c368f70f16a613f19f5598d4f12d9f44235d4/MIRACLReranking.json create mode 100644 results/WhereIsAI__UAE-Large-V1/369c368f70f16a613f19f5598d4f12d9f44235d4/MIRACLRetrieval.json create mode 100644 results/WhereIsAI__UAE-Large-V1/369c368f70f16a613f19f5598d4f12d9f44235d4/MassiveScenarioClassification.json create mode 100644 results/WhereIsAI__UAE-Large-V1/369c368f70f16a613f19f5598d4f12d9f44235d4/RUParaPhraserSTS.json create mode 100644 results/WhereIsAI__UAE-Large-V1/369c368f70f16a613f19f5598d4f12d9f44235d4/RiaNewsRetrieval.json create mode 100644 results/WhereIsAI__UAE-Large-V1/369c368f70f16a613f19f5598d4f12d9f44235d4/RuBQRetrieval.json create mode 100644 results/WhereIsAI__UAE-Large-V1/369c368f70f16a613f19f5598d4f12d9f44235d4/RuReviewsClassification.json create mode 100644 results/WhereIsAI__UAE-Large-V1/369c368f70f16a613f19f5598d4f12d9f44235d4/RuSTSBenchmarkSTS.json create mode 100644 results/WhereIsAI__UAE-Large-V1/369c368f70f16a613f19f5598d4f12d9f44235d4/RuSciBenchGRNTIClassification.json create mode 100644 results/WhereIsAI__UAE-Large-V1/369c368f70f16a613f19f5598d4f12d9f44235d4/RuSciBenchGRNTIClusteringP2P.json create mode 100644 results/WhereIsAI__UAE-Large-V1/369c368f70f16a613f19f5598d4f12d9f44235d4/RuSciBenchOECDClassification.json create mode 100644 results/WhereIsAI__UAE-Large-V1/369c368f70f16a613f19f5598d4f12d9f44235d4/RuSciBenchOECDClusteringP2P.json create mode 100644 results/WhereIsAI__UAE-Large-V1/369c368f70f16a613f19f5598d4f12d9f44235d4/STS22.json create mode 100644 results/WhereIsAI__UAE-Large-V1/369c368f70f16a613f19f5598d4f12d9f44235d4/SensitiveTopicsClassification.json create mode 100644 results/avsolatorio__GIST-Embedding-v0/bf6b2e55e92f510a570ad4d7d2da2ec8cd22590c/CEDRClassification.json create mode 100644 results/avsolatorio__GIST-Embedding-v0/bf6b2e55e92f510a570ad4d7d2da2ec8cd22590c/GeoreviewClassification.json create mode 100644 results/avsolatorio__GIST-Embedding-v0/bf6b2e55e92f510a570ad4d7d2da2ec8cd22590c/GeoreviewClusteringP2P.json create mode 100644 results/avsolatorio__GIST-Embedding-v0/bf6b2e55e92f510a570ad4d7d2da2ec8cd22590c/HeadlineClassification.json create mode 100644 results/avsolatorio__GIST-Embedding-v0/bf6b2e55e92f510a570ad4d7d2da2ec8cd22590c/InappropriatenessClassification.json create mode 100644 results/avsolatorio__GIST-Embedding-v0/bf6b2e55e92f510a570ad4d7d2da2ec8cd22590c/KinopoiskClassification.json create mode 100644 results/avsolatorio__GIST-Embedding-v0/bf6b2e55e92f510a570ad4d7d2da2ec8cd22590c/MIRACLReranking.json create mode 100644 results/avsolatorio__GIST-Embedding-v0/bf6b2e55e92f510a570ad4d7d2da2ec8cd22590c/MIRACLRetrieval.json create mode 100644 results/avsolatorio__GIST-Embedding-v0/bf6b2e55e92f510a570ad4d7d2da2ec8cd22590c/MassiveIntentClassification.json create mode 100644 results/avsolatorio__GIST-Embedding-v0/bf6b2e55e92f510a570ad4d7d2da2ec8cd22590c/MassiveScenarioClassification.json create mode 100644 results/avsolatorio__GIST-Embedding-v0/bf6b2e55e92f510a570ad4d7d2da2ec8cd22590c/RUParaPhraserSTS.json create mode 100644 results/avsolatorio__GIST-Embedding-v0/bf6b2e55e92f510a570ad4d7d2da2ec8cd22590c/RiaNewsRetrieval.json create mode 100644 results/avsolatorio__GIST-Embedding-v0/bf6b2e55e92f510a570ad4d7d2da2ec8cd22590c/RuBQReranking.json create mode 100644 results/avsolatorio__GIST-Embedding-v0/bf6b2e55e92f510a570ad4d7d2da2ec8cd22590c/RuBQRetrieval.json create mode 100644 results/avsolatorio__GIST-Embedding-v0/bf6b2e55e92f510a570ad4d7d2da2ec8cd22590c/RuReviewsClassification.json create mode 100644 results/avsolatorio__GIST-Embedding-v0/bf6b2e55e92f510a570ad4d7d2da2ec8cd22590c/RuSTSBenchmarkSTS.json create mode 100644 results/avsolatorio__GIST-Embedding-v0/bf6b2e55e92f510a570ad4d7d2da2ec8cd22590c/RuSciBenchGRNTIClassification.json create mode 100644 results/avsolatorio__GIST-Embedding-v0/bf6b2e55e92f510a570ad4d7d2da2ec8cd22590c/RuSciBenchGRNTIClusteringP2P.json create mode 100644 results/avsolatorio__GIST-Embedding-v0/bf6b2e55e92f510a570ad4d7d2da2ec8cd22590c/RuSciBenchOECDClassification.json create mode 100644 results/avsolatorio__GIST-Embedding-v0/bf6b2e55e92f510a570ad4d7d2da2ec8cd22590c/RuSciBenchOECDClusteringP2P.json create mode 100644 results/avsolatorio__GIST-Embedding-v0/bf6b2e55e92f510a570ad4d7d2da2ec8cd22590c/STS22.json create mode 100644 results/avsolatorio__GIST-Embedding-v0/bf6b2e55e92f510a570ad4d7d2da2ec8cd22590c/SensitiveTopicsClassification.json create mode 100644 results/avsolatorio__GIST-Embedding-v0/bf6b2e55e92f510a570ad4d7d2da2ec8cd22590c/TERRa.json create mode 100644 results/avsolatorio__GIST-Embedding-v0/bf6b2e55e92f510a570ad4d7d2da2ec8cd22590c/model_meta.json create mode 100644 results/avsolatorio__GIST-large-Embedding-v0/7831200e2f7819b994490c091cf3258a2b821f0c/CEDRClassification.json create mode 100644 results/avsolatorio__GIST-large-Embedding-v0/7831200e2f7819b994490c091cf3258a2b821f0c/GeoreviewClassification.json create mode 100644 results/avsolatorio__GIST-large-Embedding-v0/7831200e2f7819b994490c091cf3258a2b821f0c/GeoreviewClusteringP2P.json create mode 100644 results/avsolatorio__GIST-large-Embedding-v0/7831200e2f7819b994490c091cf3258a2b821f0c/HeadlineClassification.json create mode 100644 results/avsolatorio__GIST-large-Embedding-v0/7831200e2f7819b994490c091cf3258a2b821f0c/InappropriatenessClassification.json create mode 100644 results/avsolatorio__GIST-large-Embedding-v0/7831200e2f7819b994490c091cf3258a2b821f0c/KinopoiskClassification.json create mode 100644 results/avsolatorio__GIST-large-Embedding-v0/7831200e2f7819b994490c091cf3258a2b821f0c/MIRACLReranking.json create mode 100644 results/avsolatorio__GIST-large-Embedding-v0/7831200e2f7819b994490c091cf3258a2b821f0c/MIRACLRetrieval.json create mode 100644 results/avsolatorio__GIST-large-Embedding-v0/7831200e2f7819b994490c091cf3258a2b821f0c/MassiveIntentClassification.json create mode 100644 results/avsolatorio__GIST-large-Embedding-v0/7831200e2f7819b994490c091cf3258a2b821f0c/MassiveScenarioClassification.json create mode 100644 results/avsolatorio__GIST-large-Embedding-v0/7831200e2f7819b994490c091cf3258a2b821f0c/RUParaPhraserSTS.json create mode 100644 results/avsolatorio__GIST-large-Embedding-v0/7831200e2f7819b994490c091cf3258a2b821f0c/RiaNewsRetrieval.json create mode 100644 results/avsolatorio__GIST-large-Embedding-v0/7831200e2f7819b994490c091cf3258a2b821f0c/RuBQReranking.json create mode 100644 results/avsolatorio__GIST-large-Embedding-v0/7831200e2f7819b994490c091cf3258a2b821f0c/RuBQRetrieval.json create mode 100644 results/avsolatorio__GIST-large-Embedding-v0/7831200e2f7819b994490c091cf3258a2b821f0c/RuReviewsClassification.json create mode 100644 results/avsolatorio__GIST-large-Embedding-v0/7831200e2f7819b994490c091cf3258a2b821f0c/RuSTSBenchmarkSTS.json create mode 100644 results/avsolatorio__GIST-large-Embedding-v0/7831200e2f7819b994490c091cf3258a2b821f0c/RuSciBenchGRNTIClassification.json create mode 100644 results/avsolatorio__GIST-large-Embedding-v0/7831200e2f7819b994490c091cf3258a2b821f0c/RuSciBenchGRNTIClusteringP2P.json create mode 100644 results/avsolatorio__GIST-large-Embedding-v0/7831200e2f7819b994490c091cf3258a2b821f0c/RuSciBenchOECDClassification.json create mode 100644 results/avsolatorio__GIST-large-Embedding-v0/7831200e2f7819b994490c091cf3258a2b821f0c/RuSciBenchOECDClusteringP2P.json create mode 100644 results/avsolatorio__GIST-large-Embedding-v0/7831200e2f7819b994490c091cf3258a2b821f0c/STS22.json create mode 100644 results/avsolatorio__GIST-large-Embedding-v0/7831200e2f7819b994490c091cf3258a2b821f0c/SensitiveTopicsClassification.json create mode 100644 results/avsolatorio__GIST-large-Embedding-v0/7831200e2f7819b994490c091cf3258a2b821f0c/TERRa.json create mode 100644 results/avsolatorio__GIST-large-Embedding-v0/7831200e2f7819b994490c091cf3258a2b821f0c/model_meta.json create mode 100644 results/dunzhang__stella_en_1.5B_v5/d03be74b361d4eb24f42a2fe5bd2e29917df4604/GeoreviewClassification.json create mode 100644 results/dunzhang__stella_en_1.5B_v5/d03be74b361d4eb24f42a2fe5bd2e29917df4604/GeoreviewClusteringP2P.json create mode 100644 results/dunzhang__stella_en_1.5B_v5/d03be74b361d4eb24f42a2fe5bd2e29917df4604/HeadlineClassification.json create mode 100644 results/dunzhang__stella_en_1.5B_v5/d03be74b361d4eb24f42a2fe5bd2e29917df4604/InappropriatenessClassification.json create mode 100644 results/dunzhang__stella_en_1.5B_v5/d03be74b361d4eb24f42a2fe5bd2e29917df4604/KinopoiskClassification.json create mode 100644 results/dunzhang__stella_en_1.5B_v5/d03be74b361d4eb24f42a2fe5bd2e29917df4604/MIRACLReranking.json create mode 100644 results/dunzhang__stella_en_1.5B_v5/d03be74b361d4eb24f42a2fe5bd2e29917df4604/MIRACLRetrieval.json create mode 100644 results/dunzhang__stella_en_1.5B_v5/d03be74b361d4eb24f42a2fe5bd2e29917df4604/MassiveScenarioClassification.json create mode 100644 results/dunzhang__stella_en_1.5B_v5/d03be74b361d4eb24f42a2fe5bd2e29917df4604/RUParaPhraserSTS.json create mode 100644 results/dunzhang__stella_en_1.5B_v5/d03be74b361d4eb24f42a2fe5bd2e29917df4604/RiaNewsRetrieval.json create mode 100644 results/dunzhang__stella_en_1.5B_v5/d03be74b361d4eb24f42a2fe5bd2e29917df4604/RuBQRetrieval.json create mode 100644 results/dunzhang__stella_en_1.5B_v5/d03be74b361d4eb24f42a2fe5bd2e29917df4604/RuReviewsClassification.json create mode 100644 results/dunzhang__stella_en_1.5B_v5/d03be74b361d4eb24f42a2fe5bd2e29917df4604/RuSTSBenchmarkSTS.json create mode 100644 results/dunzhang__stella_en_1.5B_v5/d03be74b361d4eb24f42a2fe5bd2e29917df4604/RuSciBenchGRNTIClassification.json create mode 100644 results/dunzhang__stella_en_1.5B_v5/d03be74b361d4eb24f42a2fe5bd2e29917df4604/RuSciBenchGRNTIClusteringP2P.json create mode 100644 results/dunzhang__stella_en_1.5B_v5/d03be74b361d4eb24f42a2fe5bd2e29917df4604/RuSciBenchOECDClassification.json create mode 100644 results/dunzhang__stella_en_1.5B_v5/d03be74b361d4eb24f42a2fe5bd2e29917df4604/RuSciBenchOECDClusteringP2P.json create mode 100644 results/dunzhang__stella_en_1.5B_v5/d03be74b361d4eb24f42a2fe5bd2e29917df4604/STS22.json create mode 100644 results/dunzhang__stella_en_1.5B_v5/d03be74b361d4eb24f42a2fe5bd2e29917df4604/SensitiveTopicsClassification.json create mode 100644 results/dunzhang__stella_en_400M_v5/1bb50bc7bb726810eac2140e62155b88b0df198f/GeoreviewClassification.json create mode 100644 results/dunzhang__stella_en_400M_v5/1bb50bc7bb726810eac2140e62155b88b0df198f/GeoreviewClusteringP2P.json create mode 100644 results/dunzhang__stella_en_400M_v5/1bb50bc7bb726810eac2140e62155b88b0df198f/HeadlineClassification.json create mode 100644 results/dunzhang__stella_en_400M_v5/1bb50bc7bb726810eac2140e62155b88b0df198f/InappropriatenessClassification.json create mode 100644 results/dunzhang__stella_en_400M_v5/1bb50bc7bb726810eac2140e62155b88b0df198f/KinopoiskClassification.json create mode 100644 results/dunzhang__stella_en_400M_v5/1bb50bc7bb726810eac2140e62155b88b0df198f/MIRACLReranking.json create mode 100644 results/dunzhang__stella_en_400M_v5/1bb50bc7bb726810eac2140e62155b88b0df198f/MIRACLRetrieval.json create mode 100644 results/dunzhang__stella_en_400M_v5/1bb50bc7bb726810eac2140e62155b88b0df198f/MassiveScenarioClassification.json create mode 100644 results/dunzhang__stella_en_400M_v5/1bb50bc7bb726810eac2140e62155b88b0df198f/RUParaPhraserSTS.json create mode 100644 results/dunzhang__stella_en_400M_v5/1bb50bc7bb726810eac2140e62155b88b0df198f/RiaNewsRetrieval.json create mode 100644 results/dunzhang__stella_en_400M_v5/1bb50bc7bb726810eac2140e62155b88b0df198f/RuBQRetrieval.json create mode 100644 results/dunzhang__stella_en_400M_v5/1bb50bc7bb726810eac2140e62155b88b0df198f/RuReviewsClassification.json create mode 100644 results/dunzhang__stella_en_400M_v5/1bb50bc7bb726810eac2140e62155b88b0df198f/RuSTSBenchmarkSTS.json create mode 100644 results/dunzhang__stella_en_400M_v5/1bb50bc7bb726810eac2140e62155b88b0df198f/RuSciBenchGRNTIClassification.json create mode 100644 results/dunzhang__stella_en_400M_v5/1bb50bc7bb726810eac2140e62155b88b0df198f/RuSciBenchGRNTIClusteringP2P.json create mode 100644 results/dunzhang__stella_en_400M_v5/1bb50bc7bb726810eac2140e62155b88b0df198f/RuSciBenchOECDClassification.json create mode 100644 results/dunzhang__stella_en_400M_v5/1bb50bc7bb726810eac2140e62155b88b0df198f/RuSciBenchOECDClusteringP2P.json create mode 100644 results/dunzhang__stella_en_400M_v5/1bb50bc7bb726810eac2140e62155b88b0df198f/STS22.json create mode 100644 results/dunzhang__stella_en_400M_v5/1bb50bc7bb726810eac2140e62155b88b0df198f/SensitiveTopicsClassification.json create mode 100644 results/intfloat__multilingual-e5-small/fd1525a9fd15316a2d503bf26ab031a61d056e98/CEDRClassification.json create mode 100644 results/intfloat__multilingual-e5-small/fd1525a9fd15316a2d503bf26ab031a61d056e98/GeoreviewClassification.json create mode 100644 results/intfloat__multilingual-e5-small/fd1525a9fd15316a2d503bf26ab031a61d056e98/GeoreviewClusteringP2P.json create mode 100644 results/intfloat__multilingual-e5-small/fd1525a9fd15316a2d503bf26ab031a61d056e98/HeadlineClassification.json create mode 100644 results/intfloat__multilingual-e5-small/fd1525a9fd15316a2d503bf26ab031a61d056e98/InappropriatenessClassification.json create mode 100644 results/intfloat__multilingual-e5-small/fd1525a9fd15316a2d503bf26ab031a61d056e98/KinopoiskClassification.json create mode 100644 results/intfloat__multilingual-e5-small/fd1525a9fd15316a2d503bf26ab031a61d056e98/MIRACLReranking.json create mode 100644 results/intfloat__multilingual-e5-small/fd1525a9fd15316a2d503bf26ab031a61d056e98/MIRACLRetrieval.json create mode 100644 results/intfloat__multilingual-e5-small/fd1525a9fd15316a2d503bf26ab031a61d056e98/MassiveIntentClassification.json create mode 100644 results/intfloat__multilingual-e5-small/fd1525a9fd15316a2d503bf26ab031a61d056e98/MassiveScenarioClassification.json create mode 100644 results/intfloat__multilingual-e5-small/fd1525a9fd15316a2d503bf26ab031a61d056e98/RUParaPhraserSTS.json create mode 100644 results/intfloat__multilingual-e5-small/fd1525a9fd15316a2d503bf26ab031a61d056e98/RiaNewsRetrieval.json create mode 100644 results/intfloat__multilingual-e5-small/fd1525a9fd15316a2d503bf26ab031a61d056e98/RuBQReranking.json create mode 100644 results/intfloat__multilingual-e5-small/fd1525a9fd15316a2d503bf26ab031a61d056e98/RuBQRetrieval.json create mode 100644 results/intfloat__multilingual-e5-small/fd1525a9fd15316a2d503bf26ab031a61d056e98/RuReviewsClassification.json create mode 100644 results/intfloat__multilingual-e5-small/fd1525a9fd15316a2d503bf26ab031a61d056e98/RuSTSBenchmarkSTS.json create mode 100644 results/intfloat__multilingual-e5-small/fd1525a9fd15316a2d503bf26ab031a61d056e98/RuSciBenchGRNTIClassification.json create mode 100644 results/intfloat__multilingual-e5-small/fd1525a9fd15316a2d503bf26ab031a61d056e98/RuSciBenchGRNTIClusteringP2P.json create mode 100644 results/intfloat__multilingual-e5-small/fd1525a9fd15316a2d503bf26ab031a61d056e98/RuSciBenchOECDClassification.json create mode 100644 results/intfloat__multilingual-e5-small/fd1525a9fd15316a2d503bf26ab031a61d056e98/RuSciBenchOECDClusteringP2P.json create mode 100644 results/intfloat__multilingual-e5-small/fd1525a9fd15316a2d503bf26ab031a61d056e98/STS22.json create mode 100644 results/intfloat__multilingual-e5-small/fd1525a9fd15316a2d503bf26ab031a61d056e98/SensitiveTopicsClassification.json create mode 100644 results/intfloat__multilingual-e5-small/fd1525a9fd15316a2d503bf26ab031a61d056e98/TERRa.json create mode 100644 results/jinaai__jina-embeddings-v3/215a6e121fa0183376388ac6b1ae230326bfeaed/GeoreviewClassification.json create mode 100644 results/jinaai__jina-embeddings-v3/215a6e121fa0183376388ac6b1ae230326bfeaed/GeoreviewClusteringP2P.json create mode 100644 results/jinaai__jina-embeddings-v3/215a6e121fa0183376388ac6b1ae230326bfeaed/HeadlineClassification.json create mode 100644 results/jinaai__jina-embeddings-v3/215a6e121fa0183376388ac6b1ae230326bfeaed/InappropriatenessClassification.json create mode 100644 results/jinaai__jina-embeddings-v3/215a6e121fa0183376388ac6b1ae230326bfeaed/KinopoiskClassification.json create mode 100644 results/jinaai__jina-embeddings-v3/215a6e121fa0183376388ac6b1ae230326bfeaed/MIRACLReranking.json create mode 100644 results/jinaai__jina-embeddings-v3/215a6e121fa0183376388ac6b1ae230326bfeaed/MassiveScenarioClassification.json create mode 100644 results/jinaai__jina-embeddings-v3/215a6e121fa0183376388ac6b1ae230326bfeaed/RUParaPhraserSTS.json create mode 100644 results/jinaai__jina-embeddings-v3/215a6e121fa0183376388ac6b1ae230326bfeaed/RiaNewsRetrieval.json create mode 100644 results/jinaai__jina-embeddings-v3/215a6e121fa0183376388ac6b1ae230326bfeaed/RuBQRetrieval.json create mode 100644 results/jinaai__jina-embeddings-v3/215a6e121fa0183376388ac6b1ae230326bfeaed/RuReviewsClassification.json create mode 100644 results/jinaai__jina-embeddings-v3/215a6e121fa0183376388ac6b1ae230326bfeaed/RuSTSBenchmarkSTS.json create mode 100644 results/jinaai__jina-embeddings-v3/215a6e121fa0183376388ac6b1ae230326bfeaed/RuSciBenchGRNTIClassification.json create mode 100644 results/jinaai__jina-embeddings-v3/215a6e121fa0183376388ac6b1ae230326bfeaed/RuSciBenchGRNTIClusteringP2P.json create mode 100644 results/jinaai__jina-embeddings-v3/215a6e121fa0183376388ac6b1ae230326bfeaed/RuSciBenchOECDClassification.json create mode 100644 results/jinaai__jina-embeddings-v3/215a6e121fa0183376388ac6b1ae230326bfeaed/RuSciBenchOECDClusteringP2P.json create mode 100644 results/jinaai__jina-embeddings-v3/215a6e121fa0183376388ac6b1ae230326bfeaed/STS22.json create mode 100644 results/jinaai__jina-embeddings-v3/215a6e121fa0183376388ac6b1ae230326bfeaed/SensitiveTopicsClassification.json create mode 100644 results/mixedbread-ai__mxbai-embed-large-v1/990580e27d329c7408b3741ecff85876e128e203/CEDRClassification.json create mode 100644 results/mixedbread-ai__mxbai-embed-large-v1/990580e27d329c7408b3741ecff85876e128e203/GeoreviewClassification.json create mode 100644 results/mixedbread-ai__mxbai-embed-large-v1/990580e27d329c7408b3741ecff85876e128e203/GeoreviewClusteringP2P.json create mode 100644 results/mixedbread-ai__mxbai-embed-large-v1/990580e27d329c7408b3741ecff85876e128e203/HeadlineClassification.json create mode 100644 results/mixedbread-ai__mxbai-embed-large-v1/990580e27d329c7408b3741ecff85876e128e203/InappropriatenessClassification.json create mode 100644 results/mixedbread-ai__mxbai-embed-large-v1/990580e27d329c7408b3741ecff85876e128e203/KinopoiskClassification.json create mode 100644 results/mixedbread-ai__mxbai-embed-large-v1/990580e27d329c7408b3741ecff85876e128e203/MIRACLReranking.json create mode 100644 results/mixedbread-ai__mxbai-embed-large-v1/990580e27d329c7408b3741ecff85876e128e203/MIRACLRetrieval.json create mode 100644 results/mixedbread-ai__mxbai-embed-large-v1/990580e27d329c7408b3741ecff85876e128e203/MassiveScenarioClassification.json create mode 100644 results/mixedbread-ai__mxbai-embed-large-v1/990580e27d329c7408b3741ecff85876e128e203/RUParaPhraserSTS.json create mode 100644 results/mixedbread-ai__mxbai-embed-large-v1/990580e27d329c7408b3741ecff85876e128e203/RiaNewsRetrieval.json create mode 100644 results/mixedbread-ai__mxbai-embed-large-v1/990580e27d329c7408b3741ecff85876e128e203/RuBQRetrieval.json create mode 100644 results/mixedbread-ai__mxbai-embed-large-v1/990580e27d329c7408b3741ecff85876e128e203/RuReviewsClassification.json create mode 100644 results/mixedbread-ai__mxbai-embed-large-v1/990580e27d329c7408b3741ecff85876e128e203/RuSTSBenchmarkSTS.json create mode 100644 results/mixedbread-ai__mxbai-embed-large-v1/990580e27d329c7408b3741ecff85876e128e203/RuSciBenchGRNTIClassification.json create mode 100644 results/mixedbread-ai__mxbai-embed-large-v1/990580e27d329c7408b3741ecff85876e128e203/RuSciBenchGRNTIClusteringP2P.json create mode 100644 results/mixedbread-ai__mxbai-embed-large-v1/990580e27d329c7408b3741ecff85876e128e203/RuSciBenchOECDClassification.json create mode 100644 results/mixedbread-ai__mxbai-embed-large-v1/990580e27d329c7408b3741ecff85876e128e203/RuSciBenchOECDClusteringP2P.json create mode 100644 results/mixedbread-ai__mxbai-embed-large-v1/990580e27d329c7408b3741ecff85876e128e203/STS22.json create mode 100644 results/mixedbread-ai__mxbai-embed-large-v1/990580e27d329c7408b3741ecff85876e128e203/SensitiveTopicsClassification.json create mode 100644 results/nvidia__NV-Embed-v1/570834afd5fef5bf3a3c2311a2b6e0a66f6f4f2c/CEDRClassification.json create mode 100644 results/nvidia__NV-Embed-v1/570834afd5fef5bf3a3c2311a2b6e0a66f6f4f2c/GeoreviewClassification.json create mode 100644 results/nvidia__NV-Embed-v1/570834afd5fef5bf3a3c2311a2b6e0a66f6f4f2c/GeoreviewClusteringP2P.json create mode 100644 results/nvidia__NV-Embed-v1/570834afd5fef5bf3a3c2311a2b6e0a66f6f4f2c/HeadlineClassification.json create mode 100644 results/nvidia__NV-Embed-v1/570834afd5fef5bf3a3c2311a2b6e0a66f6f4f2c/InappropriatenessClassification.json create mode 100644 results/nvidia__NV-Embed-v1/570834afd5fef5bf3a3c2311a2b6e0a66f6f4f2c/KinopoiskClassification.json create mode 100644 results/nvidia__NV-Embed-v1/570834afd5fef5bf3a3c2311a2b6e0a66f6f4f2c/MIRACLReranking.json create mode 100644 results/nvidia__NV-Embed-v1/570834afd5fef5bf3a3c2311a2b6e0a66f6f4f2c/MassiveIntentClassification.json create mode 100644 results/nvidia__NV-Embed-v1/570834afd5fef5bf3a3c2311a2b6e0a66f6f4f2c/MassiveScenarioClassification.json create mode 100644 results/nvidia__NV-Embed-v1/570834afd5fef5bf3a3c2311a2b6e0a66f6f4f2c/RUParaPhraserSTS.json create mode 100644 results/nvidia__NV-Embed-v1/570834afd5fef5bf3a3c2311a2b6e0a66f6f4f2c/RiaNewsRetrieval.json create mode 100644 results/nvidia__NV-Embed-v1/570834afd5fef5bf3a3c2311a2b6e0a66f6f4f2c/RuBQReranking.json create mode 100644 results/nvidia__NV-Embed-v1/570834afd5fef5bf3a3c2311a2b6e0a66f6f4f2c/RuBQRetrieval.json create mode 100644 results/nvidia__NV-Embed-v1/570834afd5fef5bf3a3c2311a2b6e0a66f6f4f2c/RuReviewsClassification.json create mode 100644 results/nvidia__NV-Embed-v1/570834afd5fef5bf3a3c2311a2b6e0a66f6f4f2c/RuSTSBenchmarkSTS.json create mode 100644 results/nvidia__NV-Embed-v1/570834afd5fef5bf3a3c2311a2b6e0a66f6f4f2c/RuSciBenchGRNTIClassification.json create mode 100644 results/nvidia__NV-Embed-v1/570834afd5fef5bf3a3c2311a2b6e0a66f6f4f2c/RuSciBenchGRNTIClusteringP2P.json create mode 100644 results/nvidia__NV-Embed-v1/570834afd5fef5bf3a3c2311a2b6e0a66f6f4f2c/RuSciBenchOECDClassification.json create mode 100644 results/nvidia__NV-Embed-v1/570834afd5fef5bf3a3c2311a2b6e0a66f6f4f2c/RuSciBenchOECDClusteringP2P.json create mode 100644 results/nvidia__NV-Embed-v1/570834afd5fef5bf3a3c2311a2b6e0a66f6f4f2c/STS22.json create mode 100644 results/nvidia__NV-Embed-v1/570834afd5fef5bf3a3c2311a2b6e0a66f6f4f2c/SensitiveTopicsClassification.json create mode 100644 results/nvidia__NV-Embed-v1/570834afd5fef5bf3a3c2311a2b6e0a66f6f4f2c/TERRa.json create mode 100644 results/nvidia__NV-Embed-v1/570834afd5fef5bf3a3c2311a2b6e0a66f6f4f2c/model_meta.json create mode 100644 results/nvidia__NV-Embed-v2/7604d305b621f14095a1aa23d351674c2859553a/CEDRClassification.json create mode 100644 results/nvidia__NV-Embed-v2/7604d305b621f14095a1aa23d351674c2859553a/GeoreviewClassification.json create mode 100644 results/nvidia__NV-Embed-v2/7604d305b621f14095a1aa23d351674c2859553a/GeoreviewClusteringP2P.json create mode 100644 results/nvidia__NV-Embed-v2/7604d305b621f14095a1aa23d351674c2859553a/HeadlineClassification.json create mode 100644 results/nvidia__NV-Embed-v2/7604d305b621f14095a1aa23d351674c2859553a/InappropriatenessClassification.json create mode 100644 results/nvidia__NV-Embed-v2/7604d305b621f14095a1aa23d351674c2859553a/KinopoiskClassification.json create mode 100644 results/nvidia__NV-Embed-v2/7604d305b621f14095a1aa23d351674c2859553a/MIRACLReranking.json create mode 100644 results/nvidia__NV-Embed-v2/7604d305b621f14095a1aa23d351674c2859553a/MassiveIntentClassification.json create mode 100644 results/nvidia__NV-Embed-v2/7604d305b621f14095a1aa23d351674c2859553a/MassiveScenarioClassification.json create mode 100644 results/nvidia__NV-Embed-v2/7604d305b621f14095a1aa23d351674c2859553a/RUParaPhraserSTS.json create mode 100644 results/nvidia__NV-Embed-v2/7604d305b621f14095a1aa23d351674c2859553a/RiaNewsRetrieval.json create mode 100644 results/nvidia__NV-Embed-v2/7604d305b621f14095a1aa23d351674c2859553a/RuBQReranking.json create mode 100644 results/nvidia__NV-Embed-v2/7604d305b621f14095a1aa23d351674c2859553a/RuBQRetrieval.json create mode 100644 results/nvidia__NV-Embed-v2/7604d305b621f14095a1aa23d351674c2859553a/RuReviewsClassification.json create mode 100644 results/nvidia__NV-Embed-v2/7604d305b621f14095a1aa23d351674c2859553a/RuSTSBenchmarkSTS.json create mode 100644 results/nvidia__NV-Embed-v2/7604d305b621f14095a1aa23d351674c2859553a/RuSciBenchGRNTIClassification.json create mode 100644 results/nvidia__NV-Embed-v2/7604d305b621f14095a1aa23d351674c2859553a/RuSciBenchGRNTIClusteringP2P.json create mode 100644 results/nvidia__NV-Embed-v2/7604d305b621f14095a1aa23d351674c2859553a/RuSciBenchOECDClassification.json create mode 100644 results/nvidia__NV-Embed-v2/7604d305b621f14095a1aa23d351674c2859553a/RuSciBenchOECDClusteringP2P.json create mode 100644 results/nvidia__NV-Embed-v2/7604d305b621f14095a1aa23d351674c2859553a/STS22.json create mode 100644 results/nvidia__NV-Embed-v2/7604d305b621f14095a1aa23d351674c2859553a/SensitiveTopicsClassification.json create mode 100644 results/nvidia__NV-Embed-v2/7604d305b621f14095a1aa23d351674c2859553a/TERRa.json diff --git a/results/Alibaba-NLP__gte-Qwen1.5-7B-instruct/07d27e5226328010336563bc1b564a5e3436a298/CEDRClassification.json b/results/Alibaba-NLP__gte-Qwen1.5-7B-instruct/07d27e5226328010336563bc1b564a5e3436a298/CEDRClassification.json new file mode 100644 index 000000000..382efefa9 --- /dev/null +++ b/results/Alibaba-NLP__gte-Qwen1.5-7B-instruct/07d27e5226328010336563bc1b564a5e3436a298/CEDRClassification.json @@ -0,0 +1,73 @@ +{ + "dataset_revision": "c0ba03d058e3e1b2f3fd20518875a4563dd12db4", + "evaluation_time": 49.17470383644104, + "kg_co2_emissions": null, + "mteb_version": "1.19.2", + "scores": { + "test": [ + { + "accuracy": 0.5477151965993624, + "f1": 0.6036500719599494, + "hf_subset": "default", + "languages": [ + "rus-Cyrl" + ], + "lrap": 0.8649468650371996, + "main_score": 0.5477151965993624, + "scores_per_experiment": [ + { + "accuracy": 0.5451647183846972, + "f1": 0.5850362532892082, + "lrap": 0.8438363443145652 + }, + { + "accuracy": 0.628586609989373, + "f1": 0.6422497078024308, + "lrap": 0.8565887353878912 + }, + { + "accuracy": 0.5371944739638682, + "f1": 0.602381004950392, + "lrap": 0.8797555791710995 + }, + { + "accuracy": 0.5143464399574921, + "f1": 0.5742093596923625, + "lrap": 0.8505844845908667 + }, + { + "accuracy": 0.5446333687566419, + "f1": 0.6001832009513983, + "lrap": 0.8803400637619601 + }, + { + "accuracy": 0.5712008501594049, + "f1": 0.6201225698955309, + "lrap": 0.8817215727949038 + }, + { + "accuracy": 0.502125398512221, + "f1": 0.5781829471349942, + "lrap": 0.8520191285866153 + }, + { + "accuracy": 0.5371944739638682, + "f1": 0.6032409719825884, + "lrap": 0.8667375132837453 + }, + { + "accuracy": 0.5749202975557917, + "f1": 0.622622514090637, + "lrap": 0.8739107332624917 + }, + { + "accuracy": 0.5217853347502657, + "f1": 0.6082721898099509, + "lrap": 0.8639744952178583 + } + ] + } + ] + }, + "task_name": "CEDRClassification" +} \ No newline at end of file diff --git a/results/Alibaba-NLP__gte-Qwen1.5-7B-instruct/07d27e5226328010336563bc1b564a5e3436a298/GeoreviewClassification.json b/results/Alibaba-NLP__gte-Qwen1.5-7B-instruct/07d27e5226328010336563bc1b564a5e3436a298/GeoreviewClassification.json new file mode 100644 index 000000000..4bf460266 --- /dev/null +++ b/results/Alibaba-NLP__gte-Qwen1.5-7B-instruct/07d27e5226328010336563bc1b564a5e3436a298/GeoreviewClassification.json @@ -0,0 +1,73 @@ +{ + "dataset_revision": "3765c0d1de6b7d264bc459433c45e5a75513839c", + "evaluation_time": 70.92497396469116, + "kg_co2_emissions": null, + "mteb_version": "1.19.2", + "scores": { + "test": [ + { + "accuracy": 0.593115234375, + "f1": 0.5654370923929647, + "f1_weighted": 0.565366043274544, + "hf_subset": "default", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.593115234375, + "scores_per_experiment": [ + { + "accuracy": 0.57275390625, + "f1": 0.5311404233477083, + "f1_weighted": 0.5310401242904998 + }, + { + "accuracy": 0.5693359375, + "f1": 0.5233095126267937, + "f1_weighted": 0.5231911107096815 + }, + { + "accuracy": 0.59619140625, + "f1": 0.5690305785888471, + "f1_weighted": 0.5689581598276597 + }, + { + "accuracy": 0.60595703125, + "f1": 0.5901413093480931, + "f1_weighted": 0.590105109669658 + }, + { + "accuracy": 0.6025390625, + "f1": 0.5729259523761623, + "f1_weighted": 0.5728395007156636 + }, + { + "accuracy": 0.5947265625, + "f1": 0.5692650108462612, + "f1_weighted": 0.569206137591517 + }, + { + "accuracy": 0.60693359375, + "f1": 0.5888821876084315, + "f1_weighted": 0.5888208744862033 + }, + { + "accuracy": 0.58935546875, + "f1": 0.55793654500316, + "f1_weighted": 0.5578614223389805 + }, + { + "accuracy": 0.60302734375, + "f1": 0.5833953830631733, + "f1_weighted": 0.5833356693161511 + }, + { + "accuracy": 0.59033203125, + "f1": 0.5683440211210167, + "f1_weighted": 0.5683023237994256 + } + ] + } + ] + }, + "task_name": "GeoreviewClassification" +} \ No newline at end of file diff --git a/results/Alibaba-NLP__gte-Qwen1.5-7B-instruct/07d27e5226328010336563bc1b564a5e3436a298/GeoreviewClusteringP2P.json b/results/Alibaba-NLP__gte-Qwen1.5-7B-instruct/07d27e5226328010336563bc1b564a5e3436a298/GeoreviewClusteringP2P.json new file mode 100644 index 000000000..a13c11c59 --- /dev/null +++ b/results/Alibaba-NLP__gte-Qwen1.5-7B-instruct/07d27e5226328010336563bc1b564a5e3436a298/GeoreviewClusteringP2P.json @@ -0,0 +1,34 @@ +{ + "dataset_revision": "97a313c8fc85b47f13f33e7e9a95c1ad888c7fec", + "evaluation_time": 106.16331720352173, + "kg_co2_emissions": null, + "mteb_version": "1.19.2", + "scores": { + "test": [ + { + "hf_subset": "default", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.7490333015485914, + "v_measure": 0.7490333015485914, + "v_measure_std": 0.0034579399932174047, + "v_measures": { + "Level 0": [ + 0.752744253546754, + 0.7440034439559667, + 0.7466405375270816, + 0.7504486865332559, + 0.74656634640751, + 0.750591472356923, + 0.7485759760938262, + 0.7532063350425257, + 0.7439558077953369, + 0.7536001562267337 + ] + } + } + ] + }, + "task_name": "GeoreviewClusteringP2P" +} \ No newline at end of file diff --git a/results/Alibaba-NLP__gte-Qwen1.5-7B-instruct/07d27e5226328010336563bc1b564a5e3436a298/HeadlineClassification.json b/results/Alibaba-NLP__gte-Qwen1.5-7B-instruct/07d27e5226328010336563bc1b564a5e3436a298/HeadlineClassification.json new file mode 100644 index 000000000..0ed3d52d9 --- /dev/null +++ b/results/Alibaba-NLP__gte-Qwen1.5-7B-instruct/07d27e5226328010336563bc1b564a5e3436a298/HeadlineClassification.json @@ -0,0 +1,73 @@ +{ + "dataset_revision": "2fe05ee6b5832cda29f2ef7aaad7b7fe6a3609eb", + "evaluation_time": 41.733545541763306, + "kg_co2_emissions": null, + "mteb_version": "1.19.2", + "scores": { + "test": [ + { + "accuracy": 0.823193359375, + "f1": 0.8239465084764503, + "f1_weighted": 0.8239283463460406, + "hf_subset": "default", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.823193359375, + "scores_per_experiment": [ + { + "accuracy": 0.80712890625, + "f1": 0.8089034546914284, + "f1_weighted": 0.8088769259452948 + }, + { + "accuracy": 0.8349609375, + "f1": 0.8357485899262026, + "f1_weighted": 0.8357459144685893 + }, + { + "accuracy": 0.81591796875, + "f1": 0.8175058255173461, + "f1_weighted": 0.8174854105287481 + }, + { + "accuracy": 0.81640625, + "f1": 0.8184209189049058, + "f1_weighted": 0.8184018752116653 + }, + { + "accuracy": 0.81494140625, + "f1": 0.816025755172741, + "f1_weighted": 0.8160000608774359 + }, + { + "accuracy": 0.8369140625, + "f1": 0.8376501552762083, + "f1_weighted": 0.8376405057751175 + }, + { + "accuracy": 0.82080078125, + "f1": 0.8200892484658041, + "f1_weighted": 0.8200637301606719 + }, + { + "accuracy": 0.814453125, + "f1": 0.8152234211475688, + "f1_weighted": 0.8151917129263095 + }, + { + "accuracy": 0.82958984375, + "f1": 0.8291598309049161, + "f1_weighted": 0.8291522960195152 + }, + { + "accuracy": 0.8408203125, + "f1": 0.840737884757382, + "f1_weighted": 0.84072503154706 + } + ] + } + ] + }, + "task_name": "HeadlineClassification" +} \ No newline at end of file diff --git a/results/Alibaba-NLP__gte-Qwen1.5-7B-instruct/07d27e5226328010336563bc1b564a5e3436a298/InappropriatenessClassification.json b/results/Alibaba-NLP__gte-Qwen1.5-7B-instruct/07d27e5226328010336563bc1b564a5e3436a298/InappropriatenessClassification.json new file mode 100644 index 000000000..ddc3a5cab --- /dev/null +++ b/results/Alibaba-NLP__gte-Qwen1.5-7B-instruct/07d27e5226328010336563bc1b564a5e3436a298/InappropriatenessClassification.json @@ -0,0 +1,95 @@ +{ + "dataset_revision": "601651fdc45ef243751676e62dd7a19f491c0285", + "evaluation_time": 37.14429306983948, + "kg_co2_emissions": null, + "mteb_version": "1.19.2", + "scores": { + "test": [ + { + "accuracy": 0.732568359375, + "ap": 0.6710765411408649, + "ap_weighted": 0.6710765411408649, + "f1": 0.730367241384523, + "f1_weighted": 0.730367241384523, + "hf_subset": "default", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.732568359375, + "scores_per_experiment": [ + { + "accuracy": 0.73291015625, + "ap": 0.678244924916574, + "ap_weighted": 0.678244924916574, + "f1": 0.7319114485702299, + "f1_weighted": 0.7319114485702299 + }, + { + "accuracy": 0.7568359375, + "ap": 0.696101186748497, + "ap_weighted": 0.696101186748497, + "f1": 0.7567967401972707, + "f1_weighted": 0.7567967401972707 + }, + { + "accuracy": 0.75341796875, + "ap": 0.6911183505754163, + "ap_weighted": 0.6911183505754163, + "f1": 0.753417439641227, + "f1_weighted": 0.753417439641227 + }, + { + "accuracy": 0.705078125, + "ap": 0.6430150082236842, + "ap_weighted": 0.6430150082236842, + "f1": 0.7049655782998275, + "f1_weighted": 0.7049655782998275 + }, + { + "accuracy": 0.7255859375, + "ap": 0.672415796624714, + "ap_weighted": 0.672415796624714, + "f1": 0.7241059263570389, + "f1_weighted": 0.7241059263570389 + }, + { + "accuracy": 0.6845703125, + "ap": 0.6162109375, + "ap_weighted": 0.6162109375, + "f1": 0.6697390979613315, + "f1_weighted": 0.6697390979613315 + }, + { + "accuracy": 0.77001953125, + "ap": 0.7068677063282002, + "ap_weighted": 0.7068677063282002, + "f1": 0.7700071934744195, + "f1_weighted": 0.7700071934744195 + }, + { + "accuracy": 0.75, + "ap": 0.675314465408805, + "ap_weighted": 0.675314465408805, + "f1": 0.7462795199380565, + "f1_weighted": 0.7462795199380565 + }, + { + "accuracy": 0.728515625, + "ap": 0.6725068933823529, + "ap_weighted": 0.6725068933823529, + "f1": 0.7277863998385874, + "f1_weighted": 0.7277863998385874 + }, + { + "accuracy": 0.71875, + "ap": 0.6589701417004049, + "ap_weighted": 0.6589701417004049, + "f1": 0.7186630695672415, + "f1_weighted": 0.7186630695672415 + } + ] + } + ] + }, + "task_name": "InappropriatenessClassification" +} \ No newline at end of file diff --git a/results/Alibaba-NLP__gte-Qwen1.5-7B-instruct/07d27e5226328010336563bc1b564a5e3436a298/KinopoiskClassification.json b/results/Alibaba-NLP__gte-Qwen1.5-7B-instruct/07d27e5226328010336563bc1b564a5e3436a298/KinopoiskClassification.json new file mode 100644 index 000000000..789024483 --- /dev/null +++ b/results/Alibaba-NLP__gte-Qwen1.5-7B-instruct/07d27e5226328010336563bc1b564a5e3436a298/KinopoiskClassification.json @@ -0,0 +1,73 @@ +{ + "dataset_revision": "5911f26666ac11af46cb9c6849d0dc80a378af24", + "evaluation_time": 84.22461795806885, + "kg_co2_emissions": null, + "mteb_version": "1.19.2", + "scores": { + "test": [ + { + "accuracy": 0.6716666666666666, + "f1": 0.6476545663988791, + "f1_weighted": 0.6476545663988791, + "hf_subset": "default", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.6716666666666666, + "scores_per_experiment": [ + { + "accuracy": 0.6746666666666666, + "f1": 0.64998884298854, + "f1_weighted": 0.64998884298854 + }, + { + "accuracy": 0.6273333333333333, + "f1": 0.5395116792472158, + "f1_weighted": 0.5395116792472157 + }, + { + "accuracy": 0.6933333333333334, + "f1": 0.6851147871880379, + "f1_weighted": 0.6851147871880379 + }, + { + "accuracy": 0.6746666666666666, + "f1": 0.6537567797235678, + "f1_weighted": 0.6537567797235679 + }, + { + "accuracy": 0.664, + "f1": 0.6461389383302979, + "f1_weighted": 0.6461389383302979 + }, + { + "accuracy": 0.6786666666666666, + "f1": 0.6629463945304079, + "f1_weighted": 0.6629463945304079 + }, + { + "accuracy": 0.6693333333333333, + "f1": 0.6561239246851377, + "f1_weighted": 0.6561239246851377 + }, + { + "accuracy": 0.6746666666666666, + "f1": 0.664298738939659, + "f1_weighted": 0.664298738939659 + }, + { + "accuracy": 0.678, + "f1": 0.6493604358600386, + "f1_weighted": 0.6493604358600387 + }, + { + "accuracy": 0.682, + "f1": 0.6693051424958885, + "f1_weighted": 0.6693051424958885 + } + ] + } + ] + }, + "task_name": "KinopoiskClassification" +} \ No newline at end of file diff --git a/results/Alibaba-NLP__gte-Qwen1.5-7B-instruct/07d27e5226328010336563bc1b564a5e3436a298/MIRACLReranking.json b/results/Alibaba-NLP__gte-Qwen1.5-7B-instruct/07d27e5226328010336563bc1b564a5e3436a298/MIRACLReranking.json new file mode 100644 index 000000000..ea9009b4f --- /dev/null +++ b/results/Alibaba-NLP__gte-Qwen1.5-7B-instruct/07d27e5226328010336563bc1b564a5e3436a298/MIRACLReranking.json @@ -0,0 +1,130 @@ +{ + "dataset_revision": "6d1962c527217f8927fca80f890f14f36b2802af", + "evaluation_time": 3058.513674020767, + "kg_co2_emissions": null, + "mteb_version": "1.19.2", + "scores": { + "dev": [ + { + "MAP@1(MIRACL)": 0.30839, + "MAP@10(MIRACL)": 0.47796, + "MAP@100(MIRACL)": 0.49908, + "MAP@1000(MIRACL)": 0.49908, + "MAP@20(MIRACL)": 0.49089, + "MAP@3(MIRACL)": 0.41929, + "MAP@5(MIRACL)": 0.44976, + "NDCG@1(MIRACL)": 0.50682, + "NDCG@10(MIRACL)": 0.55338, + "NDCG@100(MIRACL)": 0.60257, + "NDCG@1000(MIRACL)": 0.60257, + "NDCG@20(MIRACL)": 0.57749, + "NDCG@3(MIRACL)": 0.50141, + "NDCG@5(MIRACL)": 0.51849, + "P@1(MIRACL)": 0.50682, + "P@10(MIRACL)": 0.13978, + "P@100(MIRACL)": 0.01953, + "P@1000(MIRACL)": 0.00195, + "P@20(MIRACL)": 0.0816, + "P@3(MIRACL)": 0.30446, + "P@5(MIRACL)": 0.22326, + "Recall@1(MIRACL)": 0.30839, + "Recall@10(MIRACL)": 0.64404, + "Recall@100(MIRACL)": 0.79952, + "Recall@1000(MIRACL)": 0.79952, + "Recall@20(MIRACL)": 0.70976, + "Recall@3(MIRACL)": 0.47999, + "Recall@5(MIRACL)": 0.55208, + "hf_subset": "ru", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.55338, + "nAUC_MAP@1000_diff1(MIRACL)": 0.2089962442210562, + "nAUC_MAP@1000_max(MIRACL)": 0.2919003259842182, + "nAUC_MAP@1000_std(MIRACL)": 0.11313653899226035, + "nAUC_MAP@100_diff1(MIRACL)": 0.2089962442210562, + "nAUC_MAP@100_max(MIRACL)": 0.2919003259842182, + "nAUC_MAP@100_std(MIRACL)": 0.11313653899226035, + "nAUC_MAP@10_diff1(MIRACL)": 0.222888454995082, + "nAUC_MAP@10_max(MIRACL)": 0.26810830243952344, + "nAUC_MAP@10_std(MIRACL)": 0.10456851487569355, + "nAUC_MAP@1_diff1(MIRACL)": 0.329830297887621, + "nAUC_MAP@1_max(MIRACL)": 0.17126033256438405, + "nAUC_MAP@1_std(MIRACL)": 0.011921962991838076, + "nAUC_MAP@20_diff1(MIRACL)": 0.21525905527375686, + "nAUC_MAP@20_max(MIRACL)": 0.28305684187493246, + "nAUC_MAP@20_std(MIRACL)": 0.11348945318590824, + "nAUC_MAP@3_diff1(MIRACL)": 0.24982718126885262, + "nAUC_MAP@3_max(MIRACL)": 0.2222827611042665, + "nAUC_MAP@3_std(MIRACL)": 0.0686188171932736, + "nAUC_MAP@5_diff1(MIRACL)": 0.23695295846406364, + "nAUC_MAP@5_max(MIRACL)": 0.24665776208873202, + "nAUC_MAP@5_std(MIRACL)": 0.08700740306625203, + "nAUC_NDCG@1000_diff1(MIRACL)": 0.1404888787431995, + "nAUC_NDCG@1000_max(MIRACL)": 0.3670312011441587, + "nAUC_NDCG@1000_std(MIRACL)": 0.14472410998897423, + "nAUC_NDCG@100_diff1(MIRACL)": 0.1404888787431995, + "nAUC_NDCG@100_max(MIRACL)": 0.3670312011441587, + "nAUC_NDCG@100_std(MIRACL)": 0.14472410998897423, + "nAUC_NDCG@10_diff1(MIRACL)": 0.18315726445656572, + "nAUC_NDCG@10_max(MIRACL)": 0.31366687889861145, + "nAUC_NDCG@10_std(MIRACL)": 0.13224385174174086, + "nAUC_NDCG@1_diff1(MIRACL)": 0.26296207012200273, + "nAUC_NDCG@1_max(MIRACL)": 0.36530226031318025, + "nAUC_NDCG@1_std(MIRACL)": 0.1093290487120926, + "nAUC_NDCG@20_diff1(MIRACL)": 0.16408309785943495, + "nAUC_NDCG@20_max(MIRACL)": 0.34197426643090567, + "nAUC_NDCG@20_std(MIRACL)": 0.14523474760359054, + "nAUC_NDCG@3_diff1(MIRACL)": 0.20704435696438578, + "nAUC_NDCG@3_max(MIRACL)": 0.2932234688017299, + "nAUC_NDCG@3_std(MIRACL)": 0.10967851247525288, + "nAUC_NDCG@5_diff1(MIRACL)": 0.20267432251237152, + "nAUC_NDCG@5_max(MIRACL)": 0.30015217521787424, + "nAUC_NDCG@5_std(MIRACL)": 0.11955390856968096, + "nAUC_P@1000_diff1(MIRACL)": -0.18581808095830424, + "nAUC_P@1000_max(MIRACL)": 0.28768119683086585, + "nAUC_P@1000_std(MIRACL)": 0.11453610966247148, + "nAUC_P@100_diff1(MIRACL)": -0.18581808095830377, + "nAUC_P@100_max(MIRACL)": 0.28768119683086624, + "nAUC_P@100_std(MIRACL)": 0.1145361096624716, + "nAUC_P@10_diff1(MIRACL)": -0.11490679091021795, + "nAUC_P@10_max(MIRACL)": 0.303732953660836, + "nAUC_P@10_std(MIRACL)": 0.1815144896650242, + "nAUC_P@1_diff1(MIRACL)": 0.26296207012200273, + "nAUC_P@1_max(MIRACL)": 0.36530226031318025, + "nAUC_P@1_std(MIRACL)": 0.1093290487120926, + "nAUC_P@20_diff1(MIRACL)": -0.1469460760354143, + "nAUC_P@20_max(MIRACL)": 0.3073193969322568, + "nAUC_P@20_std(MIRACL)": 0.16885272580854835, + "nAUC_P@3_diff1(MIRACL)": -0.012682613687038172, + "nAUC_P@3_max(MIRACL)": 0.3248833378603578, + "nAUC_P@3_std(MIRACL)": 0.1753789292191723, + "nAUC_P@5_diff1(MIRACL)": -0.06119474522408995, + "nAUC_P@5_max(MIRACL)": 0.3250797516073804, + "nAUC_P@5_std(MIRACL)": 0.18667527856903227, + "nAUC_Recall@1000_diff1(MIRACL)": -0.1336575788083657, + "nAUC_Recall@1000_max(MIRACL)": 0.5176985694789664, + "nAUC_Recall@1000_std(MIRACL)": 0.22392826741997982, + "nAUC_Recall@100_diff1(MIRACL)": -0.1336575788083657, + "nAUC_Recall@100_max(MIRACL)": 0.5176985694789664, + "nAUC_Recall@100_std(MIRACL)": 0.22392826741997982, + "nAUC_Recall@10_diff1(MIRACL)": 0.0926463193056263, + "nAUC_Recall@10_max(MIRACL)": 0.28291780552434215, + "nAUC_Recall@10_std(MIRACL)": 0.1497300332187107, + "nAUC_Recall@1_diff1(MIRACL)": 0.329830297887621, + "nAUC_Recall@1_max(MIRACL)": 0.17126033256438405, + "nAUC_Recall@1_std(MIRACL)": 0.011921962991838076, + "nAUC_Recall@20_diff1(MIRACL)": 0.02226157355604967, + "nAUC_Recall@20_max(MIRACL)": 0.36952005700517465, + "nAUC_Recall@20_std(MIRACL)": 0.1946681259929083, + "nAUC_Recall@3_diff1(MIRACL)": 0.19357536696091276, + "nAUC_Recall@3_max(MIRACL)": 0.19784137636238455, + "nAUC_Recall@3_std(MIRACL)": 0.08700031115205505, + "nAUC_Recall@5_diff1(MIRACL)": 0.1508838319445154, + "nAUC_Recall@5_max(MIRACL)": 0.23837577186242764, + "nAUC_Recall@5_std(MIRACL)": 0.1224704068815029 + } + ] + }, + "task_name": "MIRACLReranking" +} \ No newline at end of file diff --git a/results/Alibaba-NLP__gte-Qwen1.5-7B-instruct/07d27e5226328010336563bc1b564a5e3436a298/MassiveIntentClassification.json b/results/Alibaba-NLP__gte-Qwen1.5-7B-instruct/07d27e5226328010336563bc1b564a5e3436a298/MassiveIntentClassification.json new file mode 100644 index 000000000..0d9976f96 --- /dev/null +++ b/results/Alibaba-NLP__gte-Qwen1.5-7B-instruct/07d27e5226328010336563bc1b564a5e3436a298/MassiveIntentClassification.json @@ -0,0 +1,137 @@ +{ + "dataset_revision": "4672e20407010da34463acc759c162ca9734bca6", + "evaluation_time": 238.89698457717896, + "kg_co2_emissions": null, + "mteb_version": "1.19.2", + "scores": { + "test": [ + { + "accuracy": 0.7314727639542704, + "f1": 0.7029509613044315, + "f1_weighted": 0.7256631827900565, + "hf_subset": "ru", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.7314727639542704, + "scores_per_experiment": [ + { + "accuracy": 0.7498318762609281, + "f1": 0.716140430710699, + "f1_weighted": 0.7451098688894293 + }, + { + "accuracy": 0.7397444519166106, + "f1": 0.7096092013926623, + "f1_weighted": 0.7366102563637507 + }, + { + "accuracy": 0.7114996637525218, + "f1": 0.6915641707074874, + "f1_weighted": 0.7116819647995989 + }, + { + "accuracy": 0.7528581035642232, + "f1": 0.7102457063724665, + "f1_weighted": 0.742370987289208 + }, + { + "accuracy": 0.7431069266980498, + "f1": 0.7092041013363598, + "f1_weighted": 0.7358669124480911 + }, + { + "accuracy": 0.7078009414929388, + "f1": 0.6840768024865365, + "f1_weighted": 0.7085615317675305 + }, + { + "accuracy": 0.726630800268998, + "f1": 0.6986013055501684, + "f1_weighted": 0.7156717667858077 + }, + { + "accuracy": 0.7246133154001345, + "f1": 0.6899712907863645, + "f1_weighted": 0.7155703887651953 + }, + { + "accuracy": 0.7135171486213854, + "f1": 0.7004742109283838, + "f1_weighted": 0.7065636857122223 + }, + { + "accuracy": 0.7451244115669132, + "f1": 0.7196223927731854, + "f1_weighted": 0.7386244650797311 + } + ] + } + ], + "validation": [ + { + "accuracy": 0.7371864240039351, + "f1": 0.6919872164606259, + "f1_weighted": 0.728897197686264, + "hf_subset": "ru", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.7371864240039351, + "scores_per_experiment": [ + { + "accuracy": 0.7481554353172651, + "f1": 0.697706708330012, + "f1_weighted": 0.741921738803396 + }, + { + "accuracy": 0.7402852926709297, + "f1": 0.6929606464768286, + "f1_weighted": 0.7358916138155063 + }, + { + "accuracy": 0.735858337432366, + "f1": 0.6939703259575215, + "f1_weighted": 0.7307368868035592 + }, + { + "accuracy": 0.7515986227250369, + "f1": 0.6975390996102395, + "f1_weighted": 0.7401302137124002 + }, + { + "accuracy": 0.7476635514018691, + "f1": 0.6975070690946991, + "f1_weighted": 0.736950671981587 + }, + { + "accuracy": 0.7274963108706345, + "f1": 0.6854731694412295, + "f1_weighted": 0.7247630530178875 + }, + { + "accuracy": 0.7324151500245942, + "f1": 0.6832147716839522, + "f1_weighted": 0.7196689872914709 + }, + { + "accuracy": 0.7284800787014265, + "f1": 0.6837681985370787, + "f1_weighted": 0.7181839293197748 + }, + { + "accuracy": 0.7147073290703394, + "f1": 0.6814897084521937, + "f1_weighted": 0.7028530347559647 + }, + { + "accuracy": 0.7452041318248893, + "f1": 0.7062424670225043, + "f1_weighted": 0.7378718473610946 + } + ] + } + ] + }, + "task_name": "MassiveIntentClassification" +} \ No newline at end of file diff --git a/results/Alibaba-NLP__gte-Qwen1.5-7B-instruct/07d27e5226328010336563bc1b564a5e3436a298/MassiveScenarioClassification.json b/results/Alibaba-NLP__gte-Qwen1.5-7B-instruct/07d27e5226328010336563bc1b564a5e3436a298/MassiveScenarioClassification.json new file mode 100644 index 000000000..a589bf993 --- /dev/null +++ b/results/Alibaba-NLP__gte-Qwen1.5-7B-instruct/07d27e5226328010336563bc1b564a5e3436a298/MassiveScenarioClassification.json @@ -0,0 +1,137 @@ +{ + "dataset_revision": "fad2c6e8459f9e1c45d9315f4953d921437d70f8", + "evaluation_time": 133.87348747253418, + "kg_co2_emissions": null, + "mteb_version": "1.19.2", + "scores": { + "test": [ + { + "accuracy": 0.7545729657027572, + "f1": 0.7490898691066853, + "f1_weighted": 0.7515855273153575, + "hf_subset": "ru", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.7545729657027572, + "scores_per_experiment": [ + { + "accuracy": 0.7511768661735037, + "f1": 0.7456108309189888, + "f1_weighted": 0.7484723005511253 + }, + { + "accuracy": 0.7505043712172159, + "f1": 0.7457814471713863, + "f1_weighted": 0.7462111693089635 + }, + { + "accuracy": 0.7636180228648285, + "f1": 0.754175232640998, + "f1_weighted": 0.7607174268135466 + }, + { + "accuracy": 0.7575655682582381, + "f1": 0.7491067946517629, + "f1_weighted": 0.7551134220589979 + }, + { + "accuracy": 0.7363819771351715, + "f1": 0.7311910589063159, + "f1_weighted": 0.7306166372787689 + }, + { + "accuracy": 0.7279757901815737, + "f1": 0.7275025983852841, + "f1_weighted": 0.7248716100340834 + }, + { + "accuracy": 0.7552118359112306, + "f1": 0.7432017673921737, + "f1_weighted": 0.7523270597889727 + }, + { + "accuracy": 0.7457969065232011, + "f1": 0.7452713354966792, + "f1_weighted": 0.7453588019062662 + }, + { + "accuracy": 0.7824478816408877, + "f1": 0.7765375469569651, + "f1_weighted": 0.7774774088361666 + }, + { + "accuracy": 0.7750504371217216, + "f1": 0.772520078546299, + "f1_weighted": 0.7746894365766843 + } + ] + } + ], + "validation": [ + { + "accuracy": 0.7585833743236596, + "f1": 0.750976458122237, + "f1_weighted": 0.7552329606312337, + "hf_subset": "ru", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.7585833743236596, + "scores_per_experiment": [ + { + "accuracy": 0.7589768814559764, + "f1": 0.752446272325823, + "f1_weighted": 0.7567932005533703 + }, + { + "accuracy": 0.7461878996556812, + "f1": 0.7413218044474701, + "f1_weighted": 0.7434603528500205 + }, + { + "accuracy": 0.7722577471716675, + "f1": 0.761345597506989, + "f1_weighted": 0.7688634050629207 + }, + { + "accuracy": 0.7530742744712248, + "f1": 0.7441008750798291, + "f1_weighted": 0.7482848214919452 + }, + { + "accuracy": 0.7456960157402853, + "f1": 0.7377843033368218, + "f1_weighted": 0.7396580823759403 + }, + { + "accuracy": 0.7309394982784063, + "f1": 0.7263739502710909, + "f1_weighted": 0.7269369244142984 + }, + { + "accuracy": 0.7624200688637481, + "f1": 0.7487759384438273, + "f1_weighted": 0.7604143042265185 + }, + { + "accuracy": 0.735858337432366, + "f1": 0.7303500540265936, + "f1_weighted": 0.734421778539582 + }, + { + "accuracy": 0.7968519429414658, + "f1": 0.7875421066196677, + "f1_weighted": 0.7908719752208149 + }, + { + "accuracy": 0.7835710772257747, + "f1": 0.7797236791642576, + "f1_weighted": 0.7826247615769252 + } + ] + } + ] + }, + "task_name": "MassiveScenarioClassification" +} \ No newline at end of file diff --git a/results/Alibaba-NLP__gte-Qwen1.5-7B-instruct/07d27e5226328010336563bc1b564a5e3436a298/RUParaPhraserSTS.json b/results/Alibaba-NLP__gte-Qwen1.5-7B-instruct/07d27e5226328010336563bc1b564a5e3436a298/RUParaPhraserSTS.json new file mode 100644 index 000000000..2861d6627 --- /dev/null +++ b/results/Alibaba-NLP__gte-Qwen1.5-7B-instruct/07d27e5226328010336563bc1b564a5e3436a298/RUParaPhraserSTS.json @@ -0,0 +1,26 @@ +{ + "dataset_revision": "43265056790b8f7c59e0139acb4be0a8dad2c8f4", + "evaluation_time": 52.95315933227539, + "kg_co2_emissions": null, + "mteb_version": "1.19.2", + "scores": { + "test": [ + { + "cosine_pearson": 0.6994501757022691, + "cosine_spearman": 0.7364909320959803, + "euclidean_pearson": 0.718112255879904, + "euclidean_spearman": 0.7364909320959803, + "hf_subset": "default", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.7364909320959803, + "manhattan_pearson": 0.7181879721477015, + "manhattan_spearman": 0.7363332544480308, + "pearson": 0.6994501757022691, + "spearman": 0.7364909320959803 + } + ] + }, + "task_name": "RUParaPhraserSTS" +} \ No newline at end of file diff --git a/results/Alibaba-NLP__gte-Qwen1.5-7B-instruct/07d27e5226328010336563bc1b564a5e3436a298/RiaNewsRetrieval.json b/results/Alibaba-NLP__gte-Qwen1.5-7B-instruct/07d27e5226328010336563bc1b564a5e3436a298/RiaNewsRetrieval.json new file mode 100644 index 000000000..bf2086822 --- /dev/null +++ b/results/Alibaba-NLP__gte-Qwen1.5-7B-instruct/07d27e5226328010336563bc1b564a5e3436a298/RiaNewsRetrieval.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "82374b0bbacda6114f39ff9c5b925fa1512ca5d7", + "evaluation_time": 28827.79638028145, + "kg_co2_emissions": null, + "mteb_version": "1.19.4", + "scores": { + "test": [ + { + "hf_subset": "default", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.61105, + "map_at_1": 0.4719, + "map_at_10": 0.56564, + "map_at_100": 0.57159, + "map_at_1000": 0.57187, + "map_at_20": 0.56936, + "map_at_3": 0.54457, + "map_at_5": 0.55712, + "mrr_at_1": 0.4719, + "mrr_at_10": 0.5656386507936508, + "mrr_at_100": 0.5715907421190816, + "mrr_at_1000": 0.5718664157490458, + "mrr_at_20": 0.5693608111784156, + "mrr_at_3": 0.5445666666666666, + "mrr_at_5": 0.5571166666666667, + "nauc_map_at_1000_diff1": 0.6287499640238895, + "nauc_map_at_1000_max": 0.2034016409284853, + "nauc_map_at_1000_std": -0.024213719396550072, + "nauc_map_at_100_diff1": 0.6286832444938639, + "nauc_map_at_100_max": 0.20339278798339705, + "nauc_map_at_100_std": -0.024115897091813165, + "nauc_map_at_10_diff1": 0.6284471188698416, + "nauc_map_at_10_max": 0.20336793851550597, + "nauc_map_at_10_std": -0.026007494086294988, + "nauc_map_at_1_diff1": 0.6686374004258928, + "nauc_map_at_1_max": 0.18676159333349435, + "nauc_map_at_1_std": -0.05212337458336629, + "nauc_map_at_20_diff1": 0.6284087008703055, + "nauc_map_at_20_max": 0.20354268936083575, + "nauc_map_at_20_std": -0.024912987275978588, + "nauc_map_at_3_diff1": 0.6307094590356809, + "nauc_map_at_3_max": 0.2010374239943087, + "nauc_map_at_3_std": -0.03385505506633223, + "nauc_map_at_5_diff1": 0.6279907550563606, + "nauc_map_at_5_max": 0.20196529282308165, + "nauc_map_at_5_std": -0.029191933770232412, + "nauc_mrr_at_1000_diff1": 0.6287493903360661, + "nauc_mrr_at_1000_max": 0.20340088178138574, + "nauc_mrr_at_1000_std": -0.024214496755751477, + "nauc_mrr_at_100_diff1": 0.6286826712446165, + "nauc_mrr_at_100_max": 0.2033920294768933, + "nauc_mrr_at_100_std": -0.024116673713993626, + "nauc_mrr_at_10_diff1": 0.6284471188698416, + "nauc_mrr_at_10_max": 0.20336793851550597, + "nauc_mrr_at_10_std": -0.026007494086294988, + "nauc_mrr_at_1_diff1": 0.6686374004258928, + "nauc_mrr_at_1_max": 0.18676159333349435, + "nauc_mrr_at_1_std": -0.05212337458336629, + "nauc_mrr_at_20_diff1": 0.6284081309623656, + "nauc_mrr_at_20_max": 0.20354193566219686, + "nauc_mrr_at_20_std": -0.02491375969185557, + "nauc_mrr_at_3_diff1": 0.6307094590356809, + "nauc_mrr_at_3_max": 0.2010374239943087, + "nauc_mrr_at_3_std": -0.03385505506633223, + "nauc_mrr_at_5_diff1": 0.6279907550563606, + "nauc_mrr_at_5_max": 0.20196529282308165, + "nauc_mrr_at_5_std": -0.029191933770232412, + "nauc_ndcg_at_1000_diff1": 0.6165892576941506, + "nauc_ndcg_at_1000_max": 0.21333892985031438, + "nauc_ndcg_at_1000_std": -0.0005843485217573434, + "nauc_ndcg_at_100_diff1": 0.6142205015081649, + "nauc_ndcg_at_100_max": 0.21314468048070861, + "nauc_ndcg_at_100_std": 0.004184575062404777, + "nauc_ndcg_at_10_diff1": 0.612957070773732, + "nauc_ndcg_at_10_max": 0.21140675903403125, + "nauc_ndcg_at_10_std": -0.008762805121120773, + "nauc_ndcg_at_1_diff1": 0.6686374004258928, + "nauc_ndcg_at_1_max": 0.18676159333349435, + "nauc_ndcg_at_1_std": -0.05212337458336629, + "nauc_ndcg_at_20_diff1": 0.6123757854804859, + "nauc_ndcg_at_20_max": 0.21268533489575686, + "nauc_ndcg_at_20_std": -0.0035666036311125435, + "nauc_ndcg_at_3_diff1": 0.6180249982578259, + "nauc_ndcg_at_3_max": 0.2057248816308168, + "nauc_ndcg_at_3_std": -0.027131282756732888, + "nauc_ndcg_at_5_diff1": 0.6123891288274917, + "nauc_ndcg_at_5_max": 0.20774580758763359, + "nauc_ndcg_at_5_std": -0.01772572627091591, + "nauc_precision_at_1000_diff1": 0.46975805247471825, + "nauc_precision_at_1000_max": 0.44040103603630476, + "nauc_precision_at_1000_std": 0.46633678346960505, + "nauc_precision_at_100_diff1": 0.5026853368120134, + "nauc_precision_at_100_max": 0.3100523189652789, + "nauc_precision_at_100_std": 0.30930509103777437, + "nauc_precision_at_10_diff1": 0.548554527934185, + "nauc_precision_at_10_max": 0.24671333595263523, + "nauc_precision_at_10_std": 0.07008771278395116, + "nauc_precision_at_1_diff1": 0.6686374004258928, + "nauc_precision_at_1_max": 0.18676159333349435, + "nauc_precision_at_1_std": -0.05212337458336629, + "nauc_precision_at_20_diff1": 0.531505354951576, + "nauc_precision_at_20_max": 0.26312956602606635, + "nauc_precision_at_20_std": 0.11977453322496394, + "nauc_precision_at_3_diff1": 0.5773867475454311, + "nauc_precision_at_3_max": 0.22071139718699107, + "nauc_precision_at_3_std": -0.005370102746454722, + "nauc_precision_at_5_diff1": 0.556826582018447, + "nauc_precision_at_5_max": 0.2283534205639564, + "nauc_precision_at_5_std": 0.024599254847439068, + "nauc_recall_at_1000_diff1": 0.46975805247472324, + "nauc_recall_at_1000_max": 0.4404010360363066, + "nauc_recall_at_1000_std": 0.46633678346960866, + "nauc_recall_at_100_diff1": 0.5026853368120127, + "nauc_recall_at_100_max": 0.31005231896527796, + "nauc_recall_at_100_std": 0.3093050910377749, + "nauc_recall_at_10_diff1": 0.548554527934184, + "nauc_recall_at_10_max": 0.24671333595263586, + "nauc_recall_at_10_std": 0.07008771278395223, + "nauc_recall_at_1_diff1": 0.6686374004258928, + "nauc_recall_at_1_max": 0.18676159333349435, + "nauc_recall_at_1_std": -0.05212337458336629, + "nauc_recall_at_20_diff1": 0.5315053549515758, + "nauc_recall_at_20_max": 0.2631295660260669, + "nauc_recall_at_20_std": 0.11977453322496431, + "nauc_recall_at_3_diff1": 0.5773867475454315, + "nauc_recall_at_3_max": 0.22071139718699118, + "nauc_recall_at_3_std": -0.0053701027464555, + "nauc_recall_at_5_diff1": 0.5568265820184471, + "nauc_recall_at_5_max": 0.22835342056395666, + "nauc_recall_at_5_std": 0.02459925484743956, + "ndcg_at_1": 0.4719, + "ndcg_at_10": 0.61105, + "ndcg_at_100": 0.64036, + "ndcg_at_1000": 0.6481, + "ndcg_at_20": 0.62447, + "ndcg_at_3": 0.56806, + "ndcg_at_5": 0.59064, + "precision_at_1": 0.4719, + "precision_at_10": 0.07533, + "precision_at_100": 0.00891, + "precision_at_1000": 0.00095, + "precision_at_20": 0.04031, + "precision_at_3": 0.212, + "precision_at_5": 0.13814, + "recall_at_1": 0.4719, + "recall_at_10": 0.7533, + "recall_at_100": 0.8912, + "recall_at_1000": 0.9531, + "recall_at_20": 0.8061, + "recall_at_3": 0.636, + "recall_at_5": 0.6907 + } + ] + }, + "task_name": "RiaNewsRetrieval" +} \ No newline at end of file diff --git a/results/Alibaba-NLP__gte-Qwen1.5-7B-instruct/07d27e5226328010336563bc1b564a5e3436a298/RuBQReranking.json b/results/Alibaba-NLP__gte-Qwen1.5-7B-instruct/07d27e5226328010336563bc1b564a5e3436a298/RuBQReranking.json new file mode 100644 index 000000000..31276c97f --- /dev/null +++ b/results/Alibaba-NLP__gte-Qwen1.5-7B-instruct/07d27e5226328010336563bc1b564a5e3436a298/RuBQReranking.json @@ -0,0 +1,26 @@ +{ + "dataset_revision": "2e96b8f098fa4b0950fc58eacadeb31c0d0c7fa2", + "evaluation_time": 935.4835982322693, + "kg_co2_emissions": null, + "mteb_version": "1.19.2", + "scores": { + "test": [ + { + "hf_subset": "default", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.7028418717346855, + "map": 0.7028418717346855, + "mrr": 0.7563855372366011, + "nAUC_map_diff1": 0.3723907795868552, + "nAUC_map_max": 0.22814889043420492, + "nAUC_map_std": 0.15025167598805644, + "nAUC_mrr_diff1": 0.427561251537096, + "nAUC_mrr_max": 0.29252104240163374, + "nAUC_mrr_std": 0.16902210776192714 + } + ] + }, + "task_name": "RuBQReranking" +} \ No newline at end of file diff --git a/results/Alibaba-NLP__gte-Qwen1.5-7B-instruct/07d27e5226328010336563bc1b564a5e3436a298/RuBQRetrieval.json b/results/Alibaba-NLP__gte-Qwen1.5-7B-instruct/07d27e5226328010336563bc1b564a5e3436a298/RuBQRetrieval.json new file mode 100644 index 000000000..005b135ee --- /dev/null +++ b/results/Alibaba-NLP__gte-Qwen1.5-7B-instruct/07d27e5226328010336563bc1b564a5e3436a298/RuBQRetrieval.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "e19b6ffa60b3bc248e0b41f4cc37c26a55c2a67b", + "evaluation_time": 1340.1966524124146, + "kg_co2_emissions": null, + "mteb_version": "1.19.2", + "scores": { + "test": [ + { + "hf_subset": "default", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.65167, + "map_at_1": 0.39151, + "map_at_10": 0.57358, + "map_at_100": 0.5852, + "map_at_1000": 0.58555, + "map_at_20": 0.5812, + "map_at_3": 0.52439, + "map_at_5": 0.55277, + "mrr_at_1": 0.5585106382978723, + "mrr_at_10": 0.6664867818679876, + "mrr_at_100": 0.6709254578889642, + "mrr_at_1000": 0.6710152668820618, + "mrr_at_20": 0.6693232592644961, + "mrr_at_3": 0.6447990543735225, + "mrr_at_5": 0.658274231678487, + "nauc_map_at_1000_diff1": 0.36531591057326135, + "nauc_map_at_1000_max": 0.3035653661266291, + "nauc_map_at_1000_std": 0.005254597941377395, + "nauc_map_at_100_diff1": 0.3652588812366525, + "nauc_map_at_100_max": 0.30363406949043575, + "nauc_map_at_100_std": 0.00539823574349937, + "nauc_map_at_10_diff1": 0.3599205834203505, + "nauc_map_at_10_max": 0.3012657738696787, + "nauc_map_at_10_std": -4.3776833764746496e-05, + "nauc_map_at_1_diff1": 0.4068781443901671, + "nauc_map_at_1_max": 0.2069025852633963, + "nauc_map_at_1_std": -0.028159099303659164, + "nauc_map_at_20_diff1": 0.3632085940089257, + "nauc_map_at_20_max": 0.30369858488942186, + "nauc_map_at_20_std": 0.005490637898915787, + "nauc_map_at_3_diff1": 0.3674666430542543, + "nauc_map_at_3_max": 0.27341770449371994, + "nauc_map_at_3_std": -0.0299484654813272, + "nauc_map_at_5_diff1": 0.3639287998557502, + "nauc_map_at_5_max": 0.29046081589915973, + "nauc_map_at_5_std": -0.0150391154964749, + "nauc_mrr_at_1000_diff1": 0.43898998213997864, + "nauc_mrr_at_1000_max": 0.356688952241998, + "nauc_mrr_at_1000_std": -0.019103601325945414, + "nauc_mrr_at_100_diff1": 0.43899463746675116, + "nauc_mrr_at_100_max": 0.3567232095267896, + "nauc_mrr_at_100_std": -0.01908249410095766, + "nauc_mrr_at_10_diff1": 0.4380133613648583, + "nauc_mrr_at_10_max": 0.35963278324407727, + "nauc_mrr_at_10_std": -0.019943373571629102, + "nauc_mrr_at_1_diff1": 0.45744567048708495, + "nauc_mrr_at_1_max": 0.3185087628600214, + "nauc_mrr_at_1_std": -0.039586670000522815, + "nauc_mrr_at_20_diff1": 0.43846580304956645, + "nauc_mrr_at_20_max": 0.3575530067448174, + "nauc_mrr_at_20_std": -0.01769537099436092, + "nauc_mrr_at_3_diff1": 0.43257374543992816, + "nauc_mrr_at_3_max": 0.3526088977070129, + "nauc_mrr_at_3_std": -0.031830883555102794, + "nauc_mrr_at_5_diff1": 0.4370843589532293, + "nauc_mrr_at_5_max": 0.35847116461695416, + "nauc_mrr_at_5_std": -0.022925984021903743, + "nauc_ndcg_at_1000_diff1": 0.3846228579932152, + "nauc_ndcg_at_1000_max": 0.3385203759660593, + "nauc_ndcg_at_1000_std": 0.025326913976533756, + "nauc_ndcg_at_100_diff1": 0.382922798833745, + "nauc_ndcg_at_100_max": 0.34072439256810394, + "nauc_ndcg_at_100_std": 0.030850535620277262, + "nauc_ndcg_at_10_diff1": 0.3664216836031285, + "nauc_ndcg_at_10_max": 0.3408465736838746, + "nauc_ndcg_at_10_std": 0.02028073320293512, + "nauc_ndcg_at_1_diff1": 0.45744567048708495, + "nauc_ndcg_at_1_max": 0.3185087628600214, + "nauc_ndcg_at_1_std": -0.039586670000522815, + "nauc_ndcg_at_20_diff1": 0.374660140644635, + "nauc_ndcg_at_20_max": 0.3448991496619514, + "nauc_ndcg_at_20_std": 0.03777121024339731, + "nauc_ndcg_at_3_diff1": 0.3740162479451307, + "nauc_ndcg_at_3_max": 0.30444229908458836, + "nauc_ndcg_at_3_std": -0.027354767912133024, + "nauc_ndcg_at_5_diff1": 0.37148180990292456, + "nauc_ndcg_at_5_max": 0.3232666768938328, + "nauc_ndcg_at_5_std": -0.005629486117054672, + "nauc_precision_at_1000_diff1": -0.07692478248741134, + "nauc_precision_at_1000_max": 0.10546345110161712, + "nauc_precision_at_1000_std": 0.09072980199575012, + "nauc_precision_at_100_diff1": -0.05544005240602762, + "nauc_precision_at_100_max": 0.14687673785864958, + "nauc_precision_at_100_std": 0.12306702671036018, + "nauc_precision_at_10_diff1": 0.0069682779710068254, + "nauc_precision_at_10_max": 0.25108941153969105, + "nauc_precision_at_10_std": 0.11151018137974626, + "nauc_precision_at_1_diff1": 0.45744567048708495, + "nauc_precision_at_1_max": 0.3185087628600214, + "nauc_precision_at_1_std": -0.039586670000522815, + "nauc_precision_at_20_diff1": -0.014568797025451129, + "nauc_precision_at_20_max": 0.22056869274585322, + "nauc_precision_at_20_std": 0.1484714192270009, + "nauc_precision_at_3_diff1": 0.16018213606544024, + "nauc_precision_at_3_max": 0.30585149317022786, + "nauc_precision_at_3_std": 0.008720314924668173, + "nauc_precision_at_5_diff1": 0.08676098234046163, + "nauc_precision_at_5_max": 0.28789719223139537, + "nauc_precision_at_5_std": 0.059939597328937146, + "nauc_recall_at_1000_diff1": 0.4355537994702468, + "nauc_recall_at_1000_max": 0.6211736142405392, + "nauc_recall_at_1000_std": 0.6889712140530266, + "nauc_recall_at_100_diff1": 0.3040181700049794, + "nauc_recall_at_100_max": 0.4278044235579686, + "nauc_recall_at_100_std": 0.34883596833238506, + "nauc_recall_at_10_diff1": 0.2662561079056947, + "nauc_recall_at_10_max": 0.37801380089849934, + "nauc_recall_at_10_std": 0.11158230029055471, + "nauc_recall_at_1_diff1": 0.4068781443901671, + "nauc_recall_at_1_max": 0.2069025852633963, + "nauc_recall_at_1_std": -0.028159099303659164, + "nauc_recall_at_20_diff1": 0.2807360018857007, + "nauc_recall_at_20_max": 0.40740384224903375, + "nauc_recall_at_20_std": 0.23705892616586494, + "nauc_recall_at_3_diff1": 0.31433810757274283, + "nauc_recall_at_3_max": 0.28054691827784783, + "nauc_recall_at_3_std": -0.03140434205104806, + "nauc_recall_at_5_diff1": 0.30352635635604164, + "nauc_recall_at_5_max": 0.32593953695023975, + "nauc_recall_at_5_std": 0.02104189655512243, + "ndcg_at_1": 0.55851, + "ndcg_at_10": 0.65167, + "ndcg_at_100": 0.69101, + "ndcg_at_1000": 0.69751, + "ndcg_at_20": 0.67121, + "ndcg_at_3": 0.58256, + "ndcg_at_5": 0.61573, + "precision_at_1": 0.55851, + "precision_at_10": 0.12784, + "precision_at_100": 0.01566, + "precision_at_1000": 0.00165, + "precision_at_20": 0.07004, + "precision_at_3": 0.31678, + "precision_at_5": 0.22021, + "recall_at_1": 0.39151, + "recall_at_10": 0.78596, + "recall_at_100": 0.94161, + "recall_at_1000": 0.98523, + "recall_at_20": 0.85088, + "recall_at_3": 0.61443, + "recall_at_5": 0.69258 + } + ] + }, + "task_name": "RuBQRetrieval" +} \ No newline at end of file diff --git a/results/Alibaba-NLP__gte-Qwen1.5-7B-instruct/07d27e5226328010336563bc1b564a5e3436a298/RuReviewsClassification.json b/results/Alibaba-NLP__gte-Qwen1.5-7B-instruct/07d27e5226328010336563bc1b564a5e3436a298/RuReviewsClassification.json new file mode 100644 index 000000000..30d56d75b --- /dev/null +++ b/results/Alibaba-NLP__gte-Qwen1.5-7B-instruct/07d27e5226328010336563bc1b564a5e3436a298/RuReviewsClassification.json @@ -0,0 +1,73 @@ +{ + "dataset_revision": "f6d2c31f4dc6b88f468552750bfec05b4b41b05a", + "evaluation_time": 41.255961656570435, + "kg_co2_emissions": null, + "mteb_version": "1.19.2", + "scores": { + "test": [ + { + "accuracy": 0.72890625, + "f1": 0.7231523669494578, + "f1_weighted": 0.7231535324344908, + "hf_subset": "default", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.72890625, + "scores_per_experiment": [ + { + "accuracy": 0.7392578125, + "f1": 0.7374470623959578, + "f1_weighted": 0.7374483662714908 + }, + { + "accuracy": 0.71337890625, + "f1": 0.7025077487164691, + "f1_weighted": 0.7024996324421906 + }, + { + "accuracy": 0.7275390625, + "f1": 0.7181835375736103, + "f1_weighted": 0.7181760978819921 + }, + { + "accuracy": 0.744140625, + "f1": 0.7441710349655026, + "f1_weighted": 0.7441865289188956 + }, + { + "accuracy": 0.755859375, + "f1": 0.7578381803383026, + "f1_weighted": 0.7578548320823348 + }, + { + "accuracy": 0.7080078125, + "f1": 0.7018243857792056, + "f1_weighted": 0.7018253776860007 + }, + { + "accuracy": 0.71240234375, + "f1": 0.7081317089689931, + "f1_weighted": 0.7081352680570496 + }, + { + "accuracy": 0.74462890625, + "f1": 0.7422437769434015, + "f1_weighted": 0.7422450161497445 + }, + { + "accuracy": 0.70458984375, + "f1": 0.6812758637978081, + "f1_weighted": 0.6812528526304156 + }, + { + "accuracy": 0.7392578125, + "f1": 0.7379003700153285, + "f1_weighted": 0.7379113522247933 + } + ] + } + ] + }, + "task_name": "RuReviewsClassification" +} \ No newline at end of file diff --git a/results/Alibaba-NLP__gte-Qwen1.5-7B-instruct/07d27e5226328010336563bc1b564a5e3436a298/RuSTSBenchmarkSTS.json b/results/Alibaba-NLP__gte-Qwen1.5-7B-instruct/07d27e5226328010336563bc1b564a5e3436a298/RuSTSBenchmarkSTS.json new file mode 100644 index 000000000..02dc94efa --- /dev/null +++ b/results/Alibaba-NLP__gte-Qwen1.5-7B-instruct/07d27e5226328010336563bc1b564a5e3436a298/RuSTSBenchmarkSTS.json @@ -0,0 +1,26 @@ +{ + "dataset_revision": "7cf24f325c6da6195df55bef3d86b5e0616f3018", + "evaluation_time": 35.05771613121033, + "kg_co2_emissions": null, + "mteb_version": "1.19.2", + "scores": { + "test": [ + { + "cosine_pearson": 0.8035423362397724, + "cosine_spearman": 0.8002815903929272, + "euclidean_pearson": 0.7871602467695, + "euclidean_spearman": 0.8002815903929272, + "hf_subset": "default", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.8002815903929272, + "manhattan_pearson": 0.7872361804712056, + "manhattan_spearman": 0.8006245387025261, + "pearson": 0.8035423362397724, + "spearman": 0.8002815903929272 + } + ] + }, + "task_name": "RuSTSBenchmarkSTS" +} \ No newline at end of file diff --git a/results/Alibaba-NLP__gte-Qwen1.5-7B-instruct/07d27e5226328010336563bc1b564a5e3436a298/RuSciBenchGRNTIClassification.json b/results/Alibaba-NLP__gte-Qwen1.5-7B-instruct/07d27e5226328010336563bc1b564a5e3436a298/RuSciBenchGRNTIClassification.json new file mode 100644 index 000000000..9313b5d49 --- /dev/null +++ b/results/Alibaba-NLP__gte-Qwen1.5-7B-instruct/07d27e5226328010336563bc1b564a5e3436a298/RuSciBenchGRNTIClassification.json @@ -0,0 +1,73 @@ +{ + "dataset_revision": "673a610d6d3dd91a547a0d57ae1b56f37ebbf6a1", + "evaluation_time": 175.38222646713257, + "kg_co2_emissions": null, + "mteb_version": "1.19.2", + "scores": { + "test": [ + { + "accuracy": 0.67353515625, + "f1": 0.6605647875373758, + "f1_weighted": 0.6606887921446307, + "hf_subset": "default", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.67353515625, + "scores_per_experiment": [ + { + "accuracy": 0.69287109375, + "f1": 0.6853943231009784, + "f1_weighted": 0.6855214162196541 + }, + { + "accuracy": 0.67333984375, + "f1": 0.6613774754925439, + "f1_weighted": 0.6614810808494207 + }, + { + "accuracy": 0.67529296875, + "f1": 0.6624714325077754, + "f1_weighted": 0.6626174476178033 + }, + { + "accuracy": 0.67333984375, + "f1": 0.6590053333191753, + "f1_weighted": 0.6591454969159511 + }, + { + "accuracy": 0.67822265625, + "f1": 0.6671696203557742, + "f1_weighted": 0.6672845357084765 + }, + { + "accuracy": 0.65869140625, + "f1": 0.6492278612017663, + "f1_weighted": 0.6492791748778237 + }, + { + "accuracy": 0.6806640625, + "f1": 0.6619317114695685, + "f1_weighted": 0.6621007634082923 + }, + { + "accuracy": 0.666015625, + "f1": 0.6511894116309452, + "f1_weighted": 0.6512824516783355 + }, + { + "accuracy": 0.68310546875, + "f1": 0.670518919029475, + "f1_weighted": 0.6706549028651936 + }, + { + "accuracy": 0.65380859375, + "f1": 0.6373617872657549, + "f1_weighted": 0.6375206513053561 + } + ] + } + ] + }, + "task_name": "RuSciBenchGRNTIClassification" +} \ No newline at end of file diff --git a/results/Alibaba-NLP__gte-Qwen1.5-7B-instruct/07d27e5226328010336563bc1b564a5e3436a298/RuSciBenchGRNTIClusteringP2P.json b/results/Alibaba-NLP__gte-Qwen1.5-7B-instruct/07d27e5226328010336563bc1b564a5e3436a298/RuSciBenchGRNTIClusteringP2P.json new file mode 100644 index 000000000..c14815309 --- /dev/null +++ b/results/Alibaba-NLP__gte-Qwen1.5-7B-instruct/07d27e5226328010336563bc1b564a5e3436a298/RuSciBenchGRNTIClusteringP2P.json @@ -0,0 +1,34 @@ +{ + "dataset_revision": "673a610d6d3dd91a547a0d57ae1b56f37ebbf6a1", + "evaluation_time": 135.73782348632812, + "kg_co2_emissions": null, + "mteb_version": "1.19.2", + "scores": { + "test": [ + { + "hf_subset": "default", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.6252845532865546, + "v_measure": 0.6252845532865546, + "v_measure_std": 0.007482042082035663, + "v_measures": { + "Level 0": [ + 0.6207489349333961, + 0.6203220132070127, + 0.6280279883408472, + 0.6356338579930063, + 0.6354055280425936, + 0.63334127065613, + 0.6229906966991013, + 0.625483492226909, + 0.6195526387184621, + 0.611339112048087 + ] + } + } + ] + }, + "task_name": "RuSciBenchGRNTIClusteringP2P" +} \ No newline at end of file diff --git a/results/Alibaba-NLP__gte-Qwen1.5-7B-instruct/07d27e5226328010336563bc1b564a5e3436a298/RuSciBenchOECDClassification.json b/results/Alibaba-NLP__gte-Qwen1.5-7B-instruct/07d27e5226328010336563bc1b564a5e3436a298/RuSciBenchOECDClassification.json new file mode 100644 index 000000000..6582e6544 --- /dev/null +++ b/results/Alibaba-NLP__gte-Qwen1.5-7B-instruct/07d27e5226328010336563bc1b564a5e3436a298/RuSciBenchOECDClassification.json @@ -0,0 +1,73 @@ +{ + "dataset_revision": "26c88e99dcaba32bb45d0e1bfc21902337f6d471", + "evaluation_time": 167.4153139591217, + "kg_co2_emissions": null, + "mteb_version": "1.19.2", + "scores": { + "test": [ + { + "accuracy": 0.54384765625, + "f1": 0.5179523478811618, + "f1_weighted": 0.5179488778224148, + "hf_subset": "default", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.54384765625, + "scores_per_experiment": [ + { + "accuracy": 0.548828125, + "f1": 0.5239627248031066, + "f1_weighted": 0.5239497982202763 + }, + { + "accuracy": 0.54931640625, + "f1": 0.5271992304581379, + "f1_weighted": 0.5271890795585629 + }, + { + "accuracy": 0.5478515625, + "f1": 0.5219686975643086, + "f1_weighted": 0.5220594739324317 + }, + { + "accuracy": 0.5322265625, + "f1": 0.4922933125259625, + "f1_weighted": 0.49233519921194857 + }, + { + "accuracy": 0.55126953125, + "f1": 0.5237163168217409, + "f1_weighted": 0.5236558626083773 + }, + { + "accuracy": 0.56103515625, + "f1": 0.5399471841775259, + "f1_weighted": 0.5399500524748373 + }, + { + "accuracy": 0.53662109375, + "f1": 0.5037337634838176, + "f1_weighted": 0.5038136505311142 + }, + { + "accuracy": 0.5595703125, + "f1": 0.5426667594880292, + "f1_weighted": 0.5426221545649779 + }, + { + "accuracy": 0.51513671875, + "f1": 0.4886461135359976, + "f1_weighted": 0.4885845691306286 + }, + { + "accuracy": 0.53662109375, + "f1": 0.5153893759529918, + "f1_weighted": 0.5153289379909944 + } + ] + } + ] + }, + "task_name": "RuSciBenchOECDClassification" +} \ No newline at end of file diff --git a/results/Alibaba-NLP__gte-Qwen1.5-7B-instruct/07d27e5226328010336563bc1b564a5e3436a298/RuSciBenchOECDClusteringP2P.json b/results/Alibaba-NLP__gte-Qwen1.5-7B-instruct/07d27e5226328010336563bc1b564a5e3436a298/RuSciBenchOECDClusteringP2P.json new file mode 100644 index 000000000..189f810ca --- /dev/null +++ b/results/Alibaba-NLP__gte-Qwen1.5-7B-instruct/07d27e5226328010336563bc1b564a5e3436a298/RuSciBenchOECDClusteringP2P.json @@ -0,0 +1,34 @@ +{ + "dataset_revision": "26c88e99dcaba32bb45d0e1bfc21902337f6d471", + "evaluation_time": 128.42090034484863, + "kg_co2_emissions": null, + "mteb_version": "1.19.2", + "scores": { + "test": [ + { + "hf_subset": "default", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.5389131877733215, + "v_measure": 0.5389131877733215, + "v_measure_std": 0.007301076799965066, + "v_measures": { + "Level 0": [ + 0.5319407880397748, + 0.5351366452965421, + 0.5383348087959062, + 0.5467699170735407, + 0.5266030233104176, + 0.5361547750555381, + 0.550489123904105, + 0.5340852555806267, + 0.5414814687456001, + 0.5481360719311642 + ] + } + } + ] + }, + "task_name": "RuSciBenchOECDClusteringP2P" +} \ No newline at end of file diff --git a/results/Alibaba-NLP__gte-Qwen1.5-7B-instruct/07d27e5226328010336563bc1b564a5e3436a298/STS22.json b/results/Alibaba-NLP__gte-Qwen1.5-7B-instruct/07d27e5226328010336563bc1b564a5e3436a298/STS22.json new file mode 100644 index 000000000..5ee1306cf --- /dev/null +++ b/results/Alibaba-NLP__gte-Qwen1.5-7B-instruct/07d27e5226328010336563bc1b564a5e3436a298/STS22.json @@ -0,0 +1,26 @@ +{ + "dataset_revision": "de9d86b3b84231dc21f76c7b7af1f28e2f57f6e3", + "evaluation_time": 22.604440450668335, + "kg_co2_emissions": null, + "mteb_version": "1.19.2", + "scores": { + "test": [ + { + "cosine_pearson": 0.5058567549374162, + "cosine_spearman": 0.6037052355031338, + "euclidean_pearson": 0.5116237069159542, + "euclidean_spearman": 0.6037052355031338, + "hf_subset": "ru", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.6037052355031338, + "manhattan_pearson": 0.5174886232911461, + "manhattan_spearman": 0.6042484038697743, + "pearson": 0.5058567549374162, + "spearman": 0.6037052355031338 + } + ] + }, + "task_name": "STS22" +} \ No newline at end of file diff --git a/results/Alibaba-NLP__gte-Qwen1.5-7B-instruct/07d27e5226328010336563bc1b564a5e3436a298/SensitiveTopicsClassification.json b/results/Alibaba-NLP__gte-Qwen1.5-7B-instruct/07d27e5226328010336563bc1b564a5e3436a298/SensitiveTopicsClassification.json new file mode 100644 index 000000000..f78d1ff0a --- /dev/null +++ b/results/Alibaba-NLP__gte-Qwen1.5-7B-instruct/07d27e5226328010336563bc1b564a5e3436a298/SensitiveTopicsClassification.json @@ -0,0 +1,73 @@ +{ + "dataset_revision": "416b34a802308eac30e4192afc0ff99bb8dcc7f2", + "evaluation_time": 62.089826345443726, + "kg_co2_emissions": null, + "mteb_version": "1.19.2", + "scores": { + "test": [ + { + "accuracy": 0.356201171875, + "f1": 0.44647781989896557, + "hf_subset": "default", + "languages": [ + "rus-Cyrl" + ], + "lrap": 0.5571139865451369, + "main_score": 0.356201171875, + "scores_per_experiment": [ + { + "accuracy": 0.37451171875, + "f1": 0.45808689195543106, + "lrap": 0.5513780381944424 + }, + { + "accuracy": 0.35107421875, + "f1": 0.4430090517169707, + "lrap": 0.5574951171874984 + }, + { + "accuracy": 0.35107421875, + "f1": 0.4124270308941054, + "lrap": 0.5223117404513843 + }, + { + "accuracy": 0.365234375, + "f1": 0.4369513135160103, + "lrap": 0.5480957031249971 + }, + { + "accuracy": 0.34130859375, + "f1": 0.46223531801542667, + "lrap": 0.5410970052083304 + }, + { + "accuracy": 0.34326171875, + "f1": 0.4408524141069843, + "lrap": 0.5684611002604157 + }, + { + "accuracy": 0.35498046875, + "f1": 0.44616716252905764, + "lrap": 0.5566880967881925 + }, + { + "accuracy": 0.34912109375, + "f1": 0.44250290450356894, + "lrap": 0.5585801866319432 + }, + { + "accuracy": 0.3662109375, + "f1": 0.4479884575085484, + "lrap": 0.5710449218749987 + }, + { + "accuracy": 0.365234375, + "f1": 0.47455765424355256, + "lrap": 0.5959879557291672 + } + ] + } + ] + }, + "task_name": "SensitiveTopicsClassification" +} \ No newline at end of file diff --git a/results/Alibaba-NLP__gte-Qwen1.5-7B-instruct/07d27e5226328010336563bc1b564a5e3436a298/TERRa.json b/results/Alibaba-NLP__gte-Qwen1.5-7B-instruct/07d27e5226328010336563bc1b564a5e3436a298/TERRa.json new file mode 100644 index 000000000..3d98be81c --- /dev/null +++ b/results/Alibaba-NLP__gte-Qwen1.5-7B-instruct/07d27e5226328010336563bc1b564a5e3436a298/TERRa.json @@ -0,0 +1,58 @@ +{ + "dataset_revision": "7b58f24536063837d644aab9a023c62199b2a612", + "evaluation_time": 8.901613235473633, + "kg_co2_emissions": null, + "mteb_version": "1.19.2", + "scores": { + "dev": [ + { + "cosine_accuracy": 0.6872964169381107, + "cosine_accuracy_threshold": 0.6442170143127441, + "cosine_ap": 0.6944446593807762, + "cosine_f1": 0.7163323782234956, + "cosine_f1_threshold": 0.6143864393234253, + "cosine_precision": 0.6377551020408163, + "cosine_recall": 0.8169934640522876, + "dot_accuracy": 0.6872964169381107, + "dot_accuracy_threshold": 0.6442172527313232, + "dot_ap": 0.6944446593807762, + "dot_f1": 0.7163323782234956, + "dot_f1_threshold": 0.6143864393234253, + "dot_precision": 0.6377551020408163, + "dot_recall": 0.8169934640522876, + "euclidean_accuracy": 0.6872964169381107, + "euclidean_accuracy_threshold": 0.8435434699058533, + "euclidean_ap": 0.6944446593807762, + "euclidean_f1": 0.7163323782234956, + "euclidean_f1_threshold": 0.8781953454017639, + "euclidean_precision": 0.6377551020408163, + "euclidean_recall": 0.8169934640522876, + "hf_subset": "default", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.6949609366786602, + "manhattan_accuracy": 0.6905537459283387, + "manhattan_accuracy_threshold": 42.73845672607422, + "manhattan_ap": 0.6949609366786602, + "manhattan_f1": 0.7191011235955056, + "manhattan_f1_threshold": 44.631561279296875, + "manhattan_precision": 0.6305418719211823, + "manhattan_recall": 0.8366013071895425, + "max_accuracy": 0.6905537459283387, + "max_ap": 0.6949609366786602, + "max_f1": 0.7191011235955056, + "max_precision": 0.6377551020408163, + "max_recall": 0.8366013071895425, + "similarity_accuracy": 0.6872964169381107, + "similarity_accuracy_threshold": 0.6442170143127441, + "similarity_ap": 0.6944446593807762, + "similarity_f1": 0.7163323782234956, + "similarity_f1_threshold": 0.6143864393234253, + "similarity_precision": 0.6377551020408163, + "similarity_recall": 0.8169934640522876 + } + ] + }, + "task_name": "TERRa" +} \ No newline at end of file diff --git a/results/Alibaba-NLP__gte-Qwen1.5-7B-instruct/07d27e5226328010336563bc1b564a5e3436a298/model_meta.json b/results/Alibaba-NLP__gte-Qwen1.5-7B-instruct/07d27e5226328010336563bc1b564a5e3436a298/model_meta.json new file mode 100644 index 000000000..be5a84e55 --- /dev/null +++ b/results/Alibaba-NLP__gte-Qwen1.5-7B-instruct/07d27e5226328010336563bc1b564a5e3436a298/model_meta.json @@ -0,0 +1 @@ +{"name": "Alibaba-NLP/gte-Qwen1.5-7B-instruct", "revision": "07d27e5226328010336563bc1b564a5e3436a298", "release_date": "2024-04-20", "languages": ["eng_Latn"], "n_parameters": 7720000000, "memory_usage": null, "max_tokens": 32768, "embed_dim": 4096, "license": "apache-2.0", "open_weights": true, "public_training_data": null, "public_training_code": null, "framework": ["Sentence Transformers", "PyTorch"], "reference": "https://huggingface.co/Alibaba-NLP/gte-Qwen1.5-7B-instruct", "similarity_fn_name": "cosine", "use_instructions": true, "zero_shot_benchmarks": null, "loader": "instruct_wrapper"} \ No newline at end of file diff --git a/results/Alibaba-NLP__gte-Qwen2-1.5B-instruct/c6c1b92f4a3e1b92b326ad29dd3c8433457df8dd/CEDRClassification.json b/results/Alibaba-NLP__gte-Qwen2-1.5B-instruct/c6c1b92f4a3e1b92b326ad29dd3c8433457df8dd/CEDRClassification.json new file mode 100644 index 000000000..aa6582bd7 --- /dev/null +++ b/results/Alibaba-NLP__gte-Qwen2-1.5B-instruct/c6c1b92f4a3e1b92b326ad29dd3c8433457df8dd/CEDRClassification.json @@ -0,0 +1,73 @@ +{ + "dataset_revision": "c0ba03d058e3e1b2f3fd20518875a4563dd12db4", + "evaluation_time": 37.375492095947266, + "kg_co2_emissions": null, + "mteb_version": "1.19.2", + "scores": { + "test": [ + { + "accuracy": 0.5298087141339002, + "f1": 0.5463940656491209, + "hf_subset": "default", + "languages": [ + "rus-Cyrl" + ], + "lrap": 0.7920270988310386, + "main_score": 0.5298087141339002, + "scores_per_experiment": [ + { + "accuracy": 0.5324123273113709, + "f1": 0.5592166227592935, + "lrap": 0.8064293304994756 + }, + { + "accuracy": 0.5371944739638682, + "f1": 0.5142188501636664, + "lrap": 0.7502656748140369 + }, + { + "accuracy": 0.5159404888416578, + "f1": 0.5160018563245181, + "lrap": 0.7749468650372026 + }, + { + "accuracy": 0.5069075451647184, + "f1": 0.5878196176254, + "lrap": 0.8408076514346504 + }, + { + "accuracy": 0.5696068012752391, + "f1": 0.5584886229071322, + "lrap": 0.7752391073326336 + }, + { + "accuracy": 0.485653560042508, + "f1": 0.4924472291345112, + "lrap": 0.7523379383634526 + }, + { + "accuracy": 0.5717321997874601, + "f1": 0.5654779582641724, + "lrap": 0.8140807651434716 + }, + { + "accuracy": 0.5696068012752391, + "f1": 0.5883934067765482, + "lrap": 0.8068012752391147 + }, + { + "accuracy": 0.5223166843783209, + "f1": 0.5640478579144856, + "lrap": 0.8112646121147783 + }, + { + "accuracy": 0.4867162592986185, + "f1": 0.5178286346214805, + "lrap": 0.7880977683315701 + } + ] + } + ] + }, + "task_name": "CEDRClassification" +} \ No newline at end of file diff --git a/results/Alibaba-NLP__gte-Qwen2-1.5B-instruct/c6c1b92f4a3e1b92b326ad29dd3c8433457df8dd/GeoreviewClassification.json b/results/Alibaba-NLP__gte-Qwen2-1.5B-instruct/c6c1b92f4a3e1b92b326ad29dd3c8433457df8dd/GeoreviewClassification.json new file mode 100644 index 000000000..afd588594 --- /dev/null +++ b/results/Alibaba-NLP__gte-Qwen2-1.5B-instruct/c6c1b92f4a3e1b92b326ad29dd3c8433457df8dd/GeoreviewClassification.json @@ -0,0 +1,73 @@ +{ + "dataset_revision": "3765c0d1de6b7d264bc459433c45e5a75513839c", + "evaluation_time": 44.132378578186035, + "kg_co2_emissions": null, + "mteb_version": "1.19.2", + "scores": { + "test": [ + { + "accuracy": 0.566259765625, + "f1": 0.539998920872917, + "f1_weighted": 0.5399549345605024, + "hf_subset": "default", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.566259765625, + "scores_per_experiment": [ + { + "accuracy": 0.5322265625, + "f1": 0.48139864060427257, + "f1_weighted": 0.48132232760263777 + }, + { + "accuracy": 0.55029296875, + "f1": 0.5171318055563247, + "f1_weighted": 0.5170278436369578 + }, + { + "accuracy": 0.572265625, + "f1": 0.5486234952018896, + "f1_weighted": 0.5485959201625947 + }, + { + "accuracy": 0.58935546875, + "f1": 0.5722936933773077, + "f1_weighted": 0.5722547203160783 + }, + { + "accuracy": 0.56494140625, + "f1": 0.5388546516897115, + "f1_weighted": 0.5387814083242941 + }, + { + "accuracy": 0.56591796875, + "f1": 0.5462138892057763, + "f1_weighted": 0.5462178843455513 + }, + { + "accuracy": 0.59375, + "f1": 0.5773603022597097, + "f1_weighted": 0.5773082361244717 + }, + { + "accuracy": 0.59521484375, + "f1": 0.5751009322819041, + "f1_weighted": 0.5750472483732044 + }, + { + "accuracy": 0.56201171875, + "f1": 0.5491981623674013, + "f1_weighted": 0.5491716229160677 + }, + { + "accuracy": 0.53662109375, + "f1": 0.4938136361848744, + "f1_weighted": 0.4938221338031655 + } + ] + } + ] + }, + "task_name": "GeoreviewClassification" +} \ No newline at end of file diff --git a/results/Alibaba-NLP__gte-Qwen2-1.5B-instruct/c6c1b92f4a3e1b92b326ad29dd3c8433457df8dd/GeoreviewClusteringP2P.json b/results/Alibaba-NLP__gte-Qwen2-1.5B-instruct/c6c1b92f4a3e1b92b326ad29dd3c8433457df8dd/GeoreviewClusteringP2P.json new file mode 100644 index 000000000..a3b2ffd5f --- /dev/null +++ b/results/Alibaba-NLP__gte-Qwen2-1.5B-instruct/c6c1b92f4a3e1b92b326ad29dd3c8433457df8dd/GeoreviewClusteringP2P.json @@ -0,0 +1,34 @@ +{ + "dataset_revision": "97a313c8fc85b47f13f33e7e9a95c1ad888c7fec", + "evaluation_time": 114.29212379455566, + "kg_co2_emissions": null, + "mteb_version": "1.19.2", + "scores": { + "test": [ + { + "hf_subset": "default", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.7297991176715518, + "v_measure": 0.7297991176715518, + "v_measure_std": 0.007350022319590748, + "v_measures": { + "Level 0": [ + 0.7279130356072366, + 0.7248414099889655, + 0.7388993666959305, + 0.7345422035157355, + 0.726717152537628, + 0.7284265796154791, + 0.7278873043269999, + 0.737814237240807, + 0.7134311856739789, + 0.7375187015127566 + ] + } + } + ] + }, + "task_name": "GeoreviewClusteringP2P" +} \ No newline at end of file diff --git a/results/Alibaba-NLP__gte-Qwen2-1.5B-instruct/c6c1b92f4a3e1b92b326ad29dd3c8433457df8dd/HeadlineClassification.json b/results/Alibaba-NLP__gte-Qwen2-1.5B-instruct/c6c1b92f4a3e1b92b326ad29dd3c8433457df8dd/HeadlineClassification.json new file mode 100644 index 000000000..078f4b775 --- /dev/null +++ b/results/Alibaba-NLP__gte-Qwen2-1.5B-instruct/c6c1b92f4a3e1b92b326ad29dd3c8433457df8dd/HeadlineClassification.json @@ -0,0 +1,73 @@ +{ + "dataset_revision": "2fe05ee6b5832cda29f2ef7aaad7b7fe6a3609eb", + "evaluation_time": 38.52833533287048, + "kg_co2_emissions": null, + "mteb_version": "1.19.2", + "scores": { + "test": [ + { + "accuracy": 0.84658203125, + "f1": 0.8465626481246808, + "f1_weighted": 0.8465499669959881, + "hf_subset": "default", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.84658203125, + "scores_per_experiment": [ + { + "accuracy": 0.84814453125, + "f1": 0.8485091881722077, + "f1_weighted": 0.8484974187341153 + }, + { + "accuracy": 0.8564453125, + "f1": 0.8562652364651577, + "f1_weighted": 0.8562614677431917 + }, + { + "accuracy": 0.8486328125, + "f1": 0.8490051951829197, + "f1_weighted": 0.8489889619466362 + }, + { + "accuracy": 0.8525390625, + "f1": 0.8523523149025521, + "f1_weighted": 0.8523375830518336 + }, + { + "accuracy": 0.84814453125, + "f1": 0.8483947141083181, + "f1_weighted": 0.8483837276823756 + }, + { + "accuracy": 0.83837890625, + "f1": 0.8384119855111208, + "f1_weighted": 0.8383923360706916 + }, + { + "accuracy": 0.830078125, + "f1": 0.8300623133965269, + "f1_weighted": 0.8300509323362059 + }, + { + "accuracy": 0.849609375, + "f1": 0.8494263136535779, + "f1_weighted": 0.8494137196241152 + }, + { + "accuracy": 0.845703125, + "f1": 0.8454667379622213, + "f1_weighted": 0.8454593599599951 + }, + { + "accuracy": 0.84814453125, + "f1": 0.8477324818922055, + "f1_weighted": 0.8477141628107198 + } + ] + } + ] + }, + "task_name": "HeadlineClassification" +} \ No newline at end of file diff --git a/results/Alibaba-NLP__gte-Qwen2-1.5B-instruct/c6c1b92f4a3e1b92b326ad29dd3c8433457df8dd/InappropriatenessClassification.json b/results/Alibaba-NLP__gte-Qwen2-1.5B-instruct/c6c1b92f4a3e1b92b326ad29dd3c8433457df8dd/InappropriatenessClassification.json new file mode 100644 index 000000000..4017a0bcf --- /dev/null +++ b/results/Alibaba-NLP__gte-Qwen2-1.5B-instruct/c6c1b92f4a3e1b92b326ad29dd3c8433457df8dd/InappropriatenessClassification.json @@ -0,0 +1,95 @@ +{ + "dataset_revision": "601651fdc45ef243751676e62dd7a19f491c0285", + "evaluation_time": 34.62934637069702, + "kg_co2_emissions": null, + "mteb_version": "1.19.2", + "scores": { + "test": [ + { + "accuracy": 0.6583984375, + "ap": 0.6087707151727988, + "ap_weighted": 0.6087707151727988, + "f1": 0.6559072716864514, + "f1_weighted": 0.6559072716864514, + "hf_subset": "default", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.6583984375, + "scores_per_experiment": [ + { + "accuracy": 0.703125, + "ap": 0.6493566176470589, + "ap_weighted": 0.6493566176470589, + "f1": 0.7017311886064257, + "f1_weighted": 0.7017311886064257 + }, + { + "accuracy": 0.7041015625, + "ap": 0.6430673452524038, + "ap_weighted": 0.6430673452524038, + "f1": 0.7040835011902582, + "f1_weighted": 0.7040835011902582 + }, + { + "accuracy": 0.68603515625, + "ap": 0.6355622327055823, + "ap_weighted": 0.6355622327055823, + "f1": 0.6832804090024996, + "f1_weighted": 0.6832804090024996 + }, + { + "accuracy": 0.59716796875, + "ap": 0.557907236921408, + "ap_weighted": 0.557907236921408, + "f1": 0.5971517368897281, + "f1_weighted": 0.5971517368897281 + }, + { + "accuracy": 0.64501953125, + "ap": 0.5957410497572816, + "ap_weighted": 0.5957410497572816, + "f1": 0.64422142013121, + "f1_weighted": 0.64422142013121 + }, + { + "accuracy": 0.6181640625, + "ap": 0.5703048346742543, + "ap_weighted": 0.5703048346742543, + "f1": 0.6123881965359441, + "f1_weighted": 0.6123881965359441 + }, + { + "accuracy": 0.662109375, + "ap": 0.6207451834439528, + "ap_weighted": 0.6207451834439528, + "f1": 0.6521817665982426, + "f1_weighted": 0.6521817665982426 + }, + { + "accuracy": 0.65673828125, + "ap": 0.6002633914001305, + "ap_weighted": 0.6002633914001305, + "f1": 0.65545475017344, + "f1_weighted": 0.65545475017344 + }, + { + "accuracy": 0.68798828125, + "ap": 0.6360240409044715, + "ap_weighted": 0.6360240409044715, + "f1": 0.68599922979748, + "f1_weighted": 0.68599922979748 + }, + { + "accuracy": 0.62353515625, + "ap": 0.5787352190214441, + "ap_weighted": 0.5787352190214441, + "f1": 0.6225805179392857, + "f1_weighted": 0.6225805179392857 + } + ] + } + ] + }, + "task_name": "InappropriatenessClassification" +} \ No newline at end of file diff --git a/results/Alibaba-NLP__gte-Qwen2-1.5B-instruct/c6c1b92f4a3e1b92b326ad29dd3c8433457df8dd/KinopoiskClassification.json b/results/Alibaba-NLP__gte-Qwen2-1.5B-instruct/c6c1b92f4a3e1b92b326ad29dd3c8433457df8dd/KinopoiskClassification.json new file mode 100644 index 000000000..f2090762e --- /dev/null +++ b/results/Alibaba-NLP__gte-Qwen2-1.5B-instruct/c6c1b92f4a3e1b92b326ad29dd3c8433457df8dd/KinopoiskClassification.json @@ -0,0 +1,73 @@ +{ + "dataset_revision": "5911f26666ac11af46cb9c6849d0dc80a378af24", + "evaluation_time": 29.96605682373047, + "kg_co2_emissions": null, + "mteb_version": "1.19.2", + "scores": { + "test": [ + { + "accuracy": 0.6805333333333333, + "f1": 0.6573794252818923, + "f1_weighted": 0.6573794252818923, + "hf_subset": "default", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.6805333333333333, + "scores_per_experiment": [ + { + "accuracy": 0.678, + "f1": 0.6377280766051289, + "f1_weighted": 0.6377280766051289 + }, + { + "accuracy": 0.6746666666666666, + "f1": 0.6409336771730031, + "f1_weighted": 0.6409336771730032 + }, + { + "accuracy": 0.6866666666666666, + "f1": 0.6739421240857763, + "f1_weighted": 0.6739421240857763 + }, + { + "accuracy": 0.6826666666666666, + "f1": 0.6463135295490857, + "f1_weighted": 0.6463135295490857 + }, + { + "accuracy": 0.6713333333333333, + "f1": 0.6548412802291479, + "f1_weighted": 0.6548412802291479 + }, + { + "accuracy": 0.6806666666666666, + "f1": 0.6681098025787601, + "f1_weighted": 0.6681098025787601 + }, + { + "accuracy": 0.684, + "f1": 0.6617613091645753, + "f1_weighted": 0.6617613091645753 + }, + { + "accuracy": 0.6793333333333333, + "f1": 0.6670307794205573, + "f1_weighted": 0.6670307794205573 + }, + { + "accuracy": 0.6773333333333333, + "f1": 0.6493045481410312, + "f1_weighted": 0.6493045481410311 + }, + { + "accuracy": 0.6906666666666667, + "f1": 0.673829125871857, + "f1_weighted": 0.673829125871857 + } + ] + } + ] + }, + "task_name": "KinopoiskClassification" +} \ No newline at end of file diff --git a/results/Alibaba-NLP__gte-Qwen2-1.5B-instruct/c6c1b92f4a3e1b92b326ad29dd3c8433457df8dd/MIRACLReranking.json b/results/Alibaba-NLP__gte-Qwen2-1.5B-instruct/c6c1b92f4a3e1b92b326ad29dd3c8433457df8dd/MIRACLReranking.json new file mode 100644 index 000000000..0b2b58619 --- /dev/null +++ b/results/Alibaba-NLP__gte-Qwen2-1.5B-instruct/c6c1b92f4a3e1b92b326ad29dd3c8433457df8dd/MIRACLReranking.json @@ -0,0 +1,130 @@ +{ + "dataset_revision": "6d1962c527217f8927fca80f890f14f36b2802af", + "evaluation_time": 1758.0768535137177, + "kg_co2_emissions": null, + "mteb_version": "1.19.2", + "scores": { + "dev": [ + { + "MAP@1(MIRACL)": 0.346, + "MAP@10(MIRACL)": 0.54471, + "MAP@100(MIRACL)": 0.56152, + "MAP@1000(MIRACL)": 0.56152, + "MAP@20(MIRACL)": 0.55649, + "MAP@3(MIRACL)": 0.47676, + "MAP@5(MIRACL)": 0.51176, + "NDCG@1(MIRACL)": 0.57578, + "NDCG@10(MIRACL)": 0.61465, + "NDCG@100(MIRACL)": 0.64874, + "NDCG@1000(MIRACL)": 0.64874, + "NDCG@20(MIRACL)": 0.63527, + "NDCG@3(MIRACL)": 0.56695, + "NDCG@5(MIRACL)": 0.58104, + "P@1(MIRACL)": 0.57578, + "P@10(MIRACL)": 0.15822, + "P@100(MIRACL)": 0.01953, + "P@1000(MIRACL)": 0.00195, + "P@20(MIRACL)": 0.08877, + "P@3(MIRACL)": 0.34937, + "P@5(MIRACL)": 0.25453, + "Recall@1(MIRACL)": 0.346, + "Recall@10(MIRACL)": 0.69763, + "Recall@100(MIRACL)": 0.79952, + "Recall@1000(MIRACL)": 0.79952, + "Recall@20(MIRACL)": 0.75403, + "Recall@3(MIRACL)": 0.53613, + "Recall@5(MIRACL)": 0.60934, + "hf_subset": "ru", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.61465, + "nAUC_MAP@1000_diff1(MIRACL)": 0.05844585273238183, + "nAUC_MAP@1000_max(MIRACL)": 0.34724373194542624, + "nAUC_MAP@1000_std(MIRACL)": 0.14760716736478258, + "nAUC_MAP@100_diff1(MIRACL)": 0.05844585273238183, + "nAUC_MAP@100_max(MIRACL)": 0.34724373194542624, + "nAUC_MAP@100_std(MIRACL)": 0.14760716736478258, + "nAUC_MAP@10_diff1(MIRACL)": 0.0732187899791395, + "nAUC_MAP@10_max(MIRACL)": 0.33165340876637167, + "nAUC_MAP@10_std(MIRACL)": 0.13961470279685959, + "nAUC_MAP@1_diff1(MIRACL)": 0.26164679559463233, + "nAUC_MAP@1_max(MIRACL)": 0.1757816625864153, + "nAUC_MAP@1_std(MIRACL)": 0.03927142319306305, + "nAUC_MAP@20_diff1(MIRACL)": 0.06254085023812561, + "nAUC_MAP@20_max(MIRACL)": 0.34130843137634587, + "nAUC_MAP@20_std(MIRACL)": 0.14785631770279759, + "nAUC_MAP@3_diff1(MIRACL)": 0.12825900860644904, + "nAUC_MAP@3_max(MIRACL)": 0.27314510688262283, + "nAUC_MAP@3_std(MIRACL)": 0.09660602237538976, + "nAUC_MAP@5_diff1(MIRACL)": 0.10182116997059602, + "nAUC_MAP@5_max(MIRACL)": 0.2978782150409419, + "nAUC_MAP@5_std(MIRACL)": 0.11534764072812596, + "nAUC_NDCG@1000_diff1(MIRACL)": -0.030675764249711433, + "nAUC_NDCG@1000_max(MIRACL)": 0.4196011528049093, + "nAUC_NDCG@1000_std(MIRACL)": 0.18703000688244392, + "nAUC_NDCG@100_diff1(MIRACL)": -0.030675764249711433, + "nAUC_NDCG@100_max(MIRACL)": 0.4196011528049093, + "nAUC_NDCG@100_std(MIRACL)": 0.18703000688244392, + "nAUC_NDCG@10_diff1(MIRACL)": 0.009007660567953032, + "nAUC_NDCG@10_max(MIRACL)": 0.38840471576527, + "nAUC_NDCG@10_std(MIRACL)": 0.17528933537345287, + "nAUC_NDCG@1_diff1(MIRACL)": 0.12446653127685305, + "nAUC_NDCG@1_max(MIRACL)": 0.39998884728041634, + "nAUC_NDCG@1_std(MIRACL)": 0.14948980169454756, + "nAUC_NDCG@20_diff1(MIRACL)": -0.01594311913405895, + "nAUC_NDCG@20_max(MIRACL)": 0.40436637321725566, + "nAUC_NDCG@20_std(MIRACL)": 0.18884346313715042, + "nAUC_NDCG@3_diff1(MIRACL)": 0.0539060069241399, + "nAUC_NDCG@3_max(MIRACL)": 0.3630180722549369, + "nAUC_NDCG@3_std(MIRACL)": 0.14471287342377956, + "nAUC_NDCG@5_diff1(MIRACL)": 0.04319015283541723, + "nAUC_NDCG@5_max(MIRACL)": 0.3562966654168981, + "nAUC_NDCG@5_std(MIRACL)": 0.149283633523917, + "nAUC_P@1000_diff1(MIRACL)": -0.24754196996243197, + "nAUC_P@1000_max(MIRACL)": 0.27250891558928475, + "nAUC_P@1000_std(MIRACL)": 0.15102015255467327, + "nAUC_P@100_diff1(MIRACL)": -0.24754196996243175, + "nAUC_P@100_max(MIRACL)": 0.2725089155892854, + "nAUC_P@100_std(MIRACL)": 0.15102015255467355, + "nAUC_P@10_diff1(MIRACL)": -0.22861895405414923, + "nAUC_P@10_max(MIRACL)": 0.30596671571468875, + "nAUC_P@10_std(MIRACL)": 0.18843580349886327, + "nAUC_P@1_diff1(MIRACL)": 0.12446653127685305, + "nAUC_P@1_max(MIRACL)": 0.39998884728041634, + "nAUC_P@1_std(MIRACL)": 0.14948980169454756, + "nAUC_P@20_diff1(MIRACL)": -0.24633702067592644, + "nAUC_P@20_max(MIRACL)": 0.2868898139059664, + "nAUC_P@20_std(MIRACL)": 0.18673389653955133, + "nAUC_P@3_diff1(MIRACL)": -0.17545494495988545, + "nAUC_P@3_max(MIRACL)": 0.35260634279077574, + "nAUC_P@3_std(MIRACL)": 0.17859334573912375, + "nAUC_P@5_diff1(MIRACL)": -0.20846168857855932, + "nAUC_P@5_max(MIRACL)": 0.3233255815077814, + "nAUC_P@5_std(MIRACL)": 0.18689737668991488, + "nAUC_Recall@1000_diff1(MIRACL)": -0.33166292530333225, + "nAUC_Recall@1000_max(MIRACL)": 0.5630072103768957, + "nAUC_Recall@1000_std(MIRACL)": 0.2613299642929344, + "nAUC_Recall@100_diff1(MIRACL)": -0.33166292530333225, + "nAUC_Recall@100_max(MIRACL)": 0.5630072103768957, + "nAUC_Recall@100_std(MIRACL)": 0.2613299642929344, + "nAUC_Recall@10_diff1(MIRACL)": -0.11851664614856665, + "nAUC_Recall@10_max(MIRACL)": 0.40891368337706835, + "nAUC_Recall@10_std(MIRACL)": 0.2043278726016832, + "nAUC_Recall@1_diff1(MIRACL)": 0.26164679559463233, + "nAUC_Recall@1_max(MIRACL)": 0.1757816625864153, + "nAUC_Recall@1_std(MIRACL)": 0.03927142319306305, + "nAUC_Recall@20_diff1(MIRACL)": -0.22475522212535015, + "nAUC_Recall@20_max(MIRACL)": 0.4692190129269281, + "nAUC_Recall@20_std(MIRACL)": 0.2541055155891416, + "nAUC_Recall@3_diff1(MIRACL)": 0.05217300854332576, + "nAUC_Recall@3_max(MIRACL)": 0.2794215773373771, + "nAUC_Recall@3_std(MIRACL)": 0.10928630701331563, + "nAUC_Recall@5_diff1(MIRACL)": -0.011142669330513005, + "nAUC_Recall@5_max(MIRACL)": 0.31233821299281755, + "nAUC_Recall@5_std(MIRACL)": 0.13837108609230428 + } + ] + }, + "task_name": "MIRACLReranking" +} \ No newline at end of file diff --git a/results/Alibaba-NLP__gte-Qwen2-1.5B-instruct/c6c1b92f4a3e1b92b326ad29dd3c8433457df8dd/MassiveIntentClassification.json b/results/Alibaba-NLP__gte-Qwen2-1.5B-instruct/c6c1b92f4a3e1b92b326ad29dd3c8433457df8dd/MassiveIntentClassification.json new file mode 100644 index 000000000..cc1e4b33b --- /dev/null +++ b/results/Alibaba-NLP__gte-Qwen2-1.5B-instruct/c6c1b92f4a3e1b92b326ad29dd3c8433457df8dd/MassiveIntentClassification.json @@ -0,0 +1,137 @@ +{ + "dataset_revision": "4672e20407010da34463acc759c162ca9734bca6", + "evaluation_time": 210.42088508605957, + "kg_co2_emissions": null, + "mteb_version": "1.19.2", + "scores": { + "test": [ + { + "accuracy": 0.7336583725622058, + "f1": 0.7041486844338799, + "f1_weighted": 0.7219001292766163, + "hf_subset": "ru", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.7336583725622058, + "scores_per_experiment": [ + { + "accuracy": 0.7410894418291862, + "f1": 0.71036265711149, + "f1_weighted": 0.7297361186854204 + }, + { + "accuracy": 0.7471418964357768, + "f1": 0.7096025161995269, + "f1_weighted": 0.7382751725849083 + }, + { + "accuracy": 0.7242770679219905, + "f1": 0.6956569622589873, + "f1_weighted": 0.7138835702751832 + }, + { + "accuracy": 0.7501681237390719, + "f1": 0.709203702226879, + "f1_weighted": 0.7407402980045585 + }, + { + "accuracy": 0.7407531943510424, + "f1": 0.7087779173279923, + "f1_weighted": 0.7300548958336911 + }, + { + "accuracy": 0.7269670477471419, + "f1": 0.7118757006668955, + "f1_weighted": 0.7194101903409768 + }, + { + "accuracy": 0.7326832548755884, + "f1": 0.7001322294482387, + "f1_weighted": 0.7171119326353882 + }, + { + "accuracy": 0.7239408204438467, + "f1": 0.683906516901999, + "f1_weighted": 0.7110453438377231 + }, + { + "accuracy": 0.6997310020174848, + "f1": 0.687285247942128, + "f1_weighted": 0.6756161892733505 + }, + { + "accuracy": 0.7498318762609281, + "f1": 0.7246833942546619, + "f1_weighted": 0.7431275812949625 + } + ] + } + ], + "validation": [ + { + "accuracy": 0.7424003935071324, + "f1": 0.7002602727697711, + "f1_weighted": 0.7281309209875871, + "hf_subset": "ru", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.7424003935071324, + "scores_per_experiment": [ + { + "accuracy": 0.7560255779636006, + "f1": 0.7059355862386306, + "f1_weighted": 0.7436840115840425 + }, + { + "accuracy": 0.7570093457943925, + "f1": 0.7061836448113953, + "f1_weighted": 0.7460450031376 + }, + { + "accuracy": 0.7397934087555337, + "f1": 0.6973484287084937, + "f1_weighted": 0.7259273846794938 + }, + { + "accuracy": 0.7565174618789966, + "f1": 0.70912944118439, + "f1_weighted": 0.7420127135102002 + }, + { + "accuracy": 0.7456960157402853, + "f1": 0.6981902622782278, + "f1_weighted": 0.7327054921113475 + }, + { + "accuracy": 0.7397934087555337, + "f1": 0.705913482623903, + "f1_weighted": 0.7310716749391933 + }, + { + "accuracy": 0.7397934087555337, + "f1": 0.6917552987665041, + "f1_weighted": 0.7226417855732911 + }, + { + "accuracy": 0.7304476143630103, + "f1": 0.6915723850018337, + "f1_weighted": 0.712978333943807 + }, + { + "accuracy": 0.7112641416625677, + "f1": 0.6894183243112603, + "f1_weighted": 0.6864684003227117 + }, + { + "accuracy": 0.7476635514018691, + "f1": 0.7071558737730729, + "f1_weighted": 0.7377744100741843 + } + ] + } + ] + }, + "task_name": "MassiveIntentClassification" +} \ No newline at end of file diff --git a/results/Alibaba-NLP__gte-Qwen2-1.5B-instruct/c6c1b92f4a3e1b92b326ad29dd3c8433457df8dd/MassiveScenarioClassification.json b/results/Alibaba-NLP__gte-Qwen2-1.5B-instruct/c6c1b92f4a3e1b92b326ad29dd3c8433457df8dd/MassiveScenarioClassification.json new file mode 100644 index 000000000..4d28562d3 --- /dev/null +++ b/results/Alibaba-NLP__gte-Qwen2-1.5B-instruct/c6c1b92f4a3e1b92b326ad29dd3c8433457df8dd/MassiveScenarioClassification.json @@ -0,0 +1,137 @@ +{ + "dataset_revision": "fad2c6e8459f9e1c45d9315f4953d921437d70f8", + "evaluation_time": 131.11218643188477, + "kg_co2_emissions": null, + "mteb_version": "1.19.2", + "scores": { + "test": [ + { + "accuracy": 0.7830195023537324, + "f1": 0.7744343255677341, + "f1_weighted": 0.7805999340491574, + "hf_subset": "ru", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.7830195023537324, + "scores_per_experiment": [ + { + "accuracy": 0.7948890383322125, + "f1": 0.7880340018209849, + "f1_weighted": 0.792504173343861 + }, + { + "accuracy": 0.7827841291190316, + "f1": 0.7717606111829581, + "f1_weighted": 0.7797041272959886 + }, + { + "accuracy": 0.7817753866845999, + "f1": 0.769297118280224, + "f1_weighted": 0.7754497922223242 + }, + { + "accuracy": 0.8016139878950908, + "f1": 0.7899288902999394, + "f1_weighted": 0.7984304407649091 + }, + { + "accuracy": 0.7669804976462676, + "f1": 0.7550348736787605, + "f1_weighted": 0.7627427019624383 + }, + { + "accuracy": 0.7632817753866846, + "f1": 0.7554274492600168, + "f1_weighted": 0.7604792463903435 + }, + { + "accuracy": 0.7683254875588433, + "f1": 0.757113512393905, + "f1_weighted": 0.7661627633634723 + }, + { + "accuracy": 0.7864828513786146, + "f1": 0.7840084528456414, + "f1_weighted": 0.788638874471122 + }, + { + "accuracy": 0.7868190988567586, + "f1": 0.7840096742658867, + "f1_weighted": 0.7840384417100829 + }, + { + "accuracy": 0.7972427706792199, + "f1": 0.7897286716490226, + "f1_weighted": 0.7978487789670328 + } + ] + } + ], + "validation": [ + { + "accuracy": 0.7831775700934579, + "f1": 0.7719329305602339, + "f1_weighted": 0.7811961713256883, + "hf_subset": "ru", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.7831775700934579, + "scores_per_experiment": [ + { + "accuracy": 0.7973438268568618, + "f1": 0.7886264490986417, + "f1_weighted": 0.7936522133640386 + }, + { + "accuracy": 0.7914412198721101, + "f1": 0.7792730068980979, + "f1_weighted": 0.7898653312212526 + }, + { + "accuracy": 0.794392523364486, + "f1": 0.7771056216258827, + "f1_weighted": 0.7906276949611155 + }, + { + "accuracy": 0.7870142646335465, + "f1": 0.7728106624234612, + "f1_weighted": 0.7832769961088517 + }, + { + "accuracy": 0.7791441219872111, + "f1": 0.7677309559227985, + "f1_weighted": 0.7763075502546062 + }, + { + "accuracy": 0.7575012297097885, + "f1": 0.7498870802404436, + "f1_weighted": 0.7556708468162812 + }, + { + "accuracy": 0.7771765863256271, + "f1": 0.7579729889438993, + "f1_weighted": 0.7762030042948803 + }, + { + "accuracy": 0.778160354156419, + "f1": 0.7673291581947205, + "f1_weighted": 0.7787982318100919 + }, + { + "accuracy": 0.778652238071815, + "f1": 0.7747785508941001, + "f1_weighted": 0.775268898607672 + }, + { + "accuracy": 0.7909493359567142, + "f1": 0.7838148313602934, + "f1_weighted": 0.7922909458180922 + } + ] + } + ] + }, + "task_name": "MassiveScenarioClassification" +} \ No newline at end of file diff --git a/results/Alibaba-NLP__gte-Qwen2-1.5B-instruct/c6c1b92f4a3e1b92b326ad29dd3c8433457df8dd/RUParaPhraserSTS.json b/results/Alibaba-NLP__gte-Qwen2-1.5B-instruct/c6c1b92f4a3e1b92b326ad29dd3c8433457df8dd/RUParaPhraserSTS.json new file mode 100644 index 000000000..c5c15ca58 --- /dev/null +++ b/results/Alibaba-NLP__gte-Qwen2-1.5B-instruct/c6c1b92f4a3e1b92b326ad29dd3c8433457df8dd/RUParaPhraserSTS.json @@ -0,0 +1,26 @@ +{ + "dataset_revision": "43265056790b8f7c59e0139acb4be0a8dad2c8f4", + "evaluation_time": 49.408578872680664, + "kg_co2_emissions": null, + "mteb_version": "1.19.2", + "scores": { + "test": [ + { + "cosine_pearson": 0.6715121089349667, + "cosine_spearman": 0.7251027072326818, + "euclidean_pearson": 0.7004153159496793, + "euclidean_spearman": 0.7251032302004005, + "hf_subset": "default", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.7251027072326818, + "manhattan_pearson": 0.6986309216548198, + "manhattan_spearman": 0.7233699855486297, + "pearson": 0.6715121089349667, + "spearman": 0.7251027072326818 + } + ] + }, + "task_name": "RUParaPhraserSTS" +} \ No newline at end of file diff --git a/results/Alibaba-NLP__gte-Qwen2-1.5B-instruct/c6c1b92f4a3e1b92b326ad29dd3c8433457df8dd/RiaNewsRetrieval.json b/results/Alibaba-NLP__gte-Qwen2-1.5B-instruct/c6c1b92f4a3e1b92b326ad29dd3c8433457df8dd/RiaNewsRetrieval.json new file mode 100644 index 000000000..cf2b16d58 --- /dev/null +++ b/results/Alibaba-NLP__gte-Qwen2-1.5B-instruct/c6c1b92f4a3e1b92b326ad29dd3c8433457df8dd/RiaNewsRetrieval.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "82374b0bbacda6114f39ff9c5b925fa1512ca5d7", + "evaluation_time": 10108.248116254807, + "kg_co2_emissions": null, + "mteb_version": "1.19.4", + "scores": { + "test": [ + { + "hf_subset": "default", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.71889, + "map_at_1": 0.5837, + "map_at_10": 0.67673, + "map_at_100": 0.68089, + "map_at_1000": 0.68103, + "map_at_20": 0.67945, + "map_at_3": 0.6586, + "map_at_5": 0.66983, + "mrr_at_1": 0.5838, + "mrr_at_10": 0.676781626984127, + "mrr_at_100": 0.680935777612291, + "mrr_at_1000": 0.6810832150627617, + "mrr_at_20": 0.6795025166852412, + "mrr_at_3": 0.65865, + "mrr_at_5": 0.669875, + "nauc_map_at_1000_diff1": 0.6955742124142791, + "nauc_map_at_1000_max": 0.22593930010788726, + "nauc_map_at_1000_std": -0.05815150970819692, + "nauc_map_at_100_diff1": 0.6955140431724868, + "nauc_map_at_100_max": 0.2259497052936759, + "nauc_map_at_100_std": -0.05804012691261324, + "nauc_map_at_10_diff1": 0.6948486177470656, + "nauc_map_at_10_max": 0.22561832372989116, + "nauc_map_at_10_std": -0.059483960308164274, + "nauc_map_at_1_diff1": 0.7319377282977862, + "nauc_map_at_1_max": 0.2015248095554425, + "nauc_map_at_1_std": -0.07629133001341516, + "nauc_map_at_20_diff1": 0.6952100544527767, + "nauc_map_at_20_max": 0.22592622708874877, + "nauc_map_at_20_std": -0.05858117975823284, + "nauc_map_at_3_diff1": 0.6964801903913905, + "nauc_map_at_3_max": 0.22224112715863723, + "nauc_map_at_3_std": -0.06557313565196024, + "nauc_map_at_5_diff1": 0.6947038531654834, + "nauc_map_at_5_max": 0.22544684725819542, + "nauc_map_at_5_std": -0.06039937424924277, + "nauc_mrr_at_1000_diff1": 0.6954296311193227, + "nauc_mrr_at_1000_max": 0.2258481821647905, + "nauc_mrr_at_1000_std": -0.05817915886917972, + "nauc_mrr_at_100_diff1": 0.6953695397697215, + "nauc_mrr_at_100_max": 0.22585863976951523, + "nauc_mrr_at_100_std": -0.05806775271029188, + "nauc_mrr_at_10_diff1": 0.6947061160177611, + "nauc_mrr_at_10_max": 0.22552852562852763, + "nauc_mrr_at_10_std": -0.05951129286558239, + "nauc_mrr_at_1_diff1": 0.7317073952669194, + "nauc_mrr_at_1_max": 0.20137435892055278, + "nauc_mrr_at_1_std": -0.0763381827603582, + "nauc_mrr_at_20_diff1": 0.6950662871522489, + "nauc_mrr_at_20_max": 0.22583563697519174, + "nauc_mrr_at_20_std": -0.058608697931486896, + "nauc_mrr_at_3_diff1": 0.6963450244832373, + "nauc_mrr_at_3_max": 0.22215566769782727, + "nauc_mrr_at_3_std": -0.06559947397545805, + "nauc_mrr_at_5_diff1": 0.6945645041090036, + "nauc_mrr_at_5_max": 0.22535903049732317, + "nauc_mrr_at_5_std": -0.06042616106559084, + "nauc_ndcg_at_1000_diff1": 0.685333247276125, + "nauc_ndcg_at_1000_max": 0.23766253491764136, + "nauc_ndcg_at_1000_std": -0.04379543908560062, + "nauc_ndcg_at_100_diff1": 0.6836027819260406, + "nauc_ndcg_at_100_max": 0.2383513425844018, + "nauc_ndcg_at_100_std": -0.03976281101141971, + "nauc_ndcg_at_10_diff1": 0.6798803384225631, + "nauc_ndcg_at_10_max": 0.2366515620861918, + "nauc_ndcg_at_10_std": -0.04750581767516082, + "nauc_ndcg_at_1_diff1": 0.7319377282977862, + "nauc_ndcg_at_1_max": 0.2015248095554425, + "nauc_ndcg_at_1_std": -0.07629133001341516, + "nauc_ndcg_at_20_diff1": 0.6808895785463664, + "nauc_ndcg_at_20_max": 0.23817315440704365, + "nauc_ndcg_at_20_std": -0.04325208883730105, + "nauc_ndcg_at_3_diff1": 0.6845277361051741, + "nauc_ndcg_at_3_max": 0.22923162367527397, + "nauc_ndcg_at_3_std": -0.06096897758992976, + "nauc_ndcg_at_5_diff1": 0.6804573515323572, + "nauc_ndcg_at_5_max": 0.23565439508892008, + "nauc_ndcg_at_5_std": -0.05077785451053942, + "nauc_precision_at_1000_diff1": 0.49319155416100613, + "nauc_precision_at_1000_max": 0.5908260299828035, + "nauc_precision_at_1000_std": 0.35106574818768727, + "nauc_precision_at_100_diff1": 0.5528740559094314, + "nauc_precision_at_100_max": 0.4097560732478041, + "nauc_precision_at_100_std": 0.24965276739402667, + "nauc_precision_at_10_diff1": 0.6001251873425006, + "nauc_precision_at_10_max": 0.296426996029996, + "nauc_precision_at_10_std": 0.02129906392090402, + "nauc_precision_at_1_diff1": 0.7319377282977862, + "nauc_precision_at_1_max": 0.2015248095554425, + "nauc_precision_at_1_std": -0.07629133001341516, + "nauc_precision_at_20_diff1": 0.5842114816335258, + "nauc_precision_at_20_max": 0.32624326121547537, + "nauc_precision_at_20_std": 0.07690356879921746, + "nauc_precision_at_3_diff1": 0.6422094187275297, + "nauc_precision_at_3_max": 0.2539849114930569, + "nauc_precision_at_3_std": -0.044246875963812916, + "nauc_precision_at_5_diff1": 0.6213971746098673, + "nauc_precision_at_5_max": 0.2786243041491821, + "nauc_precision_at_5_std": -0.008763731908253832, + "nauc_recall_at_1000_diff1": 0.4931915541610098, + "nauc_recall_at_1000_max": 0.5908260299828074, + "nauc_recall_at_1000_std": 0.35106574818768893, + "nauc_recall_at_100_diff1": 0.5528740559094334, + "nauc_recall_at_100_max": 0.40975607324780505, + "nauc_recall_at_100_std": 0.24965276739402947, + "nauc_recall_at_10_diff1": 0.6001251873425009, + "nauc_recall_at_10_max": 0.29642699602999556, + "nauc_recall_at_10_std": 0.021299063920903794, + "nauc_recall_at_1_diff1": 0.7319377282977862, + "nauc_recall_at_1_max": 0.2015248095554425, + "nauc_recall_at_1_std": -0.07629133001341516, + "nauc_recall_at_20_diff1": 0.584211481633525, + "nauc_recall_at_20_max": 0.3262432612154751, + "nauc_recall_at_20_std": 0.07690356879921681, + "nauc_recall_at_3_diff1": 0.6422094187275298, + "nauc_recall_at_3_max": 0.2539849114930569, + "nauc_recall_at_3_std": -0.044246875963812014, + "nauc_recall_at_5_diff1": 0.621397174609867, + "nauc_recall_at_5_max": 0.27862430414918327, + "nauc_recall_at_5_std": -0.008763731908251915, + "ndcg_at_1": 0.5837, + "ndcg_at_10": 0.71889, + "ndcg_at_100": 0.73902, + "ndcg_at_1000": 0.74314, + "ndcg_at_20": 0.72872, + "ndcg_at_3": 0.68217, + "ndcg_at_5": 0.70237, + "precision_at_1": 0.5837, + "precision_at_10": 0.08497, + "precision_at_100": 0.00944, + "precision_at_1000": 0.00098, + "precision_at_20": 0.04442, + "precision_at_3": 0.25003, + "precision_at_5": 0.15982, + "recall_at_1": 0.5837, + "recall_at_10": 0.8497, + "recall_at_100": 0.9438, + "recall_at_1000": 0.9767, + "recall_at_20": 0.8884, + "recall_at_3": 0.7501, + "recall_at_5": 0.7991 + } + ] + }, + "task_name": "RiaNewsRetrieval" +} \ No newline at end of file diff --git a/results/Alibaba-NLP__gte-Qwen2-1.5B-instruct/c6c1b92f4a3e1b92b326ad29dd3c8433457df8dd/RuBQReranking.json b/results/Alibaba-NLP__gte-Qwen2-1.5B-instruct/c6c1b92f4a3e1b92b326ad29dd3c8433457df8dd/RuBQReranking.json new file mode 100644 index 000000000..95bafcc78 --- /dev/null +++ b/results/Alibaba-NLP__gte-Qwen2-1.5B-instruct/c6c1b92f4a3e1b92b326ad29dd3c8433457df8dd/RuBQReranking.json @@ -0,0 +1,26 @@ +{ + "dataset_revision": "2e96b8f098fa4b0950fc58eacadeb31c0d0c7fa2", + "evaluation_time": 549.4406032562256, + "kg_co2_emissions": null, + "mteb_version": "1.19.2", + "scores": { + "test": [ + { + "hf_subset": "default", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.7387568855586618, + "map": 0.7387568855586618, + "mrr": 0.7947627234861276, + "nAUC_map_diff1": 0.4185378603074729, + "nAUC_map_max": 0.2557231510377531, + "nAUC_map_std": 0.13128175699927613, + "nAUC_mrr_diff1": 0.47436093079736, + "nAUC_mrr_max": 0.32174764627694347, + "nAUC_mrr_std": 0.17102903008849993 + } + ] + }, + "task_name": "RuBQReranking" +} \ No newline at end of file diff --git a/results/Alibaba-NLP__gte-Qwen2-1.5B-instruct/c6c1b92f4a3e1b92b326ad29dd3c8433457df8dd/RuBQRetrieval.json b/results/Alibaba-NLP__gte-Qwen2-1.5B-instruct/c6c1b92f4a3e1b92b326ad29dd3c8433457df8dd/RuBQRetrieval.json new file mode 100644 index 000000000..6f3691b43 --- /dev/null +++ b/results/Alibaba-NLP__gte-Qwen2-1.5B-instruct/c6c1b92f4a3e1b92b326ad29dd3c8433457df8dd/RuBQRetrieval.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "e19b6ffa60b3bc248e0b41f4cc37c26a55c2a67b", + "evaluation_time": 784.6238145828247, + "kg_co2_emissions": null, + "mteb_version": "1.19.2", + "scores": { + "test": [ + { + "hf_subset": "default", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.7084, + "map_at_1": 0.43332, + "map_at_10": 0.63053, + "map_at_100": 0.63993, + "map_at_1000": 0.64026, + "map_at_20": 0.63709, + "map_at_3": 0.57886, + "map_at_5": 0.61114, + "mrr_at_1": 0.6199763593380615, + "mrr_at_10": 0.7245206199106908, + "mrr_at_100": 0.7273490906521782, + "mrr_at_1000": 0.7274286731850885, + "mrr_at_20": 0.7264379350659075, + "mrr_at_3": 0.7035066981875492, + "mrr_at_5": 0.7172182821118991, + "nauc_map_at_1000_diff1": 0.4098493844311728, + "nauc_map_at_1000_max": 0.31177707877202504, + "nauc_map_at_1000_std": -0.04566651903378437, + "nauc_map_at_100_diff1": 0.40952562105342194, + "nauc_map_at_100_max": 0.31169297709731714, + "nauc_map_at_100_std": -0.04523866869295098, + "nauc_map_at_10_diff1": 0.40711399489725486, + "nauc_map_at_10_max": 0.3108701521245131, + "nauc_map_at_10_std": -0.04967984333609782, + "nauc_map_at_1_diff1": 0.45449485733217426, + "nauc_map_at_1_max": 0.2272541734394137, + "nauc_map_at_1_std": -0.057269877241334324, + "nauc_map_at_20_diff1": 0.40831577007417696, + "nauc_map_at_20_max": 0.3114967925232305, + "nauc_map_at_20_std": -0.04601751526875259, + "nauc_map_at_3_diff1": 0.4083213088622995, + "nauc_map_at_3_max": 0.2918533799320895, + "nauc_map_at_3_std": -0.06555391990203349, + "nauc_map_at_5_diff1": 0.4033711250277714, + "nauc_map_at_5_max": 0.30257732148218364, + "nauc_map_at_5_std": -0.0631614517998428, + "nauc_mrr_at_1000_diff1": 0.4943221498869888, + "nauc_mrr_at_1000_max": 0.3613450471452896, + "nauc_mrr_at_1000_std": -0.054787367722350176, + "nauc_mrr_at_100_diff1": 0.49427485634008067, + "nauc_mrr_at_100_max": 0.3612967269520609, + "nauc_mrr_at_100_std": -0.05468894676941939, + "nauc_mrr_at_10_diff1": 0.4940433471901002, + "nauc_mrr_at_10_max": 0.3637456812378933, + "nauc_mrr_at_10_std": -0.053550809807821634, + "nauc_mrr_at_1_diff1": 0.524379377394298, + "nauc_mrr_at_1_max": 0.33024859326180706, + "nauc_mrr_at_1_std": -0.06920671552303741, + "nauc_mrr_at_20_diff1": 0.4942583856242923, + "nauc_mrr_at_20_max": 0.361520220767161, + "nauc_mrr_at_20_std": -0.05516540683278387, + "nauc_mrr_at_3_diff1": 0.48552814894456764, + "nauc_mrr_at_3_max": 0.36097653693631854, + "nauc_mrr_at_3_std": -0.056742983019184304, + "nauc_mrr_at_5_diff1": 0.4871754747833668, + "nauc_mrr_at_5_max": 0.3614729667028967, + "nauc_mrr_at_5_std": -0.057561200455925894, + "nauc_ndcg_at_1000_diff1": 0.4267257023809727, + "nauc_ndcg_at_1000_max": 0.3380181226065005, + "nauc_ndcg_at_1000_std": -0.029975086176097218, + "nauc_ndcg_at_100_diff1": 0.42047993425371843, + "nauc_ndcg_at_100_max": 0.33725479430617705, + "nauc_ndcg_at_100_std": -0.020038862668499628, + "nauc_ndcg_at_10_diff1": 0.41372994897265053, + "nauc_ndcg_at_10_max": 0.3402185054238364, + "nauc_ndcg_at_10_std": -0.032482077629039076, + "nauc_ndcg_at_1_diff1": 0.524379377394298, + "nauc_ndcg_at_1_max": 0.33024859326180706, + "nauc_ndcg_at_1_std": -0.06920671552303741, + "nauc_ndcg_at_20_diff1": 0.4157809742540088, + "nauc_ndcg_at_20_max": 0.33753169142968914, + "nauc_ndcg_at_20_std": -0.026101116562713023, + "nauc_ndcg_at_3_diff1": 0.4089658577309038, + "nauc_ndcg_at_3_max": 0.31276369584128266, + "nauc_ndcg_at_3_std": -0.05993983038697398, + "nauc_ndcg_at_5_diff1": 0.40360054168811305, + "nauc_ndcg_at_5_max": 0.32451996935897526, + "nauc_ndcg_at_5_std": -0.057496261019184904, + "nauc_precision_at_1000_diff1": -0.1039335981528995, + "nauc_precision_at_1000_max": 0.031305504944271374, + "nauc_precision_at_1000_std": 0.04291631348500297, + "nauc_precision_at_100_diff1": -0.09751197473832654, + "nauc_precision_at_100_max": 0.059270886848049154, + "nauc_precision_at_100_std": 0.08155218308146268, + "nauc_precision_at_10_diff1": -0.029614761682125797, + "nauc_precision_at_10_max": 0.1470431677304054, + "nauc_precision_at_10_std": 0.05620191569349491, + "nauc_precision_at_1_diff1": 0.524379377394298, + "nauc_precision_at_1_max": 0.33024859326180706, + "nauc_precision_at_1_std": -0.06920671552303741, + "nauc_precision_at_20_diff1": -0.06502174226456801, + "nauc_precision_at_20_max": 0.10619995806354679, + "nauc_precision_at_20_std": 0.07597531058913329, + "nauc_precision_at_3_diff1": 0.1136659799581031, + "nauc_precision_at_3_max": 0.2339666948885143, + "nauc_precision_at_3_std": -0.019247497680378353, + "nauc_precision_at_5_diff1": 0.029822992344513077, + "nauc_precision_at_5_max": 0.18940176550320303, + "nauc_precision_at_5_std": 0.0009625435506027025, + "nauc_recall_at_1000_diff1": 0.2776275954706104, + "nauc_recall_at_1000_max": 0.6315289106882999, + "nauc_recall_at_1000_std": 0.7288279620071334, + "nauc_recall_at_100_diff1": 0.18254957860285773, + "nauc_recall_at_100_max": 0.348275786234804, + "nauc_recall_at_100_std": 0.40769676520874576, + "nauc_recall_at_10_diff1": 0.28734906439784297, + "nauc_recall_at_10_max": 0.350678923614868, + "nauc_recall_at_10_std": 0.05565517533474942, + "nauc_recall_at_1_diff1": 0.45449485733217426, + "nauc_recall_at_1_max": 0.2272541734394137, + "nauc_recall_at_1_std": -0.057269877241334324, + "nauc_recall_at_20_diff1": 0.25111298116248404, + "nauc_recall_at_20_max": 0.33170412942756444, + "nauc_recall_at_20_std": 0.12211197873972823, + "nauc_recall_at_3_diff1": 0.32051875578747546, + "nauc_recall_at_3_max": 0.2803433904916248, + "nauc_recall_at_3_std": -0.04820404247875975, + "nauc_recall_at_5_diff1": 0.28342025003580124, + "nauc_recall_at_5_max": 0.30065661133125327, + "nauc_recall_at_5_std": -0.038787962818204645, + "ndcg_at_1": 0.61998, + "ndcg_at_10": 0.7084, + "ndcg_at_100": 0.73744, + "ndcg_at_1000": 0.74323, + "ndcg_at_20": 0.72441, + "ndcg_at_3": 0.63919, + "ndcg_at_5": 0.67598, + "precision_at_1": 0.61998, + "precision_at_10": 0.13712, + "precision_at_100": 0.01584, + "precision_at_1000": 0.00166, + "precision_at_20": 0.07376, + "precision_at_3": 0.34634, + "precision_at_5": 0.24173, + "recall_at_1": 0.43332, + "recall_at_10": 0.84009, + "recall_at_100": 0.95086, + "recall_at_1000": 0.98937, + "recall_at_20": 0.89235, + "recall_at_3": 0.66979, + "recall_at_5": 0.75729 + } + ] + }, + "task_name": "RuBQRetrieval" +} \ No newline at end of file diff --git a/results/Alibaba-NLP__gte-Qwen2-1.5B-instruct/c6c1b92f4a3e1b92b326ad29dd3c8433457df8dd/RuReviewsClassification.json b/results/Alibaba-NLP__gte-Qwen2-1.5B-instruct/c6c1b92f4a3e1b92b326ad29dd3c8433457df8dd/RuReviewsClassification.json new file mode 100644 index 000000000..6609d87ea --- /dev/null +++ b/results/Alibaba-NLP__gte-Qwen2-1.5B-instruct/c6c1b92f4a3e1b92b326ad29dd3c8433457df8dd/RuReviewsClassification.json @@ -0,0 +1,73 @@ +{ + "dataset_revision": "f6d2c31f4dc6b88f468552750bfec05b4b41b05a", + "evaluation_time": 36.28613591194153, + "kg_co2_emissions": null, + "mteb_version": "1.19.2", + "scores": { + "test": [ + { + "accuracy": 0.698095703125, + "f1": 0.6896768789132586, + "f1_weighted": 0.6896807821830067, + "hf_subset": "default", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.698095703125, + "scores_per_experiment": [ + { + "accuracy": 0.7275390625, + "f1": 0.7220694639014127, + "f1_weighted": 0.7220625619911872 + }, + { + "accuracy": 0.65478515625, + "f1": 0.6491362610344539, + "f1_weighted": 0.6491672807222408 + }, + { + "accuracy": 0.7109375, + "f1": 0.6952384221984715, + "f1_weighted": 0.6952186235993608 + }, + { + "accuracy": 0.68408203125, + "f1": 0.6839828621347107, + "f1_weighted": 0.684040452528405 + }, + { + "accuracy": 0.7373046875, + "f1": 0.7395317495184309, + "f1_weighted": 0.7395381702528763 + }, + { + "accuracy": 0.68212890625, + "f1": 0.6788792738133856, + "f1_weighted": 0.6788869502680129 + }, + { + "accuracy": 0.68017578125, + "f1": 0.674774307090292, + "f1_weighted": 0.6747852833857639 + }, + { + "accuracy": 0.70166015625, + "f1": 0.6879877165733109, + "f1_weighted": 0.6879796687559976 + }, + { + "accuracy": 0.6689453125, + "f1": 0.634339983172118, + "f1_weighted": 0.6342962843130933 + }, + { + "accuracy": 0.7333984375, + "f1": 0.7308287496959999, + "f1_weighted": 0.730832546013129 + } + ] + } + ] + }, + "task_name": "RuReviewsClassification" +} \ No newline at end of file diff --git a/results/Alibaba-NLP__gte-Qwen2-1.5B-instruct/c6c1b92f4a3e1b92b326ad29dd3c8433457df8dd/RuSTSBenchmarkSTS.json b/results/Alibaba-NLP__gte-Qwen2-1.5B-instruct/c6c1b92f4a3e1b92b326ad29dd3c8433457df8dd/RuSTSBenchmarkSTS.json new file mode 100644 index 000000000..9bc9bf381 --- /dev/null +++ b/results/Alibaba-NLP__gte-Qwen2-1.5B-instruct/c6c1b92f4a3e1b92b326ad29dd3c8433457df8dd/RuSTSBenchmarkSTS.json @@ -0,0 +1,26 @@ +{ + "dataset_revision": "7cf24f325c6da6195df55bef3d86b5e0616f3018", + "evaluation_time": 33.171607971191406, + "kg_co2_emissions": null, + "mteb_version": "1.19.2", + "scores": { + "test": [ + { + "cosine_pearson": 0.7969696535182529, + "cosine_spearman": 0.7863818865780526, + "euclidean_pearson": 0.7856831649296786, + "euclidean_spearman": 0.7863818865780526, + "hf_subset": "default", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.7863818865780526, + "manhattan_pearson": 0.7868536524192116, + "manhattan_spearman": 0.7884445888242286, + "pearson": 0.7969696535182529, + "spearman": 0.7863818865780526 + } + ] + }, + "task_name": "RuSTSBenchmarkSTS" +} \ No newline at end of file diff --git a/results/Alibaba-NLP__gte-Qwen2-1.5B-instruct/c6c1b92f4a3e1b92b326ad29dd3c8433457df8dd/RuSciBenchGRNTIClassification.json b/results/Alibaba-NLP__gte-Qwen2-1.5B-instruct/c6c1b92f4a3e1b92b326ad29dd3c8433457df8dd/RuSciBenchGRNTIClassification.json new file mode 100644 index 000000000..7ff1cdc0f --- /dev/null +++ b/results/Alibaba-NLP__gte-Qwen2-1.5B-instruct/c6c1b92f4a3e1b92b326ad29dd3c8433457df8dd/RuSciBenchGRNTIClassification.json @@ -0,0 +1,73 @@ +{ + "dataset_revision": "673a610d6d3dd91a547a0d57ae1b56f37ebbf6a1", + "evaluation_time": 65.67443513870239, + "kg_co2_emissions": null, + "mteb_version": "1.19.2", + "scores": { + "test": [ + { + "accuracy": 0.6673828125, + "f1": 0.6564611808368113, + "f1_weighted": 0.6565678384778197, + "hf_subset": "default", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.6673828125, + "scores_per_experiment": [ + { + "accuracy": 0.68798828125, + "f1": 0.6810050944295518, + "f1_weighted": 0.6811133014806365 + }, + { + "accuracy": 0.66357421875, + "f1": 0.6511976944015947, + "f1_weighted": 0.6513117011798647 + }, + { + "accuracy": 0.65478515625, + "f1": 0.6411231743506376, + "f1_weighted": 0.6412676104371658 + }, + { + "accuracy": 0.67236328125, + "f1": 0.66135007733751, + "f1_weighted": 0.661463776056001 + }, + { + "accuracy": 0.671875, + "f1": 0.6629382495623598, + "f1_weighted": 0.6630279677814571 + }, + { + "accuracy": 0.66259765625, + "f1": 0.6530275032620626, + "f1_weighted": 0.6530952276222671 + }, + { + "accuracy": 0.67822265625, + "f1": 0.665913133789097, + "f1_weighted": 0.6660414948471616 + }, + { + "accuracy": 0.6591796875, + "f1": 0.6484716936319798, + "f1_weighted": 0.6485803505158273 + }, + { + "accuracy": 0.671875, + "f1": 0.6620798628890334, + "f1_weighted": 0.6621870228217818 + }, + { + "accuracy": 0.6513671875, + "f1": 0.6375053247142862, + "f1_weighted": 0.6375899320360338 + } + ] + } + ] + }, + "task_name": "RuSciBenchGRNTIClassification" +} \ No newline at end of file diff --git a/results/Alibaba-NLP__gte-Qwen2-1.5B-instruct/c6c1b92f4a3e1b92b326ad29dd3c8433457df8dd/RuSciBenchGRNTIClusteringP2P.json b/results/Alibaba-NLP__gte-Qwen2-1.5B-instruct/c6c1b92f4a3e1b92b326ad29dd3c8433457df8dd/RuSciBenchGRNTIClusteringP2P.json new file mode 100644 index 000000000..cae0bb09f --- /dev/null +++ b/results/Alibaba-NLP__gte-Qwen2-1.5B-instruct/c6c1b92f4a3e1b92b326ad29dd3c8433457df8dd/RuSciBenchGRNTIClusteringP2P.json @@ -0,0 +1,34 @@ +{ + "dataset_revision": "673a610d6d3dd91a547a0d57ae1b56f37ebbf6a1", + "evaluation_time": 115.56734299659729, + "kg_co2_emissions": null, + "mteb_version": "1.19.2", + "scores": { + "test": [ + { + "hf_subset": "default", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.6140423807896046, + "v_measure": 0.6140423807896046, + "v_measure_std": 0.00557206978100521, + "v_measures": { + "Level 0": [ + 0.6127788469180457, + 0.618699078427273, + 0.606794452399866, + 0.6088767832666402, + 0.6078227897415304, + 0.6127429855670982, + 0.6256263602773254, + 0.6162643186875588, + 0.6118913377965314, + 0.6189268548141774 + ] + } + } + ] + }, + "task_name": "RuSciBenchGRNTIClusteringP2P" +} \ No newline at end of file diff --git a/results/Alibaba-NLP__gte-Qwen2-1.5B-instruct/c6c1b92f4a3e1b92b326ad29dd3c8433457df8dd/RuSciBenchOECDClassification.json b/results/Alibaba-NLP__gte-Qwen2-1.5B-instruct/c6c1b92f4a3e1b92b326ad29dd3c8433457df8dd/RuSciBenchOECDClassification.json new file mode 100644 index 000000000..7f088de87 --- /dev/null +++ b/results/Alibaba-NLP__gte-Qwen2-1.5B-instruct/c6c1b92f4a3e1b92b326ad29dd3c8433457df8dd/RuSciBenchOECDClassification.json @@ -0,0 +1,73 @@ +{ + "dataset_revision": "26c88e99dcaba32bb45d0e1bfc21902337f6d471", + "evaluation_time": 74.62318015098572, + "kg_co2_emissions": null, + "mteb_version": "1.19.2", + "scores": { + "test": [ + { + "accuracy": 0.531494140625, + "f1": 0.5092538189981415, + "f1_weighted": 0.5092330972697919, + "hf_subset": "default", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.531494140625, + "scores_per_experiment": [ + { + "accuracy": 0.53515625, + "f1": 0.5120307795631852, + "f1_weighted": 0.5120687483399415 + }, + { + "accuracy": 0.5322265625, + "f1": 0.5083554434216956, + "f1_weighted": 0.5083144296186541 + }, + { + "accuracy": 0.5439453125, + "f1": 0.527283707549943, + "f1_weighted": 0.5273521003997528 + }, + { + "accuracy": 0.513671875, + "f1": 0.4757018495567095, + "f1_weighted": 0.4756329470679142 + }, + { + "accuracy": 0.53125, + "f1": 0.5074090398183793, + "f1_weighted": 0.5074314808861585 + }, + { + "accuracy": 0.54296875, + "f1": 0.5268638850045184, + "f1_weighted": 0.5268322149028193 + }, + { + "accuracy": 0.53076171875, + "f1": 0.5021852239085048, + "f1_weighted": 0.5021507495182084 + }, + { + "accuracy": 0.54248046875, + "f1": 0.5325792283658487, + "f1_weighted": 0.5325468536479304 + }, + { + "accuracy": 0.51513671875, + "f1": 0.48825906305928374, + "f1_weighted": 0.4881892459697636 + }, + { + "accuracy": 0.52734375, + "f1": 0.5118699697333468, + "f1_weighted": 0.5118122023467767 + } + ] + } + ] + }, + "task_name": "RuSciBenchOECDClassification" +} \ No newline at end of file diff --git a/results/Alibaba-NLP__gte-Qwen2-1.5B-instruct/c6c1b92f4a3e1b92b326ad29dd3c8433457df8dd/RuSciBenchOECDClusteringP2P.json b/results/Alibaba-NLP__gte-Qwen2-1.5B-instruct/c6c1b92f4a3e1b92b326ad29dd3c8433457df8dd/RuSciBenchOECDClusteringP2P.json new file mode 100644 index 000000000..19fbec125 --- /dev/null +++ b/results/Alibaba-NLP__gte-Qwen2-1.5B-instruct/c6c1b92f4a3e1b92b326ad29dd3c8433457df8dd/RuSciBenchOECDClusteringP2P.json @@ -0,0 +1,34 @@ +{ + "dataset_revision": "26c88e99dcaba32bb45d0e1bfc21902337f6d471", + "evaluation_time": 87.11343717575073, + "kg_co2_emissions": null, + "mteb_version": "1.19.2", + "scores": { + "test": [ + { + "hf_subset": "default", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.5293031451453222, + "v_measure": 0.5293031451453222, + "v_measure_std": 0.006817383450301443, + "v_measures": { + "Level 0": [ + 0.5275782837034745, + 0.5317326590619295, + 0.5272979993924329, + 0.5208098266788344, + 0.5348989142344105, + 0.5210363368983464, + 0.544040470280087, + 0.5222786185801435, + 0.5306171116181606, + 0.532741231005403 + ] + } + } + ] + }, + "task_name": "RuSciBenchOECDClusteringP2P" +} \ No newline at end of file diff --git a/results/Alibaba-NLP__gte-Qwen2-1.5B-instruct/c6c1b92f4a3e1b92b326ad29dd3c8433457df8dd/STS22.json b/results/Alibaba-NLP__gte-Qwen2-1.5B-instruct/c6c1b92f4a3e1b92b326ad29dd3c8433457df8dd/STS22.json new file mode 100644 index 000000000..ed876dc8f --- /dev/null +++ b/results/Alibaba-NLP__gte-Qwen2-1.5B-instruct/c6c1b92f4a3e1b92b326ad29dd3c8433457df8dd/STS22.json @@ -0,0 +1,26 @@ +{ + "dataset_revision": "de9d86b3b84231dc21f76c7b7af1f28e2f57f6e3", + "evaluation_time": 7.6240458488464355, + "kg_co2_emissions": null, + "mteb_version": "1.19.2", + "scores": { + "test": [ + { + "cosine_pearson": 0.5498167422150342, + "cosine_spearman": 0.6208272980605856, + "euclidean_pearson": 0.5513938985046544, + "euclidean_spearman": 0.6208272980605856, + "hf_subset": "ru", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.6208272980605856, + "manhattan_pearson": 0.5495505582267963, + "manhattan_spearman": 0.6197925120287682, + "pearson": 0.5498167422150342, + "spearman": 0.6208272980605856 + } + ] + }, + "task_name": "STS22" +} \ No newline at end of file diff --git a/results/Alibaba-NLP__gte-Qwen2-1.5B-instruct/c6c1b92f4a3e1b92b326ad29dd3c8433457df8dd/SensitiveTopicsClassification.json b/results/Alibaba-NLP__gte-Qwen2-1.5B-instruct/c6c1b92f4a3e1b92b326ad29dd3c8433457df8dd/SensitiveTopicsClassification.json new file mode 100644 index 000000000..712249306 --- /dev/null +++ b/results/Alibaba-NLP__gte-Qwen2-1.5B-instruct/c6c1b92f4a3e1b92b326ad29dd3c8433457df8dd/SensitiveTopicsClassification.json @@ -0,0 +1,73 @@ +{ + "dataset_revision": "416b34a802308eac30e4192afc0ff99bb8dcc7f2", + "evaluation_time": 55.321635007858276, + "kg_co2_emissions": null, + "mteb_version": "1.19.2", + "scores": { + "test": [ + { + "accuracy": 0.343212890625, + "f1": 0.4444830510959751, + "hf_subset": "default", + "languages": [ + "rus-Cyrl" + ], + "lrap": 0.5518161349826368, + "main_score": 0.343212890625, + "scores_per_experiment": [ + { + "accuracy": 0.34326171875, + "f1": 0.4201625594418228, + "lrap": 0.5236545138888846 + }, + { + "accuracy": 0.3056640625, + "f1": 0.42621370107155704, + "lrap": 0.5270928276909683 + }, + { + "accuracy": 0.3349609375, + "f1": 0.40866798099165463, + "lrap": 0.5254584418402736 + }, + { + "accuracy": 0.365234375, + "f1": 0.4658714499469487, + "lrap": 0.5608384874131928 + }, + { + "accuracy": 0.345703125, + "f1": 0.4493586128248626, + "lrap": 0.5564778645833316 + }, + { + "accuracy": 0.35400390625, + "f1": 0.458588454181207, + "lrap": 0.5751817491319442 + }, + { + "accuracy": 0.34423828125, + "f1": 0.4647278198948714, + "lrap": 0.5674709743923604 + }, + { + "accuracy": 0.34521484375, + "f1": 0.4383361383624593, + "lrap": 0.5567559136284704 + }, + { + "accuracy": 0.36083984375, + "f1": 0.4573978470370404, + "lrap": 0.5700819227430541 + }, + { + "accuracy": 0.3330078125, + "f1": 0.4555059472073264, + "lrap": 0.5551486545138872 + } + ] + } + ] + }, + "task_name": "SensitiveTopicsClassification" +} \ No newline at end of file diff --git a/results/Alibaba-NLP__gte-Qwen2-1.5B-instruct/c6c1b92f4a3e1b92b326ad29dd3c8433457df8dd/TERRa.json b/results/Alibaba-NLP__gte-Qwen2-1.5B-instruct/c6c1b92f4a3e1b92b326ad29dd3c8433457df8dd/TERRa.json new file mode 100644 index 000000000..abb0ffe5a --- /dev/null +++ b/results/Alibaba-NLP__gte-Qwen2-1.5B-instruct/c6c1b92f4a3e1b92b326ad29dd3c8433457df8dd/TERRa.json @@ -0,0 +1,58 @@ +{ + "dataset_revision": "7b58f24536063837d644aab9a023c62199b2a612", + "evaluation_time": 8.094063997268677, + "kg_co2_emissions": null, + "mteb_version": "1.19.2", + "scores": { + "dev": [ + { + "cosine_accuracy": 0.6221498371335505, + "cosine_accuracy_threshold": 0.7010552883148193, + "cosine_ap": 0.60546979190962, + "cosine_f1": 0.6869158878504672, + "cosine_f1_threshold": 0.5825852155685425, + "cosine_precision": 0.5345454545454545, + "cosine_recall": 0.9607843137254902, + "dot_accuracy": 0.6221498371335505, + "dot_accuracy_threshold": 0.7010552883148193, + "dot_ap": 0.60546979190962, + "dot_f1": 0.6869158878504672, + "dot_f1_threshold": 0.5825852751731873, + "dot_precision": 0.5345454545454545, + "dot_recall": 0.9607843137254902, + "euclidean_accuracy": 0.6221498371335505, + "euclidean_accuracy_threshold": 0.7732328176498413, + "euclidean_ap": 0.60546979190962, + "euclidean_f1": 0.6869158878504672, + "euclidean_f1_threshold": 0.9136881232261658, + "euclidean_precision": 0.5345454545454545, + "euclidean_recall": 0.9607843137254902, + "hf_subset": "default", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.60546979190962, + "manhattan_accuracy": 0.6091205211726385, + "manhattan_accuracy_threshold": 23.17253875732422, + "manhattan_ap": 0.6018193054663248, + "manhattan_f1": 0.6804597701149424, + "manhattan_f1_threshold": 27.389568328857422, + "manhattan_precision": 0.524822695035461, + "manhattan_recall": 0.9673202614379085, + "max_accuracy": 0.6221498371335505, + "max_ap": 0.60546979190962, + "max_f1": 0.6869158878504672, + "max_precision": 0.5345454545454545, + "max_recall": 0.9673202614379085, + "similarity_accuracy": 0.6221498371335505, + "similarity_accuracy_threshold": 0.7010552883148193, + "similarity_ap": 0.60546979190962, + "similarity_f1": 0.6869158878504672, + "similarity_f1_threshold": 0.5825852155685425, + "similarity_precision": 0.5345454545454545, + "similarity_recall": 0.9607843137254902 + } + ] + }, + "task_name": "TERRa" +} \ No newline at end of file diff --git a/results/Alibaba-NLP__gte-Qwen2-1.5B-instruct/c6c1b92f4a3e1b92b326ad29dd3c8433457df8dd/model_meta.json b/results/Alibaba-NLP__gte-Qwen2-1.5B-instruct/c6c1b92f4a3e1b92b326ad29dd3c8433457df8dd/model_meta.json new file mode 100644 index 000000000..7ea978283 --- /dev/null +++ b/results/Alibaba-NLP__gte-Qwen2-1.5B-instruct/c6c1b92f4a3e1b92b326ad29dd3c8433457df8dd/model_meta.json @@ -0,0 +1 @@ +{"name": "Alibaba-NLP/gte-Qwen2-1.5B-instruct", "revision": "c6c1b92f4a3e1b92b326ad29dd3c8433457df8dd", "release_date": "2024-07-29", "languages": ["eng_Latn"], "n_parameters": 1780000000, "memory_usage": null, "max_tokens": 131072, "embed_dim": 8960, "license": "apache-2.0", "open_weights": true, "public_training_data": null, "public_training_code": null, "framework": ["Sentence Transformers", "PyTorch"], "reference": "https://huggingface.co/Alibaba-NLP/gte-Qwen2-1.5B-instruct", "similarity_fn_name": "cosine", "use_instructions": true, "zero_shot_benchmarks": null, "loader": "instruct_wrapper"} \ No newline at end of file diff --git a/results/Alibaba-NLP__gte-Qwen2-7B-instruct/e26182b2122f4435e8b3ebecbf363990f409b45b/CEDRClassification.json b/results/Alibaba-NLP__gte-Qwen2-7B-instruct/e26182b2122f4435e8b3ebecbf363990f409b45b/CEDRClassification.json index 7832041be..cd4e5bb26 100644 --- a/results/Alibaba-NLP__gte-Qwen2-7B-instruct/e26182b2122f4435e8b3ebecbf363990f409b45b/CEDRClassification.json +++ b/results/Alibaba-NLP__gte-Qwen2-7B-instruct/e26182b2122f4435e8b3ebecbf363990f409b45b/CEDRClassification.json @@ -6,64 +6,64 @@ "scores": { "test": [ { - "accuracy": 0.5307120085015941, - "f1": 0.5917230343930433, + "accuracy": 0.5278427205100956, + "f1": 0.5921989224887071, "hf_subset": "default", "languages": [ "rus-Cyrl" ], - "lrap": 0.8592109458023435, - "main_score": 0.5307120085015941, + "lrap": 0.8563602550478271, + "main_score": 0.5278427205100956, "scores_per_experiment": [ { - "accuracy": 0.5642933049946866, - "f1": 0.6070292680114152, - "lrap": 0.8628055260361375 + "accuracy": 0.5472901168969182, + "f1": 0.6231915731616979, + "lrap": 0.8695005313496335 }, { - "accuracy": 0.5765143464399575, - "f1": 0.5857093544461771, - "lrap": 0.8492029755579235 + "accuracy": 0.502125398512221, + "f1": 0.5965022757364782, + "lrap": 0.8441551540913984 }, { - "accuracy": 0.6036131774707758, - "f1": 0.62644976226418, - "lrap": 0.8632837407013871 + "accuracy": 0.5308182784272051, + "f1": 0.5907759632792869, + "lrap": 0.8546758767268918 }, { - "accuracy": 0.48831030818278426, - "f1": 0.5777644438978535, - "lrap": 0.8585547290116954 + "accuracy": 0.5557917109458024, + "f1": 0.6087947463545857, + "lrap": 0.8740701381509085 }, { - "accuracy": 0.5090329436769394, - "f1": 0.6004348584160987, - "lrap": 0.8719978746014928 + "accuracy": 0.5866099893730075, + "f1": 0.6140356617930539, + "lrap": 0.8620085015940544 }, { - "accuracy": 0.5302869287991498, - "f1": 0.5946970891286132, - "lrap": 0.8562167906482523 + "accuracy": 0.4914984059511158, + "f1": 0.5860852482396078, + "lrap": 0.8597768331562216 }, { - "accuracy": 0.5297555791710946, - "f1": 0.5857774068968041, - "lrap": 0.8560042507970299 + "accuracy": 0.49415515409139216, + "f1": 0.5662619403508564, + "lrap": 0.849734325185978 }, { - "accuracy": 0.5074388947927736, - "f1": 0.5779301896872774, - "lrap": 0.8629383634431511 + "accuracy": 0.5377258235919234, + "f1": 0.5785200123327568, + "lrap": 0.8406482465462339 }, { - "accuracy": 0.5132837407013815, - "f1": 0.5894929342611266, - "lrap": 0.849787460148784 + "accuracy": 0.5292242295430393, + "f1": 0.6004224638445306, + "lrap": 0.8604675876726942 }, { - "accuracy": 0.48459086078639746, - "f1": 0.5719450369208869, - "lrap": 0.8613177470775822 + "accuracy": 0.5031880977683315, + "f1": 0.5573993397942165, + "lrap": 0.8485653560042566 } ] } diff --git a/results/Alibaba-NLP__gte-Qwen2-7B-instruct/e26182b2122f4435e8b3ebecbf363990f409b45b/GeoreviewClassification.json b/results/Alibaba-NLP__gte-Qwen2-7B-instruct/e26182b2122f4435e8b3ebecbf363990f409b45b/GeoreviewClassification.json new file mode 100644 index 000000000..72a1e8d67 --- /dev/null +++ b/results/Alibaba-NLP__gte-Qwen2-7B-instruct/e26182b2122f4435e8b3ebecbf363990f409b45b/GeoreviewClassification.json @@ -0,0 +1,73 @@ +{ + "dataset_revision": "3765c0d1de6b7d264bc459433c45e5a75513839c", + "evaluation_time": 453.26696705818176, + "kg_co2_emissions": null, + "mteb_version": "1.19.0", + "scores": { + "test": [ + { + "accuracy": 0.600146484375, + "f1": 0.5785872759937656, + "f1_weighted": 0.5785346078310617, + "hf_subset": "default", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.600146484375, + "scores_per_experiment": [ + { + "accuracy": 0.5888671875, + "f1": 0.561057257563129, + "f1_weighted": 0.560990135820034 + }, + { + "accuracy": 0.58837890625, + "f1": 0.554613821793309, + "f1_weighted": 0.5545248124700248 + }, + { + "accuracy": 0.609375, + "f1": 0.5876741838282947, + "f1_weighted": 0.5876103740996099 + }, + { + "accuracy": 0.60986328125, + "f1": 0.5874157051227198, + "f1_weighted": 0.5873556740591956 + }, + { + "accuracy": 0.5966796875, + "f1": 0.5765319732980858, + "f1_weighted": 0.5764661791524901 + }, + { + "accuracy": 0.5693359375, + "f1": 0.5596541863773145, + "f1_weighted": 0.5597054274104073 + }, + { + "accuracy": 0.61328125, + "f1": 0.5977133891466978, + "f1_weighted": 0.5976513645857018 + }, + { + "accuracy": 0.60595703125, + "f1": 0.5750309442619234, + "f1_weighted": 0.5749562602742928 + }, + { + "accuracy": 0.611328125, + "f1": 0.5931787267140416, + "f1_weighted": 0.5931291831938417 + }, + { + "accuracy": 0.6083984375, + "f1": 0.5930025718321394, + "f1_weighted": 0.5929566672450189 + } + ] + } + ] + }, + "task_name": "GeoreviewClassification" +} \ No newline at end of file diff --git a/results/Alibaba-NLP__gte-Qwen2-7B-instruct/e26182b2122f4435e8b3ebecbf363990f409b45b/GeoreviewClusteringP2P.json b/results/Alibaba-NLP__gte-Qwen2-7B-instruct/e26182b2122f4435e8b3ebecbf363990f409b45b/GeoreviewClusteringP2P.json new file mode 100644 index 000000000..d620415ed --- /dev/null +++ b/results/Alibaba-NLP__gte-Qwen2-7B-instruct/e26182b2122f4435e8b3ebecbf363990f409b45b/GeoreviewClusteringP2P.json @@ -0,0 +1,34 @@ +{ + "dataset_revision": "97a313c8fc85b47f13f33e7e9a95c1ad888c7fec", + "evaluation_time": 372.32498264312744, + "kg_co2_emissions": null, + "mteb_version": "1.19.0", + "scores": { + "test": [ + { + "hf_subset": "default", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.7831573824829378, + "v_measure": 0.7831573824829378, + "v_measure_std": 0.0031176989278815246, + "v_measures": { + "Level 0": [ + 0.7825843607194956, + 0.7849570432841764, + 0.7891721456884293, + 0.7862151642070009, + 0.7838741734961716, + 0.7823980581391391, + 0.7778534604374112, + 0.7803934305692242, + 0.7799595239299798, + 0.78416646435835 + ] + } + } + ] + }, + "task_name": "GeoreviewClusteringP2P" +} \ No newline at end of file diff --git a/results/Alibaba-NLP__gte-Qwen2-7B-instruct/e26182b2122f4435e8b3ebecbf363990f409b45b/HeadlineClassification.json b/results/Alibaba-NLP__gte-Qwen2-7B-instruct/e26182b2122f4435e8b3ebecbf363990f409b45b/HeadlineClassification.json new file mode 100644 index 000000000..cbea8045b --- /dev/null +++ b/results/Alibaba-NLP__gte-Qwen2-7B-instruct/e26182b2122f4435e8b3ebecbf363990f409b45b/HeadlineClassification.json @@ -0,0 +1,73 @@ +{ + "dataset_revision": "2fe05ee6b5832cda29f2ef7aaad7b7fe6a3609eb", + "evaluation_time": 134.56600427627563, + "kg_co2_emissions": null, + "mteb_version": "1.19.0", + "scores": { + "test": [ + { + "accuracy": 0.76376953125, + "f1": 0.7678292655168574, + "f1_weighted": 0.7677914449429452, + "hf_subset": "default", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.76376953125, + "scores_per_experiment": [ + { + "accuracy": 0.765625, + "f1": 0.7712627436907281, + "f1_weighted": 0.7712322526359106 + }, + { + "accuracy": 0.77197265625, + "f1": 0.774913236114959, + "f1_weighted": 0.7748773489461512 + }, + { + "accuracy": 0.74951171875, + "f1": 0.7551607063589735, + "f1_weighted": 0.7551204826205611 + }, + { + "accuracy": 0.75634765625, + "f1": 0.762757017754281, + "f1_weighted": 0.7627145153671681 + }, + { + "accuracy": 0.75634765625, + "f1": 0.7620127891714809, + "f1_weighted": 0.7619696533603255 + }, + { + "accuracy": 0.75927734375, + "f1": 0.7643488366801355, + "f1_weighted": 0.7642998875782314 + }, + { + "accuracy": 0.7705078125, + "f1": 0.7735130202890201, + "f1_weighted": 0.7734671689226726 + }, + { + "accuracy": 0.76611328125, + "f1": 0.7671860750171898, + "f1_weighted": 0.7671701757864748 + }, + { + "accuracy": 0.76708984375, + "f1": 0.7700081895326587, + "f1_weighted": 0.7699714722946819 + }, + { + "accuracy": 0.77490234375, + "f1": 0.7771300405591465, + "f1_weighted": 0.7770914919172744 + } + ] + } + ] + }, + "task_name": "HeadlineClassification" +} \ No newline at end of file diff --git a/results/Alibaba-NLP__gte-Qwen2-7B-instruct/e26182b2122f4435e8b3ebecbf363990f409b45b/InappropriatenessClassification.json b/results/Alibaba-NLP__gte-Qwen2-7B-instruct/e26182b2122f4435e8b3ebecbf363990f409b45b/InappropriatenessClassification.json new file mode 100644 index 000000000..02020ef3b --- /dev/null +++ b/results/Alibaba-NLP__gte-Qwen2-7B-instruct/e26182b2122f4435e8b3ebecbf363990f409b45b/InappropriatenessClassification.json @@ -0,0 +1,95 @@ +{ + "dataset_revision": "601651fdc45ef243751676e62dd7a19f491c0285", + "evaluation_time": 147.0128161907196, + "kg_co2_emissions": null, + "mteb_version": "1.19.0", + "scores": { + "test": [ + { + "accuracy": 0.7640625, + "ap": 0.7114187629824069, + "ap_weighted": 0.7114187629824069, + "f1": 0.763516672336194, + "f1_weighted": 0.763516672336194, + "hf_subset": "default", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.7640625, + "scores_per_experiment": [ + { + "accuracy": 0.7802734375, + "ap": 0.7301126433165548, + "ap_weighted": 0.7301126433165548, + "f1": 0.7793845172743646, + "f1_weighted": 0.7793845172743646 + }, + { + "accuracy": 0.775390625, + "ap": 0.7217431006493507, + "ap_weighted": 0.7217431006493507, + "f1": 0.7748538347118182, + "f1_weighted": 0.7748538347118182 + }, + { + "accuracy": 0.7763671875, + "ap": 0.7256689562360179, + "ap_weighted": 0.7256689562360179, + "f1": 0.7754624642481311, + "f1_weighted": 0.7754624642481311 + }, + { + "accuracy": 0.6171875, + "ap": 0.5721938467117988, + "ap_weighted": 0.5721938467117988, + "f1": 0.61717837282116, + "f1_weighted": 0.61717837282116 + }, + { + "accuracy": 0.76318359375, + "ap": 0.7132120253164557, + "ap_weighted": 0.7132120253164557, + "f1": 0.7618192931456145, + "f1_weighted": 0.7618192931456145 + }, + { + "accuracy": 0.7744140625, + "ap": 0.7223177945640177, + "ap_weighted": 0.7223177945640177, + "f1": 0.7736626813830322, + "f1_weighted": 0.7736626813830322 + }, + { + "accuracy": 0.783203125, + "ap": 0.7248965992647058, + "ap_weighted": 0.7248965992647058, + "f1": 0.783128461241253, + "f1_weighted": 0.783128461241253 + }, + { + "accuracy": 0.80224609375, + "ap": 0.74220889499148, + "ap_weighted": 0.74220889499148, + "f1": 0.8022456694152414, + "f1_weighted": 0.8022456694152414 + }, + { + "accuracy": 0.7880859375, + "ap": 0.7336899640690928, + "ap_weighted": 0.7336899640690928, + "f1": 0.7877937070016006, + "f1_weighted": 0.7877937070016006 + }, + { + "accuracy": 0.7802734375, + "ap": 0.7281438047045952, + "ap_weighted": 0.7281438047045952, + "f1": 0.7796377221197244, + "f1_weighted": 0.7796377221197244 + } + ] + } + ] + }, + "task_name": "InappropriatenessClassification" +} \ No newline at end of file diff --git a/results/Alibaba-NLP__gte-Qwen2-7B-instruct/e26182b2122f4435e8b3ebecbf363990f409b45b/KinopoiskClassification.json b/results/Alibaba-NLP__gte-Qwen2-7B-instruct/e26182b2122f4435e8b3ebecbf363990f409b45b/KinopoiskClassification.json new file mode 100644 index 000000000..74ce76d08 --- /dev/null +++ b/results/Alibaba-NLP__gte-Qwen2-7B-instruct/e26182b2122f4435e8b3ebecbf363990f409b45b/KinopoiskClassification.json @@ -0,0 +1,73 @@ +{ + "dataset_revision": "5911f26666ac11af46cb9c6849d0dc80a378af24", + "evaluation_time": 678.8495149612427, + "kg_co2_emissions": null, + "mteb_version": "1.19.0", + "scores": { + "test": [ + { + "accuracy": 0.7104, + "f1": 0.6868140081592491, + "f1_weighted": 0.6868140081592491, + "hf_subset": "default", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.7104, + "scores_per_experiment": [ + { + "accuracy": 0.7206666666666667, + "f1": 0.7018263144837388, + "f1_weighted": 0.7018263144837389 + }, + { + "accuracy": 0.674, + "f1": 0.6137094129444546, + "f1_weighted": 0.6137094129444546 + }, + { + "accuracy": 0.724, + "f1": 0.7088585032177687, + "f1_weighted": 0.7088585032177688 + }, + { + "accuracy": 0.7066666666666667, + "f1": 0.6787231134875021, + "f1_weighted": 0.6787231134875021 + }, + { + "accuracy": 0.718, + "f1": 0.6974977978810166, + "f1_weighted": 0.6974977978810166 + }, + { + "accuracy": 0.718, + "f1": 0.6980814763498486, + "f1_weighted": 0.6980814763498486 + }, + { + "accuracy": 0.706, + "f1": 0.693251717688398, + "f1_weighted": 0.6932517176883979 + }, + { + "accuracy": 0.72, + "f1": 0.7064541713547663, + "f1_weighted": 0.7064541713547663 + }, + { + "accuracy": 0.7146666666666667, + "f1": 0.6873062863365641, + "f1_weighted": 0.6873062863365641 + }, + { + "accuracy": 0.702, + "f1": 0.6824312878484325, + "f1_weighted": 0.6824312878484327 + } + ] + } + ] + }, + "task_name": "KinopoiskClassification" +} \ No newline at end of file diff --git a/results/Alibaba-NLP__gte-Qwen2-7B-instruct/e26182b2122f4435e8b3ebecbf363990f409b45b/MIRACLReranking.json b/results/Alibaba-NLP__gte-Qwen2-7B-instruct/e26182b2122f4435e8b3ebecbf363990f409b45b/MIRACLReranking.json new file mode 100644 index 000000000..3278802db --- /dev/null +++ b/results/Alibaba-NLP__gte-Qwen2-7B-instruct/e26182b2122f4435e8b3ebecbf363990f409b45b/MIRACLReranking.json @@ -0,0 +1,130 @@ +{ + "dataset_revision": "6d1962c527217f8927fca80f890f14f36b2802af", + "evaluation_time": 24058.83981180191, + "kg_co2_emissions": null, + "mteb_version": "1.19.0", + "scores": { + "dev": [ + { + "MAP@1(MIRACL)": 0.28782, + "MAP@10(MIRACL)": 0.46967, + "MAP@100(MIRACL)": 0.49014, + "MAP@1000(MIRACL)": 0.49014, + "MAP@20(MIRACL)": 0.48176, + "MAP@3(MIRACL)": 0.40793, + "MAP@5(MIRACL)": 0.44228, + "NDCG@1(MIRACL)": 0.47394, + "NDCG@10(MIRACL)": 0.54954, + "NDCG@100(MIRACL)": 0.59624, + "NDCG@1000(MIRACL)": 0.59624, + "NDCG@20(MIRACL)": 0.57207, + "NDCG@3(MIRACL)": 0.49407, + "NDCG@5(MIRACL)": 0.51505, + "P@1(MIRACL)": 0.47394, + "P@10(MIRACL)": 0.14154, + "P@100(MIRACL)": 0.01953, + "P@1000(MIRACL)": 0.00195, + "P@20(MIRACL)": 0.08176, + "P@3(MIRACL)": 0.30527, + "P@5(MIRACL)": 0.22727, + "Recall@1(MIRACL)": 0.28782, + "Recall@10(MIRACL)": 0.65557, + "Recall@100(MIRACL)": 0.79952, + "Recall@1000(MIRACL)": 0.79952, + "Recall@20(MIRACL)": 0.71651, + "Recall@3(MIRACL)": 0.48312, + "Recall@5(MIRACL)": 0.56338, + "hf_subset": "ru", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.54954, + "nAUC_MAP@1000_diff1(MIRACL)": 0.1563474206583911, + "nAUC_MAP@1000_max(MIRACL)": 0.27620321099087075, + "nAUC_MAP@1000_std(MIRACL)": 0.1339039004613247, + "nAUC_MAP@100_diff1(MIRACL)": 0.1563474206583911, + "nAUC_MAP@100_max(MIRACL)": 0.27620321099087075, + "nAUC_MAP@100_std(MIRACL)": 0.1339039004613247, + "nAUC_MAP@10_diff1(MIRACL)": 0.17623240707115395, + "nAUC_MAP@10_max(MIRACL)": 0.25737560739428833, + "nAUC_MAP@10_std(MIRACL)": 0.12132178228251274, + "nAUC_MAP@1_diff1(MIRACL)": 0.3396966076709825, + "nAUC_MAP@1_max(MIRACL)": 0.18632032787956443, + "nAUC_MAP@1_std(MIRACL)": 0.034852842560540095, + "nAUC_MAP@20_diff1(MIRACL)": 0.16357855080922978, + "nAUC_MAP@20_max(MIRACL)": 0.26716444038091436, + "nAUC_MAP@20_std(MIRACL)": 0.13026812466087037, + "nAUC_MAP@3_diff1(MIRACL)": 0.23258023795732752, + "nAUC_MAP@3_max(MIRACL)": 0.22640344046807617, + "nAUC_MAP@3_std(MIRACL)": 0.06519137422277979, + "nAUC_MAP@5_diff1(MIRACL)": 0.20251008120660338, + "nAUC_MAP@5_max(MIRACL)": 0.23564010989910592, + "nAUC_MAP@5_std(MIRACL)": 0.09210354713417973, + "nAUC_NDCG@1000_diff1(MIRACL)": 0.04714638989238691, + "nAUC_NDCG@1000_max(MIRACL)": 0.34554843632205057, + "nAUC_NDCG@1000_std(MIRACL)": 0.19164253929823796, + "nAUC_NDCG@100_diff1(MIRACL)": 0.04714638989238691, + "nAUC_NDCG@100_max(MIRACL)": 0.34554843632205057, + "nAUC_NDCG@100_std(MIRACL)": 0.19164253929823796, + "nAUC_NDCG@10_diff1(MIRACL)": 0.10151652889558566, + "nAUC_NDCG@10_max(MIRACL)": 0.2977152716604701, + "nAUC_NDCG@10_std(MIRACL)": 0.16894640444356285, + "nAUC_NDCG@1_diff1(MIRACL)": 0.2289445822952089, + "nAUC_NDCG@1_max(MIRACL)": 0.3302580564190035, + "nAUC_NDCG@1_std(MIRACL)": 0.16068383331688468, + "nAUC_NDCG@20_diff1(MIRACL)": 0.07300216474325952, + "nAUC_NDCG@20_max(MIRACL)": 0.3163638651365428, + "nAUC_NDCG@20_std(MIRACL)": 0.1847841821437998, + "nAUC_NDCG@3_diff1(MIRACL)": 0.1559447908356773, + "nAUC_NDCG@3_max(MIRACL)": 0.288075651390397, + "nAUC_NDCG@3_std(MIRACL)": 0.12211681864987331, + "nAUC_NDCG@5_diff1(MIRACL)": 0.13730945320388213, + "nAUC_NDCG@5_max(MIRACL)": 0.2753905288222436, + "nAUC_NDCG@5_std(MIRACL)": 0.1330458883994935, + "nAUC_P@1000_diff1(MIRACL)": -0.2754175646329511, + "nAUC_P@1000_max(MIRACL)": 0.24470338742551506, + "nAUC_P@1000_std(MIRACL)": 0.18179845136935743, + "nAUC_P@100_diff1(MIRACL)": -0.27541756463295064, + "nAUC_P@100_max(MIRACL)": 0.24470338742551542, + "nAUC_P@100_std(MIRACL)": 0.18179845136935777, + "nAUC_P@10_diff1(MIRACL)": -0.2295851486680459, + "nAUC_P@10_max(MIRACL)": 0.26409268005433834, + "nAUC_P@10_std(MIRACL)": 0.24128894533978945, + "nAUC_P@1_diff1(MIRACL)": 0.2289445822952089, + "nAUC_P@1_max(MIRACL)": 0.3302580564190035, + "nAUC_P@1_std(MIRACL)": 0.16068383331688468, + "nAUC_P@20_diff1(MIRACL)": -0.2609864743404855, + "nAUC_P@20_max(MIRACL)": 0.25089897872107947, + "nAUC_P@20_std(MIRACL)": 0.22676914682497107, + "nAUC_P@3_diff1(MIRACL)": -0.0921932712964439, + "nAUC_P@3_max(MIRACL)": 0.28780553435500017, + "nAUC_P@3_std(MIRACL)": 0.17934491884923143, + "nAUC_P@5_diff1(MIRACL)": -0.17012636126492298, + "nAUC_P@5_max(MIRACL)": 0.2566790717740838, + "nAUC_P@5_std(MIRACL)": 0.20532752415535488, + "nAUC_Recall@1000_diff1(MIRACL)": -0.3405317145310675, + "nAUC_Recall@1000_max(MIRACL)": 0.4992623835230486, + "nAUC_Recall@1000_std(MIRACL)": 0.330322270264166, + "nAUC_Recall@100_diff1(MIRACL)": -0.3405317145310675, + "nAUC_Recall@100_max(MIRACL)": 0.4992623835230486, + "nAUC_Recall@100_std(MIRACL)": 0.330322270264166, + "nAUC_Recall@10_diff1(MIRACL)": -0.0444217130663372, + "nAUC_Recall@10_max(MIRACL)": 0.2669942490827101, + "nAUC_Recall@10_std(MIRACL)": 0.20944124722744123, + "nAUC_Recall@1_diff1(MIRACL)": 0.3396966076709825, + "nAUC_Recall@1_max(MIRACL)": 0.18632032787956443, + "nAUC_Recall@1_std(MIRACL)": 0.034852842560540095, + "nAUC_Recall@20_diff1(MIRACL)": -0.15334949193949884, + "nAUC_Recall@20_max(MIRACL)": 0.3220488829047202, + "nAUC_Recall@20_std(MIRACL)": 0.267448401283707, + "nAUC_Recall@3_diff1(MIRACL)": 0.14653742924054286, + "nAUC_Recall@3_max(MIRACL)": 0.2140253382446473, + "nAUC_Recall@3_std(MIRACL)": 0.0700967950346427, + "nAUC_Recall@5_diff1(MIRACL)": 0.0729117084855482, + "nAUC_Recall@5_max(MIRACL)": 0.21602937332331054, + "nAUC_Recall@5_std(MIRACL)": 0.11278916921661897 + } + ] + }, + "task_name": "MIRACLReranking" +} \ No newline at end of file diff --git a/results/Alibaba-NLP__gte-Qwen2-7B-instruct/e26182b2122f4435e8b3ebecbf363990f409b45b/MassiveScenarioClassification.json b/results/Alibaba-NLP__gte-Qwen2-7B-instruct/e26182b2122f4435e8b3ebecbf363990f409b45b/MassiveScenarioClassification.json new file mode 100644 index 000000000..0bcf9a78d --- /dev/null +++ b/results/Alibaba-NLP__gte-Qwen2-7B-instruct/e26182b2122f4435e8b3ebecbf363990f409b45b/MassiveScenarioClassification.json @@ -0,0 +1,137 @@ +{ + "dataset_revision": "fad2c6e8459f9e1c45d9315f4953d921437d70f8", + "evaluation_time": 348.6101727485657, + "kg_co2_emissions": null, + "mteb_version": "1.19.0", + "scores": { + "test": [ + { + "accuracy": 0.8742770679219906, + "f1": 0.8657929115220291, + "f1_weighted": 0.8721634824090809, + "hf_subset": "ru", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.8742770679219906, + "scores_per_experiment": [ + { + "accuracy": 0.8826496301277741, + "f1": 0.8745190941798237, + "f1_weighted": 0.8814348562423552 + }, + { + "accuracy": 0.8735709482178884, + "f1": 0.8643568325090288, + "f1_weighted": 0.8709792678351427 + }, + { + "accuracy": 0.8638197713517148, + "f1": 0.8545573549713423, + "f1_weighted": 0.8609202602222957 + }, + { + "accuracy": 0.8759246805648958, + "f1": 0.8617349512761722, + "f1_weighted": 0.8736578511755067 + }, + { + "accuracy": 0.8823133826496301, + "f1": 0.8722093558537751, + "f1_weighted": 0.879099135525846 + }, + { + "accuracy": 0.8668459986550101, + "f1": 0.8591478255584136, + "f1_weighted": 0.8652445404051274 + }, + { + "accuracy": 0.8732347007397444, + "f1": 0.8624559964958555, + "f1_weighted": 0.8691054052105809 + }, + { + "accuracy": 0.8702084734364492, + "f1": 0.866537842799181, + "f1_weighted": 0.8697639542498959 + }, + { + "accuracy": 0.8581035642232683, + "f1": 0.8531640003126318, + "f1_weighted": 0.8562544874516554 + }, + { + "accuracy": 0.8960995292535306, + "f1": 0.889245861264066, + "f1_weighted": 0.8951750657724027 + } + ] + } + ], + "validation": [ + { + "accuracy": 0.8675848499754059, + "f1": 0.8554629390972843, + "f1_weighted": 0.8643538702684053, + "hf_subset": "ru", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.8675848499754059, + "scores_per_experiment": [ + { + "accuracy": 0.8839153959665519, + "f1": 0.8749100267713424, + "f1_weighted": 0.8824078315059192 + }, + { + "accuracy": 0.8652238071815052, + "f1": 0.8534500370315077, + "f1_weighted": 0.8624793049176476 + }, + { + "accuracy": 0.8612887358583374, + "f1": 0.8491228381079821, + "f1_weighted": 0.8567239333883814 + }, + { + "accuracy": 0.8726020659124447, + "f1": 0.8562395512162817, + "f1_weighted": 0.8687632432751772 + }, + { + "accuracy": 0.8809640924741761, + "f1": 0.8655336244135785, + "f1_weighted": 0.877230127380317 + }, + { + "accuracy": 0.8519429414658141, + "f1": 0.8394940276571649, + "f1_weighted": 0.8491416750488167 + }, + { + "accuracy": 0.8652238071815052, + "f1": 0.8479347109956085, + "f1_weighted": 0.8605089335955822 + }, + { + "accuracy": 0.8593212001967536, + "f1": 0.8520226965958042, + "f1_weighted": 0.8570036727091412 + }, + { + "accuracy": 0.8558780127889818, + "f1": 0.8470518212159359, + "f1_weighted": 0.8516202920318172 + }, + { + "accuracy": 0.8794884407279882, + "f1": 0.8688700569676385, + "f1_weighted": 0.8776596888312521 + } + ] + } + ] + }, + "task_name": "MassiveScenarioClassification" +} \ No newline at end of file diff --git a/results/Alibaba-NLP__gte-Qwen2-7B-instruct/e26182b2122f4435e8b3ebecbf363990f409b45b/RUParaPhraserSTS.json b/results/Alibaba-NLP__gte-Qwen2-7B-instruct/e26182b2122f4435e8b3ebecbf363990f409b45b/RUParaPhraserSTS.json new file mode 100644 index 000000000..3fcc13224 --- /dev/null +++ b/results/Alibaba-NLP__gte-Qwen2-7B-instruct/e26182b2122f4435e8b3ebecbf363990f409b45b/RUParaPhraserSTS.json @@ -0,0 +1,26 @@ +{ + "dataset_revision": "43265056790b8f7c59e0139acb4be0a8dad2c8f4", + "evaluation_time": 175.14431071281433, + "kg_co2_emissions": null, + "mteb_version": "1.19.2", + "scores": { + "test": [ + { + "cosine_pearson": 0.7214136901742709, + "cosine_spearman": 0.7744532126532119, + "euclidean_pearson": 0.752043804833285, + "euclidean_spearman": 0.7744532123269988, + "hf_subset": "default", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.7744532126532119, + "manhattan_pearson": 0.7510134244772281, + "manhattan_spearman": 0.7729112684732905, + "pearson": 0.7214136901742709, + "spearman": 0.7744532126532119 + } + ] + }, + "task_name": "RUParaPhraserSTS" +} \ No newline at end of file diff --git a/results/Alibaba-NLP__gte-Qwen2-7B-instruct/e26182b2122f4435e8b3ebecbf363990f409b45b/RiaNewsRetrieval.json b/results/Alibaba-NLP__gte-Qwen2-7B-instruct/e26182b2122f4435e8b3ebecbf363990f409b45b/RiaNewsRetrieval.json new file mode 100644 index 000000000..42fa7387a --- /dev/null +++ b/results/Alibaba-NLP__gte-Qwen2-7B-instruct/e26182b2122f4435e8b3ebecbf363990f409b45b/RiaNewsRetrieval.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "82374b0bbacda6114f39ff9c5b925fa1512ca5d7", + "evaluation_time": 250838.25524187088, + "kg_co2_emissions": null, + "mteb_version": "1.19.4", + "scores": { + "test": [ + { + "hf_subset": "default", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.76264, + "map_at_1": 0.6363, + "map_at_10": 0.72411, + "map_at_100": 0.7272, + "map_at_1000": 0.72732, + "map_at_20": 0.72614, + "map_at_3": 0.70852, + "map_at_5": 0.71823, + "mrr_at_1": 0.6363, + "mrr_at_10": 0.7241098809523809, + "mrr_at_100": 0.7271979011039543, + "mrr_at_1000": 0.7273179254669859, + "mrr_at_20": 0.7261395932516057, + "mrr_at_3": 0.7085166666666667, + "mrr_at_5": 0.7182316666666667, + "nauc_map_at_1000_diff1": 0.7092100013246324, + "nauc_map_at_1000_max": 0.3208867172842324, + "nauc_map_at_1000_std": -0.13280801488919186, + "nauc_map_at_100_diff1": 0.7091562349706422, + "nauc_map_at_100_max": 0.320937488312634, + "nauc_map_at_100_std": -0.13278592242690812, + "nauc_map_at_10_diff1": 0.7087671777082333, + "nauc_map_at_10_max": 0.3212228503342337, + "nauc_map_at_10_std": -0.13391485448306703, + "nauc_map_at_1_diff1": 0.7452410522141724, + "nauc_map_at_1_max": 0.28782222120749595, + "nauc_map_at_1_std": -0.13706654337741186, + "nauc_map_at_20_diff1": 0.7087692324449961, + "nauc_map_at_20_max": 0.3211011020147596, + "nauc_map_at_20_std": -0.13320716868250196, + "nauc_map_at_3_diff1": 0.70786699297963, + "nauc_map_at_3_max": 0.3177266732318444, + "nauc_map_at_3_std": -0.14084658740172756, + "nauc_map_at_5_diff1": 0.7087880335458725, + "nauc_map_at_5_max": 0.3204265058352077, + "nauc_map_at_5_std": -0.13932460690261497, + "nauc_mrr_at_1000_diff1": 0.7092100013246324, + "nauc_mrr_at_1000_max": 0.3208867172842324, + "nauc_mrr_at_1000_std": -0.13280801488919186, + "nauc_mrr_at_100_diff1": 0.7091562349706422, + "nauc_mrr_at_100_max": 0.320937488312634, + "nauc_mrr_at_100_std": -0.13278592242690812, + "nauc_mrr_at_10_diff1": 0.7087671777082333, + "nauc_mrr_at_10_max": 0.3212228503342337, + "nauc_mrr_at_10_std": -0.13391485448306703, + "nauc_mrr_at_1_diff1": 0.7452410522141724, + "nauc_mrr_at_1_max": 0.28782222120749595, + "nauc_mrr_at_1_std": -0.13706654337741186, + "nauc_mrr_at_20_diff1": 0.7087692324449961, + "nauc_mrr_at_20_max": 0.3211011020147596, + "nauc_mrr_at_20_std": -0.13320716868250196, + "nauc_mrr_at_3_diff1": 0.70786699297963, + "nauc_mrr_at_3_max": 0.3177266732318444, + "nauc_mrr_at_3_std": -0.14084658740172756, + "nauc_mrr_at_5_diff1": 0.7087880335458725, + "nauc_mrr_at_5_max": 0.3204265058352077, + "nauc_mrr_at_5_std": -0.13932460690261497, + "nauc_ndcg_at_1000_diff1": 0.6983503627464738, + "nauc_ndcg_at_1000_max": 0.3362940931937794, + "nauc_ndcg_at_1000_std": -0.11846915079017828, + "nauc_ndcg_at_100_diff1": 0.6966507263236568, + "nauc_ndcg_at_100_max": 0.3384626044974317, + "nauc_ndcg_at_100_std": -0.116606050883595, + "nauc_ndcg_at_10_diff1": 0.6940021959106214, + "nauc_ndcg_at_10_max": 0.3392411122413264, + "nauc_ndcg_at_10_std": -0.12339023140215387, + "nauc_ndcg_at_1_diff1": 0.7452410522141724, + "nauc_ndcg_at_1_max": 0.28782222120749595, + "nauc_ndcg_at_1_std": -0.13706654337741186, + "nauc_ndcg_at_20_diff1": 0.6936890238278103, + "nauc_ndcg_at_20_max": 0.3394952614567259, + "nauc_ndcg_at_20_std": -0.12019483072432446, + "nauc_ndcg_at_3_diff1": 0.6938889772235773, + "nauc_ndcg_at_3_max": 0.3299414560925414, + "nauc_ndcg_at_3_std": -0.1412958088557437, + "nauc_ndcg_at_5_diff1": 0.6948162129076433, + "nauc_ndcg_at_5_max": 0.3365054254184425, + "nauc_ndcg_at_5_std": -0.13800201216164026, + "nauc_precision_at_1000_diff1": 0.5050542815996398, + "nauc_precision_at_1000_max": 0.6855860988463227, + "nauc_precision_at_1000_std": 0.33458157274208816, + "nauc_precision_at_100_diff1": 0.5548520009206096, + "nauc_precision_at_100_max": 0.5735102324162823, + "nauc_precision_at_100_std": 0.16390951838317158, + "nauc_precision_at_10_diff1": 0.6084210789182641, + "nauc_precision_at_10_max": 0.4481211529491943, + "nauc_precision_at_10_std": -0.04954312684422696, + "nauc_precision_at_1_diff1": 0.7452410522141724, + "nauc_precision_at_1_max": 0.28782222120749595, + "nauc_precision_at_1_std": -0.13706654337741186, + "nauc_precision_at_20_diff1": 0.5813507694744582, + "nauc_precision_at_20_max": 0.4831599306389209, + "nauc_precision_at_20_std": -0.0005069736227828523, + "nauc_precision_at_3_diff1": 0.6417040266170304, + "nauc_precision_at_3_max": 0.3759689789121299, + "nauc_precision_at_3_std": -0.1425747620899018, + "nauc_precision_at_5_diff1": 0.6327949328153416, + "nauc_precision_at_5_max": 0.41064792712156045, + "nauc_precision_at_5_std": -0.13052811113986146, + "nauc_recall_at_1000_diff1": 0.5050542815996344, + "nauc_recall_at_1000_max": 0.6855860988463183, + "nauc_recall_at_1000_std": 0.3345815727420872, + "nauc_recall_at_100_diff1": 0.5548520009206097, + "nauc_recall_at_100_max": 0.57351023241628, + "nauc_recall_at_100_std": 0.16390951838317228, + "nauc_recall_at_10_diff1": 0.6084210789182622, + "nauc_recall_at_10_max": 0.44812115294919425, + "nauc_recall_at_10_std": -0.04954312684422914, + "nauc_recall_at_1_diff1": 0.7452410522141724, + "nauc_recall_at_1_max": 0.28782222120749595, + "nauc_recall_at_1_std": -0.13706654337741186, + "nauc_recall_at_20_diff1": 0.5813507694744565, + "nauc_recall_at_20_max": 0.4831599306389225, + "nauc_recall_at_20_std": -0.0005069736227799365, + "nauc_recall_at_3_diff1": 0.6417040266170294, + "nauc_recall_at_3_max": 0.37596897891212816, + "nauc_recall_at_3_std": -0.1425747620899025, + "nauc_recall_at_5_diff1": 0.6327949328153424, + "nauc_recall_at_5_max": 0.41064792712156245, + "nauc_recall_at_5_std": -0.1305281111398608, + "ndcg_at_1": 0.6363, + "ndcg_at_10": 0.76264, + "ndcg_at_100": 0.77739, + "ndcg_at_1000": 0.78069, + "ndcg_at_20": 0.76995, + "ndcg_at_3": 0.73102, + "ndcg_at_5": 0.7485, + "precision_at_1": 0.6363, + "precision_at_10": 0.08817, + "precision_at_100": 0.0095, + "precision_at_1000": 0.00098, + "precision_at_20": 0.04552, + "precision_at_3": 0.26527, + "precision_at_5": 0.16764, + "recall_at_1": 0.6363, + "recall_at_10": 0.8817, + "recall_at_100": 0.9501, + "recall_at_1000": 0.9764, + "recall_at_20": 0.9104, + "recall_at_3": 0.7958, + "recall_at_5": 0.8382 + } + ] + }, + "task_name": "RiaNewsRetrieval" +} \ No newline at end of file diff --git a/results/Alibaba-NLP__gte-Qwen2-7B-instruct/e26182b2122f4435e8b3ebecbf363990f409b45b/RuBQRetrieval.json b/results/Alibaba-NLP__gte-Qwen2-7B-instruct/e26182b2122f4435e8b3ebecbf363990f409b45b/RuBQRetrieval.json new file mode 100644 index 000000000..1e2d2b1e1 --- /dev/null +++ b/results/Alibaba-NLP__gte-Qwen2-7B-instruct/e26182b2122f4435e8b3ebecbf363990f409b45b/RuBQRetrieval.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "e19b6ffa60b3bc248e0b41f4cc37c26a55c2a67b", + "evaluation_time": 10754.438274145126, + "kg_co2_emissions": null, + "mteb_version": "1.19.2", + "scores": { + "test": [ + { + "hf_subset": "default", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.72721, + "map_at_1": 0.4312, + "map_at_10": 0.64619, + "map_at_100": 0.65518, + "map_at_1000": 0.65534, + "map_at_20": 0.65266, + "map_at_3": 0.58546, + "map_at_5": 0.62172, + "mrr_at_1": 0.6087470449172577, + "mrr_at_10": 0.7270284532252617, + "mrr_at_100": 0.7296288105650283, + "mrr_at_1000": 0.7296652102752164, + "mrr_at_20": 0.7287535458338327, + "mrr_at_3": 0.7058707643814026, + "mrr_at_5": 0.7185185185185186, + "nauc_map_at_1000_diff1": 0.3447169288633134, + "nauc_map_at_1000_max": 0.2638946322407822, + "nauc_map_at_1000_std": -0.1310777725332977, + "nauc_map_at_100_diff1": 0.3446683562335147, + "nauc_map_at_100_max": 0.26409532294796706, + "nauc_map_at_100_std": -0.13082521835419522, + "nauc_map_at_10_diff1": 0.34092481413268905, + "nauc_map_at_10_max": 0.26303999195230143, + "nauc_map_at_10_std": -0.13486478824888185, + "nauc_map_at_1_diff1": 0.38370308796413183, + "nauc_map_at_1_max": 0.16919635163548577, + "nauc_map_at_1_std": -0.11187137488765482, + "nauc_map_at_20_diff1": 0.3434116550665268, + "nauc_map_at_20_max": 0.26467730256319827, + "nauc_map_at_20_std": -0.13097489484725233, + "nauc_map_at_3_diff1": 0.3384634965672529, + "nauc_map_at_3_max": 0.23474358724777586, + "nauc_map_at_3_std": -0.15118611498587806, + "nauc_map_at_5_diff1": 0.33411730694228164, + "nauc_map_at_5_max": 0.2502394850443781, + "nauc_map_at_5_std": -0.15231880355585192, + "nauc_mrr_at_1000_diff1": 0.41886433572248444, + "nauc_mrr_at_1000_max": 0.3054599501301002, + "nauc_mrr_at_1000_std": -0.1458286584152427, + "nauc_mrr_at_100_diff1": 0.41885075900152263, + "nauc_mrr_at_100_max": 0.305539656538424, + "nauc_mrr_at_100_std": -0.14572616786679835, + "nauc_mrr_at_10_diff1": 0.41775269703219237, + "nauc_mrr_at_10_max": 0.30686587325386316, + "nauc_mrr_at_10_std": -0.14694790038384506, + "nauc_mrr_at_1_diff1": 0.44428690481948596, + "nauc_mrr_at_1_max": 0.27246108999745855, + "nauc_mrr_at_1_std": -0.14614351147097357, + "nauc_mrr_at_20_diff1": 0.4184403142594404, + "nauc_mrr_at_20_max": 0.30639856339110866, + "nauc_mrr_at_20_std": -0.1449977659388187, + "nauc_mrr_at_3_diff1": 0.41372610628772566, + "nauc_mrr_at_3_max": 0.30416387148671564, + "nauc_mrr_at_3_std": -0.15888060977773283, + "nauc_mrr_at_5_diff1": 0.41488985223343416, + "nauc_mrr_at_5_max": 0.30619399540925823, + "nauc_mrr_at_5_std": -0.15373415421229114, + "nauc_ndcg_at_1000_diff1": 0.36232626067916784, + "nauc_ndcg_at_1000_max": 0.29031703290905964, + "nauc_ndcg_at_1000_std": -0.12159318511495983, + "nauc_ndcg_at_100_diff1": 0.3618716232436015, + "nauc_ndcg_at_100_max": 0.29521093769166706, + "nauc_ndcg_at_100_std": -0.11630952506985018, + "nauc_ndcg_at_10_diff1": 0.34765117619950164, + "nauc_ndcg_at_10_max": 0.2965944304633793, + "nauc_ndcg_at_10_std": -0.12505128762815876, + "nauc_ndcg_at_1_diff1": 0.44428690481948596, + "nauc_ndcg_at_1_max": 0.27246108999745855, + "nauc_ndcg_at_1_std": -0.14614351147097357, + "nauc_ndcg_at_20_diff1": 0.35531735346248094, + "nauc_ndcg_at_20_max": 0.3010176774254564, + "nauc_ndcg_at_20_std": -0.11255287639605269, + "nauc_ndcg_at_3_diff1": 0.3420886304443521, + "nauc_ndcg_at_3_max": 0.25838484366256576, + "nauc_ndcg_at_3_std": -0.16250275304401024, + "nauc_ndcg_at_5_diff1": 0.33548773624451445, + "nauc_ndcg_at_5_max": 0.27317032592218815, + "nauc_ndcg_at_5_std": -0.15962583588049808, + "nauc_precision_at_1000_diff1": -0.0916007934050276, + "nauc_precision_at_1000_max": 0.061047945777383004, + "nauc_precision_at_1000_std": 0.07835606446834495, + "nauc_precision_at_100_diff1": -0.0798517674467194, + "nauc_precision_at_100_max": 0.08935480915294748, + "nauc_precision_at_100_std": 0.09113103389286961, + "nauc_precision_at_10_diff1": -0.04233260693188345, + "nauc_precision_at_10_max": 0.15837069669423517, + "nauc_precision_at_10_std": 0.05070386041092458, + "nauc_precision_at_1_diff1": 0.44428690481948596, + "nauc_precision_at_1_max": 0.27246108999745855, + "nauc_precision_at_1_std": -0.14614351147097357, + "nauc_precision_at_20_diff1": -0.05873431407887623, + "nauc_precision_at_20_max": 0.13452703995129695, + "nauc_precision_at_20_std": 0.08634200979554917, + "nauc_precision_at_3_diff1": 0.08792700232120207, + "nauc_precision_at_3_max": 0.21828437894778335, + "nauc_precision_at_3_std": -0.09576528672313146, + "nauc_precision_at_5_diff1": 0.00601059781594497, + "nauc_precision_at_5_max": 0.18631811545940238, + "nauc_precision_at_5_std": -0.04604478344459906, + "nauc_recall_at_1000_diff1": 0.428220744238768, + "nauc_recall_at_1000_max": 0.5684819166676462, + "nauc_recall_at_1000_std": 0.797099400423435, + "nauc_recall_at_100_diff1": 0.3069302578444726, + "nauc_recall_at_100_max": 0.6027928698219505, + "nauc_recall_at_100_std": 0.35053843932928613, + "nauc_recall_at_10_diff1": 0.22602805603232898, + "nauc_recall_at_10_max": 0.35913848064145565, + "nauc_recall_at_10_std": -0.04657662466837492, + "nauc_recall_at_1_diff1": 0.38370308796413183, + "nauc_recall_at_1_max": 0.16919635163548577, + "nauc_recall_at_1_std": -0.11187137488765482, + "nauc_recall_at_20_diff1": 0.2454854897294018, + "nauc_recall_at_20_max": 0.44671576753271947, + "nauc_recall_at_20_std": 0.0936532163576088, + "nauc_recall_at_3_diff1": 0.26240334243374963, + "nauc_recall_at_3_max": 0.2268011880779738, + "nauc_recall_at_3_std": -0.17934844172947384, + "nauc_recall_at_5_diff1": 0.23018760106446154, + "nauc_recall_at_5_max": 0.2630808715013529, + "nauc_recall_at_5_std": -0.1742550737116759, + "ndcg_at_1": 0.60875, + "ndcg_at_10": 0.72721, + "ndcg_at_100": 0.75403, + "ndcg_at_1000": 0.75667, + "ndcg_at_20": 0.74221, + "ndcg_at_3": 0.64732, + "ndcg_at_5": 0.68744, + "precision_at_1": 0.60875, + "precision_at_10": 0.14439, + "precision_at_100": 0.01639, + "precision_at_1000": 0.00167, + "precision_at_20": 0.0771, + "precision_at_3": 0.35244, + "precision_at_5": 0.24858, + "recall_at_1": 0.4312, + "recall_at_10": 0.87758, + "recall_at_100": 0.97942, + "recall_at_1000": 0.99695, + "recall_at_20": 0.926, + "recall_at_3": 0.68357, + "recall_at_5": 0.77699 + } + ] + }, + "task_name": "RuBQRetrieval" +} \ No newline at end of file diff --git a/results/Alibaba-NLP__gte-Qwen2-7B-instruct/e26182b2122f4435e8b3ebecbf363990f409b45b/RuReviewsClassification.json b/results/Alibaba-NLP__gte-Qwen2-7B-instruct/e26182b2122f4435e8b3ebecbf363990f409b45b/RuReviewsClassification.json new file mode 100644 index 000000000..e55562562 --- /dev/null +++ b/results/Alibaba-NLP__gte-Qwen2-7B-instruct/e26182b2122f4435e8b3ebecbf363990f409b45b/RuReviewsClassification.json @@ -0,0 +1,73 @@ +{ + "dataset_revision": "f6d2c31f4dc6b88f468552750bfec05b4b41b05a", + "evaluation_time": 191.76958346366882, + "kg_co2_emissions": null, + "mteb_version": "1.19.0", + "scores": { + "test": [ + { + "accuracy": 0.748486328125, + "f1": 0.7454639412847763, + "f1_weighted": 0.7454682073224734, + "hf_subset": "default", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.748486328125, + "scores_per_experiment": [ + { + "accuracy": 0.74951171875, + "f1": 0.7456733319247455, + "f1_weighted": 0.7456723362771012 + }, + { + "accuracy": 0.74169921875, + "f1": 0.742974691093561, + "f1_weighted": 0.7430017044526305 + }, + { + "accuracy": 0.748046875, + "f1": 0.7412380269844728, + "f1_weighted": 0.7412380304288668 + }, + { + "accuracy": 0.7509765625, + "f1": 0.7484609393238836, + "f1_weighted": 0.7484685534871924 + }, + { + "accuracy": 0.7578125, + "f1": 0.7581913452770745, + "f1_weighted": 0.7581967751402336 + }, + { + "accuracy": 0.75, + "f1": 0.7477398602792508, + "f1_weighted": 0.7477424508205288 + }, + { + "accuracy": 0.7451171875, + "f1": 0.7410496116626378, + "f1_weighted": 0.7410517362292125 + }, + { + "accuracy": 0.7568359375, + "f1": 0.7557545747176411, + "f1_weighted": 0.7557582060377059 + }, + { + "accuracy": 0.7314453125, + "f1": 0.7190902047007594, + "f1_weighted": 0.7190816360684438 + }, + { + "accuracy": 0.75341796875, + "f1": 0.7544668268837377, + "f1_weighted": 0.7544706442828187 + } + ] + } + ] + }, + "task_name": "RuReviewsClassification" +} \ No newline at end of file diff --git a/results/Alibaba-NLP__gte-Qwen2-7B-instruct/e26182b2122f4435e8b3ebecbf363990f409b45b/RuSTSBenchmarkSTS.json b/results/Alibaba-NLP__gte-Qwen2-7B-instruct/e26182b2122f4435e8b3ebecbf363990f409b45b/RuSTSBenchmarkSTS.json new file mode 100644 index 000000000..9a65a0d2d --- /dev/null +++ b/results/Alibaba-NLP__gte-Qwen2-7B-instruct/e26182b2122f4435e8b3ebecbf363990f409b45b/RuSTSBenchmarkSTS.json @@ -0,0 +1,26 @@ +{ + "dataset_revision": "7cf24f325c6da6195df55bef3d86b5e0616f3018", + "evaluation_time": 106.68932390213013, + "kg_co2_emissions": null, + "mteb_version": "1.19.2", + "scores": { + "test": [ + { + "cosine_pearson": 0.8416361804136121, + "cosine_spearman": 0.8381677798722024, + "euclidean_pearson": 0.8222911693812988, + "euclidean_spearman": 0.8381610470028282, + "hf_subset": "default", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.8381677798722024, + "manhattan_pearson": 0.8243306801018795, + "manhattan_spearman": 0.8413001762150166, + "pearson": 0.8416361804136121, + "spearman": 0.8381677798722024 + } + ] + }, + "task_name": "RuSTSBenchmarkSTS" +} \ No newline at end of file diff --git a/results/Alibaba-NLP__gte-Qwen2-7B-instruct/e26182b2122f4435e8b3ebecbf363990f409b45b/RuSciBenchGRNTIClassification.json b/results/Alibaba-NLP__gte-Qwen2-7B-instruct/e26182b2122f4435e8b3ebecbf363990f409b45b/RuSciBenchGRNTIClassification.json new file mode 100644 index 000000000..b57d35948 --- /dev/null +++ b/results/Alibaba-NLP__gte-Qwen2-7B-instruct/e26182b2122f4435e8b3ebecbf363990f409b45b/RuSciBenchGRNTIClassification.json @@ -0,0 +1,73 @@ +{ + "dataset_revision": "673a610d6d3dd91a547a0d57ae1b56f37ebbf6a1", + "evaluation_time": 1293.1949758529663, + "kg_co2_emissions": null, + "mteb_version": "1.19.0", + "scores": { + "test": [ + { + "accuracy": 0.710107421875, + "f1": 0.701648725803443, + "f1_weighted": 0.7017400052510243, + "hf_subset": "default", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.710107421875, + "scores_per_experiment": [ + { + "accuracy": 0.71728515625, + "f1": 0.7108764312793586, + "f1_weighted": 0.7109639882167645 + }, + { + "accuracy": 0.7080078125, + "f1": 0.699738185002647, + "f1_weighted": 0.699825965639565 + }, + { + "accuracy": 0.7099609375, + "f1": 0.7012021489841622, + "f1_weighted": 0.7013362130760885 + }, + { + "accuracy": 0.70361328125, + "f1": 0.6941508465981153, + "f1_weighted": 0.6942357519145657 + }, + { + "accuracy": 0.7109375, + "f1": 0.7048367172562011, + "f1_weighted": 0.7048818242224332 + }, + { + "accuracy": 0.70458984375, + "f1": 0.6950303162292225, + "f1_weighted": 0.6951287094150025 + }, + { + "accuracy": 0.71044921875, + "f1": 0.6986752257329953, + "f1_weighted": 0.6988003586450718 + }, + { + "accuracy": 0.70849609375, + "f1": 0.699328322317944, + "f1_weighted": 0.699386279051781 + }, + { + "accuracy": 0.7216796875, + "f1": 0.7143102130127655, + "f1_weighted": 0.7144266911286569 + }, + { + "accuracy": 0.7060546875, + "f1": 0.698338851621017, + "f1_weighted": 0.6984142712003144 + } + ] + } + ] + }, + "task_name": "RuSciBenchGRNTIClassification" +} \ No newline at end of file diff --git a/results/Alibaba-NLP__gte-Qwen2-7B-instruct/e26182b2122f4435e8b3ebecbf363990f409b45b/RuSciBenchGRNTIClusteringP2P.json b/results/Alibaba-NLP__gte-Qwen2-7B-instruct/e26182b2122f4435e8b3ebecbf363990f409b45b/RuSciBenchGRNTIClusteringP2P.json new file mode 100644 index 000000000..3d533aabd --- /dev/null +++ b/results/Alibaba-NLP__gte-Qwen2-7B-instruct/e26182b2122f4435e8b3ebecbf363990f409b45b/RuSciBenchGRNTIClusteringP2P.json @@ -0,0 +1,34 @@ +{ + "dataset_revision": "673a610d6d3dd91a547a0d57ae1b56f37ebbf6a1", + "evaluation_time": 679.6372694969177, + "kg_co2_emissions": null, + "mteb_version": "1.19.0", + "scores": { + "test": [ + { + "hf_subset": "default", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.6519268070119939, + "v_measure": 0.6519268070119939, + "v_measure_std": 0.007900076230809345, + "v_measures": { + "Level 0": [ + 0.6447945843834892, + 0.6567396334972696, + 0.6518928823461254, + 0.6565098766059223, + 0.6452957140982045, + 0.6553086469851236, + 0.6358151264890465, + 0.6523450875361834, + 0.6663760598818013, + 0.6541904582967716 + ] + } + } + ] + }, + "task_name": "RuSciBenchGRNTIClusteringP2P" +} \ No newline at end of file diff --git a/results/Alibaba-NLP__gte-Qwen2-7B-instruct/e26182b2122f4435e8b3ebecbf363990f409b45b/RuSciBenchOECDClassification.json b/results/Alibaba-NLP__gte-Qwen2-7B-instruct/e26182b2122f4435e8b3ebecbf363990f409b45b/RuSciBenchOECDClassification.json new file mode 100644 index 000000000..2ac728635 --- /dev/null +++ b/results/Alibaba-NLP__gte-Qwen2-7B-instruct/e26182b2122f4435e8b3ebecbf363990f409b45b/RuSciBenchOECDClassification.json @@ -0,0 +1,73 @@ +{ + "dataset_revision": "26c88e99dcaba32bb45d0e1bfc21902337f6d471", + "evaluation_time": 1287.459352016449, + "kg_co2_emissions": null, + "mteb_version": "1.19.0", + "scores": { + "test": [ + { + "accuracy": 0.576806640625, + "f1": 0.558697156214207, + "f1_weighted": 0.5587030998753857, + "hf_subset": "default", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.576806640625, + "scores_per_experiment": [ + { + "accuracy": 0.580078125, + "f1": 0.5616682239825546, + "f1_weighted": 0.56169577008473 + }, + { + "accuracy": 0.58642578125, + "f1": 0.5701055641110215, + "f1_weighted": 0.5700933785802509 + }, + { + "accuracy": 0.57861328125, + "f1": 0.5652514158431722, + "f1_weighted": 0.5653105419445209 + }, + { + "accuracy": 0.564453125, + "f1": 0.5358482557199754, + "f1_weighted": 0.5357955531874012 + }, + { + "accuracy": 0.583984375, + "f1": 0.5637111209211457, + "f1_weighted": 0.5637318715762852 + }, + { + "accuracy": 0.58544921875, + "f1": 0.5694719546985216, + "f1_weighted": 0.5694742184896898 + }, + { + "accuracy": 0.5712890625, + "f1": 0.5469931911736399, + "f1_weighted": 0.547003944145249 + }, + { + "accuracy": 0.587890625, + "f1": 0.5758821807883708, + "f1_weighted": 0.5759186645524731 + }, + { + "accuracy": 0.5576171875, + "f1": 0.5386393655682765, + "f1_weighted": 0.5386000765225613 + }, + { + "accuracy": 0.572265625, + "f1": 0.5594002893353909, + "f1_weighted": 0.559406979670696 + } + ] + } + ] + }, + "task_name": "RuSciBenchOECDClassification" +} \ No newline at end of file diff --git a/results/Alibaba-NLP__gte-Qwen2-7B-instruct/e26182b2122f4435e8b3ebecbf363990f409b45b/RuSciBenchOECDClusteringP2P.json b/results/Alibaba-NLP__gte-Qwen2-7B-instruct/e26182b2122f4435e8b3ebecbf363990f409b45b/RuSciBenchOECDClusteringP2P.json new file mode 100644 index 000000000..c086e0f8e --- /dev/null +++ b/results/Alibaba-NLP__gte-Qwen2-7B-instruct/e26182b2122f4435e8b3ebecbf363990f409b45b/RuSciBenchOECDClusteringP2P.json @@ -0,0 +1,34 @@ +{ + "dataset_revision": "26c88e99dcaba32bb45d0e1bfc21902337f6d471", + "evaluation_time": 658.6950669288635, + "kg_co2_emissions": null, + "mteb_version": "1.19.0", + "scores": { + "test": [ + { + "hf_subset": "default", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.5559965921284723, + "v_measure": 0.5559965921284723, + "v_measure_std": 0.005031001800641698, + "v_measures": { + "Level 0": [ + 0.5551144928714475, + 0.5628365252177606, + 0.5513262416874603, + 0.5614933026465954, + 0.5649183884099191, + 0.5536784637652664, + 0.552375941162476, + 0.5556248350117649, + 0.5536615170870941, + 0.5489362134249387 + ] + } + } + ] + }, + "task_name": "RuSciBenchOECDClusteringP2P" +} \ No newline at end of file diff --git a/results/Alibaba-NLP__gte-Qwen2-7B-instruct/e26182b2122f4435e8b3ebecbf363990f409b45b/STS22.json b/results/Alibaba-NLP__gte-Qwen2-7B-instruct/e26182b2122f4435e8b3ebecbf363990f409b45b/STS22.json new file mode 100644 index 000000000..5b2e408fc --- /dev/null +++ b/results/Alibaba-NLP__gte-Qwen2-7B-instruct/e26182b2122f4435e8b3ebecbf363990f409b45b/STS22.json @@ -0,0 +1,26 @@ +{ + "dataset_revision": "de9d86b3b84231dc21f76c7b7af1f28e2f57f6e3", + "evaluation_time": 197.66416430473328, + "kg_co2_emissions": null, + "mteb_version": "1.19.2", + "scores": { + "test": [ + { + "cosine_pearson": 0.5714229432286685, + "cosine_spearman": 0.6382109566019848, + "euclidean_pearson": 0.5677514368499141, + "euclidean_spearman": 0.6381828494732269, + "hf_subset": "ru", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.6382109566019848, + "manhattan_pearson": 0.5731336581682932, + "manhattan_spearman": 0.639101332231655, + "pearson": 0.5714229432286685, + "spearman": 0.6382109566019848 + } + ] + }, + "task_name": "STS22" +} \ No newline at end of file diff --git a/results/Alibaba-NLP__gte-Qwen2-7B-instruct/e26182b2122f4435e8b3ebecbf363990f409b45b/SensitiveTopicsClassification.json b/results/Alibaba-NLP__gte-Qwen2-7B-instruct/e26182b2122f4435e8b3ebecbf363990f409b45b/SensitiveTopicsClassification.json new file mode 100644 index 000000000..7a6dc8265 --- /dev/null +++ b/results/Alibaba-NLP__gte-Qwen2-7B-instruct/e26182b2122f4435e8b3ebecbf363990f409b45b/SensitiveTopicsClassification.json @@ -0,0 +1,73 @@ +{ + "dataset_revision": "416b34a802308eac30e4192afc0ff99bb8dcc7f2", + "evaluation_time": 232.20746874809265, + "kg_co2_emissions": null, + "mteb_version": "1.19.0", + "scores": { + "test": [ + { + "accuracy": 0.357568359375, + "f1": 0.44401063725164747, + "hf_subset": "default", + "languages": [ + "rus-Cyrl" + ], + "lrap": 0.5657721625434015, + "main_score": 0.357568359375, + "scores_per_experiment": [ + { + "accuracy": 0.35693359375, + "f1": 0.4336466228027721, + "lrap": 0.5473904079861084 + }, + { + "accuracy": 0.35791015625, + "f1": 0.4746327536722153, + "lrap": 0.5837673611111117 + }, + { + "accuracy": 0.33837890625, + "f1": 0.4005930426494919, + "lrap": 0.5191379123263831 + }, + { + "accuracy": 0.3525390625, + "f1": 0.4359109607148087, + "lrap": 0.5552029079861093 + }, + { + "accuracy": 0.35205078125, + "f1": 0.4566950412276132, + "lrap": 0.5716010199652769 + }, + { + "accuracy": 0.35888671875, + "f1": 0.43537715295775997, + "lrap": 0.5726182725694435 + }, + { + "accuracy": 0.369140625, + "f1": 0.44476874370722136, + "lrap": 0.5789116753472221 + }, + { + "accuracy": 0.34765625, + "f1": 0.42773027274239583, + "lrap": 0.5599636501736103 + }, + { + "accuracy": 0.38037109375, + "f1": 0.4611689384497001, + "lrap": 0.5740695529513872 + }, + { + "accuracy": 0.36181640625, + "f1": 0.46958284359249625, + "lrap": 0.595058865017362 + } + ] + } + ] + }, + "task_name": "SensitiveTopicsClassification" +} \ No newline at end of file diff --git a/results/Alibaba-NLP__gte-Qwen2-7B-instruct/e26182b2122f4435e8b3ebecbf363990f409b45b/model_meta.json b/results/Alibaba-NLP__gte-Qwen2-7B-instruct/e26182b2122f4435e8b3ebecbf363990f409b45b/model_meta.json index 5ab177033..be1a793f8 100644 --- a/results/Alibaba-NLP__gte-Qwen2-7B-instruct/e26182b2122f4435e8b3ebecbf363990f409b45b/model_meta.json +++ b/results/Alibaba-NLP__gte-Qwen2-7B-instruct/e26182b2122f4435e8b3ebecbf363990f409b45b/model_meta.json @@ -1 +1,23 @@ -{"name": "Alibaba-NLP/gte-Qwen2-7B-instruct", "revision": "e26182b2122f4435e8b3ebecbf363990f409b45b", "release_date": "2024-06-15", "languages": null, "n_parameters": 7613000000, "memory_usage": null, "max_tokens": null, "embed_dim": 3584, "license": "apache-2.0", "open_weights": true, "public_training_data": null, "public_training_code": null, "framework": ["Sentence Transformers", "PyTorch"], "reference": "https://huggingface.co/Alibaba-NLP/gte-Qwen2-7B-instruct", "similarity_fn_name": "cosine", "use_instructions": true, "zero_shot_benchmarks": null, "loader": "instruct_wrapper"} \ No newline at end of file +{ + "name": "Alibaba-NLP/gte-Qwen2-7B-instruct", + "revision": "e26182b2122f4435e8b3ebecbf363990f409b45b", + "release_date": "2024-06-15", + "languages": null, + "n_parameters": 7613000000, + "memory_usage": null, + "max_tokens": null, + "embed_dim": 3584, + "license": "apache-2.0", + "open_weights": true, + "public_training_data": null, + "public_training_code": null, + "framework": [ + "Sentence Transformers", + "PyTorch" + ], + "reference": "https://huggingface.co/Alibaba-NLP/gte-Qwen2-7B-instruct", + "similarity_fn_name": "cosine", + "use_instructions": true, + "zero_shot_benchmarks": null, + "loader": "instruct_wrapper" +} \ No newline at end of file diff --git a/results/BAAI__bge-large-en-v1.5/d4aa6901d3a41ba39fb536a557fa166f842b0e09/CEDRClassification.json b/results/BAAI__bge-large-en-v1.5/d4aa6901d3a41ba39fb536a557fa166f842b0e09/CEDRClassification.json new file mode 100644 index 000000000..a4e98a41a --- /dev/null +++ b/results/BAAI__bge-large-en-v1.5/d4aa6901d3a41ba39fb536a557fa166f842b0e09/CEDRClassification.json @@ -0,0 +1,73 @@ +{ + "dataset_revision": "c0ba03d058e3e1b2f3fd20518875a4563dd12db4", + "evaluation_time": 22.414844751358032, + "kg_co2_emissions": null, + "mteb_version": "1.19.0", + "scores": { + "test": [ + { + "accuracy": 0.3615302869287992, + "f1": 0.1458134437856463, + "hf_subset": "default", + "languages": [ + "rus-Cyrl" + ], + "lrap": 0.5823007438894916, + "main_score": 0.3615302869287992, + "scores_per_experiment": [ + { + "accuracy": 0.36238044633368754, + "f1": 0.1738226893370129, + "lrap": 0.6019128586610102 + }, + { + "accuracy": 0.357066950053135, + "f1": 0.13135345640276855, + "lrap": 0.565674814027643 + }, + { + "accuracy": 0.39744952178533477, + "f1": 0.17923859425659738, + "lrap": 0.600106269925623 + }, + { + "accuracy": 0.3746014877789586, + "f1": 0.16629131348953713, + "lrap": 0.5979808714134021 + }, + { + "accuracy": 0.3294367693942614, + "f1": 0.11044571046095641, + "lrap": 0.5551009564293434 + }, + { + "accuracy": 0.3421891604675877, + "f1": 0.11876763396960596, + "lrap": 0.5668969181721699 + }, + { + "accuracy": 0.34112646121147716, + "f1": 0.11563938856369067, + "lrap": 0.5606801275239235 + }, + { + "accuracy": 0.33846971307120083, + "f1": 0.11660133114358975, + "lrap": 0.5576514346440089 + }, + { + "accuracy": 0.4022316684378321, + "f1": 0.16335656216519193, + "lrap": 0.6137088204038377 + }, + { + "accuracy": 0.3703506907545165, + "f1": 0.18261775806751232, + "lrap": 0.6032943676939544 + } + ] + } + ] + }, + "task_name": "CEDRClassification" +} \ No newline at end of file diff --git a/results/BAAI__bge-large-en-v1.5/d4aa6901d3a41ba39fb536a557fa166f842b0e09/GeoreviewClassification.json b/results/BAAI__bge-large-en-v1.5/d4aa6901d3a41ba39fb536a557fa166f842b0e09/GeoreviewClassification.json new file mode 100644 index 000000000..652854716 --- /dev/null +++ b/results/BAAI__bge-large-en-v1.5/d4aa6901d3a41ba39fb536a557fa166f842b0e09/GeoreviewClassification.json @@ -0,0 +1,73 @@ +{ + "dataset_revision": "3765c0d1de6b7d264bc459433c45e5a75513839c", + "evaluation_time": 47.31564259529114, + "kg_co2_emissions": null, + "mteb_version": "1.19.0", + "scores": { + "test": [ + { + "accuracy": 0.286376953125, + "f1": 0.2817106143810647, + "f1_weighted": 0.28172267414733076, + "hf_subset": "default", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.286376953125, + "scores_per_experiment": [ + { + "accuracy": 0.298828125, + "f1": 0.2967334793436584, + "f1_weighted": 0.29678412525616205 + }, + { + "accuracy": 0.333984375, + "f1": 0.33054997393486335, + "f1_weighted": 0.3305402247998043 + }, + { + "accuracy": 0.30029296875, + "f1": 0.2982352544146397, + "f1_weighted": 0.29824375148703497 + }, + { + "accuracy": 0.27294921875, + "f1": 0.26262761379236854, + "f1_weighted": 0.262678616939202 + }, + { + "accuracy": 0.283203125, + "f1": 0.2726404304031727, + "f1_weighted": 0.27264408631601267 + }, + { + "accuracy": 0.27880859375, + "f1": 0.2755171253729271, + "f1_weighted": 0.27555178807037517 + }, + { + "accuracy": 0.27978515625, + "f1": 0.2804512253764763, + "f1_weighted": 0.28044801265743097 + }, + { + "accuracy": 0.27294921875, + "f1": 0.26878885990512613, + "f1_weighted": 0.26876666300796803 + }, + { + "accuracy": 0.2392578125, + "f1": 0.23451148706643732, + "f1_weighted": 0.23452825922802942 + }, + { + "accuracy": 0.3037109375, + "f1": 0.2970506942009775, + "f1_weighted": 0.2970412137112885 + } + ] + } + ] + }, + "task_name": "GeoreviewClassification" +} \ No newline at end of file diff --git a/results/BAAI__bge-large-en-v1.5/d4aa6901d3a41ba39fb536a557fa166f842b0e09/GeoreviewClusteringP2P.json b/results/BAAI__bge-large-en-v1.5/d4aa6901d3a41ba39fb536a557fa166f842b0e09/GeoreviewClusteringP2P.json new file mode 100644 index 000000000..b90b21993 --- /dev/null +++ b/results/BAAI__bge-large-en-v1.5/d4aa6901d3a41ba39fb536a557fa166f842b0e09/GeoreviewClusteringP2P.json @@ -0,0 +1,34 @@ +{ + "dataset_revision": "97a313c8fc85b47f13f33e7e9a95c1ad888c7fec", + "evaluation_time": 99.4278826713562, + "kg_co2_emissions": null, + "mteb_version": "1.19.0", + "scores": { + "test": [ + { + "hf_subset": "default", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.21872509542751403, + "v_measure": 0.21872509542751403, + "v_measure_std": 0.012745190261776533, + "v_measures": { + "Level 0": [ + 0.22865500165990196, + 0.2336219166469655, + 0.19542311825492456, + 0.21177515676780265, + 0.23280902574306558, + 0.2182752031490612, + 0.22922135317836384, + 0.21630833886416523, + 0.19906202642859197, + 0.222099813582298 + ] + } + } + ] + }, + "task_name": "GeoreviewClusteringP2P" +} \ No newline at end of file diff --git a/results/BAAI__bge-large-en-v1.5/d4aa6901d3a41ba39fb536a557fa166f842b0e09/HeadlineClassification.json b/results/BAAI__bge-large-en-v1.5/d4aa6901d3a41ba39fb536a557fa166f842b0e09/HeadlineClassification.json new file mode 100644 index 000000000..f7fced2d5 --- /dev/null +++ b/results/BAAI__bge-large-en-v1.5/d4aa6901d3a41ba39fb536a557fa166f842b0e09/HeadlineClassification.json @@ -0,0 +1,73 @@ +{ + "dataset_revision": "2fe05ee6b5832cda29f2ef7aaad7b7fe6a3609eb", + "evaluation_time": 19.69994330406189, + "kg_co2_emissions": null, + "mteb_version": "1.19.0", + "scores": { + "test": [ + { + "accuracy": 0.335595703125, + "f1": 0.33458908440732343, + "f1_weighted": 0.3345860076294259, + "hf_subset": "default", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.335595703125, + "scores_per_experiment": [ + { + "accuracy": 0.3681640625, + "f1": 0.3656101890433243, + "f1_weighted": 0.3656331568387766 + }, + { + "accuracy": 0.32958984375, + "f1": 0.32505324340540115, + "f1_weighted": 0.32505657842044205 + }, + { + "accuracy": 0.29638671875, + "f1": 0.2975852771400779, + "f1_weighted": 0.2975630659668219 + }, + { + "accuracy": 0.38818359375, + "f1": 0.39004055208453314, + "f1_weighted": 0.3900600949288251 + }, + { + "accuracy": 0.35009765625, + "f1": 0.3509839144598706, + "f1_weighted": 0.3509723420225092 + }, + { + "accuracy": 0.359375, + "f1": 0.3582394974207763, + "f1_weighted": 0.3582512853806675 + }, + { + "accuracy": 0.302734375, + "f1": 0.3005163832563791, + "f1_weighted": 0.3004982777771801 + }, + { + "accuracy": 0.28125, + "f1": 0.2792003137255434, + "f1_weighted": 0.2792057002111239 + }, + { + "accuracy": 0.34130859375, + "f1": 0.33977513924252817, + "f1_weighted": 0.33977107394764444 + }, + { + "accuracy": 0.3388671875, + "f1": 0.3388863342948001, + "f1_weighted": 0.33884850080026857 + } + ] + } + ] + }, + "task_name": "HeadlineClassification" +} \ No newline at end of file diff --git a/results/BAAI__bge-large-en-v1.5/d4aa6901d3a41ba39fb536a557fa166f842b0e09/InappropriatenessClassification.json b/results/BAAI__bge-large-en-v1.5/d4aa6901d3a41ba39fb536a557fa166f842b0e09/InappropriatenessClassification.json new file mode 100644 index 000000000..99f988f7e --- /dev/null +++ b/results/BAAI__bge-large-en-v1.5/d4aa6901d3a41ba39fb536a557fa166f842b0e09/InappropriatenessClassification.json @@ -0,0 +1,95 @@ +{ + "dataset_revision": "601651fdc45ef243751676e62dd7a19f491c0285", + "evaluation_time": 20.5714328289032, + "kg_co2_emissions": null, + "mteb_version": "1.19.0", + "scores": { + "test": [ + { + "accuracy": 0.518115234375, + "ap": 0.509894761844196, + "ap_weighted": 0.509894761844196, + "f1": 0.5156041585979931, + "f1_weighted": 0.5156041585979931, + "hf_subset": "default", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.518115234375, + "scores_per_experiment": [ + { + "accuracy": 0.52099609375, + "ap": 0.5110412671104091, + "ap_weighted": 0.5110412671104091, + "f1": 0.5167040128167657, + "f1_weighted": 0.5167040128167657 + }, + { + "accuracy": 0.521484375, + "ap": 0.5110954432922272, + "ap_weighted": 0.5110954432922272, + "f1": 0.5099650658689535, + "f1_weighted": 0.5099650658689535 + }, + { + "accuracy": 0.51708984375, + "ap": 0.5088355654761905, + "ap_weighted": 0.5088355654761905, + "f1": 0.5170869653640113, + "f1_weighted": 0.5170869653640113 + }, + { + "accuracy": 0.4775390625, + "ap": 0.4893995331554878, + "ap_weighted": 0.4893995331554878, + "f1": 0.47230324069614665, + "f1_weighted": 0.47230324069614665 + }, + { + "accuracy": 0.546875, + "ap": 0.5256520669291338, + "ap_weighted": 0.5256520669291338, + "f1": 0.5468680857557031, + "f1_weighted": 0.5468680857557031 + }, + { + "accuracy": 0.54150390625, + "ap": 0.5224334745888941, + "ap_weighted": 0.5224334745888941, + "f1": 0.5414355748258272, + "f1_weighted": 0.5414355748258272 + }, + { + "accuracy": 0.5439453125, + "ap": 0.5239621321680081, + "ap_weighted": 0.5239621321680081, + "f1": 0.5438474327777624, + "f1_weighted": 0.5438474327777624 + }, + { + "accuracy": 0.4873046875, + "ap": 0.49381871377268144, + "ap_weighted": 0.49381871377268144, + "f1": 0.4871794871794872, + "f1_weighted": 0.4871794871794872 + }, + { + "accuracy": 0.52294921875, + "ap": 0.5119744301726135, + "ap_weighted": 0.5119744301726135, + "f1": 0.5226049136790941, + "f1_weighted": 0.5226049136790941 + }, + { + "accuracy": 0.50146484375, + "ap": 0.5007349917763158, + "ap_weighted": 0.5007349917763158, + "f1": 0.4980468070161793, + "f1_weighted": 0.4980468070161793 + } + ] + } + ] + }, + "task_name": "InappropriatenessClassification" +} \ No newline at end of file diff --git a/results/BAAI__bge-large-en-v1.5/d4aa6901d3a41ba39fb536a557fa166f842b0e09/KinopoiskClassification.json b/results/BAAI__bge-large-en-v1.5/d4aa6901d3a41ba39fb536a557fa166f842b0e09/KinopoiskClassification.json new file mode 100644 index 000000000..bdb16cd7c --- /dev/null +++ b/results/BAAI__bge-large-en-v1.5/d4aa6901d3a41ba39fb536a557fa166f842b0e09/KinopoiskClassification.json @@ -0,0 +1,73 @@ +{ + "dataset_revision": "5911f26666ac11af46cb9c6849d0dc80a378af24", + "evaluation_time": 48.485769748687744, + "kg_co2_emissions": null, + "mteb_version": "1.19.0", + "scores": { + "test": [ + { + "accuracy": 0.3568, + "f1": 0.35192968966395366, + "f1_weighted": 0.35192968966395366, + "hf_subset": "default", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.3568, + "scores_per_experiment": [ + { + "accuracy": 0.328, + "f1": 0.32647610024859186, + "f1_weighted": 0.32647610024859186 + }, + { + "accuracy": 0.3526666666666667, + "f1": 0.3521702978216255, + "f1_weighted": 0.3521702978216255 + }, + { + "accuracy": 0.37266666666666665, + "f1": 0.36447277595428645, + "f1_weighted": 0.36447277595428645 + }, + { + "accuracy": 0.36733333333333335, + "f1": 0.3676295655617487, + "f1_weighted": 0.3676295655617487 + }, + { + "accuracy": 0.372, + "f1": 0.3695874247856494, + "f1_weighted": 0.3695874247856493 + }, + { + "accuracy": 0.35933333333333334, + "f1": 0.3480512798900026, + "f1_weighted": 0.34805127989000256 + }, + { + "accuracy": 0.3626666666666667, + "f1": 0.3498708871345315, + "f1_weighted": 0.3498708871345315 + }, + { + "accuracy": 0.36333333333333334, + "f1": 0.3606955874220448, + "f1_weighted": 0.3606955874220448 + }, + { + "accuracy": 0.3406666666666667, + "f1": 0.3359419062686464, + "f1_weighted": 0.3359419062686464 + }, + { + "accuracy": 0.34933333333333333, + "f1": 0.34440107155240923, + "f1_weighted": 0.3444010715524092 + } + ] + } + ] + }, + "task_name": "KinopoiskClassification" +} \ No newline at end of file diff --git a/results/BAAI__bge-large-en-v1.5/d4aa6901d3a41ba39fb536a557fa166f842b0e09/MIRACLReranking.json b/results/BAAI__bge-large-en-v1.5/d4aa6901d3a41ba39fb536a557fa166f842b0e09/MIRACLReranking.json new file mode 100644 index 000000000..99ea159d4 --- /dev/null +++ b/results/BAAI__bge-large-en-v1.5/d4aa6901d3a41ba39fb536a557fa166f842b0e09/MIRACLReranking.json @@ -0,0 +1,130 @@ +{ + "dataset_revision": "6d1962c527217f8927fca80f890f14f36b2802af", + "evaluation_time": 2237.675261735916, + "kg_co2_emissions": null, + "mteb_version": "1.19.0", + "scores": { + "dev": [ + { + "MAP@1(MIRACL)": 0.11473, + "MAP@10(MIRACL)": 0.19495, + "MAP@100(MIRACL)": 0.22743, + "MAP@1000(MIRACL)": 0.22743, + "MAP@20(MIRACL)": 0.20996, + "MAP@3(MIRACL)": 0.15618, + "MAP@5(MIRACL)": 0.17454, + "NDCG@1(MIRACL)": 0.19487, + "NDCG@10(MIRACL)": 0.25964, + "NDCG@100(MIRACL)": 0.37773, + "NDCG@1000(MIRACL)": 0.37773, + "NDCG@20(MIRACL)": 0.30172, + "NDCG@3(MIRACL)": 0.20021, + "NDCG@5(MIRACL)": 0.22239, + "P@1(MIRACL)": 0.19487, + "P@10(MIRACL)": 0.07674, + "P@100(MIRACL)": 0.01953, + "P@1000(MIRACL)": 0.00195, + "P@20(MIRACL)": 0.05513, + "P@3(MIRACL)": 0.12216, + "P@5(MIRACL)": 0.10265, + "Recall@1(MIRACL)": 0.11473, + "Recall@10(MIRACL)": 0.35845, + "Recall@100(MIRACL)": 0.79952, + "Recall@1000(MIRACL)": 0.79952, + "Recall@20(MIRACL)": 0.48819, + "Recall@3(MIRACL)": 0.19608, + "Recall@5(MIRACL)": 0.25924, + "hf_subset": "ru", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.25964, + "nAUC_MAP@1000_diff1(MIRACL)": 0.23913399181154177, + "nAUC_MAP@1000_max(MIRACL)": 0.13803682168332826, + "nAUC_MAP@1000_std(MIRACL)": 0.0804762751885424, + "nAUC_MAP@100_diff1(MIRACL)": 0.23913399181154177, + "nAUC_MAP@100_max(MIRACL)": 0.13803682168332826, + "nAUC_MAP@100_std(MIRACL)": 0.0804762751885424, + "nAUC_MAP@10_diff1(MIRACL)": 0.257542754851533, + "nAUC_MAP@10_max(MIRACL)": 0.12014240334175647, + "nAUC_MAP@10_std(MIRACL)": 0.07228538102913817, + "nAUC_MAP@1_diff1(MIRACL)": 0.3268861974662907, + "nAUC_MAP@1_max(MIRACL)": 0.10136228746499974, + "nAUC_MAP@1_std(MIRACL)": 0.0246584917313928, + "nAUC_MAP@20_diff1(MIRACL)": 0.24858180825190235, + "nAUC_MAP@20_max(MIRACL)": 0.1319853730176018, + "nAUC_MAP@20_std(MIRACL)": 0.07754422626737477, + "nAUC_MAP@3_diff1(MIRACL)": 0.2908812112968638, + "nAUC_MAP@3_max(MIRACL)": 0.1042765313262766, + "nAUC_MAP@3_std(MIRACL)": 0.04701597009243808, + "nAUC_MAP@5_diff1(MIRACL)": 0.27627814761351016, + "nAUC_MAP@5_max(MIRACL)": 0.11178200837568693, + "nAUC_MAP@5_std(MIRACL)": 0.06041666018681761, + "nAUC_NDCG@1000_diff1(MIRACL)": 0.1574366438448174, + "nAUC_NDCG@1000_max(MIRACL)": 0.18083622890008338, + "nAUC_NDCG@1000_std(MIRACL)": 0.10315447484595551, + "nAUC_NDCG@100_diff1(MIRACL)": 0.1574366438448174, + "nAUC_NDCG@100_max(MIRACL)": 0.18083622890008338, + "nAUC_NDCG@100_std(MIRACL)": 0.10315447484595551, + "nAUC_NDCG@10_diff1(MIRACL)": 0.21236035211637772, + "nAUC_NDCG@10_max(MIRACL)": 0.13351267488141455, + "nAUC_NDCG@10_std(MIRACL)": 0.0863923318366973, + "nAUC_NDCG@1_diff1(MIRACL)": 0.24942542008647825, + "nAUC_NDCG@1_max(MIRACL)": 0.14478078479184048, + "nAUC_NDCG@1_std(MIRACL)": 0.06926118873985203, + "nAUC_NDCG@20_diff1(MIRACL)": 0.19825423683643145, + "nAUC_NDCG@20_max(MIRACL)": 0.16135288486028238, + "nAUC_NDCG@20_std(MIRACL)": 0.09718606397687699, + "nAUC_NDCG@3_diff1(MIRACL)": 0.2568186789419343, + "nAUC_NDCG@3_max(MIRACL)": 0.12487349573850647, + "nAUC_NDCG@3_std(MIRACL)": 0.061666447284566175, + "nAUC_NDCG@5_diff1(MIRACL)": 0.24028112070771454, + "nAUC_NDCG@5_max(MIRACL)": 0.12570115747291338, + "nAUC_NDCG@5_std(MIRACL)": 0.07096205911196991, + "nAUC_P@1000_diff1(MIRACL)": -0.07626201034239827, + "nAUC_P@1000_max(MIRACL)": 0.14566660017699712, + "nAUC_P@1000_std(MIRACL)": 0.05233556997512764, + "nAUC_P@100_diff1(MIRACL)": -0.07626201034239775, + "nAUC_P@100_max(MIRACL)": 0.1456666001769976, + "nAUC_P@100_std(MIRACL)": 0.05233556997512759, + "nAUC_P@10_diff1(MIRACL)": 0.04535316679931853, + "nAUC_P@10_max(MIRACL)": 0.15534985000862403, + "nAUC_P@10_std(MIRACL)": 0.13328383531188498, + "nAUC_P@1_diff1(MIRACL)": 0.24942542008647825, + "nAUC_P@1_max(MIRACL)": 0.14478078479184048, + "nAUC_P@1_std(MIRACL)": 0.06926118873985203, + "nAUC_P@20_diff1(MIRACL)": 0.004377574575121997, + "nAUC_P@20_max(MIRACL)": 0.1873814860730414, + "nAUC_P@20_std(MIRACL)": 0.11880917917727048, + "nAUC_P@3_diff1(MIRACL)": 0.18321809454963456, + "nAUC_P@3_max(MIRACL)": 0.16103549255676572, + "nAUC_P@3_std(MIRACL)": 0.1028182358274775, + "nAUC_P@5_diff1(MIRACL)": 0.12487324639413754, + "nAUC_P@5_max(MIRACL)": 0.15931370465586733, + "nAUC_P@5_std(MIRACL)": 0.11610331176303595, + "nAUC_Recall@1000_diff1(MIRACL)": -0.15367453460864197, + "nAUC_Recall@1000_max(MIRACL)": 0.34139012706052824, + "nAUC_Recall@1000_std(MIRACL)": 0.2046707924627386, + "nAUC_Recall@100_diff1(MIRACL)": -0.15367453460864197, + "nAUC_Recall@100_max(MIRACL)": 0.34139012706052824, + "nAUC_Recall@100_std(MIRACL)": 0.2046707924627386, + "nAUC_Recall@10_diff1(MIRACL)": 0.13952225006852098, + "nAUC_Recall@10_max(MIRACL)": 0.11776559961776821, + "nAUC_Recall@10_std(MIRACL)": 0.09545637281349739, + "nAUC_Recall@1_diff1(MIRACL)": 0.3268861974662907, + "nAUC_Recall@1_max(MIRACL)": 0.10136228746499974, + "nAUC_Recall@1_std(MIRACL)": 0.0246584917313928, + "nAUC_Recall@20_diff1(MIRACL)": 0.10110263949438449, + "nAUC_Recall@20_max(MIRACL)": 0.18412221725217856, + "nAUC_Recall@20_std(MIRACL)": 0.12361639804169113, + "nAUC_Recall@3_diff1(MIRACL)": 0.24827296631996545, + "nAUC_Recall@3_max(MIRACL)": 0.09670931072800784, + "nAUC_Recall@3_std(MIRACL)": 0.04290473719028078, + "nAUC_Recall@5_diff1(MIRACL)": 0.19794876475488996, + "nAUC_Recall@5_max(MIRACL)": 0.10507099245315263, + "nAUC_Recall@5_std(MIRACL)": 0.06797697072847177 + } + ] + }, + "task_name": "MIRACLReranking" +} \ No newline at end of file diff --git a/results/BAAI__bge-large-en-v1.5/d4aa6901d3a41ba39fb536a557fa166f842b0e09/MIRACLRetrieval.json b/results/BAAI__bge-large-en-v1.5/d4aa6901d3a41ba39fb536a557fa166f842b0e09/MIRACLRetrieval.json new file mode 100644 index 000000000..f3a18b686 --- /dev/null +++ b/results/BAAI__bge-large-en-v1.5/d4aa6901d3a41ba39fb536a557fa166f842b0e09/MIRACLRetrieval.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "main", + "evaluation_time": 86606.05354309082, + "kg_co2_emissions": null, + "mteb_version": "1.19.4", + "scores": { + "dev": [ + { + "hf_subset": "ru", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.10725, + "map_at_1": 0.04305, + "map_at_10": 0.07331, + "map_at_100": 0.08326, + "map_at_1000": 0.08449, + "map_at_20": 0.078, + "map_at_3": 0.05927, + "map_at_5": 0.06641, + "mrr_at_1": 0.09105431309904154, + "mrr_at_10": 0.13922771185151375, + "mrr_at_100": 0.14803465400524043, + "mrr_at_1000": 0.14880584417022055, + "mrr_at_20": 0.1442100449873462, + "mrr_at_3": 0.1216719914802982, + "mrr_at_5": 0.1314164004259851, + "nauc_map_at_1000_diff1": 0.32414949611276933, + "nauc_map_at_1000_max": 0.34638341022930075, + "nauc_map_at_1000_std": 0.22247643961794453, + "nauc_map_at_100_diff1": 0.32487425524582336, + "nauc_map_at_100_max": 0.3442575562271872, + "nauc_map_at_100_std": 0.2190604671124487, + "nauc_map_at_10_diff1": 0.33672669633321334, + "nauc_map_at_10_max": 0.31785455186201866, + "nauc_map_at_10_std": 0.17497569457188997, + "nauc_map_at_1_diff1": 0.41682683282981453, + "nauc_map_at_1_max": 0.32242722192485945, + "nauc_map_at_1_std": 0.13029436319937457, + "nauc_map_at_20_diff1": 0.33193125759539144, + "nauc_map_at_20_max": 0.3276614755734197, + "nauc_map_at_20_std": 0.19186924304217196, + "nauc_map_at_3_diff1": 0.36811483035181636, + "nauc_map_at_3_max": 0.3154363573787111, + "nauc_map_at_3_std": 0.1460674281604091, + "nauc_map_at_5_diff1": 0.34110119843272263, + "nauc_map_at_5_max": 0.30727795888756104, + "nauc_map_at_5_std": 0.15560080667533074, + "nauc_mrr_at_1000_diff1": 0.28807432016736345, + "nauc_mrr_at_1000_max": 0.29375612199532236, + "nauc_mrr_at_1000_std": 0.18956601816353347, + "nauc_mrr_at_100_diff1": 0.2877993836935633, + "nauc_mrr_at_100_max": 0.29359902432295, + "nauc_mrr_at_100_std": 0.1891102733873815, + "nauc_mrr_at_10_diff1": 0.2888368616770747, + "nauc_mrr_at_10_max": 0.28538484411752696, + "nauc_mrr_at_10_std": 0.1757279551601589, + "nauc_mrr_at_1_diff1": 0.3449060066178827, + "nauc_mrr_at_1_max": 0.2778881405715573, + "nauc_mrr_at_1_std": 0.14107599279180383, + "nauc_mrr_at_20_diff1": 0.28870326636271887, + "nauc_mrr_at_20_max": 0.2919190520887219, + "nauc_mrr_at_20_std": 0.1838374791459164, + "nauc_mrr_at_3_diff1": 0.30826158428706923, + "nauc_mrr_at_3_max": 0.28718336453598015, + "nauc_mrr_at_3_std": 0.1577291328017996, + "nauc_mrr_at_5_diff1": 0.29287665057181067, + "nauc_mrr_at_5_max": 0.2755392561802687, + "nauc_mrr_at_5_std": 0.1600053176773477, + "nauc_ndcg_at_1000_diff1": 0.2671222632680834, + "nauc_ndcg_at_1000_max": 0.3975796902514001, + "nauc_ndcg_at_1000_std": 0.3487039793848241, + "nauc_ndcg_at_100_diff1": 0.2710524503406301, + "nauc_ndcg_at_100_max": 0.3788214594730649, + "nauc_ndcg_at_100_std": 0.31641736106699536, + "nauc_ndcg_at_10_diff1": 0.2977477805232909, + "nauc_ndcg_at_10_max": 0.3110716879599296, + "nauc_ndcg_at_10_std": 0.1981262436161795, + "nauc_ndcg_at_1_diff1": 0.3449060066178827, + "nauc_ndcg_at_1_max": 0.2778881405715573, + "nauc_ndcg_at_1_std": 0.14107599279180383, + "nauc_ndcg_at_20_diff1": 0.2935791073033311, + "nauc_ndcg_at_20_max": 0.3357486310586011, + "nauc_ndcg_at_20_std": 0.2339500838962517, + "nauc_ndcg_at_3_diff1": 0.33736667561386546, + "nauc_ndcg_at_3_max": 0.3053346162396636, + "nauc_ndcg_at_3_std": 0.15413243563381435, + "nauc_ndcg_at_5_diff1": 0.3067656788715382, + "nauc_ndcg_at_5_max": 0.29063579665893235, + "nauc_ndcg_at_5_std": 0.1629045427207625, + "nauc_precision_at_1000_diff1": 0.09255761433222197, + "nauc_precision_at_1000_max": 0.36190991096649494, + "nauc_precision_at_1000_std": 0.404489631490746, + "nauc_precision_at_100_diff1": 0.12574551832954653, + "nauc_precision_at_100_max": 0.41046865354290846, + "nauc_precision_at_100_std": 0.4343156374451467, + "nauc_precision_at_10_diff1": 0.23260588782884475, + "nauc_precision_at_10_max": 0.33240592249843154, + "nauc_precision_at_10_std": 0.26578940227193704, + "nauc_precision_at_1_diff1": 0.3449060066178827, + "nauc_precision_at_1_max": 0.2778881405715573, + "nauc_precision_at_1_std": 0.14107599279180383, + "nauc_precision_at_20_diff1": 0.206107373424545, + "nauc_precision_at_20_max": 0.36682257398734724, + "nauc_precision_at_20_std": 0.31618010928098894, + "nauc_precision_at_3_diff1": 0.3019858556612891, + "nauc_precision_at_3_max": 0.3126657340845122, + "nauc_precision_at_3_std": 0.17679510978726765, + "nauc_precision_at_5_diff1": 0.2553941256333017, + "nauc_precision_at_5_max": 0.29590224994264547, + "nauc_precision_at_5_std": 0.19793608696707987, + "nauc_recall_at_1000_diff1": 0.15013812673488833, + "nauc_recall_at_1000_max": 0.4503941828078171, + "nauc_recall_at_1000_std": 0.5409815771464656, + "nauc_recall_at_100_diff1": 0.16928476122023414, + "nauc_recall_at_100_max": 0.39298342241670653, + "nauc_recall_at_100_std": 0.43602266136472984, + "nauc_recall_at_10_diff1": 0.23508430611101677, + "nauc_recall_at_10_max": 0.2837312238991305, + "nauc_recall_at_10_std": 0.21033080726982337, + "nauc_recall_at_1_diff1": 0.41682683282981453, + "nauc_recall_at_1_max": 0.32242722192485945, + "nauc_recall_at_1_std": 0.13029436319937457, + "nauc_recall_at_20_diff1": 0.23110896606554654, + "nauc_recall_at_20_max": 0.3278661819634108, + "nauc_recall_at_20_std": 0.2768048152321431, + "nauc_recall_at_3_diff1": 0.32033962611218664, + "nauc_recall_at_3_max": 0.2935863654878428, + "nauc_recall_at_3_std": 0.15051699338084165, + "nauc_recall_at_5_diff1": 0.2524737896157351, + "nauc_recall_at_5_max": 0.25610544032715066, + "nauc_recall_at_5_std": 0.15761045936562915, + "ndcg_at_1": 0.09105, + "ndcg_at_10": 0.10725, + "ndcg_at_100": 0.15369, + "ndcg_at_1000": 0.18212, + "ndcg_at_20": 0.12301, + "ndcg_at_3": 0.08605, + "ndcg_at_5": 0.09303, + "precision_at_1": 0.09105, + "precision_at_10": 0.03466, + "precision_at_100": 0.00832, + "precision_at_1000": 0.00134, + "precision_at_20": 0.02384, + "precision_at_3": 0.05964, + "precision_at_5": 0.04792, + "recall_at_1": 0.04305, + "recall_at_10": 0.14359, + "recall_at_100": 0.3166, + "recall_at_1000": 0.48962, + "recall_at_20": 0.19116, + "recall_at_3": 0.07829, + "recall_at_5": 0.10485 + } + ] + }, + "task_name": "MIRACLRetrieval" +} \ No newline at end of file diff --git a/results/BAAI__bge-large-en-v1.5/d4aa6901d3a41ba39fb536a557fa166f842b0e09/MassiveScenarioClassification.json b/results/BAAI__bge-large-en-v1.5/d4aa6901d3a41ba39fb536a557fa166f842b0e09/MassiveScenarioClassification.json new file mode 100644 index 000000000..197ce83f3 --- /dev/null +++ b/results/BAAI__bge-large-en-v1.5/d4aa6901d3a41ba39fb536a557fa166f842b0e09/MassiveScenarioClassification.json @@ -0,0 +1,137 @@ +{ + "dataset_revision": "fad2c6e8459f9e1c45d9315f4953d921437d70f8", + "evaluation_time": 56.431074142456055, + "kg_co2_emissions": null, + "mteb_version": "1.19.0", + "scores": { + "test": [ + { + "accuracy": 0.38217888365837255, + "f1": 0.3637086729457913, + "f1_weighted": 0.3709887125137692, + "hf_subset": "ru", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.38217888365837255, + "scores_per_experiment": [ + { + "accuracy": 0.3910558170813719, + "f1": 0.38229467935230854, + "f1_weighted": 0.3788974908014846 + }, + { + "accuracy": 0.34936112979152656, + "f1": 0.33669475298615914, + "f1_weighted": 0.3379198533157962 + }, + { + "accuracy": 0.39710827168796237, + "f1": 0.37287433650118956, + "f1_weighted": 0.3842282017415017 + }, + { + "accuracy": 0.3940820443846671, + "f1": 0.37081882334807514, + "f1_weighted": 0.3873828654688617 + }, + { + "accuracy": 0.38063214525891054, + "f1": 0.35467415893614507, + "f1_weighted": 0.36007686892902624 + }, + { + "accuracy": 0.3782784129119032, + "f1": 0.3595705054974931, + "f1_weighted": 0.36994766029476434 + }, + { + "accuracy": 0.3813046402151984, + "f1": 0.3596070159236462, + "f1_weighted": 0.373702629254836 + }, + { + "accuracy": 0.40248823133826495, + "f1": 0.3946962827558542, + "f1_weighted": 0.39155378130335877 + }, + { + "accuracy": 0.35137861466039005, + "f1": 0.33408540585498814, + "f1_weighted": 0.34116055931323575 + }, + { + "accuracy": 0.3960995292535306, + "f1": 0.37177076830205347, + "f1_weighted": 0.3850172147148267 + } + ] + } + ], + "validation": [ + { + "accuracy": 0.3771765863256272, + "f1": 0.3635169861152871, + "f1_weighted": 0.3651668357641755, + "hf_subset": "ru", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.3771765863256272, + "scores_per_experiment": [ + { + "accuracy": 0.3890801770782095, + "f1": 0.38786946530965116, + "f1_weighted": 0.38358799696389945 + }, + { + "accuracy": 0.3487456960157403, + "f1": 0.33975201861048165, + "f1_weighted": 0.3402491816165025 + }, + { + "accuracy": 0.4033448106246926, + "f1": 0.37953020137778964, + "f1_weighted": 0.38847041375176256 + }, + { + "accuracy": 0.37579931136251843, + "f1": 0.3616111997998951, + "f1_weighted": 0.36420617031772523 + }, + { + "accuracy": 0.3767830791933104, + "f1": 0.365372162520146, + "f1_weighted": 0.3533325590689296 + }, + { + "accuracy": 0.367929168716183, + "f1": 0.35099776147609085, + "f1_weighted": 0.3609846347306369 + }, + { + "accuracy": 0.3753074274471225, + "f1": 0.3549610390671737, + "f1_weighted": 0.36424613347091717 + }, + { + "accuracy": 0.38957206099360553, + "f1": 0.3846241499110977, + "f1_weighted": 0.3730252571378179 + }, + { + "accuracy": 0.3295622233152976, + "f1": 0.319572804089438, + "f1_weighted": 0.3209851109409843 + }, + { + "accuracy": 0.41564190850959176, + "f1": 0.39087905899110686, + "f1_weighted": 0.40258089964257987 + } + ] + } + ] + }, + "task_name": "MassiveScenarioClassification" +} \ No newline at end of file diff --git a/results/BAAI__bge-large-en-v1.5/d4aa6901d3a41ba39fb536a557fa166f842b0e09/RUParaPhraserSTS.json b/results/BAAI__bge-large-en-v1.5/d4aa6901d3a41ba39fb536a557fa166f842b0e09/RUParaPhraserSTS.json new file mode 100644 index 000000000..4c8f13d12 --- /dev/null +++ b/results/BAAI__bge-large-en-v1.5/d4aa6901d3a41ba39fb536a557fa166f842b0e09/RUParaPhraserSTS.json @@ -0,0 +1,26 @@ +{ + "dataset_revision": "43265056790b8f7c59e0139acb4be0a8dad2c8f4", + "evaluation_time": 21.42787003517151, + "kg_co2_emissions": null, + "mteb_version": "1.19.0", + "scores": { + "test": [ + { + "cosine_pearson": 0.4871998139520565, + "cosine_spearman": 0.49107601175437415, + "euclidean_pearson": 0.5024227601350222, + "euclidean_spearman": 0.49107601175437415, + "hf_subset": "default", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.49107601175437415, + "manhattan_pearson": 0.5020034671984516, + "manhattan_spearman": 0.49028741961036487, + "pearson": 0.4871998139520565, + "spearman": 0.49107601175437415 + } + ] + }, + "task_name": "RUParaPhraserSTS" +} \ No newline at end of file diff --git a/results/BAAI__bge-large-en-v1.5/d4aa6901d3a41ba39fb536a557fa166f842b0e09/RiaNewsRetrieval.json b/results/BAAI__bge-large-en-v1.5/d4aa6901d3a41ba39fb536a557fa166f842b0e09/RiaNewsRetrieval.json new file mode 100644 index 000000000..1dac3ffcf --- /dev/null +++ b/results/BAAI__bge-large-en-v1.5/d4aa6901d3a41ba39fb536a557fa166f842b0e09/RiaNewsRetrieval.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "82374b0bbacda6114f39ff9c5b925fa1512ca5d7", + "evaluation_time": 16214.654376029968, + "kg_co2_emissions": null, + "mteb_version": "1.19.0", + "scores": { + "test": [ + { + "hf_subset": "default", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.29087, + "map_at_1": 0.2058, + "map_at_10": 0.26075, + "map_at_100": 0.26711, + "map_at_1000": 0.26773, + "map_at_20": 0.26429, + "map_at_3": 0.24473, + "map_at_5": 0.25345, + "mrr_at_1": 0.206, + "mrr_at_10": 0.2608965079365079, + "mrr_at_100": 0.26726235053599845, + "mrr_at_1000": 0.26787530081624905, + "mrr_at_20": 0.2644397286047286, + "mrr_at_3": 0.24486666666666665, + "mrr_at_5": 0.25359166666666666, + "nauc_map_at_1000_diff1": 0.48301987615753705, + "nauc_map_at_1000_max": 0.37628930754285445, + "nauc_map_at_1000_std": 0.16047570446074538, + "nauc_map_at_100_diff1": 0.48297490470604393, + "nauc_map_at_100_max": 0.3762447828345044, + "nauc_map_at_100_std": 0.16037910873405764, + "nauc_map_at_10_diff1": 0.48583152633434445, + "nauc_map_at_10_max": 0.37516546061787726, + "nauc_map_at_10_std": 0.15621703923237593, + "nauc_map_at_1_diff1": 0.5492255275617551, + "nauc_map_at_1_max": 0.37088981076965777, + "nauc_map_at_1_std": 0.12455415911159717, + "nauc_map_at_20_diff1": 0.48385847525815057, + "nauc_map_at_20_max": 0.3762034000030319, + "nauc_map_at_20_std": 0.1589799529326884, + "nauc_map_at_3_diff1": 0.4970425646355963, + "nauc_map_at_3_max": 0.3744891419049221, + "nauc_map_at_3_std": 0.14463381321480678, + "nauc_map_at_5_diff1": 0.4915137211788807, + "nauc_map_at_5_max": 0.37533013235511914, + "nauc_map_at_5_std": 0.15196730190602897, + "nauc_mrr_at_1000_diff1": 0.4824827815396758, + "nauc_mrr_at_1000_max": 0.37695205151111877, + "nauc_mrr_at_1000_std": 0.16082252861586996, + "nauc_mrr_at_100_diff1": 0.482438286857085, + "nauc_mrr_at_100_max": 0.3769069487380003, + "nauc_mrr_at_100_std": 0.160725651613742, + "nauc_mrr_at_10_diff1": 0.4852967916180098, + "nauc_mrr_at_10_max": 0.37582647122946583, + "nauc_mrr_at_10_std": 0.15656562661702994, + "nauc_mrr_at_1_diff1": 0.5483303066184517, + "nauc_mrr_at_1_max": 0.37257987814413984, + "nauc_mrr_at_1_std": 0.12517482071102815, + "nauc_mrr_at_20_diff1": 0.48332236790130606, + "nauc_mrr_at_20_max": 0.376864875901049, + "nauc_mrr_at_20_std": 0.15932625479716606, + "nauc_mrr_at_3_diff1": 0.49650671448201467, + "nauc_mrr_at_3_max": 0.37515582762918004, + "nauc_mrr_at_3_std": 0.14497618651944902, + "nauc_mrr_at_5_diff1": 0.49097564003502575, + "nauc_mrr_at_5_max": 0.3759893777622305, + "nauc_mrr_at_5_std": 0.15229656757989316, + "nauc_ndcg_at_1000_diff1": 0.4472294005626283, + "nauc_ndcg_at_1000_max": 0.3825337724495491, + "nauc_ndcg_at_1000_std": 0.19552495440512707, + "nauc_ndcg_at_100_diff1": 0.44662245209756646, + "nauc_ndcg_at_100_max": 0.3806695678493896, + "nauc_ndcg_at_100_std": 0.19352029541058377, + "nauc_ndcg_at_10_diff1": 0.45937397619623466, + "nauc_ndcg_at_10_max": 0.3762773931242269, + "nauc_ndcg_at_10_std": 0.17366081505958822, + "nauc_ndcg_at_1_diff1": 0.5492255275617551, + "nauc_ndcg_at_1_max": 0.37088981076965777, + "nauc_ndcg_at_1_std": 0.12455415911159717, + "nauc_ndcg_at_20_diff1": 0.4525604772309586, + "nauc_ndcg_at_20_max": 0.3799060703262353, + "nauc_ndcg_at_20_std": 0.18298132467094141, + "nauc_ndcg_at_3_diff1": 0.4817245152677164, + "nauc_ndcg_at_3_max": 0.3752139204499108, + "nauc_ndcg_at_3_std": 0.15084531544679936, + "nauc_ndcg_at_5_diff1": 0.4721984400641255, + "nauc_ndcg_at_5_max": 0.37683025133979553, + "nauc_ndcg_at_5_std": 0.16395136651029116, + "nauc_precision_at_1000_diff1": 0.2923433784054819, + "nauc_precision_at_1000_max": 0.42319283777412414, + "nauc_precision_at_1000_std": 0.374196830226086, + "nauc_precision_at_100_diff1": 0.32469417065194467, + "nauc_precision_at_100_max": 0.3980782685128241, + "nauc_precision_at_100_std": 0.31842601201237253, + "nauc_precision_at_10_diff1": 0.38650324706822226, + "nauc_precision_at_10_max": 0.37897940550914244, + "nauc_precision_at_10_std": 0.22333617462866176, + "nauc_precision_at_1_diff1": 0.5492255275617551, + "nauc_precision_at_1_max": 0.37088981076965777, + "nauc_precision_at_1_std": 0.12455415911159717, + "nauc_precision_at_20_diff1": 0.3616673423902463, + "nauc_precision_at_20_max": 0.39204164978444456, + "nauc_precision_at_20_std": 0.25655792016063994, + "nauc_precision_at_3_diff1": 0.44107345784339075, + "nauc_precision_at_3_max": 0.37704119439125316, + "nauc_precision_at_3_std": 0.1674213302266038, + "nauc_precision_at_5_diff1": 0.42066351502171234, + "nauc_precision_at_5_max": 0.38094861464626933, + "nauc_precision_at_5_std": 0.19714911009719172, + "nauc_recall_at_1000_diff1": 0.2923433784054822, + "nauc_recall_at_1000_max": 0.42319283777412464, + "nauc_recall_at_1000_std": 0.3741968302260875, + "nauc_recall_at_100_diff1": 0.3246941706519445, + "nauc_recall_at_100_max": 0.39807826851282435, + "nauc_recall_at_100_std": 0.3184260120123729, + "nauc_recall_at_10_diff1": 0.38650324706822214, + "nauc_recall_at_10_max": 0.3789794055091422, + "nauc_recall_at_10_std": 0.22333617462866176, + "nauc_recall_at_1_diff1": 0.5492255275617551, + "nauc_recall_at_1_max": 0.37088981076965777, + "nauc_recall_at_1_std": 0.12455415911159717, + "nauc_recall_at_20_diff1": 0.36166734239024584, + "nauc_recall_at_20_max": 0.3920416497844443, + "nauc_recall_at_20_std": 0.2565579201606399, + "nauc_recall_at_3_diff1": 0.4410734578433906, + "nauc_recall_at_3_max": 0.37704119439125305, + "nauc_recall_at_3_std": 0.16742133022660352, + "nauc_recall_at_5_diff1": 0.42066351502171234, + "nauc_recall_at_5_max": 0.38094861464626945, + "nauc_recall_at_5_std": 0.19714911009719205, + "ndcg_at_1": 0.2058, + "ndcg_at_10": 0.29087, + "ndcg_at_100": 0.32494, + "ndcg_at_1000": 0.34415, + "ndcg_at_20": 0.30374, + "ndcg_at_3": 0.25751, + "ndcg_at_5": 0.27334, + "precision_at_1": 0.2058, + "precision_at_10": 0.03871, + "precision_at_100": 0.00554, + "precision_at_1000": 0.00071, + "precision_at_20": 0.0219, + "precision_at_3": 0.09817, + "precision_at_5": 0.06664, + "recall_at_1": 0.2058, + "recall_at_10": 0.3871, + "recall_at_100": 0.5538, + "recall_at_1000": 0.7114, + "recall_at_20": 0.4379, + "recall_at_3": 0.2945, + "recall_at_5": 0.3332 + } + ] + }, + "task_name": "RiaNewsRetrieval" +} \ No newline at end of file diff --git a/results/BAAI__bge-large-en-v1.5/d4aa6901d3a41ba39fb536a557fa166f842b0e09/RuBQRetrieval.json b/results/BAAI__bge-large-en-v1.5/d4aa6901d3a41ba39fb536a557fa166f842b0e09/RuBQRetrieval.json new file mode 100644 index 000000000..fc963d136 --- /dev/null +++ b/results/BAAI__bge-large-en-v1.5/d4aa6901d3a41ba39fb536a557fa166f842b0e09/RuBQRetrieval.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "e19b6ffa60b3bc248e0b41f4cc37c26a55c2a67b", + "evaluation_time": 878.4462375640869, + "kg_co2_emissions": null, + "mteb_version": "1.19.0", + "scores": { + "test": [ + { + "hf_subset": "default", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.21052, + "map_at_1": 0.11069, + "map_at_10": 0.16955, + "map_at_100": 0.17729, + "map_at_1000": 0.17824, + "map_at_20": 0.174, + "map_at_3": 0.15364, + "map_at_5": 0.16169, + "mrr_at_1": 0.16962174940898345, + "mrr_at_10": 0.23115313895455739, + "mrr_at_100": 0.2383673568592099, + "mrr_at_1000": 0.23909383080531318, + "mrr_at_20": 0.23550735620030025, + "mrr_at_3": 0.21463750985027583, + "mrr_at_5": 0.2227935382190701, + "nauc_map_at_1000_diff1": 0.35625763517990783, + "nauc_map_at_1000_max": 0.36334895268783923, + "nauc_map_at_1000_std": 0.17401136771710868, + "nauc_map_at_100_diff1": 0.3562175387616592, + "nauc_map_at_100_max": 0.36337476531346935, + "nauc_map_at_100_std": 0.17386706141445168, + "nauc_map_at_10_diff1": 0.3585950818245034, + "nauc_map_at_10_max": 0.36221175790946264, + "nauc_map_at_10_std": 0.16900887288119543, + "nauc_map_at_1_diff1": 0.418754440044197, + "nauc_map_at_1_max": 0.3201202687853332, + "nauc_map_at_1_std": 0.08821257320418381, + "nauc_map_at_20_diff1": 0.35689808225439323, + "nauc_map_at_20_max": 0.36243842250308184, + "nauc_map_at_20_std": 0.17169988245329745, + "nauc_map_at_3_diff1": 0.36569462559187865, + "nauc_map_at_3_max": 0.35992233412330416, + "nauc_map_at_3_std": 0.1564014218977745, + "nauc_map_at_5_diff1": 0.36180401206363916, + "nauc_map_at_5_max": 0.3650904409917985, + "nauc_map_at_5_std": 0.16593195213296325, + "nauc_mrr_at_1000_diff1": 0.36366660610052254, + "nauc_mrr_at_1000_max": 0.36309091599301585, + "nauc_mrr_at_1000_std": 0.1517748772303864, + "nauc_mrr_at_100_diff1": 0.36364412086481007, + "nauc_mrr_at_100_max": 0.3631534856113731, + "nauc_mrr_at_100_std": 0.1517936117214468, + "nauc_mrr_at_10_diff1": 0.3671001303950863, + "nauc_mrr_at_10_max": 0.36352016956088856, + "nauc_mrr_at_10_std": 0.1491410307420657, + "nauc_mrr_at_1_diff1": 0.4297645318671107, + "nauc_mrr_at_1_max": 0.34823991704052554, + "nauc_mrr_at_1_std": 0.10295800302509564, + "nauc_mrr_at_20_diff1": 0.36428111675175673, + "nauc_mrr_at_20_max": 0.3634363771365113, + "nauc_mrr_at_20_std": 0.1512078707305061, + "nauc_mrr_at_3_diff1": 0.3735595403526666, + "nauc_mrr_at_3_max": 0.36646073930942374, + "nauc_mrr_at_3_std": 0.1409014886759411, + "nauc_mrr_at_5_diff1": 0.36881041448402585, + "nauc_mrr_at_5_max": 0.36677497691402633, + "nauc_mrr_at_5_std": 0.14589574916260442, + "nauc_ndcg_at_1000_diff1": 0.3275381806379634, + "nauc_ndcg_at_1000_max": 0.3650867732214024, + "nauc_ndcg_at_1000_std": 0.20188056206854516, + "nauc_ndcg_at_100_diff1": 0.32794930733348276, + "nauc_ndcg_at_100_max": 0.3667597778633193, + "nauc_ndcg_at_100_std": 0.20032260857908518, + "nauc_ndcg_at_10_diff1": 0.3426319348761798, + "nauc_ndcg_at_10_max": 0.3663870443083238, + "nauc_ndcg_at_10_std": 0.1846299269257484, + "nauc_ndcg_at_1_diff1": 0.43324949297655896, + "nauc_ndcg_at_1_max": 0.34511698109787897, + "nauc_ndcg_at_1_std": 0.09917640181738709, + "nauc_ndcg_at_20_diff1": 0.33501579169233203, + "nauc_ndcg_at_20_max": 0.3657177055500923, + "nauc_ndcg_at_20_std": 0.19272036615925336, + "nauc_ndcg_at_3_diff1": 0.35499417790124194, + "nauc_ndcg_at_3_max": 0.3699896074864231, + "nauc_ndcg_at_3_std": 0.16629425400828377, + "nauc_ndcg_at_5_diff1": 0.3475558605805017, + "nauc_ndcg_at_5_max": 0.3737026863914172, + "nauc_ndcg_at_5_std": 0.17864180221010315, + "nauc_precision_at_1000_diff1": 0.13632285035249758, + "nauc_precision_at_1000_max": 0.22864728492090627, + "nauc_precision_at_1000_std": 0.19785626559735933, + "nauc_precision_at_100_diff1": 0.20807226405556506, + "nauc_precision_at_100_max": 0.3089476508485861, + "nauc_precision_at_100_std": 0.23593911297146616, + "nauc_precision_at_10_diff1": 0.2979357528809485, + "nauc_precision_at_10_max": 0.37678006214857784, + "nauc_precision_at_10_std": 0.22943972098226037, + "nauc_precision_at_1_diff1": 0.43324949297655896, + "nauc_precision_at_1_max": 0.34511698109787897, + "nauc_precision_at_1_std": 0.09917640181738709, + "nauc_precision_at_20_diff1": 0.26428162294992225, + "nauc_precision_at_20_max": 0.35988813772230016, + "nauc_precision_at_20_std": 0.24170392950254266, + "nauc_precision_at_3_diff1": 0.32947910553301785, + "nauc_precision_at_3_max": 0.3979704777187416, + "nauc_precision_at_3_std": 0.2017913712999419, + "nauc_precision_at_5_diff1": 0.3131190649363651, + "nauc_precision_at_5_max": 0.4001143884788561, + "nauc_precision_at_5_std": 0.216076289691353, + "nauc_recall_at_1000_diff1": 0.17404385256680147, + "nauc_recall_at_1000_max": 0.29434613540671595, + "nauc_recall_at_1000_std": 0.2662756573099805, + "nauc_recall_at_100_diff1": 0.2052131316378911, + "nauc_recall_at_100_max": 0.3229551882214468, + "nauc_recall_at_100_std": 0.25190613793956235, + "nauc_recall_at_10_diff1": 0.2757262526534067, + "nauc_recall_at_10_max": 0.3452937875639342, + "nauc_recall_at_10_std": 0.22082824043434454, + "nauc_recall_at_1_diff1": 0.418754440044197, + "nauc_recall_at_1_max": 0.3201202687853332, + "nauc_recall_at_1_std": 0.08821257320418381, + "nauc_recall_at_20_diff1": 0.24833517954354356, + "nauc_recall_at_20_max": 0.3346196705958076, + "nauc_recall_at_20_std": 0.23788109321247558, + "nauc_recall_at_3_diff1": 0.30934482134023855, + "nauc_recall_at_3_max": 0.3710904835218233, + "nauc_recall_at_3_std": 0.19483930957527482, + "nauc_recall_at_5_diff1": 0.2957857293435367, + "nauc_recall_at_5_max": 0.37792780026164846, + "nauc_recall_at_5_std": 0.2210559979936048, + "ndcg_at_1": 0.16903, + "ndcg_at_10": 0.21052, + "ndcg_at_100": 0.24861, + "ndcg_at_1000": 0.27554, + "ndcg_at_20": 0.22563, + "ndcg_at_3": 0.17994, + "ndcg_at_5": 0.19225, + "precision_at_1": 0.16903, + "precision_at_10": 0.04326, + "precision_at_100": 0.00708, + "precision_at_1000": 0.00104, + "precision_at_20": 0.02592, + "precision_at_3": 0.10185, + "precision_at_5": 0.0708, + "recall_at_1": 0.11069, + "recall_at_10": 0.27372, + "recall_at_100": 0.43638, + "recall_at_1000": 0.63159, + "recall_at_20": 0.32614, + "recall_at_3": 0.1927, + "recall_at_5": 0.22298 + } + ] + }, + "task_name": "RuBQRetrieval" +} \ No newline at end of file diff --git a/results/BAAI__bge-large-en-v1.5/d4aa6901d3a41ba39fb536a557fa166f842b0e09/RuReviewsClassification.json b/results/BAAI__bge-large-en-v1.5/d4aa6901d3a41ba39fb536a557fa166f842b0e09/RuReviewsClassification.json new file mode 100644 index 000000000..ac9d9cdfc --- /dev/null +++ b/results/BAAI__bge-large-en-v1.5/d4aa6901d3a41ba39fb536a557fa166f842b0e09/RuReviewsClassification.json @@ -0,0 +1,73 @@ +{ + "dataset_revision": "f6d2c31f4dc6b88f468552750bfec05b4b41b05a", + "evaluation_time": 23.58547329902649, + "kg_co2_emissions": null, + "mteb_version": "1.19.0", + "scores": { + "test": [ + { + "accuracy": 0.446240234375, + "f1": 0.44233390006306167, + "f1_weighted": 0.4423235413370941, + "hf_subset": "default", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.446240234375, + "scores_per_experiment": [ + { + "accuracy": 0.4580078125, + "f1": 0.4433776569842889, + "f1_weighted": 0.44335532446380954 + }, + { + "accuracy": 0.431640625, + "f1": 0.4315599982254279, + "f1_weighted": 0.4315458675300349 + }, + { + "accuracy": 0.451171875, + "f1": 0.4511312516691271, + "f1_weighted": 0.4510922412788168 + }, + { + "accuracy": 0.486328125, + "f1": 0.48673621920203997, + "f1_weighted": 0.4867365878108246 + }, + { + "accuracy": 0.4755859375, + "f1": 0.47531941484547885, + "f1_weighted": 0.4753070184593787 + }, + { + "accuracy": 0.45361328125, + "f1": 0.4515139219706321, + "f1_weighted": 0.451496663840141 + }, + { + "accuracy": 0.3955078125, + "f1": 0.3744914837860341, + "f1_weighted": 0.3745192923793761 + }, + { + "accuracy": 0.478515625, + "f1": 0.47727125532819153, + "f1_weighted": 0.47725582345108375 + }, + { + "accuracy": 0.35498046875, + "f1": 0.35473538902613416, + "f1_weighted": 0.3547450960503908 + }, + { + "accuracy": 0.47705078125, + "f1": 0.4772024095932624, + "f1_weighted": 0.47718149810708493 + } + ] + } + ] + }, + "task_name": "RuReviewsClassification" +} \ No newline at end of file diff --git a/results/BAAI__bge-large-en-v1.5/d4aa6901d3a41ba39fb536a557fa166f842b0e09/RuSTSBenchmarkSTS.json b/results/BAAI__bge-large-en-v1.5/d4aa6901d3a41ba39fb536a557fa166f842b0e09/RuSTSBenchmarkSTS.json new file mode 100644 index 000000000..745b09679 --- /dev/null +++ b/results/BAAI__bge-large-en-v1.5/d4aa6901d3a41ba39fb536a557fa166f842b0e09/RuSTSBenchmarkSTS.json @@ -0,0 +1,26 @@ +{ + "dataset_revision": "7cf24f325c6da6195df55bef3d86b5e0616f3018", + "evaluation_time": 14.971743822097778, + "kg_co2_emissions": null, + "mteb_version": "1.19.0", + "scores": { + "test": [ + { + "cosine_pearson": 0.6112455793191856, + "cosine_spearman": 0.603270996059513, + "euclidean_pearson": 0.6018556071247612, + "euclidean_spearman": 0.6032697066617156, + "hf_subset": "default", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.603270996059513, + "manhattan_pearson": 0.6019661384366517, + "manhattan_spearman": 0.6032602411298505, + "pearson": 0.6112455793191856, + "spearman": 0.603270996059513 + } + ] + }, + "task_name": "RuSTSBenchmarkSTS" +} \ No newline at end of file diff --git a/results/BAAI__bge-large-en-v1.5/d4aa6901d3a41ba39fb536a557fa166f842b0e09/RuSciBenchGRNTIClassification.json b/results/BAAI__bge-large-en-v1.5/d4aa6901d3a41ba39fb536a557fa166f842b0e09/RuSciBenchGRNTIClassification.json new file mode 100644 index 000000000..d11643d8c --- /dev/null +++ b/results/BAAI__bge-large-en-v1.5/d4aa6901d3a41ba39fb536a557fa166f842b0e09/RuSciBenchGRNTIClassification.json @@ -0,0 +1,73 @@ +{ + "dataset_revision": "673a610d6d3dd91a547a0d57ae1b56f37ebbf6a1", + "evaluation_time": 111.3483555316925, + "kg_co2_emissions": null, + "mteb_version": "1.19.0", + "scores": { + "test": [ + { + "accuracy": 0.2205078125, + "f1": 0.20650702008786012, + "f1_weighted": 0.20662006217519924, + "hf_subset": "default", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.2205078125, + "scores_per_experiment": [ + { + "accuracy": 0.2392578125, + "f1": 0.22168764846207664, + "f1_weighted": 0.22177230750829802 + }, + { + "accuracy": 0.2080078125, + "f1": 0.19399134251430677, + "f1_weighted": 0.19420502031194525 + }, + { + "accuracy": 0.205078125, + "f1": 0.19491243876419295, + "f1_weighted": 0.1949242740415256 + }, + { + "accuracy": 0.22412109375, + "f1": 0.20843960462173494, + "f1_weighted": 0.20854068561036151 + }, + { + "accuracy": 0.20849609375, + "f1": 0.19541290523113822, + "f1_weighted": 0.19547049014321707 + }, + { + "accuracy": 0.22509765625, + "f1": 0.20915039429901913, + "f1_weighted": 0.20924666653914134 + }, + { + "accuracy": 0.24365234375, + "f1": 0.23848591838973565, + "f1_weighted": 0.2387001185754376 + }, + { + "accuracy": 0.208984375, + "f1": 0.18786971557855622, + "f1_weighted": 0.18804177399946814 + }, + { + "accuracy": 0.2099609375, + "f1": 0.19498052324534892, + "f1_weighted": 0.1950464906057493 + }, + { + "accuracy": 0.232421875, + "f1": 0.22013970977249192, + "f1_weighted": 0.2202527944168488 + } + ] + } + ] + }, + "task_name": "RuSciBenchGRNTIClassification" +} \ No newline at end of file diff --git a/results/BAAI__bge-large-en-v1.5/d4aa6901d3a41ba39fb536a557fa166f842b0e09/RuSciBenchGRNTIClusteringP2P.json b/results/BAAI__bge-large-en-v1.5/d4aa6901d3a41ba39fb536a557fa166f842b0e09/RuSciBenchGRNTIClusteringP2P.json new file mode 100644 index 000000000..966202cdb --- /dev/null +++ b/results/BAAI__bge-large-en-v1.5/d4aa6901d3a41ba39fb536a557fa166f842b0e09/RuSciBenchGRNTIClusteringP2P.json @@ -0,0 +1,34 @@ +{ + "dataset_revision": "673a610d6d3dd91a547a0d57ae1b56f37ebbf6a1", + "evaluation_time": 108.29422974586487, + "kg_co2_emissions": null, + "mteb_version": "1.19.0", + "scores": { + "test": [ + { + "hf_subset": "default", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.19005480904556968, + "v_measure": 0.19005480904556968, + "v_measure_std": 0.005015135457890354, + "v_measures": { + "Level 0": [ + 0.1848128130225392, + 0.18650388182631525, + 0.20060778462114687, + 0.1976829601002126, + 0.190125271414806, + 0.1888647152481913, + 0.18686936405110155, + 0.1898860951301267, + 0.19061436469009896, + 0.18458084035115835 + ] + } + } + ] + }, + "task_name": "RuSciBenchGRNTIClusteringP2P" +} \ No newline at end of file diff --git a/results/BAAI__bge-large-en-v1.5/d4aa6901d3a41ba39fb536a557fa166f842b0e09/RuSciBenchOECDClassification.json b/results/BAAI__bge-large-en-v1.5/d4aa6901d3a41ba39fb536a557fa166f842b0e09/RuSciBenchOECDClassification.json new file mode 100644 index 000000000..5c63a7261 --- /dev/null +++ b/results/BAAI__bge-large-en-v1.5/d4aa6901d3a41ba39fb536a557fa166f842b0e09/RuSciBenchOECDClassification.json @@ -0,0 +1,73 @@ +{ + "dataset_revision": "26c88e99dcaba32bb45d0e1bfc21902337f6d471", + "evaluation_time": 108.94598937034607, + "kg_co2_emissions": null, + "mteb_version": "1.19.0", + "scores": { + "test": [ + { + "accuracy": 0.165283203125, + "f1": 0.15145394812348753, + "f1_weighted": 0.15142412071058084, + "hf_subset": "default", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.165283203125, + "scores_per_experiment": [ + { + "accuracy": 0.1455078125, + "f1": 0.1279919584899787, + "f1_weighted": 0.12784429088353177 + }, + { + "accuracy": 0.16748046875, + "f1": 0.15376856164375072, + "f1_weighted": 0.153725092410371 + }, + { + "accuracy": 0.16064453125, + "f1": 0.14974632083049383, + "f1_weighted": 0.14969538745012098 + }, + { + "accuracy": 0.162109375, + "f1": 0.15065664474044893, + "f1_weighted": 0.15063009482135553 + }, + { + "accuracy": 0.16552734375, + "f1": 0.14637273007318247, + "f1_weighted": 0.14635133237786485 + }, + { + "accuracy": 0.17041015625, + "f1": 0.15984591875418203, + "f1_weighted": 0.1599163985702103 + }, + { + "accuracy": 0.1787109375, + "f1": 0.16577660429001348, + "f1_weighted": 0.1657238249605765 + }, + { + "accuracy": 0.15234375, + "f1": 0.1349832460237668, + "f1_weighted": 0.13501111395272247 + }, + { + "accuracy": 0.1728515625, + "f1": 0.16173677066427916, + "f1_weighted": 0.1617152211366516 + }, + { + "accuracy": 0.17724609375, + "f1": 0.16366072572477913, + "f1_weighted": 0.16362845054240333 + } + ] + } + ] + }, + "task_name": "RuSciBenchOECDClassification" +} \ No newline at end of file diff --git a/results/BAAI__bge-large-en-v1.5/d4aa6901d3a41ba39fb536a557fa166f842b0e09/RuSciBenchOECDClusteringP2P.json b/results/BAAI__bge-large-en-v1.5/d4aa6901d3a41ba39fb536a557fa166f842b0e09/RuSciBenchOECDClusteringP2P.json new file mode 100644 index 000000000..ba15d60fe --- /dev/null +++ b/results/BAAI__bge-large-en-v1.5/d4aa6901d3a41ba39fb536a557fa166f842b0e09/RuSciBenchOECDClusteringP2P.json @@ -0,0 +1,34 @@ +{ + "dataset_revision": "26c88e99dcaba32bb45d0e1bfc21902337f6d471", + "evaluation_time": 107.02520823478699, + "kg_co2_emissions": null, + "mteb_version": "1.19.0", + "scores": { + "test": [ + { + "hf_subset": "default", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.15984730532206398, + "v_measure": 0.15984730532206398, + "v_measure_std": 0.0052610329905498805, + "v_measures": { + "Level 0": [ + 0.15505917855764398, + 0.16432232121757, + 0.16288911686319982, + 0.15268167876781455, + 0.1518034171035897, + 0.1644716593438242, + 0.1577351282332397, + 0.16320955966094777, + 0.15817269834709322, + 0.16812829512571706 + ] + } + } + ] + }, + "task_name": "RuSciBenchOECDClusteringP2P" +} \ No newline at end of file diff --git a/results/BAAI__bge-large-en-v1.5/d4aa6901d3a41ba39fb536a557fa166f842b0e09/STS22.json b/results/BAAI__bge-large-en-v1.5/d4aa6901d3a41ba39fb536a557fa166f842b0e09/STS22.json new file mode 100644 index 000000000..7a956e2ca --- /dev/null +++ b/results/BAAI__bge-large-en-v1.5/d4aa6901d3a41ba39fb536a557fa166f842b0e09/STS22.json @@ -0,0 +1,26 @@ +{ + "dataset_revision": "de9d86b3b84231dc21f76c7b7af1f28e2f57f6e3", + "evaluation_time": 12.712047100067139, + "kg_co2_emissions": null, + "mteb_version": "1.19.0", + "scores": { + "test": [ + { + "cosine_pearson": 0.1577177890518716, + "cosine_spearman": 0.25185016085202866, + "euclidean_pearson": 0.16491469615153642, + "euclidean_spearman": 0.2517943719620769, + "hf_subset": "ru", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.25185016085202866, + "manhattan_pearson": 0.1663178228371507, + "manhattan_spearman": 0.2569447271052511, + "pearson": 0.1577177890518716, + "spearman": 0.25185016085202866 + } + ] + }, + "task_name": "STS22" +} \ No newline at end of file diff --git a/results/BAAI__bge-large-en-v1.5/d4aa6901d3a41ba39fb536a557fa166f842b0e09/SensitiveTopicsClassification.json b/results/BAAI__bge-large-en-v1.5/d4aa6901d3a41ba39fb536a557fa166f842b0e09/SensitiveTopicsClassification.json new file mode 100644 index 000000000..010330e10 --- /dev/null +++ b/results/BAAI__bge-large-en-v1.5/d4aa6901d3a41ba39fb536a557fa166f842b0e09/SensitiveTopicsClassification.json @@ -0,0 +1,73 @@ +{ + "dataset_revision": "416b34a802308eac30e4192afc0ff99bb8dcc7f2", + "evaluation_time": 32.578866720199585, + "kg_co2_emissions": null, + "mteb_version": "1.19.0", + "scores": { + "test": [ + { + "accuracy": 0.179736328125, + "f1": 0.04039328608900143, + "hf_subset": "default", + "languages": [ + "rus-Cyrl" + ], + "lrap": 0.26560329861110465, + "main_score": 0.179736328125, + "scores_per_experiment": [ + { + "accuracy": 0.17333984375, + "f1": 0.019137519932623456, + "lrap": 0.25622558593749367 + }, + { + "accuracy": 0.1884765625, + "f1": 0.06628925400413806, + "lrap": 0.27383083767360455 + }, + { + "accuracy": 0.177734375, + "f1": 0.042062771760862844, + "lrap": 0.26641167534721577 + }, + { + "accuracy": 0.17236328125, + "f1": 0.014940649505633313, + "lrap": 0.2570258246527714 + }, + { + "accuracy": 0.17333984375, + "f1": 0.030785174123042047, + "lrap": 0.26196289062499367 + }, + { + "accuracy": 0.1865234375, + "f1": 0.05816612138299049, + "lrap": 0.2726236979166601 + }, + { + "accuracy": 0.1767578125, + "f1": 0.02578170053141273, + "lrap": 0.2650417751736047 + }, + { + "accuracy": 0.18115234375, + "f1": 0.033162168504233286, + "lrap": 0.2650417751736048 + }, + { + "accuracy": 0.181640625, + "f1": 0.04628311016560868, + "lrap": 0.2643364800347158 + }, + { + "accuracy": 0.18603515625, + "f1": 0.06732439097946934, + "lrap": 0.2735324435763824 + } + ] + } + ] + }, + "task_name": "SensitiveTopicsClassification" +} \ No newline at end of file diff --git a/results/BAAI__bge-large-en-v1.5/d4aa6901d3a41ba39fb536a557fa166f842b0e09/model_meta.json b/results/BAAI__bge-large-en-v1.5/d4aa6901d3a41ba39fb536a557fa166f842b0e09/model_meta.json index 47c138e08..9283f5610 100644 --- a/results/BAAI__bge-large-en-v1.5/d4aa6901d3a41ba39fb536a557fa166f842b0e09/model_meta.json +++ b/results/BAAI__bge-large-en-v1.5/d4aa6901d3a41ba39fb536a557fa166f842b0e09/model_meta.json @@ -1 +1,25 @@ -{"name": "BAAI/bge-large-en-v1.5", "revision": "d4aa6901d3a41ba39fb536a557fa166f842b0e09", "release_date": "2023-09-12", "languages": ["eng_Latn"], "n_parameters": 1340000000, "memory_usage": null, "max_tokens": 512, "embed_dim": 1024, "license": "mit", "open_weights": true, "public_training_data": null, "public_training_code": null, "framework": ["Sentence Transformers", "PyTorch"], "reference": "https://huggingface.co/BAAI/bge-large-en-v1.5", "similarity_fn_name": "cosine", "use_instructions": false, "zero_shot_benchmarks": null, "loader": "sentence_transformers_loader"} \ No newline at end of file +{ + "name": "BAAI/bge-large-en-v1.5", + "revision": "d4aa6901d3a41ba39fb536a557fa166f842b0e09", + "release_date": "2023-09-12", + "languages": [ + "eng_Latn" + ], + "n_parameters": 1340000000, + "memory_usage": null, + "max_tokens": 512, + "embed_dim": 1024, + "license": "mit", + "open_weights": true, + "public_training_data": null, + "public_training_code": null, + "framework": [ + "Sentence Transformers", + "PyTorch" + ], + "reference": "https://huggingface.co/BAAI/bge-large-en-v1.5", + "similarity_fn_name": "cosine", + "use_instructions": false, + "zero_shot_benchmarks": null, + "loader": "sentence_transformers_loader" +} \ No newline at end of file diff --git a/results/GritLM__GritLM-7B/13f00a0e36500c80ce12870ea513846a066004af/CEDRClassification.json b/results/GritLM__GritLM-7B/13f00a0e36500c80ce12870ea513846a066004af/CEDRClassification.json index 5cdb0e476..5657c4acb 100644 --- a/results/GritLM__GritLM-7B/13f00a0e36500c80ce12870ea513846a066004af/CEDRClassification.json +++ b/results/GritLM__GritLM-7B/13f00a0e36500c80ce12870ea513846a066004af/CEDRClassification.json @@ -6,64 +6,64 @@ "scores": { "test": [ { - "accuracy": 0.4268331562167906, - "f1": 0.3390945952415643, + "accuracy": 0.5067481402763019, + "f1": 0.558164806912173, "hf_subset": "default", "languages": [ "rus-Cyrl" ], - "lrap": 0.6651222104144641, - "main_score": 0.4268331562167906, + "lrap": 0.829187035069082, + "main_score": 0.5067481402763019, "scores_per_experiment": [ { - "accuracy": 0.3884165781083953, - "f1": 0.2804607818140036, - "lrap": 0.6260361317747205 + "accuracy": 0.5536663124335813, + "f1": 0.5961258663627271, + "lrap": 0.8411264612114828 }, { - "accuracy": 0.4314558979808714, - "f1": 0.33355965920567177, - "lrap": 0.6728480340063872 + "accuracy": 0.5026567481402763, + "f1": 0.5526126630349404, + "lrap": 0.818597236981941 }, { - "accuracy": 0.43358129649309246, - "f1": 0.3305699868807631, - "lrap": 0.6441551540914032 + "accuracy": 0.4628055260361318, + "f1": 0.5373377590903264, + "lrap": 0.8115302869288061 }, { - "accuracy": 0.4165781083953241, - "f1": 0.36085144649122225, - "lrap": 0.6580233793836461 + "accuracy": 0.5164718384697131, + "f1": 0.5612016842656354, + "lrap": 0.8409139213602613 }, { - "accuracy": 0.487778958554729, - "f1": 0.3644028302014223, - "lrap": 0.7116365568544206 + "accuracy": 0.4723698193411265, + "f1": 0.5379887319193427, + "lrap": 0.8242826780021321 }, { - "accuracy": 0.4351753453772582, - "f1": 0.3395071879671313, - "lrap": 0.6680658873538905 + "accuracy": 0.5324123273113709, + "f1": 0.5747294951807358, + "lrap": 0.8439426142401759 }, { - "accuracy": 0.412327311370882, - "f1": 0.34329340475559833, - "lrap": 0.6608926673751445 + "accuracy": 0.4867162592986185, + "f1": 0.5528110520146972, + "lrap": 0.8523379383634486 }, { - "accuracy": 0.4070138150903294, - "f1": 0.3024433664772063, - "lrap": 0.6655685441020306 + "accuracy": 0.5260361317747078, + "f1": 0.564912510141887, + "lrap": 0.8302869287991567 }, { - "accuracy": 0.4463336875664187, - "f1": 0.3610523618904392, - "lrap": 0.6724229543039428 + "accuracy": 0.5371944739638682, + "f1": 0.5761977201639873, + "lrap": 0.816365568544109 }, { - "accuracy": 0.40967056323060574, - "f1": 0.3748049267321848, - "lrap": 0.6715727948990549 + "accuracy": 0.4771519659936238, + "f1": 0.5277305869474496, + "lrap": 0.8124867162593056 } ] } diff --git a/results/GritLM__GritLM-7B/13f00a0e36500c80ce12870ea513846a066004af/GeoreviewClassification.json b/results/GritLM__GritLM-7B/13f00a0e36500c80ce12870ea513846a066004af/GeoreviewClassification.json index 19d342138..90756341d 100644 --- a/results/GritLM__GritLM-7B/13f00a0e36500c80ce12870ea513846a066004af/GeoreviewClassification.json +++ b/results/GritLM__GritLM-7B/13f00a0e36500c80ce12870ea513846a066004af/GeoreviewClassification.json @@ -6,64 +6,64 @@ "scores": { "test": [ { - "accuracy": 0.534716796875, - "f1": 0.5156386894496995, - "f1_weighted": 0.5156098889631766, + "accuracy": 0.577001953125, + "f1": 0.5535459921379807, + "f1_weighted": 0.5534836355374633, "hf_subset": "default", "languages": [ "rus-Cyrl" ], - "main_score": 0.534716796875, + "main_score": 0.577001953125, "scores_per_experiment": [ { - "accuracy": 0.52734375, - "f1": 0.5015618649563032, - "f1_weighted": 0.5015476131146306 + "accuracy": 0.56982421875, + "f1": 0.532468735346517, + "f1_weighted": 0.5323990946055268 }, { - "accuracy": 0.5439453125, - "f1": 0.5268084702493628, - "f1_weighted": 0.5267427155400863 + "accuracy": 0.568359375, + "f1": 0.5400540759187279, + "f1_weighted": 0.5399525115125363 }, { - "accuracy": 0.53857421875, - "f1": 0.5124472098391717, - "f1_weighted": 0.5124249241868093 + "accuracy": 0.57958984375, + "f1": 0.5545035836084236, + "f1_weighted": 0.5544533184204532 }, { - "accuracy": 0.52587890625, - "f1": 0.5127576707722727, - "f1_weighted": 0.5127651968317497 + "accuracy": 0.5888671875, + "f1": 0.574574637756059, + "f1_weighted": 0.5745164938647425 }, { - "accuracy": 0.54833984375, - "f1": 0.5296442022930707, - "f1_weighted": 0.5296045802109672 + "accuracy": 0.58251953125, + "f1": 0.5700287702164261, + "f1_weighted": 0.5699689196116826 }, { - "accuracy": 0.5146484375, - "f1": 0.503861863615976, - "f1_weighted": 0.5038497018133786 + "accuracy": 0.5703125, + "f1": 0.5510744984258008, + "f1_weighted": 0.5510102720175564 }, { - "accuracy": 0.546875, - "f1": 0.5261589198911442, - "f1_weighted": 0.5261049715815689 + "accuracy": 0.5869140625, + "f1": 0.5609594874132412, + "f1_weighted": 0.5608919718879246 }, { - "accuracy": 0.537109375, - "f1": 0.5177628854038879, - "f1_weighted": 0.5177183070829026 + "accuracy": 0.58447265625, + "f1": 0.5621435730220392, + "f1_weighted": 0.5620959679309617 }, { - "accuracy": 0.52001953125, - "f1": 0.5065990108521519, - "f1_weighted": 0.50657279854163 + "accuracy": 0.56591796875, + "f1": 0.5468057571970385, + "f1_weighted": 0.5467527102270716 }, { - "accuracy": 0.54443359375, - "f1": 0.5187847966236547, - "f1_weighted": 0.5187680807280429 + "accuracy": 0.5732421875, + "f1": 0.542846802475534, + "f1_weighted": 0.5427950952961769 } ] } diff --git a/results/GritLM__GritLM-7B/13f00a0e36500c80ce12870ea513846a066004af/GeoreviewClusteringP2P.json b/results/GritLM__GritLM-7B/13f00a0e36500c80ce12870ea513846a066004af/GeoreviewClusteringP2P.json index 8d04b7d69..fb1e4f6dd 100644 --- a/results/GritLM__GritLM-7B/13f00a0e36500c80ce12870ea513846a066004af/GeoreviewClusteringP2P.json +++ b/results/GritLM__GritLM-7B/13f00a0e36500c80ce12870ea513846a066004af/GeoreviewClusteringP2P.json @@ -10,21 +10,21 @@ "languages": [ "rus-Cyrl" ], - "main_score": 0.740574715825668, - "v_measure": 0.740574715825668, - "v_measure_std": 0.006241797033532162, + "main_score": 0.7745853792148774, + "v_measure": 0.7745853792148774, + "v_measure_std": 0.006853086743158207, "v_measures": { "Level 0": [ - 0.7333573833199961, - 0.7365297846873942, - 0.7470436176654318, - 0.7434982025914605, - 0.7417383404557991, - 0.7484082725016754, - 0.7436322004319716, - 0.7475006150278336, - 0.7305475185547514, - 0.7334912230203668 + 0.7764658718508038, + 0.7619757108408939, + 0.7810897085614994, + 0.7790867293679788, + 0.78002906397694, + 0.7777393934182731, + 0.7717762108307444, + 0.7639889302573775, + 0.7707471938920466, + 0.7829549791522159 ] } } diff --git a/results/GritLM__GritLM-7B/13f00a0e36500c80ce12870ea513846a066004af/HeadlineClassification.json b/results/GritLM__GritLM-7B/13f00a0e36500c80ce12870ea513846a066004af/HeadlineClassification.json index 2210cf89e..6eec34b16 100644 --- a/results/GritLM__GritLM-7B/13f00a0e36500c80ce12870ea513846a066004af/HeadlineClassification.json +++ b/results/GritLM__GritLM-7B/13f00a0e36500c80ce12870ea513846a066004af/HeadlineClassification.json @@ -6,64 +6,64 @@ "scores": { "test": [ { - "accuracy": 0.856640625, - "f1": 0.85655177870273, - "f1_weighted": 0.8565417396210966, + "accuracy": 0.860791015625, + "f1": 0.8610913011888389, + "f1_weighted": 0.8610780445644284, "hf_subset": "default", "languages": [ "rus-Cyrl" ], - "main_score": 0.856640625, + "main_score": 0.860791015625, "scores_per_experiment": [ { - "accuracy": 0.8505859375, - "f1": 0.8510884884418957, - "f1_weighted": 0.8510734107100054 + "accuracy": 0.85693359375, + "f1": 0.8580712845137968, + "f1_weighted": 0.8580533627338289 }, { - "accuracy": 0.86279296875, - "f1": 0.862634949677982, - "f1_weighted": 0.8626311146821234 + "accuracy": 0.87060546875, + "f1": 0.870404691634029, + "f1_weighted": 0.8704069219117399 }, { - "accuracy": 0.84912109375, - "f1": 0.8484616454497608, - "f1_weighted": 0.8484457335227277 + "accuracy": 0.8623046875, + "f1": 0.8625628571050958, + "f1_weighted": 0.862545717156161 }, { - "accuracy": 0.865234375, - "f1": 0.8657064565723097, - "f1_weighted": 0.8656960231578561 + "accuracy": 0.8662109375, + "f1": 0.8667344628411026, + "f1_weighted": 0.8667152003973787 }, { - "accuracy": 0.86767578125, - "f1": 0.8682408735039889, - "f1_weighted": 0.8682317971849683 + "accuracy": 0.859375, + "f1": 0.8600100881882357, + "f1_weighted": 0.8599966539935013 }, { - "accuracy": 0.853515625, - "f1": 0.8531110222373887, - "f1_weighted": 0.8530992679505259 + "accuracy": 0.86376953125, + "f1": 0.8641743490938604, + "f1_weighted": 0.864161448189619 }, { - "accuracy": 0.83984375, - "f1": 0.8397178057061078, - "f1_weighted": 0.8397160431405494 + "accuracy": 0.8447265625, + "f1": 0.8451338855807656, + "f1_weighted": 0.8451286088071308 }, { - "accuracy": 0.86767578125, - "f1": 0.867507142434622, - "f1_weighted": 0.8674937258034945 + "accuracy": 0.8642578125, + "f1": 0.8643218037597333, + "f1_weighted": 0.8643045311646452 }, { - "accuracy": 0.8447265625, - "f1": 0.8441363022471196, - "f1_weighted": 0.8441315211367281 + "accuracy": 0.8505859375, + "f1": 0.8506898188937985, + "f1_weighted": 0.8506822537342947 }, { - "accuracy": 0.865234375, - "f1": 0.8649131007561254, - "f1_weighted": 0.8648987589219874 + "accuracy": 0.869140625, + "f1": 0.8688097702779726, + "f1_weighted": 0.8687857475559841 } ] } diff --git a/results/GritLM__GritLM-7B/13f00a0e36500c80ce12870ea513846a066004af/InappropriatenessClassification.json b/results/GritLM__GritLM-7B/13f00a0e36500c80ce12870ea513846a066004af/InappropriatenessClassification.json index 87a42b3a6..2f34c753f 100644 --- a/results/GritLM__GritLM-7B/13f00a0e36500c80ce12870ea513846a066004af/InappropriatenessClassification.json +++ b/results/GritLM__GritLM-7B/13f00a0e36500c80ce12870ea513846a066004af/InappropriatenessClassification.json @@ -6,86 +6,86 @@ "scores": { "test": [ { - "accuracy": 0.6529296875, - "ap": 0.6018879744044455, - "ap_weighted": 0.6018879744044455, - "f1": 0.6508322137568193, - "f1_weighted": 0.6508322137568193, + "accuracy": 0.678466796875, + "ap": 0.6223127837753338, + "ap_weighted": 0.6223127837753338, + "f1": 0.6762766473817112, + "f1_weighted": 0.6762766473817112, "hf_subset": "default", "languages": [ "rus-Cyrl" ], - "main_score": 0.6529296875, + "main_score": 0.678466796875, "scores_per_experiment": [ { - "accuracy": 0.69189453125, - "ap": 0.6335417575398804, - "ap_weighted": 0.6335417575398804, - "f1": 0.6918621328355266, - "f1_weighted": 0.6918621328355266 + "accuracy": 0.71337890625, + "ap": 0.6586663661858975, + "ap_weighted": 0.6586663661858975, + "f1": 0.7122724634559348, + "f1_weighted": 0.7122724634559348 }, { - "accuracy": 0.6953125, - "ap": 0.636103592519685, - "ap_weighted": 0.636103592519685, - "f1": 0.6953078507667658, - "f1_weighted": 0.6953078507667658 + "accuracy": 0.68701171875, + "ap": 0.6288590125246791, + "ap_weighted": 0.6288590125246791, + "f1": 0.6870026892007335, + "f1_weighted": 0.6870026892007335 }, { - "accuracy": 0.6875, - "ap": 0.6408704188481675, - "ap_weighted": 0.6408704188481675, - "f1": 0.6823809025314149, - "f1_weighted": 0.6823809025314149 + "accuracy": 0.6982421875, + "ap": 0.6535037478885135, + "ap_weighted": 0.6535037478885135, + "f1": 0.6923256438081236, + "f1_weighted": 0.6923256438081236 }, { - "accuracy": 0.57080078125, - "ap": 0.5406116156408629, - "ap_weighted": 0.5406116156408629, - "f1": 0.5706450822759013, - "f1_weighted": 0.5706450822759013 + "accuracy": 0.6064453125, + "ap": 0.5639856794990723, + "ap_weighted": 0.5639856794990723, + "f1": 0.6061715116806174, + "f1_weighted": 0.6061715116806174 }, { - "accuracy": 0.65673828125, - "ap": 0.6045465538826743, - "ap_weighted": 0.6045465538826743, - "f1": 0.6564131507385448, - "f1_weighted": 0.6564131507385448 + "accuracy": 0.720703125, + "ap": 0.6561120269495413, + "ap_weighted": 0.6561120269495413, + "f1": 0.7204127592991607, + "f1_weighted": 0.7204127592991607 }, { - "accuracy": 0.62353515625, - "ap": 0.576038991152968, - "ap_weighted": 0.576038991152968, - "f1": 0.6230821507267508, - "f1_weighted": 0.6230821507267508 + "accuracy": 0.6591796875, + "ap": 0.6012117513020834, + "ap_weighted": 0.6012117513020834, + "f1": 0.6566439156367214, + "f1_weighted": 0.6566439156367214 }, { - "accuracy": 0.669921875, - "ap": 0.6153165907340863, - "ap_weighted": 0.6153165907340863, - "f1": 0.669725015768867, - "f1_weighted": 0.669725015768867 + "accuracy": 0.69091796875, + "ap": 0.6288141790940571, + "ap_weighted": 0.6288141790940571, + "f1": 0.690251474274475, + "f1_weighted": 0.690251474274475 }, { - "accuracy": 0.6640625, - "ap": 0.601171875, - "ap_weighted": 0.601171875, - "f1": 0.6496052966641201, - "f1_weighted": 0.6496052966641201 + "accuracy": 0.68212890625, + "ap": 0.6156960999818709, + "ap_weighted": 0.6156960999818709, + "f1": 0.6722820632508242, + "f1_weighted": 0.6722820632508242 }, { - "accuracy": 0.6240234375, - "ap": 0.5776999424178286, - "ap_weighted": 0.5776999424178286, - "f1": 0.6239875781610642, - "f1_weighted": 0.6239875781610642 + "accuracy": 0.6611328125, + "ap": 0.6041363655252661, + "ap_weighted": 0.6041363655252661, + "f1": 0.6602567044533174, + "f1_weighted": 0.6602567044533174 }, { - "accuracy": 0.6455078125, - "ap": 0.5929784063083022, - "ap_weighted": 0.5929784063083022, - "f1": 0.6453129770992366, - "f1_weighted": 0.6453129770992366 + "accuracy": 0.66552734375, + "ap": 0.612142608802356, + "ap_weighted": 0.612142608802356, + "f1": 0.6651472487572033, + "f1_weighted": 0.6651472487572033 } ] } diff --git a/results/GritLM__GritLM-7B/13f00a0e36500c80ce12870ea513846a066004af/KinopoiskClassification.json b/results/GritLM__GritLM-7B/13f00a0e36500c80ce12870ea513846a066004af/KinopoiskClassification.json index 37dcb091b..fe4a58669 100644 --- a/results/GritLM__GritLM-7B/13f00a0e36500c80ce12870ea513846a066004af/KinopoiskClassification.json +++ b/results/GritLM__GritLM-7B/13f00a0e36500c80ce12870ea513846a066004af/KinopoiskClassification.json @@ -6,64 +6,64 @@ "scores": { "test": [ { - "accuracy": 0.6425333333333333, - "f1": 0.6159124064651188, - "f1_weighted": 0.6159124064651188, + "accuracy": 0.6914666666666667, + "f1": 0.6642884178770301, + "f1_weighted": 0.6642884178770301, "hf_subset": "default", "languages": [ "rus-Cyrl" ], - "main_score": 0.6425333333333333, + "main_score": 0.6914666666666667, "scores_per_experiment": [ { - "accuracy": 0.5873333333333334, - "f1": 0.5718361265158024, - "f1_weighted": 0.5718361265158024 + "accuracy": 0.6733333333333333, + "f1": 0.642832565001915, + "f1_weighted": 0.6428325650019151 }, { - "accuracy": 0.6073333333333333, - "f1": 0.5651081078842642, - "f1_weighted": 0.5651081078842642 + "accuracy": 0.6373333333333333, + "f1": 0.5687524088450674, + "f1_weighted": 0.5687524088450673 }, { - "accuracy": 0.6486666666666666, - "f1": 0.6233924980025897, - "f1_weighted": 0.6233924980025897 + "accuracy": 0.7093333333333334, + "f1": 0.6877318611220148, + "f1_weighted": 0.6877318611220148 }, { - "accuracy": 0.654, - "f1": 0.6268986628104868, - "f1_weighted": 0.6268986628104868 + "accuracy": 0.69, + "f1": 0.6619129578635038, + "f1_weighted": 0.6619129578635038 }, { - "accuracy": 0.638, - "f1": 0.6203801789755566, - "f1_weighted": 0.6203801789755566 + "accuracy": 0.694, + "f1": 0.6780558926212752, + "f1_weighted": 0.6780558926212752 }, { - "accuracy": 0.672, - "f1": 0.6496274460853835, - "f1_weighted": 0.6496274460853835 + "accuracy": 0.7033333333333334, + "f1": 0.6837738970762802, + "f1_weighted": 0.6837738970762801 }, { - "accuracy": 0.6466666666666666, - "f1": 0.6106816417177391, - "f1_weighted": 0.6106816417177391 + "accuracy": 0.684, + "f1": 0.6547264739229025, + "f1_weighted": 0.6547264739229025 }, { - "accuracy": 0.662, - "f1": 0.642178052304648, - "f1_weighted": 0.6421780523046481 + "accuracy": 0.7026666666666667, + "f1": 0.6889107616392005, + "f1_weighted": 0.6889107616392005 }, { - "accuracy": 0.662, - "f1": 0.6332511037278649, - "f1_weighted": 0.6332511037278649 + "accuracy": 0.708, + "f1": 0.6825875780282934, + "f1_weighted": 0.6825875780282934 }, { - "accuracy": 0.6473333333333333, - "f1": 0.6157702466268525, - "f1_weighted": 0.6157702466268525 + "accuracy": 0.7126666666666667, + "f1": 0.6935997826498485, + "f1_weighted": 0.6935997826498485 } ] } diff --git a/results/GritLM__GritLM-7B/13f00a0e36500c80ce12870ea513846a066004af/MIRACLReranking.json b/results/GritLM__GritLM-7B/13f00a0e36500c80ce12870ea513846a066004af/MIRACLReranking.json new file mode 100644 index 000000000..a36b469c5 --- /dev/null +++ b/results/GritLM__GritLM-7B/13f00a0e36500c80ce12870ea513846a066004af/MIRACLReranking.json @@ -0,0 +1,130 @@ +{ + "dataset_revision": "6d1962c527217f8927fca80f890f14f36b2802af", + "evaluation_time": 3327.1696689128876, + "kg_co2_emissions": null, + "mteb_version": "1.19.0", + "scores": { + "dev": [ + { + "MAP@1(MIRACL)": 0.37192, + "MAP@10(MIRACL)": 0.57721, + "MAP@100(MIRACL)": 0.59328, + "MAP@1000(MIRACL)": 0.59328, + "MAP@20(MIRACL)": 0.58944, + "MAP@3(MIRACL)": 0.50433, + "MAP@5(MIRACL)": 0.54734, + "NDCG@1(MIRACL)": 0.61026, + "NDCG@10(MIRACL)": 0.64227, + "NDCG@100(MIRACL)": 0.67129, + "NDCG@1000(MIRACL)": 0.67129, + "NDCG@20(MIRACL)": 0.66208, + "NDCG@3(MIRACL)": 0.59165, + "NDCG@5(MIRACL)": 0.61283, + "P@1(MIRACL)": 0.61026, + "P@10(MIRACL)": 0.16271, + "P@100(MIRACL)": 0.01953, + "P@1000(MIRACL)": 0.00195, + "P@20(MIRACL)": 0.09138, + "P@3(MIRACL)": 0.3598, + "P@5(MIRACL)": 0.26929, + "Recall@1(MIRACL)": 0.37192, + "Recall@10(MIRACL)": 0.71763, + "Recall@100(MIRACL)": 0.79952, + "Recall@1000(MIRACL)": 0.79952, + "Recall@20(MIRACL)": 0.76994, + "Recall@3(MIRACL)": 0.55569, + "Recall@5(MIRACL)": 0.63772, + "hf_subset": "ru", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.64227, + "nAUC_MAP@1000_diff1(MIRACL)": 0.18872885501720585, + "nAUC_MAP@1000_max(MIRACL)": 0.44458168829872846, + "nAUC_MAP@1000_std(MIRACL)": 0.21388869413292924, + "nAUC_MAP@100_diff1(MIRACL)": 0.18872885501720585, + "nAUC_MAP@100_max(MIRACL)": 0.44458168829872846, + "nAUC_MAP@100_std(MIRACL)": 0.21388869413292924, + "nAUC_MAP@10_diff1(MIRACL)": 0.2023879419641266, + "nAUC_MAP@10_max(MIRACL)": 0.4246095472029897, + "nAUC_MAP@10_std(MIRACL)": 0.20080317617402443, + "nAUC_MAP@1_diff1(MIRACL)": 0.4011884909379283, + "nAUC_MAP@1_max(MIRACL)": 0.2471065895147548, + "nAUC_MAP@1_std(MIRACL)": 0.07120959022651682, + "nAUC_MAP@20_diff1(MIRACL)": 0.19343643048980902, + "nAUC_MAP@20_max(MIRACL)": 0.44094681989153706, + "nAUC_MAP@20_std(MIRACL)": 0.21478456824200937, + "nAUC_MAP@3_diff1(MIRACL)": 0.27665620839690985, + "nAUC_MAP@3_max(MIRACL)": 0.34386563694687033, + "nAUC_MAP@3_std(MIRACL)": 0.1272761557478349, + "nAUC_MAP@5_diff1(MIRACL)": 0.233793841338357, + "nAUC_MAP@5_max(MIRACL)": 0.38761314873799974, + "nAUC_MAP@5_std(MIRACL)": 0.1633693632110916, + "nAUC_NDCG@1000_diff1(MIRACL)": 0.10906732273903746, + "nAUC_NDCG@1000_max(MIRACL)": 0.5300053284482074, + "nAUC_NDCG@1000_std(MIRACL)": 0.2688312502486628, + "nAUC_NDCG@100_diff1(MIRACL)": 0.10906732273903746, + "nAUC_NDCG@100_max(MIRACL)": 0.5300053284482074, + "nAUC_NDCG@100_std(MIRACL)": 0.2688312502486628, + "nAUC_NDCG@10_diff1(MIRACL)": 0.14052974876067997, + "nAUC_NDCG@10_max(MIRACL)": 0.493088060202041, + "nAUC_NDCG@10_std(MIRACL)": 0.2478390816090102, + "nAUC_NDCG@1_diff1(MIRACL)": 0.2425677275533581, + "nAUC_NDCG@1_max(MIRACL)": 0.5144581862707664, + "nAUC_NDCG@1_std(MIRACL)": 0.2107229097437294, + "nAUC_NDCG@20_diff1(MIRACL)": 0.12146064345890883, + "nAUC_NDCG@20_max(MIRACL)": 0.5196459483479215, + "nAUC_NDCG@20_std(MIRACL)": 0.27099950191795363, + "nAUC_NDCG@3_diff1(MIRACL)": 0.19901084569535568, + "nAUC_NDCG@3_max(MIRACL)": 0.44446074382798856, + "nAUC_NDCG@3_std(MIRACL)": 0.1864691369451806, + "nAUC_NDCG@5_diff1(MIRACL)": 0.17646326347011831, + "nAUC_NDCG@5_max(MIRACL)": 0.45752657431505034, + "nAUC_NDCG@5_std(MIRACL)": 0.208128625087784, + "nAUC_P@1000_diff1(MIRACL)": -0.2780185821200513, + "nAUC_P@1000_max(MIRACL)": 0.28115706069915325, + "nAUC_P@1000_std(MIRACL)": 0.20234666179777056, + "nAUC_P@100_diff1(MIRACL)": -0.27801858212005115, + "nAUC_P@100_max(MIRACL)": 0.2811570606991533, + "nAUC_P@100_std(MIRACL)": 0.20234666179777097, + "nAUC_P@10_diff1(MIRACL)": -0.2651497249443695, + "nAUC_P@10_max(MIRACL)": 0.33015283596142714, + "nAUC_P@10_std(MIRACL)": 0.24273149339628214, + "nAUC_P@1_diff1(MIRACL)": 0.2425677275533581, + "nAUC_P@1_max(MIRACL)": 0.5144581862707664, + "nAUC_P@1_std(MIRACL)": 0.2107229097437294, + "nAUC_P@20_diff1(MIRACL)": -0.27098244224997264, + "nAUC_P@20_max(MIRACL)": 0.30578176174665733, + "nAUC_P@20_std(MIRACL)": 0.24319403683847374, + "nAUC_P@3_diff1(MIRACL)": -0.13284529898224817, + "nAUC_P@3_max(MIRACL)": 0.3883227827212337, + "nAUC_P@3_std(MIRACL)": 0.2094681546048287, + "nAUC_P@5_diff1(MIRACL)": -0.21681173905507617, + "nAUC_P@5_max(MIRACL)": 0.3539485028251882, + "nAUC_P@5_std(MIRACL)": 0.22385684148131207, + "nAUC_Recall@1000_diff1(MIRACL)": -0.13958510988712858, + "nAUC_Recall@1000_max(MIRACL)": 0.695732159810508, + "nAUC_Recall@1000_std(MIRACL)": 0.39649387573561623, + "nAUC_Recall@100_diff1(MIRACL)": -0.13958510988712858, + "nAUC_Recall@100_max(MIRACL)": 0.695732159810508, + "nAUC_Recall@100_std(MIRACL)": 0.39649387573561623, + "nAUC_Recall@10_diff1(MIRACL)": 0.010378278089324592, + "nAUC_Recall@10_max(MIRACL)": 0.5234665068638968, + "nAUC_Recall@10_std(MIRACL)": 0.30140256427061435, + "nAUC_Recall@1_diff1(MIRACL)": 0.4011884909379283, + "nAUC_Recall@1_max(MIRACL)": 0.2471065895147548, + "nAUC_Recall@1_std(MIRACL)": 0.07120959022651682, + "nAUC_Recall@20_diff1(MIRACL)": -0.06931801984685573, + "nAUC_Recall@20_max(MIRACL)": 0.6252190064439774, + "nAUC_Recall@20_std(MIRACL)": 0.386613784588266, + "nAUC_Recall@3_diff1(MIRACL)": 0.22168336771267008, + "nAUC_Recall@3_max(MIRACL)": 0.34258493382442223, + "nAUC_Recall@3_std(MIRACL)": 0.1305757257481243, + "nAUC_Recall@5_diff1(MIRACL)": 0.12650271368245275, + "nAUC_Recall@5_max(MIRACL)": 0.40879630555010543, + "nAUC_Recall@5_std(MIRACL)": 0.19551432320681342 + } + ] + }, + "task_name": "MIRACLReranking" +} \ No newline at end of file diff --git a/results/GritLM__GritLM-7B/13f00a0e36500c80ce12870ea513846a066004af/MIRACLRetrieval.json b/results/GritLM__GritLM-7B/13f00a0e36500c80ce12870ea513846a066004af/MIRACLRetrieval.json new file mode 100644 index 000000000..e06011d50 --- /dev/null +++ b/results/GritLM__GritLM-7B/13f00a0e36500c80ce12870ea513846a066004af/MIRACLRetrieval.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "main", + "evaluation_time": 337420.53926754, + "kg_co2_emissions": null, + "mteb_version": "1.19.4", + "scores": { + "dev": [ + { + "hf_subset": "ru", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.68304, + "map_at_1": 0.33407, + "map_at_10": 0.58079, + "map_at_100": 0.60802, + "map_at_1000": 0.6086, + "map_at_20": 0.59876, + "map_at_3": 0.49322, + "map_at_5": 0.54164, + "mrr_at_1": 0.6573482428115016, + "mrr_at_10": 0.763011562452457, + "mrr_at_100": 0.765171735763222, + "mrr_at_1000": 0.7652109598789325, + "mrr_at_20": 0.7646630158181026, + "mrr_at_3": 0.7456070287539937, + "mrr_at_5": 0.7571485623003196, + "nauc_map_at_1000_diff1": 0.31327376735139456, + "nauc_map_at_1000_max": 0.27220605917173984, + "nauc_map_at_1000_std": 0.025372510112681396, + "nauc_map_at_100_diff1": 0.3132562736896674, + "nauc_map_at_100_max": 0.27256006810349703, + "nauc_map_at_100_std": 0.025701756209609315, + "nauc_map_at_10_diff1": 0.30837312680691475, + "nauc_map_at_10_max": 0.25978762231706776, + "nauc_map_at_10_std": -0.0004667527330805698, + "nauc_map_at_1_diff1": 0.39365127092175695, + "nauc_map_at_1_max": 0.16861672106931977, + "nauc_map_at_1_std": -0.07468988346935397, + "nauc_map_at_20_diff1": 0.3126977728369778, + "nauc_map_at_20_max": 0.27081176648289557, + "nauc_map_at_20_std": 0.018122044106511657, + "nauc_map_at_3_diff1": 0.3182337388605749, + "nauc_map_at_3_max": 0.20758282274978015, + "nauc_map_at_3_std": -0.05596339473239914, + "nauc_map_at_5_diff1": 0.30659468476572554, + "nauc_map_at_5_max": 0.23125701131135495, + "nauc_map_at_5_std": -0.0355645038678688, + "nauc_mrr_at_1000_diff1": 0.41655933314791843, + "nauc_mrr_at_1000_max": 0.3860147154033262, + "nauc_mrr_at_1000_std": 0.17658610961038462, + "nauc_mrr_at_100_diff1": 0.41649475791056595, + "nauc_mrr_at_100_max": 0.3861127235056782, + "nauc_mrr_at_100_std": 0.176726052043403, + "nauc_mrr_at_10_diff1": 0.41701202318365604, + "nauc_mrr_at_10_max": 0.38842138555638983, + "nauc_mrr_at_10_std": 0.17924697670703102, + "nauc_mrr_at_1_diff1": 0.4308079006571138, + "nauc_mrr_at_1_max": 0.3398523912184161, + "nauc_mrr_at_1_std": 0.13950290395204198, + "nauc_mrr_at_20_diff1": 0.4165502241925315, + "nauc_mrr_at_20_max": 0.3870348093721698, + "nauc_mrr_at_20_std": 0.1774164905845436, + "nauc_mrr_at_3_diff1": 0.41487116601908824, + "nauc_mrr_at_3_max": 0.3926485266471638, + "nauc_mrr_at_3_std": 0.17141639765927694, + "nauc_mrr_at_5_diff1": 0.4149349132605035, + "nauc_mrr_at_5_max": 0.3916187498102159, + "nauc_mrr_at_5_std": 0.18189328005917277, + "nauc_ndcg_at_1000_diff1": 0.33187373240581813, + "nauc_ndcg_at_1000_max": 0.32335488166581966, + "nauc_ndcg_at_1000_std": 0.1020264469577908, + "nauc_ndcg_at_100_diff1": 0.3301297877597755, + "nauc_ndcg_at_100_max": 0.32809518933505316, + "nauc_ndcg_at_100_std": 0.11013977499332274, + "nauc_ndcg_at_10_diff1": 0.3190301763797242, + "nauc_ndcg_at_10_max": 0.3077848414839524, + "nauc_ndcg_at_10_std": 0.05664542894163085, + "nauc_ndcg_at_1_diff1": 0.4308079006571138, + "nauc_ndcg_at_1_max": 0.3398523912184161, + "nauc_ndcg_at_1_std": 0.13950290395204198, + "nauc_ndcg_at_20_diff1": 0.32839893058202124, + "nauc_ndcg_at_20_max": 0.3281039687820589, + "nauc_ndcg_at_20_std": 0.09100895886625812, + "nauc_ndcg_at_3_diff1": 0.3130268310003486, + "nauc_ndcg_at_3_max": 0.2768232662704616, + "nauc_ndcg_at_3_std": 0.03984810374748364, + "nauc_ndcg_at_5_diff1": 0.31296929441449506, + "nauc_ndcg_at_5_max": 0.27910041586333806, + "nauc_ndcg_at_5_std": 0.030070262981992824, + "nauc_precision_at_1000_diff1": -0.13439424402682798, + "nauc_precision_at_1000_max": 0.04116872217431826, + "nauc_precision_at_1000_std": 0.2315132576734252, + "nauc_precision_at_100_diff1": -0.12262651973807857, + "nauc_precision_at_100_max": 0.07143989721240905, + "nauc_precision_at_100_std": 0.26129055322730194, + "nauc_precision_at_10_diff1": -0.08518456790489645, + "nauc_precision_at_10_max": 0.16877369376578144, + "nauc_precision_at_10_std": 0.25219505091433353, + "nauc_precision_at_1_diff1": 0.4308079006571138, + "nauc_precision_at_1_max": 0.3398523912184161, + "nauc_precision_at_1_std": 0.13950290395204198, + "nauc_precision_at_20_diff1": -0.09564250815054394, + "nauc_precision_at_20_max": 0.13915634299622515, + "nauc_precision_at_20_std": 0.2741736337710519, + "nauc_precision_at_3_diff1": 0.023824661854890038, + "nauc_precision_at_3_max": 0.22794211155082097, + "nauc_precision_at_3_std": 0.16693572315738514, + "nauc_precision_at_5_diff1": -0.05042533742430008, + "nauc_precision_at_5_max": 0.18488974245961376, + "nauc_precision_at_5_std": 0.18993506620925418, + "nauc_recall_at_1000_diff1": 0.20153264962623274, + "nauc_recall_at_1000_max": 0.631658647030604, + "nauc_recall_at_1000_std": 0.904653315036784, + "nauc_recall_at_100_diff1": 0.21456449124136237, + "nauc_recall_at_100_max": 0.46150733250597503, + "nauc_recall_at_100_std": 0.4634067017585428, + "nauc_recall_at_10_diff1": 0.2259336169833242, + "nauc_recall_at_10_max": 0.2757159090150717, + "nauc_recall_at_10_std": 0.041482749465821644, + "nauc_recall_at_1_diff1": 0.39365127092175695, + "nauc_recall_at_1_max": 0.16861672106931977, + "nauc_recall_at_1_std": -0.07468988346935397, + "nauc_recall_at_20_diff1": 0.24437812494328126, + "nauc_recall_at_20_max": 0.35595197167925857, + "nauc_recall_at_20_std": 0.15324965439295687, + "nauc_recall_at_3_diff1": 0.2670465950636703, + "nauc_recall_at_3_max": 0.19677042873617728, + "nauc_recall_at_3_std": -0.06291908225293155, + "nauc_recall_at_5_diff1": 0.23050492794697033, + "nauc_recall_at_5_max": 0.2134100199690657, + "nauc_recall_at_5_std": -0.038178900942984415, + "ndcg_at_1": 0.65735, + "ndcg_at_10": 0.68304, + "ndcg_at_100": 0.7427, + "ndcg_at_1000": 0.74893, + "ndcg_at_20": 0.71508, + "ndcg_at_3": 0.62359, + "ndcg_at_5": 0.64306, + "precision_at_1": 0.65735, + "precision_at_10": 0.19936, + "precision_at_100": 0.02709, + "precision_at_1000": 0.00283, + "precision_at_20": 0.11565, + "precision_at_3": 0.42173, + "precision_at_5": 0.31805, + "recall_at_1": 0.33407, + "recall_at_10": 0.77913, + "recall_at_100": 0.96052, + "recall_at_1000": 0.99405, + "recall_at_20": 0.86434, + "recall_at_3": 0.56429, + "recall_at_5": 0.66494 + } + ] + }, + "task_name": "MIRACLRetrieval" +} \ No newline at end of file diff --git a/results/GritLM__GritLM-7B/13f00a0e36500c80ce12870ea513846a066004af/RiaNewsRetrieval.json b/results/GritLM__GritLM-7B/13f00a0e36500c80ce12870ea513846a066004af/RiaNewsRetrieval.json new file mode 100644 index 000000000..e1ad3b83e --- /dev/null +++ b/results/GritLM__GritLM-7B/13f00a0e36500c80ce12870ea513846a066004af/RiaNewsRetrieval.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "82374b0bbacda6114f39ff9c5b925fa1512ca5d7", + "evaluation_time": 26724.571657896042, + "kg_co2_emissions": null, + "mteb_version": "1.19.4", + "scores": { + "test": [ + { + "hf_subset": "default", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.8276, + "map_at_1": 0.7218, + "map_at_10": 0.79638, + "map_at_100": 0.79849, + "map_at_1000": 0.79857, + "map_at_20": 0.79783, + "map_at_3": 0.78467, + "map_at_5": 0.79232, + "mrr_at_1": 0.7218, + "mrr_at_10": 0.7963939285714285, + "mrr_at_100": 0.7985087848085632, + "mrr_at_1000": 0.7985878087693301, + "mrr_at_20": 0.7978419962820118, + "mrr_at_3": 0.7846833333333333, + "mrr_at_5": 0.7923333333333333, + "nauc_map_at_1000_diff1": 0.7685551724182931, + "nauc_map_at_1000_max": 0.28535538553703593, + "nauc_map_at_1000_std": -0.13894913172429815, + "nauc_map_at_100_diff1": 0.7685417274996087, + "nauc_map_at_100_max": 0.28536796436852063, + "nauc_map_at_100_std": -0.13883213041100312, + "nauc_map_at_10_diff1": 0.7677171651197945, + "nauc_map_at_10_max": 0.28486881416681464, + "nauc_map_at_10_std": -0.1405853742590068, + "nauc_map_at_1_diff1": 0.7913553266993422, + "nauc_map_at_1_max": 0.2667436936993964, + "nauc_map_at_1_std": -0.1551647814607289, + "nauc_map_at_20_diff1": 0.7683711150400182, + "nauc_map_at_20_max": 0.28550655772662853, + "nauc_map_at_20_std": -0.13884984807905204, + "nauc_map_at_3_diff1": 0.7681341544640481, + "nauc_map_at_3_max": 0.2839446676840796, + "nauc_map_at_3_std": -0.14853210550422866, + "nauc_map_at_5_diff1": 0.7664598785456492, + "nauc_map_at_5_max": 0.2841353485044683, + "nauc_map_at_5_std": -0.14419352922737608, + "nauc_mrr_at_1000_diff1": 0.768550207923453, + "nauc_mrr_at_1000_max": 0.2852984120101035, + "nauc_mrr_at_1000_std": -0.1390110314149599, + "nauc_mrr_at_100_diff1": 0.768536764091206, + "nauc_mrr_at_100_max": 0.28531101660776204, + "nauc_mrr_at_100_std": -0.13889399387306933, + "nauc_mrr_at_10_diff1": 0.7677121938310604, + "nauc_mrr_at_10_max": 0.2848124819041456, + "nauc_mrr_at_10_std": -0.14064666448688243, + "nauc_mrr_at_1_diff1": 0.7913553266993422, + "nauc_mrr_at_1_max": 0.2667436936993964, + "nauc_mrr_at_1_std": -0.1551647814607289, + "nauc_mrr_at_20_diff1": 0.7683661562224086, + "nauc_mrr_at_20_max": 0.285449829064035, + "nauc_mrr_at_20_std": -0.13891148689099508, + "nauc_mrr_at_3_diff1": 0.7681294843837785, + "nauc_mrr_at_3_max": 0.28389132774965564, + "nauc_mrr_at_3_std": -0.14859065710339403, + "nauc_mrr_at_5_diff1": 0.7664549140008935, + "nauc_mrr_at_5_max": 0.28408013493759443, + "nauc_mrr_at_5_std": -0.1442538190412962, + "nauc_ndcg_at_1000_diff1": 0.7625745204841022, + "nauc_ndcg_at_1000_max": 0.29551270584412115, + "nauc_ndcg_at_1000_std": -0.1223425605943076, + "nauc_ndcg_at_100_diff1": 0.7621136930645617, + "nauc_ndcg_at_100_max": 0.2962258839391911, + "nauc_ndcg_at_100_std": -0.11775919249057419, + "nauc_ndcg_at_10_diff1": 0.7585832477121311, + "nauc_ndcg_at_10_max": 0.2942752841351498, + "nauc_ndcg_at_10_std": -0.12539506674698242, + "nauc_ndcg_at_1_diff1": 0.7913553266993422, + "nauc_ndcg_at_1_max": 0.2667436936993964, + "nauc_ndcg_at_1_std": -0.1551647814607289, + "nauc_ndcg_at_20_diff1": 0.7608811811229103, + "nauc_ndcg_at_20_max": 0.2972974374082699, + "nauc_ndcg_at_20_std": -0.11737064613695848, + "nauc_ndcg_at_3_diff1": 0.7594066172236434, + "nauc_ndcg_at_3_max": 0.2912381374805187, + "nauc_ndcg_at_3_std": -0.14467678054622676, + "nauc_ndcg_at_5_diff1": 0.7554322780182783, + "nauc_ndcg_at_5_max": 0.2918811995876501, + "nauc_ndcg_at_5_std": -0.1354421862977647, + "nauc_precision_at_1000_diff1": 0.6209596583731548, + "nauc_precision_at_1000_max": 0.6521083923765557, + "nauc_precision_at_1000_std": 0.40587068160597, + "nauc_precision_at_100_diff1": 0.6816978404264991, + "nauc_precision_at_100_max": 0.4847971446643206, + "nauc_precision_at_100_std": 0.290982199331348, + "nauc_precision_at_10_diff1": 0.698432150638036, + "nauc_precision_at_10_max": 0.36079744397759544, + "nauc_precision_at_10_std": -0.008851370409274388, + "nauc_precision_at_1_diff1": 0.7913553266993422, + "nauc_precision_at_1_max": 0.2667436936993964, + "nauc_precision_at_1_std": -0.1551647814607289, + "nauc_precision_at_20_diff1": 0.6998107709632133, + "nauc_precision_at_20_max": 0.4168868257782681, + "nauc_precision_at_20_std": 0.1192192685244594, + "nauc_precision_at_3_diff1": 0.7234864413082248, + "nauc_precision_at_3_max": 0.32165394145592224, + "nauc_precision_at_3_std": -0.12815509350162624, + "nauc_precision_at_5_diff1": 0.698626620062485, + "nauc_precision_at_5_max": 0.3314140775862442, + "nauc_precision_at_5_std": -0.08758743808080392, + "nauc_recall_at_1000_diff1": 0.6209596583731529, + "nauc_recall_at_1000_max": 0.6521083923765636, + "nauc_recall_at_1000_std": 0.405870681605975, + "nauc_recall_at_100_diff1": 0.6816978404264898, + "nauc_recall_at_100_max": 0.4847971446643141, + "nauc_recall_at_100_std": 0.2909821993313444, + "nauc_recall_at_10_diff1": 0.6984321506380333, + "nauc_recall_at_10_max": 0.3607974439775918, + "nauc_recall_at_10_std": -0.008851370409275241, + "nauc_recall_at_1_diff1": 0.7913553266993422, + "nauc_recall_at_1_max": 0.2667436936993964, + "nauc_recall_at_1_std": -0.1551647814607289, + "nauc_recall_at_20_diff1": 0.6998107709632089, + "nauc_recall_at_20_max": 0.41688682577827035, + "nauc_recall_at_20_std": 0.11921926852446021, + "nauc_recall_at_3_diff1": 0.7234864413082236, + "nauc_recall_at_3_max": 0.3216539414559216, + "nauc_recall_at_3_std": -0.1281550935016285, + "nauc_recall_at_5_diff1": 0.6986266200624814, + "nauc_recall_at_5_max": 0.33141407758624625, + "nauc_recall_at_5_std": -0.0875874380808035, + "ndcg_at_1": 0.7218, + "ndcg_at_10": 0.8276, + "ndcg_at_100": 0.83754, + "ndcg_at_1000": 0.83973, + "ndcg_at_20": 0.83282, + "ndcg_at_3": 0.80422, + "ndcg_at_5": 0.8179, + "precision_at_1": 0.7218, + "precision_at_10": 0.09232, + "precision_at_100": 0.00969, + "precision_at_1000": 0.00099, + "precision_at_20": 0.04719, + "precision_at_3": 0.28683, + "precision_at_5": 0.1787, + "recall_at_1": 0.7218, + "recall_at_10": 0.9232, + "recall_at_100": 0.969, + "recall_at_1000": 0.9864, + "recall_at_20": 0.9437, + "recall_at_3": 0.8605, + "recall_at_5": 0.8935 + } + ] + }, + "task_name": "RiaNewsRetrieval" +} \ No newline at end of file diff --git a/results/GritLM__GritLM-7B/13f00a0e36500c80ce12870ea513846a066004af/RuBQReranking.json b/results/GritLM__GritLM-7B/13f00a0e36500c80ce12870ea513846a066004af/RuBQReranking.json index 207473fdb..8f4a68dd4 100644 --- a/results/GritLM__GritLM-7B/13f00a0e36500c80ce12870ea513846a066004af/RuBQReranking.json +++ b/results/GritLM__GritLM-7B/13f00a0e36500c80ce12870ea513846a066004af/RuBQReranking.json @@ -10,15 +10,15 @@ "languages": [ "rus-Cyrl" ], - "main_score": 0.7242713776227294, - "map": 0.7242713776227294, - "mrr": 0.7700469743022934, - "nAUC_map_diff1": 0.3181193615547211, - "nAUC_map_max": -0.004842024797418193, - "nAUC_map_std": -0.018188448801014237, - "nAUC_mrr_diff1": 0.3331256805441882, - "nAUC_mrr_max": 0.010706577714197451, - "nAUC_mrr_std": -0.0063704348629141155 + "main_score": 0.7574687449941541, + "map": 0.7574687449941541, + "mrr": 0.8044428581662624, + "nAUC_map_diff1": 0.3955038614005378, + "nAUC_map_max": 0.12918856749521054, + "nAUC_map_std": 0.04475564991098927, + "nAUC_mrr_diff1": 0.4305288527485588, + "nAUC_mrr_max": 0.17679709351088221, + "nAUC_mrr_std": 0.07209152773343802 } ] }, diff --git a/results/GritLM__GritLM-7B/13f00a0e36500c80ce12870ea513846a066004af/RuBQRetrieval.json b/results/GritLM__GritLM-7B/13f00a0e36500c80ce12870ea513846a066004af/RuBQRetrieval.json index a1bdd356e..80a21eb7c 100644 --- a/results/GritLM__GritLM-7B/13f00a0e36500c80ce12870ea513846a066004af/RuBQRetrieval.json +++ b/results/GritLM__GritLM-7B/13f00a0e36500c80ce12870ea513846a066004af/RuBQRetrieval.json @@ -10,147 +10,147 @@ "languages": [ "rus-Cyrl" ], - "main_score": 0.70942, - "map_at_1": 0.3914, - "map_at_10": 0.62241, - "map_at_100": 0.63215, - "map_at_1000": 0.63227, - "map_at_20": 0.62972, - "map_at_3": 0.56019, - "map_at_5": 0.59943, - "mrr_at_1": 0.5543735224586288, - "mrr_at_10": 0.6959111974182906, - "mrr_at_100": 0.6986464709841909, - "mrr_at_1000": 0.6986720117361649, - "mrr_at_20": 0.6978271266868045, - "mrr_at_3": 0.6735618597320732, - "mrr_at_5": 0.6888100866824286, - "nauc_map_at_1000_diff1": 0.25759529562031197, - "nauc_map_at_1000_max": 0.05094092264230687, - "nauc_map_at_1000_std": -0.2537968270390769, - "nauc_map_at_100_diff1": 0.2574314865098614, - "nauc_map_at_100_max": 0.05105165004881788, - "nauc_map_at_100_std": -0.2536070457670585, - "nauc_map_at_10_diff1": 0.25685890262273114, - "nauc_map_at_10_max": 0.05179489008310951, - "nauc_map_at_10_std": -0.2568727269868421, - "nauc_map_at_1_diff1": 0.3116411284344674, - "nauc_map_at_1_max": 0.015890703466955838, - "nauc_map_at_1_std": -0.2125533602986318, - "nauc_map_at_20_diff1": 0.2571645323654548, - "nauc_map_at_20_max": 0.052767766446166924, - "nauc_map_at_20_std": -0.2533163808303952, - "nauc_map_at_3_diff1": 0.26200836330329635, - "nauc_map_at_3_max": 0.030920144159953155, - "nauc_map_at_3_std": -0.2702449292412035, - "nauc_map_at_5_diff1": 0.2581121097345037, - "nauc_map_at_5_max": 0.04452979302911522, - "nauc_map_at_5_std": -0.2641707478369312, - "nauc_mrr_at_1000_diff1": 0.2837872936673349, - "nauc_mrr_at_1000_max": 0.0563661722380757, - "nauc_mrr_at_1000_std": -0.26303216090509296, - "nauc_mrr_at_100_diff1": 0.2837404371117496, - "nauc_mrr_at_100_max": 0.05642876457175256, - "nauc_mrr_at_100_std": -0.262970027759236, - "nauc_mrr_at_10_diff1": 0.28273789402983956, - "nauc_mrr_at_10_max": 0.05822303220210209, - "nauc_mrr_at_10_std": -0.26266877042479814, - "nauc_mrr_at_1_diff1": 0.3035368640426485, - "nauc_mrr_at_1_max": 0.02632381600179968, - "nauc_mrr_at_1_std": -0.25570125766815116, - "nauc_mrr_at_20_diff1": 0.28343082258354446, - "nauc_mrr_at_20_max": 0.0571196279219444, - "nauc_mrr_at_20_std": -0.2626420344053813, - "nauc_mrr_at_3_diff1": 0.2812411353076279, - "nauc_mrr_at_3_max": 0.04970836259261729, - "nauc_mrr_at_3_std": -0.27561702034871355, - "nauc_mrr_at_5_diff1": 0.28381049006897235, - "nauc_mrr_at_5_max": 0.05863680618294814, - "nauc_mrr_at_5_std": -0.26747078003629177, - "nauc_ndcg_at_1000_diff1": 0.25972886691258346, - "nauc_ndcg_at_1000_max": 0.06539796723864891, - "nauc_ndcg_at_1000_std": -0.24570391349926982, - "nauc_ndcg_at_100_diff1": 0.2573786622454451, - "nauc_ndcg_at_100_max": 0.06818097425654232, - "nauc_ndcg_at_100_std": -0.24153388472282025, - "nauc_ndcg_at_10_diff1": 0.2520060536906403, - "nauc_ndcg_at_10_max": 0.07811505289464594, - "nauc_ndcg_at_10_std": -0.2464136751709898, - "nauc_ndcg_at_1_diff1": 0.3035368640426485, - "nauc_ndcg_at_1_max": 0.02632381600179968, - "nauc_ndcg_at_1_std": -0.25570125766815116, - "nauc_ndcg_at_20_diff1": 0.25484422033739834, - "nauc_ndcg_at_20_max": 0.0787254512701679, - "nauc_ndcg_at_20_std": -0.23725894180541857, - "nauc_ndcg_at_3_diff1": 0.2518840902971601, - "nauc_ndcg_at_3_max": 0.03429694276574871, - "nauc_ndcg_at_3_std": -0.28179290888565367, - "nauc_ndcg_at_5_diff1": 0.25368462389495083, - "nauc_ndcg_at_5_max": 0.06216613141478157, - "nauc_ndcg_at_5_std": -0.2672243289299006, - "nauc_precision_at_1000_diff1": -0.129353493617144, - "nauc_precision_at_1000_max": 0.01867297930650377, - "nauc_precision_at_1000_std": 0.06526048320760017, - "nauc_precision_at_100_diff1": -0.13148188261412586, - "nauc_precision_at_100_max": 0.028455047505829514, - "nauc_precision_at_100_std": 0.0746296873452419, - "nauc_precision_at_10_diff1": -0.09192307254413215, - "nauc_precision_at_10_max": 0.06692271848873407, - "nauc_precision_at_10_std": 0.005974450070111703, - "nauc_precision_at_1_diff1": 0.3035368640426485, - "nauc_precision_at_1_max": 0.02632381600179968, - "nauc_precision_at_1_std": -0.25570125766815116, - "nauc_precision_at_20_diff1": -0.11496673273018226, - "nauc_precision_at_20_max": 0.05817713915278614, - "nauc_precision_at_20_std": 0.05769256064723617, - "nauc_precision_at_3_diff1": 0.016390227125436047, - "nauc_precision_at_3_max": 0.02614289798410267, - "nauc_precision_at_3_std": -0.18036238609792438, - "nauc_precision_at_5_diff1": -0.04269499387765357, - "nauc_precision_at_5_max": 0.05648974662553826, - "nauc_precision_at_5_std": -0.08486128401824687, - "nauc_recall_at_1000_diff1": 0.3903552566382808, - "nauc_recall_at_1000_max": 0.7302753511828539, - "nauc_recall_at_1000_std": 0.5851139310993769, - "nauc_recall_at_100_diff1": 0.07728219482586643, - "nauc_recall_at_100_max": 0.43701108276178785, - "nauc_recall_at_100_std": 0.30416912335498475, - "nauc_recall_at_10_diff1": 0.15109684958593406, - "nauc_recall_at_10_max": 0.207307810554185, - "nauc_recall_at_10_std": -0.1369138086811323, - "nauc_recall_at_1_diff1": 0.3116411284344674, - "nauc_recall_at_1_max": 0.015890703466955838, - "nauc_recall_at_1_std": -0.2125533602986318, - "nauc_recall_at_20_diff1": 0.14944589484828946, - "nauc_recall_at_20_max": 0.3076888115872027, - "nauc_recall_at_20_std": -0.0053940039859413915, - "nauc_recall_at_3_diff1": 0.209528241308026, - "nauc_recall_at_3_max": 0.04320976434704012, - "nauc_recall_at_3_std": -0.2726477532366196, - "nauc_recall_at_5_diff1": 0.19031169123605582, - "nauc_recall_at_5_max": 0.10340603217994729, - "nauc_recall_at_5_std": -0.2445398882790516, - "ndcg_at_1": 0.55437, - "ndcg_at_10": 0.70942, - "ndcg_at_100": 0.73732, - "ndcg_at_1000": 0.73915, - "ndcg_at_20": 0.72632, - "ndcg_at_3": 0.62578, - "ndcg_at_5": 0.67174, - "precision_at_1": 0.55437, - "precision_at_10": 0.14474, - "precision_at_100": 0.01652, + "main_score": 0.76308, + "map_at_1": 0.4573, + "map_at_10": 0.68559, + "map_at_100": 0.69269, + "map_at_1000": 0.69279, + "map_at_20": 0.69088, + "map_at_3": 0.62536, + "map_at_5": 0.66272, + "mrr_at_1": 0.642434988179669, + "mrr_at_10": 0.7565595800968141, + "mrr_at_100": 0.7587362252813463, + "mrr_at_1000": 0.7587684353683409, + "mrr_at_20": 0.7583483104988195, + "mrr_at_3": 0.7375886524822695, + "mrr_at_5": 0.7503841607565013, + "nauc_map_at_1000_diff1": 0.39234048617816986, + "nauc_map_at_1000_max": 0.19736437167329918, + "nauc_map_at_1000_std": -0.20795890640006043, + "nauc_map_at_100_diff1": 0.3921287293174344, + "nauc_map_at_100_max": 0.19746448780375403, + "nauc_map_at_100_std": -0.2077064795685256, + "nauc_map_at_10_diff1": 0.38909530775106727, + "nauc_map_at_10_max": 0.19818574153331936, + "nauc_map_at_10_std": -0.20898067188664174, + "nauc_map_at_1_diff1": 0.44362720616770623, + "nauc_map_at_1_max": 0.150140661012972, + "nauc_map_at_1_std": -0.18067731022979855, + "nauc_map_at_20_diff1": 0.390234718100367, + "nauc_map_at_20_max": 0.19728319417155726, + "nauc_map_at_20_std": -0.2078478143815166, + "nauc_map_at_3_diff1": 0.3889187095531059, + "nauc_map_at_3_max": 0.16580538200440798, + "nauc_map_at_3_std": -0.22256374857313632, + "nauc_map_at_5_diff1": 0.38627535331535673, + "nauc_map_at_5_max": 0.17781409881361873, + "nauc_map_at_5_std": -0.21916006013173472, + "nauc_mrr_at_1000_diff1": 0.4584444042870276, + "nauc_mrr_at_1000_max": 0.2460202404274903, + "nauc_mrr_at_1000_std": -0.22156309317944234, + "nauc_mrr_at_100_diff1": 0.4583789750851367, + "nauc_mrr_at_100_max": 0.24610685385565295, + "nauc_mrr_at_100_std": -0.22149437396659763, + "nauc_mrr_at_10_diff1": 0.45623320989100247, + "nauc_mrr_at_10_max": 0.24915660189677122, + "nauc_mrr_at_10_std": -0.2193983036209973, + "nauc_mrr_at_1_diff1": 0.4840917297107521, + "nauc_mrr_at_1_max": 0.20824808887073123, + "nauc_mrr_at_1_std": -0.2257971919145046, + "nauc_mrr_at_20_diff1": 0.45784831639843226, + "nauc_mrr_at_20_max": 0.24667142241450835, + "nauc_mrr_at_20_std": -0.2211779179092714, + "nauc_mrr_at_3_diff1": 0.4593363985102777, + "nauc_mrr_at_3_max": 0.24800426996621217, + "nauc_mrr_at_3_std": -0.22715438504209823, + "nauc_mrr_at_5_diff1": 0.4557304324844246, + "nauc_mrr_at_5_max": 0.24769148011753664, + "nauc_mrr_at_5_std": -0.22414086702434674, + "nauc_ndcg_at_1000_diff1": 0.40548042307924315, + "nauc_ndcg_at_1000_max": 0.22674457337453877, + "nauc_ndcg_at_1000_std": -0.19829275096008953, + "nauc_ndcg_at_100_diff1": 0.40140349744670756, + "nauc_ndcg_at_100_max": 0.22835164575979336, + "nauc_ndcg_at_100_std": -0.19394113332227206, + "nauc_ndcg_at_10_diff1": 0.38914173115891865, + "nauc_ndcg_at_10_max": 0.235931781220058, + "nauc_ndcg_at_10_std": -0.19598271063964948, + "nauc_ndcg_at_1_diff1": 0.4840917297107521, + "nauc_ndcg_at_1_max": 0.20824808887073123, + "nauc_ndcg_at_1_std": -0.2257971919145046, + "nauc_ndcg_at_20_diff1": 0.3928937233319363, + "nauc_ndcg_at_20_max": 0.23005333547246132, + "nauc_ndcg_at_20_std": -0.1943882934712011, + "nauc_ndcg_at_3_diff1": 0.38698824829501327, + "nauc_ndcg_at_3_max": 0.18248234792393908, + "nauc_ndcg_at_3_std": -0.23027236395832845, + "nauc_ndcg_at_5_diff1": 0.3840731756747734, + "nauc_ndcg_at_5_max": 0.19902043999737418, + "nauc_ndcg_at_5_std": -0.21867727258662267, + "nauc_precision_at_1000_diff1": -0.14563780450456498, + "nauc_precision_at_1000_max": 0.051293346284225286, + "nauc_precision_at_1000_std": 0.07789082218800825, + "nauc_precision_at_100_diff1": -0.1467025471343973, + "nauc_precision_at_100_max": 0.05954247639908044, + "nauc_precision_at_100_std": 0.0864922387014887, + "nauc_precision_at_10_diff1": -0.1121757286082742, + "nauc_precision_at_10_max": 0.10569700378713055, + "nauc_precision_at_10_std": 0.0430820248122203, + "nauc_precision_at_1_diff1": 0.4840917297107521, + "nauc_precision_at_1_max": 0.20824808887073123, + "nauc_precision_at_1_std": -0.2257971919145046, + "nauc_precision_at_20_diff1": -0.14108372830131083, + "nauc_precision_at_20_max": 0.07878382664819399, + "nauc_precision_at_20_std": 0.07281747554282438, + "nauc_precision_at_3_diff1": 0.031356132098102465, + "nauc_precision_at_3_max": 0.10653407007492809, + "nauc_precision_at_3_std": -0.10213703786397461, + "nauc_precision_at_5_diff1": -0.05105748400071351, + "nauc_precision_at_5_max": 0.09024405960706532, + "nauc_precision_at_5_std": -0.03474043940917473, + "nauc_recall_at_1000_diff1": 0.6497935578911636, + "nauc_recall_at_1000_max": 0.867763675213603, + "nauc_recall_at_1000_std": 0.8994808067600393, + "nauc_recall_at_100_diff1": 0.11473421056422282, + "nauc_recall_at_100_max": 0.4658973369152973, + "nauc_recall_at_100_std": 0.3753498939798389, + "nauc_recall_at_10_diff1": 0.21175009718132237, + "nauc_recall_at_10_max": 0.33295803614648806, + "nauc_recall_at_10_std": -0.07316624459056616, + "nauc_recall_at_1_diff1": 0.44362720616770623, + "nauc_recall_at_1_max": 0.150140661012972, + "nauc_recall_at_1_std": -0.18067731022979855, + "nauc_recall_at_20_diff1": 0.13758324233799163, + "nauc_recall_at_20_max": 0.3321150634353552, + "nauc_recall_at_20_std": 0.008515525723854287, + "nauc_recall_at_3_diff1": 0.30604424650769274, + "nauc_recall_at_3_max": 0.14976906647944233, + "nauc_recall_at_3_std": -0.2115366991694914, + "nauc_recall_at_5_diff1": 0.25938842937696244, + "nauc_recall_at_5_max": 0.16574933063535352, + "nauc_recall_at_5_std": -0.19394686939790792, + "ndcg_at_1": 0.64243, + "ndcg_at_10": 0.76308, + "ndcg_at_100": 0.78306, + "ndcg_at_1000": 0.7848, + "ndcg_at_20": 0.77574, + "ndcg_at_3": 0.68631, + "ndcg_at_5": 0.72699, + "precision_at_1": 0.64243, + "precision_at_10": 0.15095, + "precision_at_100": 0.01654, "precision_at_1000": 0.00167, - "precision_at_20": 0.07796, - "precision_at_3": 0.34968, - "precision_at_5": 0.25071, - "recall_at_1": 0.3914, - "recall_at_10": 0.88187, - "recall_at_100": 0.98524, - "recall_at_1000": 0.9969, - "recall_at_20": 0.93656, - "recall_at_3": 0.67914, - "recall_at_5": 0.78479 + "precision_at_20": 0.07943, + "precision_at_3": 0.37727, + "precision_at_5": 0.26407, + "recall_at_1": 0.4573, + "recall_at_10": 0.91205, + "recall_at_100": 0.98489, + "recall_at_1000": 0.9964, + "recall_at_20": 0.95375, + "recall_at_3": 0.72482, + "recall_at_5": 0.82022 } ] }, diff --git a/results/GritLM__GritLM-7B/13f00a0e36500c80ce12870ea513846a066004af/RuReviewsClassification.json b/results/GritLM__GritLM-7B/13f00a0e36500c80ce12870ea513846a066004af/RuReviewsClassification.json index c72e92ef0..a147f167c 100644 --- a/results/GritLM__GritLM-7B/13f00a0e36500c80ce12870ea513846a066004af/RuReviewsClassification.json +++ b/results/GritLM__GritLM-7B/13f00a0e36500c80ce12870ea513846a066004af/RuReviewsClassification.json @@ -6,64 +6,64 @@ "scores": { "test": [ { - "accuracy": 0.68583984375, - "f1": 0.6805981470260111, - "f1_weighted": 0.68061138241352, + "accuracy": 0.7205078125, + "f1": 0.7090897157590046, + "f1_weighted": 0.7090850262421948, "hf_subset": "default", "languages": [ "rus-Cyrl" ], - "main_score": 0.68583984375, + "main_score": 0.7205078125, "scores_per_experiment": [ { - "accuracy": 0.71728515625, - "f1": 0.7147954674921443, - "f1_weighted": 0.7148121962759411 + "accuracy": 0.72607421875, + "f1": 0.7189457428692826, + "f1_weighted": 0.7189401703485927 }, { - "accuracy": 0.6572265625, - "f1": 0.6501278680784873, - "f1_weighted": 0.650149784881378 + "accuracy": 0.70849609375, + "f1": 0.70504965045246, + "f1_weighted": 0.7050745899741977 }, { - "accuracy": 0.689453125, - "f1": 0.6892170853460348, - "f1_weighted": 0.6892316154367997 + "accuracy": 0.72216796875, + "f1": 0.710147734105211, + "f1_weighted": 0.710138231914375 }, { - "accuracy": 0.697265625, - "f1": 0.7000293712774012, - "f1_weighted": 0.7000605279662862 + "accuracy": 0.7490234375, + "f1": 0.7475624943760284, + "f1_weighted": 0.7475734482497749 }, { - "accuracy": 0.69482421875, - "f1": 0.6966409581488603, - "f1_weighted": 0.6966498754241388 + "accuracy": 0.74169921875, + "f1": 0.7392024735620063, + "f1_weighted": 0.7392007630655564 }, { - "accuracy": 0.6826171875, - "f1": 0.6686825365823482, - "f1_weighted": 0.6686820158437423 + "accuracy": 0.70947265625, + "f1": 0.6941449564101775, + "f1_weighted": 0.6941369244966837 }, { - "accuracy": 0.65234375, - "f1": 0.6526591221185526, - "f1_weighted": 0.652696753789956 + "accuracy": 0.71337890625, + "f1": 0.700032469090735, + "f1_weighted": 0.700023868216842 }, { - "accuracy": 0.67138671875, - "f1": 0.6626802334252339, - "f1_weighted": 0.6626889597388353 + "accuracy": 0.71728515625, + "f1": 0.7061467836516039, + "f1_weighted": 0.7061354485405109 }, { - "accuracy": 0.677734375, - "f1": 0.6544077130836846, - "f1_weighted": 0.654381191052859 + "accuracy": 0.6806640625, + "f1": 0.6370552048449957, + "f1_weighted": 0.637012195191758 }, { - "accuracy": 0.71826171875, - "f1": 0.7167411147073638, - "f1_weighted": 0.7167609037252647 + "accuracy": 0.73681640625, + "f1": 0.7326096482275468, + "f1_weighted": 0.7326146224236576 } ] } diff --git a/results/GritLM__GritLM-7B/13f00a0e36500c80ce12870ea513846a066004af/RuSciBenchGRNTIClassification.json b/results/GritLM__GritLM-7B/13f00a0e36500c80ce12870ea513846a066004af/RuSciBenchGRNTIClassification.json index 2ebf298db..3604c9116 100644 --- a/results/GritLM__GritLM-7B/13f00a0e36500c80ce12870ea513846a066004af/RuSciBenchGRNTIClassification.json +++ b/results/GritLM__GritLM-7B/13f00a0e36500c80ce12870ea513846a066004af/RuSciBenchGRNTIClassification.json @@ -6,64 +6,64 @@ "scores": { "test": [ { - "accuracy": 0.64560546875, - "f1": 0.6347522808293438, - "f1_weighted": 0.6348384394589267, + "accuracy": 0.67978515625, + "f1": 0.6688184674102005, + "f1_weighted": 0.6689054490395934, "hf_subset": "default", "languages": [ "rus-Cyrl" ], - "main_score": 0.64560546875, + "main_score": 0.67978515625, "scores_per_experiment": [ { - "accuracy": 0.66650390625, - "f1": 0.6571923243946676, - "f1_weighted": 0.6572589706386203 + "accuracy": 0.68603515625, + "f1": 0.6783063581296379, + "f1_weighted": 0.6783796222204301 }, { - "accuracy": 0.6455078125, - "f1": 0.6344074317690216, - "f1_weighted": 0.6344956422225593 + "accuracy": 0.68505859375, + "f1": 0.6750873839407513, + "f1_weighted": 0.6751629317328599 }, { - "accuracy": 0.6357421875, - "f1": 0.625020851264137, - "f1_weighted": 0.6251456093387753 + "accuracy": 0.6640625, + "f1": 0.6501272466052858, + "f1_weighted": 0.6502768334056338 }, { - "accuracy": 0.669921875, - "f1": 0.6613002308428682, - "f1_weighted": 0.6613750556186949 + "accuracy": 0.69482421875, + "f1": 0.6856261386454664, + "f1_weighted": 0.6856808097339715 }, { - "accuracy": 0.64794921875, - "f1": 0.640486942956998, - "f1_weighted": 0.640583384426727 + "accuracy": 0.68701171875, + "f1": 0.6772417429618586, + "f1_weighted": 0.6773019779893052 }, { - "accuracy": 0.6123046875, - "f1": 0.597516108205184, - "f1_weighted": 0.5975894559897497 + "accuracy": 0.6611328125, + "f1": 0.6489748497334507, + "f1_weighted": 0.6490842908806768 }, { - "accuracy": 0.6337890625, - "f1": 0.6213127813159909, - "f1_weighted": 0.6214492551643144 + "accuracy": 0.6767578125, + "f1": 0.6632264145484237, + "f1_weighted": 0.6633594422154904 }, { - "accuracy": 0.6455078125, - "f1": 0.6320480041355927, - "f1_weighted": 0.6320822547455862 + "accuracy": 0.68115234375, + "f1": 0.6689096500394116, + "f1_weighted": 0.6689690654907257 }, { - "accuracy": 0.654296875, - "f1": 0.6446294484578006, - "f1_weighted": 0.644717374946289 + "accuracy": 0.68994140625, + "f1": 0.6808141401142505, + "f1_weighted": 0.6808846692629896 }, { - "accuracy": 0.64453125, - "f1": 0.6336086849511774, - "f1_weighted": 0.6336873914979503 + "accuracy": 0.671875, + "f1": 0.6598707493834678, + "f1_weighted": 0.6599548474638515 } ] } diff --git a/results/GritLM__GritLM-7B/13f00a0e36500c80ce12870ea513846a066004af/RuSciBenchGRNTIClusteringP2P.json b/results/GritLM__GritLM-7B/13f00a0e36500c80ce12870ea513846a066004af/RuSciBenchGRNTIClusteringP2P.json index a58e59cfc..ddb18dcd2 100644 --- a/results/GritLM__GritLM-7B/13f00a0e36500c80ce12870ea513846a066004af/RuSciBenchGRNTIClusteringP2P.json +++ b/results/GritLM__GritLM-7B/13f00a0e36500c80ce12870ea513846a066004af/RuSciBenchGRNTIClusteringP2P.json @@ -10,21 +10,21 @@ "languages": [ "rus-Cyrl" ], - "main_score": 0.6001043796693875, - "v_measure": 0.6001043796693875, - "v_measure_std": 0.01117772796276811, + "main_score": 0.6157193537707601, + "v_measure": 0.6157193537707601, + "v_measure_std": 0.008886081715286254, "v_measures": { "Level 0": [ - 0.601909550601499, - 0.6115753148767875, - 0.5944019540838528, - 0.6137747708166417, - 0.6095723169482973, - 0.6025788716499394, - 0.5888461083291029, - 0.5785472370109127, - 0.6101140532772653, - 0.5897236190995763 + 0.6190901096177724, + 0.6168106456057155, + 0.6000401985555471, + 0.6363803490122687, + 0.6179309170324397, + 0.6063468588439251, + 0.6126485928127284, + 0.6169749059400433, + 0.6149810567332614, + 0.6159899035538997 ] } } diff --git a/results/GritLM__GritLM-7B/13f00a0e36500c80ce12870ea513846a066004af/RuSciBenchOECDClassification.json b/results/GritLM__GritLM-7B/13f00a0e36500c80ce12870ea513846a066004af/RuSciBenchOECDClassification.json index 484d28b36..4261ec478 100644 --- a/results/GritLM__GritLM-7B/13f00a0e36500c80ce12870ea513846a066004af/RuSciBenchOECDClassification.json +++ b/results/GritLM__GritLM-7B/13f00a0e36500c80ce12870ea513846a066004af/RuSciBenchOECDClassification.json @@ -6,64 +6,64 @@ "scores": { "test": [ { - "accuracy": 0.511962890625, - "f1": 0.49078257208161097, - "f1_weighted": 0.4907845081846152, + "accuracy": 0.54453125, + "f1": 0.5224158504244392, + "f1_weighted": 0.5223945496886111, "hf_subset": "default", "languages": [ "rus-Cyrl" ], - "main_score": 0.511962890625, + "main_score": 0.54453125, "scores_per_experiment": [ { - "accuracy": 0.49658203125, - "f1": 0.47056081615342843, - "f1_weighted": 0.4707379596149285 + "accuracy": 0.5380859375, + "f1": 0.5143092744212872, + "f1_weighted": 0.5144282052864618 }, { - "accuracy": 0.4990234375, - "f1": 0.47566950591403473, - "f1_weighted": 0.4755355168649508 + "accuracy": 0.5517578125, + "f1": 0.5334676604852822, + "f1_weighted": 0.5333798473373353 }, { - "accuracy": 0.509765625, - "f1": 0.4913412218531345, - "f1_weighted": 0.49140767546434466 + "accuracy": 0.55126953125, + "f1": 0.534249403730069, + "f1_weighted": 0.5342689286199402 }, { - "accuracy": 0.50439453125, - "f1": 0.47033253713123774, - "f1_weighted": 0.4701886908840734 + "accuracy": 0.52294921875, + "f1": 0.4840212560016881, + "f1_weighted": 0.4839338288253472 }, { - "accuracy": 0.5166015625, - "f1": 0.49075436268154365, - "f1_weighted": 0.49082421518520974 + "accuracy": 0.55029296875, + "f1": 0.5267270262756203, + "f1_weighted": 0.5267644334445204 }, { - "accuracy": 0.52978515625, - "f1": 0.5155359360240024, - "f1_weighted": 0.5155393799408292 + "accuracy": 0.560546875, + "f1": 0.5429816534543871, + "f1_weighted": 0.5429814096833294 }, { - "accuracy": 0.515625, - "f1": 0.4963670787020541, - "f1_weighted": 0.49639782712821445 + "accuracy": 0.54833984375, + "f1": 0.5252446357266669, + "f1_weighted": 0.5252086769877454 }, { - "accuracy": 0.53173828125, - "f1": 0.5205668059503759, - "f1_weighted": 0.5206284512266982 + "accuracy": 0.55517578125, + "f1": 0.5411301229674745, + "f1_weighted": 0.5411462626499789 }, { - "accuracy": 0.49267578125, - "f1": 0.46731815352899764, - "f1_weighted": 0.4672247242585677 + "accuracy": 0.52294921875, + "f1": 0.4965692004976708, + "f1_weighted": 0.4964231186306385 }, { - "accuracy": 0.5234375, - "f1": 0.5093793028773014, - "f1_weighted": 0.5093606412783352 + "accuracy": 0.5439453125, + "f1": 0.525458270684245, + "f1_weighted": 0.525410785420815 } ] } diff --git a/results/GritLM__GritLM-7B/13f00a0e36500c80ce12870ea513846a066004af/RuSciBenchOECDClusteringP2P.json b/results/GritLM__GritLM-7B/13f00a0e36500c80ce12870ea513846a066004af/RuSciBenchOECDClusteringP2P.json index 99e739d0a..13bde888f 100644 --- a/results/GritLM__GritLM-7B/13f00a0e36500c80ce12870ea513846a066004af/RuSciBenchOECDClusteringP2P.json +++ b/results/GritLM__GritLM-7B/13f00a0e36500c80ce12870ea513846a066004af/RuSciBenchOECDClusteringP2P.json @@ -10,21 +10,21 @@ "languages": [ "rus-Cyrl" ], - "main_score": 0.5165997655043515, - "v_measure": 0.5165997655043515, - "v_measure_std": 0.010625183144227331, + "main_score": 0.5388087304724207, + "v_measure": 0.5388087304724207, + "v_measure_std": 0.007159466545698467, "v_measures": { "Level 0": [ - 0.51110507685522, - 0.5250304117590894, - 0.5186151660156529, - 0.5282656725805043, - 0.5079301646743243, - 0.5301655957620985, - 0.4949014787496403, - 0.5113584133571815, - 0.5269062516101722, - 0.5117194236796315 + 0.543581943498442, + 0.5487300662195526, + 0.5437123780319851, + 0.5363538275002241, + 0.5388428660809423, + 0.5323049697683667, + 0.5217259902706479, + 0.5437723007334423, + 0.5386686400794564, + 0.5403943225411473 ] } } diff --git a/results/GritLM__GritLM-7B/13f00a0e36500c80ce12870ea513846a066004af/STS22.json b/results/GritLM__GritLM-7B/13f00a0e36500c80ce12870ea513846a066004af/STS22.json new file mode 100644 index 000000000..b58839c4a --- /dev/null +++ b/results/GritLM__GritLM-7B/13f00a0e36500c80ce12870ea513846a066004af/STS22.json @@ -0,0 +1,26 @@ +{ + "dataset_revision": "d31f33a128469b20e357535c39b82fb3c3f6f2bd", + "evaluation_time": 21.497711420059204, + "kg_co2_emissions": null, + "mteb_version": "1.19.4", + "scores": { + "test": [ + { + "cosine_pearson": 0.6602737493784214, + "cosine_spearman": 0.6844681472215942, + "euclidean_pearson": 0.6486636598033109, + "euclidean_spearman": 0.6844681472215942, + "hf_subset": "ru", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.6844681472215942, + "manhattan_pearson": 0.6475205677155285, + "manhattan_spearman": 0.6850400086703208, + "pearson": 0.6602737493784214, + "spearman": 0.6844681472215942 + } + ] + }, + "task_name": "STS22" +} \ No newline at end of file diff --git a/results/GritLM__GritLM-7B/13f00a0e36500c80ce12870ea513846a066004af/SensitiveTopicsClassification.json b/results/GritLM__GritLM-7B/13f00a0e36500c80ce12870ea513846a066004af/SensitiveTopicsClassification.json index e2d0f2002..51f023d08 100644 --- a/results/GritLM__GritLM-7B/13f00a0e36500c80ce12870ea513846a066004af/SensitiveTopicsClassification.json +++ b/results/GritLM__GritLM-7B/13f00a0e36500c80ce12870ea513846a066004af/SensitiveTopicsClassification.json @@ -6,64 +6,64 @@ "scores": { "test": [ { - "accuracy": 0.28515625, - "f1": 0.27687989769002586, + "accuracy": 0.332421875, + "f1": 0.4090465744271413, "hf_subset": "default", "languages": [ "rus-Cyrl" ], - "lrap": 0.4152377658420061, - "main_score": 0.28515625, + "lrap": 0.5319118923611074, + "main_score": 0.332421875, "scores_per_experiment": [ { - "accuracy": 0.25, - "f1": 0.20721102416746126, - "lrap": 0.37175835503471466 + "accuracy": 0.31982421875, + "f1": 0.36613098043190645, + "lrap": 0.491075303819437 }, { - "accuracy": 0.27880859375, - "f1": 0.28616500844654175, - "lrap": 0.4227973090277701 + "accuracy": 0.31689453125, + "f1": 0.4136446375785791, + "lrap": 0.5266113281249959 }, { - "accuracy": 0.2890625, - "f1": 0.2645680775441892, - "lrap": 0.404690212673603 + "accuracy": 0.31298828125, + "f1": 0.3686166629177195, + "lrap": 0.4991590711805484 }, { - "accuracy": 0.30224609375, - "f1": 0.2793207188602188, - "lrap": 0.4257269965277702 + "accuracy": 0.33544921875, + "f1": 0.39899367834536237, + "lrap": 0.5114746093749939 }, { - "accuracy": 0.27490234375, - "f1": 0.27710516719184575, - "lrap": 0.4010484483506865 + "accuracy": 0.33251953125, + "f1": 0.42638336881965705, + "lrap": 0.5359768337673576 }, { - "accuracy": 0.29052734375, - "f1": 0.29126555740108917, - "lrap": 0.4232855902777702 + "accuracy": 0.35693359375, + "f1": 0.43454464136665494, + "lrap": 0.5677354600694441 }, { - "accuracy": 0.29052734375, - "f1": 0.25819403503975563, - "lrap": 0.41643608940971466 + "accuracy": 0.35498046875, + "f1": 0.41620922816949574, + "lrap": 0.5410427517361083 }, { - "accuracy": 0.28271484375, - "f1": 0.293388723349701, - "lrap": 0.41739908854165875 + "accuracy": 0.32470703125, + "f1": 0.40743156814065934, + "lrap": 0.5290052625868018 }, { - "accuracy": 0.30859375, - "f1": 0.3095153657982328, - "lrap": 0.4410264756944367 + "accuracy": 0.34619140625, + "f1": 0.4354545780082219, + "lrap": 0.5702175564236106 }, { - "accuracy": 0.2841796875, - "f1": 0.3020652991012232, - "lrap": 0.4282090928819369 + "accuracy": 0.32373046875, + "f1": 0.4230564004931569, + "lrap": 0.5468207465277756 } ] } diff --git a/results/GritLM__GritLM-7B/13f00a0e36500c80ce12870ea513846a066004af/model_meta.json b/results/GritLM__GritLM-7B/13f00a0e36500c80ce12870ea513846a066004af/model_meta.json index 01167804b..33e5b06ef 100644 --- a/results/GritLM__GritLM-7B/13f00a0e36500c80ce12870ea513846a066004af/model_meta.json +++ b/results/GritLM__GritLM-7B/13f00a0e36500c80ce12870ea513846a066004af/model_meta.json @@ -1 +1,21 @@ -{"name": "GritLM/GritLM-7B", "revision": "13f00a0e36500c80ce12870ea513846a066004af", "release_date": "2024-02-15", "languages": ["eng_Latn", "fra_Latn", "deu_Latn", "ita_Latn", "spa_Latn"], "n_parameters": null, "memory_usage": null, "max_tokens": null, "embed_dim": null, "license": null, "open_source": true, "similarity_fn_name": null, "framework": [], "loader": "gritlm_loader"} \ No newline at end of file +{ + "name": "GritLM/GritLM-7B", + "revision": "13f00a0e36500c80ce12870ea513846a066004af", + "release_date": "2024-02-15", + "languages": [ + "eng_Latn", + "fra_Latn", + "deu_Latn", + "ita_Latn", + "spa_Latn" + ], + "n_parameters": null, + "memory_usage": null, + "max_tokens": null, + "embed_dim": null, + "license": null, + "open_source": true, + "similarity_fn_name": null, + "framework": [], + "loader": "gritlm_loader" +} \ No newline at end of file diff --git a/results/Linq-AI-Research__Linq-Embed-Mistral/0c1a0b0589177079acc552433cad51d7c9132379/CEDRClassification.json b/results/Linq-AI-Research__Linq-Embed-Mistral/0c1a0b0589177079acc552433cad51d7c9132379/CEDRClassification.json new file mode 100644 index 000000000..0e8bcec1d --- /dev/null +++ b/results/Linq-AI-Research__Linq-Embed-Mistral/0c1a0b0589177079acc552433cad51d7c9132379/CEDRClassification.json @@ -0,0 +1,73 @@ +{ + "dataset_revision": "c0ba03d058e3e1b2f3fd20518875a4563dd12db4", + "evaluation_time": 55.17633938789368, + "kg_co2_emissions": null, + "mteb_version": "1.19.2", + "scores": { + "test": [ + { + "accuracy": 0.515356004250797, + "f1": 0.5684741527563759, + "hf_subset": "default", + "languages": [ + "rus-Cyrl" + ], + "lrap": 0.8401354941551604, + "main_score": 0.515356004250797, + "scores_per_experiment": [ + { + "accuracy": 0.4861849096705632, + "f1": 0.5560517243464084, + "lrap": 0.8518065887353937 + }, + { + "accuracy": 0.5292242295430393, + "f1": 0.5742548438822221, + "lrap": 0.8303400637619621 + }, + { + "accuracy": 0.5212539851222104, + "f1": 0.5800514705069072, + "lrap": 0.8565887353878909 + }, + { + "accuracy": 0.5377258235919234, + "f1": 0.5851956722251994, + "lrap": 0.8545164718384753 + }, + { + "accuracy": 0.49946865037194477, + "f1": 0.5593348760863107, + "lrap": 0.8290648246546295 + }, + { + "accuracy": 0.48990435706695007, + "f1": 0.5709477593985572, + "lrap": 0.8412592986184971 + }, + { + "accuracy": 0.48034006376195537, + "f1": 0.5503475013638145, + "lrap": 0.8366631243358192 + }, + { + "accuracy": 0.5223166843783209, + "f1": 0.5518381515125145, + "lrap": 0.8139213602550549 + }, + { + "accuracy": 0.5297555791710946, + "f1": 0.5652508524711325, + "lrap": 0.844792773645065 + }, + { + "accuracy": 0.5573857598299681, + "f1": 0.5914686757706923, + "lrap": 0.842401700318816 + } + ] + } + ] + }, + "task_name": "CEDRClassification" +} \ No newline at end of file diff --git a/results/Linq-AI-Research__Linq-Embed-Mistral/0c1a0b0589177079acc552433cad51d7c9132379/GeoreviewClassification.json b/results/Linq-AI-Research__Linq-Embed-Mistral/0c1a0b0589177079acc552433cad51d7c9132379/GeoreviewClassification.json new file mode 100644 index 000000000..1adcbc697 --- /dev/null +++ b/results/Linq-AI-Research__Linq-Embed-Mistral/0c1a0b0589177079acc552433cad51d7c9132379/GeoreviewClassification.json @@ -0,0 +1,73 @@ +{ + "dataset_revision": "3765c0d1de6b7d264bc459433c45e5a75513839c", + "evaluation_time": 76.93086171150208, + "kg_co2_emissions": null, + "mteb_version": "1.19.2", + "scores": { + "test": [ + { + "accuracy": 0.599609375, + "f1": 0.573744210230603, + "f1_weighted": 0.5736772364805091, + "hf_subset": "default", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.599609375, + "scores_per_experiment": [ + { + "accuracy": 0.591796875, + "f1": 0.5570677355566541, + "f1_weighted": 0.5569919441192186 + }, + { + "accuracy": 0.576171875, + "f1": 0.5399709325354622, + "f1_weighted": 0.5398718801750412 + }, + { + "accuracy": 0.595703125, + "f1": 0.5644738678251409, + "f1_weighted": 0.5644041427960278 + }, + { + "accuracy": 0.61572265625, + "f1": 0.6000347399728095, + "f1_weighted": 0.5999733715424803 + }, + { + "accuracy": 0.62060546875, + "f1": 0.6117456047130962, + "f1_weighted": 0.6116999038133838 + }, + { + "accuracy": 0.6142578125, + "f1": 0.6058271295101784, + "f1_weighted": 0.6058029342373414 + }, + { + "accuracy": 0.591796875, + "f1": 0.5536622195556016, + "f1_weighted": 0.5535753410007996 + }, + { + "accuracy": 0.59228515625, + "f1": 0.5597650682583435, + "f1_weighted": 0.5596869727738149 + }, + { + "accuracy": 0.60107421875, + "f1": 0.5830711432069657, + "f1_weighted": 0.5830229412111916 + }, + { + "accuracy": 0.5966796875, + "f1": 0.561823661171777, + "f1_weighted": 0.5617429331357914 + } + ] + } + ] + }, + "task_name": "GeoreviewClassification" +} \ No newline at end of file diff --git a/results/Linq-AI-Research__Linq-Embed-Mistral/0c1a0b0589177079acc552433cad51d7c9132379/GeoreviewClusteringP2P.json b/results/Linq-AI-Research__Linq-Embed-Mistral/0c1a0b0589177079acc552433cad51d7c9132379/GeoreviewClusteringP2P.json new file mode 100644 index 000000000..3a84aee6a --- /dev/null +++ b/results/Linq-AI-Research__Linq-Embed-Mistral/0c1a0b0589177079acc552433cad51d7c9132379/GeoreviewClusteringP2P.json @@ -0,0 +1,34 @@ +{ + "dataset_revision": "97a313c8fc85b47f13f33e7e9a95c1ad888c7fec", + "evaluation_time": 115.02593684196472, + "kg_co2_emissions": null, + "mteb_version": "1.19.2", + "scores": { + "test": [ + { + "hf_subset": "default", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.7488155523631818, + "v_measure": 0.7488155523631818, + "v_measure_std": 0.005660927055902265, + "v_measures": { + "Level 0": [ + 0.7463576533094954, + 0.7359436476702423, + 0.7511276682206199, + 0.7478272554693972, + 0.7530636187833275, + 0.7551403356552593, + 0.7433926568540048, + 0.7521554903456417, + 0.7477930186503288, + 0.7553541786735003 + ] + } + } + ] + }, + "task_name": "GeoreviewClusteringP2P" +} \ No newline at end of file diff --git a/results/Linq-AI-Research__Linq-Embed-Mistral/0c1a0b0589177079acc552433cad51d7c9132379/HeadlineClassification.json b/results/Linq-AI-Research__Linq-Embed-Mistral/0c1a0b0589177079acc552433cad51d7c9132379/HeadlineClassification.json new file mode 100644 index 000000000..3ce10855b --- /dev/null +++ b/results/Linq-AI-Research__Linq-Embed-Mistral/0c1a0b0589177079acc552433cad51d7c9132379/HeadlineClassification.json @@ -0,0 +1,73 @@ +{ + "dataset_revision": "2fe05ee6b5832cda29f2ef7aaad7b7fe6a3609eb", + "evaluation_time": 48.553874015808105, + "kg_co2_emissions": null, + "mteb_version": "1.19.2", + "scores": { + "test": [ + { + "accuracy": 0.8650390625, + "f1": 0.8652060441128555, + "f1_weighted": 0.8651928706128537, + "hf_subset": "default", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.8650390625, + "scores_per_experiment": [ + { + "accuracy": 0.869140625, + "f1": 0.8698436706427253, + "f1_weighted": 0.8698372712979698 + }, + { + "accuracy": 0.869140625, + "f1": 0.869351673032866, + "f1_weighted": 0.8693508799548921 + }, + { + "accuracy": 0.86328125, + "f1": 0.863452196643291, + "f1_weighted": 0.8634291891505759 + }, + { + "accuracy": 0.87744140625, + "f1": 0.8779081351972625, + "f1_weighted": 0.8778994907133049 + }, + { + "accuracy": 0.86865234375, + "f1": 0.8690518784792145, + "f1_weighted": 0.8690356645562841 + }, + { + "accuracy": 0.86181640625, + "f1": 0.8619216293340536, + "f1_weighted": 0.8618995780967509 + }, + { + "accuracy": 0.8486328125, + "f1": 0.8486111893012973, + "f1_weighted": 0.8485988518512564 + }, + { + "accuracy": 0.86328125, + "f1": 0.8631148007208799, + "f1_weighted": 0.8631019451846378 + }, + { + "accuracy": 0.857421875, + "f1": 0.8575957402201312, + "f1_weighted": 0.8575788829711262 + }, + { + "accuracy": 0.87158203125, + "f1": 0.8712095275568336, + "f1_weighted": 0.8711969523517387 + } + ] + } + ] + }, + "task_name": "HeadlineClassification" +} \ No newline at end of file diff --git a/results/Linq-AI-Research__Linq-Embed-Mistral/0c1a0b0589177079acc552433cad51d7c9132379/InappropriatenessClassification.json b/results/Linq-AI-Research__Linq-Embed-Mistral/0c1a0b0589177079acc552433cad51d7c9132379/InappropriatenessClassification.json new file mode 100644 index 000000000..04ea0e097 --- /dev/null +++ b/results/Linq-AI-Research__Linq-Embed-Mistral/0c1a0b0589177079acc552433cad51d7c9132379/InappropriatenessClassification.json @@ -0,0 +1,95 @@ +{ + "dataset_revision": "601651fdc45ef243751676e62dd7a19f491c0285", + "evaluation_time": 42.00459599494934, + "kg_co2_emissions": null, + "mteb_version": "1.19.2", + "scores": { + "test": [ + { + "accuracy": 0.7041015625, + "ap": 0.6448609804516195, + "ap_weighted": 0.6448609804516195, + "f1": 0.7034460316773077, + "f1_weighted": 0.7034460316773077, + "hf_subset": "default", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.7041015625, + "scores_per_experiment": [ + { + "accuracy": 0.7236328125, + "ap": 0.6627228814612326, + "ap_weighted": 0.6627228814612326, + "f1": 0.7236114621433578, + "f1_weighted": 0.7236114621433578 + }, + { + "accuracy": 0.734375, + "ap": 0.666878312720848, + "ap_weighted": 0.666878312720848, + "f1": 0.7336342597020065, + "f1_weighted": 0.7336342597020065 + }, + { + "accuracy": 0.7216796875, + "ap": 0.6755201638817481, + "ap_weighted": 0.6755201638817481, + "f1": 0.7176052569701203, + "f1_weighted": 0.7176052569701203 + }, + { + "accuracy": 0.65673828125, + "ap": 0.6008102593945138, + "ap_weighted": 0.6008102593945138, + "f1": 0.6559665176784603, + "f1_weighted": 0.6559665176784603 + }, + { + "accuracy": 0.69873046875, + "ap": 0.6411006836945304, + "ap_weighted": 0.6411006836945304, + "f1": 0.6985130314636654, + "f1_weighted": 0.6985130314636654 + }, + { + "accuracy": 0.70458984375, + "ap": 0.6428451321546831, + "ap_weighted": 0.6428451321546831, + "f1": 0.7045131241779876, + "f1_weighted": 0.7045131241779876 + }, + { + "accuracy": 0.7060546875, + "ap": 0.6446724961087165, + "ap_weighted": 0.6446724961087165, + "f1": 0.7060266520168321, + "f1_weighted": 0.7060266520168321 + }, + { + "accuracy": 0.7255859375, + "ap": 0.6593200683593751, + "ap_weighted": 0.6593200683593751, + "f1": 0.7249816491313923, + "f1_weighted": 0.7249816491313923 + }, + { + "accuracy": 0.6630859375, + "ap": 0.6080881457115009, + "ap_weighted": 0.6080881457115009, + "f1": 0.6630856161934053, + "f1_weighted": 0.6630856161934053 + }, + { + "accuracy": 0.70654296875, + "ap": 0.6466516610290467, + "ap_weighted": 0.6466516610290467, + "f1": 0.706522747295849, + "f1_weighted": 0.706522747295849 + } + ] + } + ] + }, + "task_name": "InappropriatenessClassification" +} \ No newline at end of file diff --git a/results/Linq-AI-Research__Linq-Embed-Mistral/0c1a0b0589177079acc552433cad51d7c9132379/KinopoiskClassification.json b/results/Linq-AI-Research__Linq-Embed-Mistral/0c1a0b0589177079acc552433cad51d7c9132379/KinopoiskClassification.json new file mode 100644 index 000000000..e2e32678f --- /dev/null +++ b/results/Linq-AI-Research__Linq-Embed-Mistral/0c1a0b0589177079acc552433cad51d7c9132379/KinopoiskClassification.json @@ -0,0 +1,73 @@ +{ + "dataset_revision": "5911f26666ac11af46cb9c6849d0dc80a378af24", + "evaluation_time": 83.33914184570312, + "kg_co2_emissions": null, + "mteb_version": "1.19.2", + "scores": { + "test": [ + { + "accuracy": 0.6955333333333333, + "f1": 0.6703422407738616, + "f1_weighted": 0.6703422407738616, + "hf_subset": "default", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.6955333333333333, + "scores_per_experiment": [ + { + "accuracy": 0.6993333333333334, + "f1": 0.6721214215228208, + "f1_weighted": 0.6721214215228208 + }, + { + "accuracy": 0.6606666666666666, + "f1": 0.592948635072373, + "f1_weighted": 0.592948635072373 + }, + { + "accuracy": 0.7066666666666667, + "f1": 0.6889241710202928, + "f1_weighted": 0.6889241710202928 + }, + { + "accuracy": 0.6926666666666667, + "f1": 0.6720468789035391, + "f1_weighted": 0.6720468789035391 + }, + { + "accuracy": 0.6993333333333334, + "f1": 0.6784360011148444, + "f1_weighted": 0.6784360011148444 + }, + { + "accuracy": 0.7026666666666667, + "f1": 0.6794179926887453, + "f1_weighted": 0.6794179926887451 + }, + { + "accuracy": 0.6786666666666666, + "f1": 0.6625253165647061, + "f1_weighted": 0.662525316564706 + }, + { + "accuracy": 0.7093333333333334, + "f1": 0.6949103770785093, + "f1_weighted": 0.6949103770785092 + }, + { + "accuracy": 0.704, + "f1": 0.6787823736278303, + "f1_weighted": 0.6787823736278304 + }, + { + "accuracy": 0.702, + "f1": 0.6833092401449545, + "f1_weighted": 0.6833092401449545 + } + ] + } + ] + }, + "task_name": "KinopoiskClassification" +} \ No newline at end of file diff --git a/results/Linq-AI-Research__Linq-Embed-Mistral/0c1a0b0589177079acc552433cad51d7c9132379/MIRACLReranking.json b/results/Linq-AI-Research__Linq-Embed-Mistral/0c1a0b0589177079acc552433cad51d7c9132379/MIRACLReranking.json new file mode 100644 index 000000000..066a7973a --- /dev/null +++ b/results/Linq-AI-Research__Linq-Embed-Mistral/0c1a0b0589177079acc552433cad51d7c9132379/MIRACLReranking.json @@ -0,0 +1,130 @@ +{ + "dataset_revision": "6d1962c527217f8927fca80f890f14f36b2802af", + "evaluation_time": 3333.2513914108276, + "kg_co2_emissions": null, + "mteb_version": "1.19.2", + "scores": { + "dev": [ + { + "MAP@1(MIRACL)": 0.35817, + "MAP@10(MIRACL)": 0.55645, + "MAP@100(MIRACL)": 0.57399, + "MAP@1000(MIRACL)": 0.57399, + "MAP@20(MIRACL)": 0.56905, + "MAP@3(MIRACL)": 0.49031, + "MAP@5(MIRACL)": 0.52676, + "NDCG@1(MIRACL)": 0.57739, + "NDCG@10(MIRACL)": 0.62302, + "NDCG@100(MIRACL)": 0.65711, + "NDCG@1000(MIRACL)": 0.65711, + "NDCG@20(MIRACL)": 0.64484, + "NDCG@3(MIRACL)": 0.57585, + "NDCG@5(MIRACL)": 0.59127, + "P@1(MIRACL)": 0.57739, + "P@10(MIRACL)": 0.15702, + "P@100(MIRACL)": 0.01953, + "P@1000(MIRACL)": 0.00195, + "P@20(MIRACL)": 0.08933, + "P@3(MIRACL)": 0.35044, + "P@5(MIRACL)": 0.25501, + "Recall@1(MIRACL)": 0.35817, + "Recall@10(MIRACL)": 0.70071, + "Recall@100(MIRACL)": 0.79952, + "Recall@1000(MIRACL)": 0.79952, + "Recall@20(MIRACL)": 0.75948, + "Recall@3(MIRACL)": 0.54858, + "Recall@5(MIRACL)": 0.61816, + "hf_subset": "ru", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.62302, + "nAUC_MAP@1000_diff1(MIRACL)": 0.26076369417682466, + "nAUC_MAP@1000_max(MIRACL)": 0.38787633766125673, + "nAUC_MAP@1000_std(MIRACL)": 0.1928301838041367, + "nAUC_MAP@100_diff1(MIRACL)": 0.26076369417682466, + "nAUC_MAP@100_max(MIRACL)": 0.38787633766125673, + "nAUC_MAP@100_std(MIRACL)": 0.1928301838041367, + "nAUC_MAP@10_diff1(MIRACL)": 0.27160020370218285, + "nAUC_MAP@10_max(MIRACL)": 0.36156514953735036, + "nAUC_MAP@10_std(MIRACL)": 0.17608274403799126, + "nAUC_MAP@1_diff1(MIRACL)": 0.42155415997327145, + "nAUC_MAP@1_max(MIRACL)": 0.17778038644253222, + "nAUC_MAP@1_std(MIRACL)": 0.028351310288876486, + "nAUC_MAP@20_diff1(MIRACL)": 0.2631809798619513, + "nAUC_MAP@20_max(MIRACL)": 0.3798143085040427, + "nAUC_MAP@20_std(MIRACL)": 0.19034702253438238, + "nAUC_MAP@3_diff1(MIRACL)": 0.317527764775182, + "nAUC_MAP@3_max(MIRACL)": 0.28339653529232184, + "nAUC_MAP@3_std(MIRACL)": 0.11442525800675575, + "nAUC_MAP@5_diff1(MIRACL)": 0.28918510415656906, + "nAUC_MAP@5_max(MIRACL)": 0.3175874494052273, + "nAUC_MAP@5_std(MIRACL)": 0.14813066240964687, + "nAUC_NDCG@1000_diff1(MIRACL)": 0.2016728393548598, + "nAUC_NDCG@1000_max(MIRACL)": 0.49001183848176216, + "nAUC_NDCG@1000_std(MIRACL)": 0.2540051715073903, + "nAUC_NDCG@100_diff1(MIRACL)": 0.2016728393548598, + "nAUC_NDCG@100_max(MIRACL)": 0.49001183848176216, + "nAUC_NDCG@100_std(MIRACL)": 0.2540051715073903, + "nAUC_NDCG@10_diff1(MIRACL)": 0.22840086523130845, + "nAUC_NDCG@10_max(MIRACL)": 0.43639344820503034, + "nAUC_NDCG@10_std(MIRACL)": 0.22780249357880153, + "nAUC_NDCG@1_diff1(MIRACL)": 0.3233651066590005, + "nAUC_NDCG@1_max(MIRACL)": 0.4350445025068634, + "nAUC_NDCG@1_std(MIRACL)": 0.1934973330222114, + "nAUC_NDCG@20_diff1(MIRACL)": 0.2104430945511106, + "nAUC_NDCG@20_max(MIRACL)": 0.46957464179314584, + "nAUC_NDCG@20_std(MIRACL)": 0.2492763856409847, + "nAUC_NDCG@3_diff1(MIRACL)": 0.2531092602623836, + "nAUC_NDCG@3_max(MIRACL)": 0.3916383316071886, + "nAUC_NDCG@3_std(MIRACL)": 0.19331735877854925, + "nAUC_NDCG@5_diff1(MIRACL)": 0.24596918445891258, + "nAUC_NDCG@5_max(MIRACL)": 0.3907708077128183, + "nAUC_NDCG@5_std(MIRACL)": 0.19897163153991634, + "nAUC_P@1000_diff1(MIRACL)": -0.20915890222056732, + "nAUC_P@1000_max(MIRACL)": 0.35372096163689576, + "nAUC_P@1000_std(MIRACL)": 0.2286857616489929, + "nAUC_P@100_diff1(MIRACL)": -0.20915890222056696, + "nAUC_P@100_max(MIRACL)": 0.3537209616368961, + "nAUC_P@100_std(MIRACL)": 0.22868576164899332, + "nAUC_P@10_diff1(MIRACL)": -0.17673653781358772, + "nAUC_P@10_max(MIRACL)": 0.377274370528821, + "nAUC_P@10_std(MIRACL)": 0.2699316075120122, + "nAUC_P@1_diff1(MIRACL)": 0.3233651066590005, + "nAUC_P@1_max(MIRACL)": 0.4350445025068634, + "nAUC_P@1_std(MIRACL)": 0.1934973330222114, + "nAUC_P@20_diff1(MIRACL)": -0.20386800860777546, + "nAUC_P@20_max(MIRACL)": 0.36692704739670323, + "nAUC_P@20_std(MIRACL)": 0.26448103256733035, + "nAUC_P@3_diff1(MIRACL)": -0.07507866876194401, + "nAUC_P@3_max(MIRACL)": 0.4091899841681543, + "nAUC_P@3_std(MIRACL)": 0.2715365615820828, + "nAUC_P@5_diff1(MIRACL)": -0.13832561090228374, + "nAUC_P@5_max(MIRACL)": 0.374903117582736, + "nAUC_P@5_std(MIRACL)": 0.2734996674587488, + "nAUC_Recall@1000_diff1(MIRACL)": 0.0030149109707095943, + "nAUC_Recall@1000_max(MIRACL)": 0.7071414759306255, + "nAUC_Recall@1000_std(MIRACL)": 0.3849994217760461, + "nAUC_Recall@100_diff1(MIRACL)": 0.0030149109707095943, + "nAUC_Recall@100_max(MIRACL)": 0.7071414759306255, + "nAUC_Recall@100_std(MIRACL)": 0.3849994217760461, + "nAUC_Recall@10_diff1(MIRACL)": 0.13053649336274073, + "nAUC_Recall@10_max(MIRACL)": 0.4749446689555434, + "nAUC_Recall@10_std(MIRACL)": 0.27519146481173423, + "nAUC_Recall@1_diff1(MIRACL)": 0.42155415997327145, + "nAUC_Recall@1_max(MIRACL)": 0.17778038644253222, + "nAUC_Recall@1_std(MIRACL)": 0.028351310288876486, + "nAUC_Recall@20_diff1(MIRACL)": 0.059335608102135964, + "nAUC_Recall@20_max(MIRACL)": 0.5888839040904134, + "nAUC_Recall@20_std(MIRACL)": 0.34457210198004823, + "nAUC_Recall@3_diff1(MIRACL)": 0.24861854094986283, + "nAUC_Recall@3_max(MIRACL)": 0.28907627414386405, + "nAUC_Recall@3_std(MIRACL)": 0.1452249948621654, + "nAUC_Recall@5_diff1(MIRACL)": 0.19465729075988866, + "nAUC_Recall@5_max(MIRACL)": 0.34262054218901233, + "nAUC_Recall@5_std(MIRACL)": 0.19471831295243394 + } + ] + }, + "task_name": "MIRACLReranking" +} \ No newline at end of file diff --git a/results/Linq-AI-Research__Linq-Embed-Mistral/0c1a0b0589177079acc552433cad51d7c9132379/MIRACLRetrieval.json b/results/Linq-AI-Research__Linq-Embed-Mistral/0c1a0b0589177079acc552433cad51d7c9132379/MIRACLRetrieval.json new file mode 100644 index 000000000..4f7eb0395 --- /dev/null +++ b/results/Linq-AI-Research__Linq-Embed-Mistral/0c1a0b0589177079acc552433cad51d7c9132379/MIRACLRetrieval.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "main", + "evaluation_time": 336133.4713406563, + "kg_co2_emissions": null, + "mteb_version": "1.19.4", + "scores": { + "dev": [ + { + "hf_subset": "ru", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.59138, + "map_at_1": 0.29045, + "map_at_10": 0.49112, + "map_at_100": 0.51823, + "map_at_1000": 0.51924, + "map_at_20": 0.50723, + "map_at_3": 0.42002, + "map_at_5": 0.45823, + "mrr_at_1": 0.5527156549520766, + "mrr_at_10": 0.6731686444545869, + "mrr_at_100": 0.6769992046379915, + "mrr_at_1000": 0.6770985849971942, + "mrr_at_20": 0.6757110575121572, + "mrr_at_3": 0.6501597444089456, + "mrr_at_5": 0.6656549520766774, + "nauc_map_at_1000_diff1": 0.30818284170751103, + "nauc_map_at_1000_max": 0.12180816789515145, + "nauc_map_at_1000_std": -0.03014328572392891, + "nauc_map_at_100_diff1": 0.30818327219797853, + "nauc_map_at_100_max": 0.12166658551238059, + "nauc_map_at_100_std": -0.030468559231628528, + "nauc_map_at_10_diff1": 0.3037002688831374, + "nauc_map_at_10_max": 0.1015524272130985, + "nauc_map_at_10_std": -0.06461209372623694, + "nauc_map_at_1_diff1": 0.34715005481168953, + "nauc_map_at_1_max": 0.03989738870380578, + "nauc_map_at_1_std": -0.13154878167980924, + "nauc_map_at_20_diff1": 0.30766683715174475, + "nauc_map_at_20_max": 0.11423035895419638, + "nauc_map_at_20_std": -0.04404871738342385, + "nauc_map_at_3_diff1": 0.3215548609991913, + "nauc_map_at_3_max": 0.07981490279284395, + "nauc_map_at_3_std": -0.10034077818097321, + "nauc_map_at_5_diff1": 0.31096779942585456, + "nauc_map_at_5_max": 0.09159339647102434, + "nauc_map_at_5_std": -0.08872556637585298, + "nauc_mrr_at_1000_diff1": 0.3732495239802774, + "nauc_mrr_at_1000_max": 0.22151631209027392, + "nauc_mrr_at_1000_std": 0.10449048749430274, + "nauc_mrr_at_100_diff1": 0.37318814106642156, + "nauc_mrr_at_100_max": 0.22164807336494324, + "nauc_mrr_at_100_std": 0.10463653191567347, + "nauc_mrr_at_10_diff1": 0.3727767743007139, + "nauc_mrr_at_10_max": 0.22258303217088857, + "nauc_mrr_at_10_std": 0.10632950543951146, + "nauc_mrr_at_1_diff1": 0.39180919878730025, + "nauc_mrr_at_1_max": 0.17254581891420434, + "nauc_mrr_at_1_std": 0.048370081999197796, + "nauc_mrr_at_20_diff1": 0.37293242050929565, + "nauc_mrr_at_20_max": 0.22160852089313837, + "nauc_mrr_at_20_std": 0.10512403135406963, + "nauc_mrr_at_3_diff1": 0.37003381637435745, + "nauc_mrr_at_3_max": 0.2338126517429297, + "nauc_mrr_at_3_std": 0.10719366766913534, + "nauc_mrr_at_5_diff1": 0.36905339350295047, + "nauc_mrr_at_5_max": 0.22189445538162098, + "nauc_mrr_at_5_std": 0.10066981019309659, + "nauc_ndcg_at_1000_diff1": 0.3223526308530261, + "nauc_ndcg_at_1000_max": 0.18186251218695984, + "nauc_ndcg_at_1000_std": 0.06453762485914287, + "nauc_ndcg_at_100_diff1": 0.3207772549303248, + "nauc_ndcg_at_100_max": 0.18419078056191823, + "nauc_ndcg_at_100_std": 0.06869254890016625, + "nauc_ndcg_at_10_diff1": 0.30470165798239135, + "nauc_ndcg_at_10_max": 0.13399630999004436, + "nauc_ndcg_at_10_std": -0.013983254527217362, + "nauc_ndcg_at_1_diff1": 0.39180919878730025, + "nauc_ndcg_at_1_max": 0.17254581891420434, + "nauc_ndcg_at_1_std": 0.048370081999197796, + "nauc_ndcg_at_20_diff1": 0.31318699231220215, + "nauc_ndcg_at_20_max": 0.15616854956199147, + "nauc_ndcg_at_20_std": 0.023120356876913748, + "nauc_ndcg_at_3_diff1": 0.3088670770866889, + "nauc_ndcg_at_3_max": 0.16161586002461675, + "nauc_ndcg_at_3_std": 0.011773889464422793, + "nauc_ndcg_at_5_diff1": 0.3069123237807126, + "nauc_ndcg_at_5_max": 0.13606973635815822, + "nauc_ndcg_at_5_std": -0.026973684685462317, + "nauc_precision_at_1000_diff1": -0.10374202937540754, + "nauc_precision_at_1000_max": 0.18207676370390027, + "nauc_precision_at_1000_std": 0.35695731027262595, + "nauc_precision_at_100_diff1": -0.08261389054842568, + "nauc_precision_at_100_max": 0.21005723870612036, + "nauc_precision_at_100_std": 0.3930465127650968, + "nauc_precision_at_10_diff1": -0.03360341662429267, + "nauc_precision_at_10_max": 0.19180329315637626, + "nauc_precision_at_10_std": 0.27955259639515656, + "nauc_precision_at_1_diff1": 0.39180919878730025, + "nauc_precision_at_1_max": 0.17254581891420434, + "nauc_precision_at_1_std": 0.048370081999197796, + "nauc_precision_at_20_diff1": -0.04656916071146582, + "nauc_precision_at_20_max": 0.2218232373067936, + "nauc_precision_at_20_std": 0.35461849894689146, + "nauc_precision_at_3_diff1": 0.10647596610284339, + "nauc_precision_at_3_max": 0.2238116727442021, + "nauc_precision_at_3_std": 0.18989593488513454, + "nauc_precision_at_5_diff1": 0.033107416051553304, + "nauc_precision_at_5_max": 0.2010423800717645, + "nauc_precision_at_5_std": 0.21655605793461627, + "nauc_recall_at_1000_diff1": 0.37695545713697937, + "nauc_recall_at_1000_max": 0.41637905310783285, + "nauc_recall_at_1000_std": 0.6804571779917068, + "nauc_recall_at_100_diff1": 0.2697458520870636, + "nauc_recall_at_100_max": 0.2971615105400482, + "nauc_recall_at_100_std": 0.34807743456589135, + "nauc_recall_at_10_diff1": 0.21548035376459954, + "nauc_recall_at_10_max": 0.0717757420628144, + "nauc_recall_at_10_std": -0.05455759656578351, + "nauc_recall_at_1_diff1": 0.34715005481168953, + "nauc_recall_at_1_max": 0.03989738870380578, + "nauc_recall_at_1_std": -0.13154878167980924, + "nauc_recall_at_20_diff1": 0.2210398748070612, + "nauc_recall_at_20_max": 0.10640912000490665, + "nauc_recall_at_20_std": 0.023622867582424725, + "nauc_recall_at_3_diff1": 0.27920052057707095, + "nauc_recall_at_3_max": 0.08631794330465158, + "nauc_recall_at_3_std": -0.09476752714366642, + "nauc_recall_at_5_diff1": 0.24397045962410813, + "nauc_recall_at_5_max": 0.07547655756979113, + "nauc_recall_at_5_std": -0.10896850932352087, + "ndcg_at_1": 0.55272, + "ndcg_at_10": 0.59138, + "ndcg_at_100": 0.66199, + "ndcg_at_1000": 0.67401, + "ndcg_at_20": 0.62406, + "ndcg_at_3": 0.5353, + "ndcg_at_5": 0.55325, + "precision_at_1": 0.55272, + "precision_at_10": 0.17005, + "precision_at_100": 0.02535, + "precision_at_1000": 0.00277, + "precision_at_20": 0.10056, + "precision_at_3": 0.35863, + "precision_at_5": 0.26821, + "recall_at_1": 0.29045, + "recall_at_10": 0.68476, + "recall_at_100": 0.91005, + "recall_at_1000": 0.97621, + "recall_at_20": 0.7745, + "recall_at_3": 0.48833, + "recall_at_5": 0.57744 + } + ] + }, + "task_name": "MIRACLRetrieval" +} \ No newline at end of file diff --git a/results/Linq-AI-Research__Linq-Embed-Mistral/0c1a0b0589177079acc552433cad51d7c9132379/MassiveIntentClassification.json b/results/Linq-AI-Research__Linq-Embed-Mistral/0c1a0b0589177079acc552433cad51d7c9132379/MassiveIntentClassification.json new file mode 100644 index 000000000..e26f76e66 --- /dev/null +++ b/results/Linq-AI-Research__Linq-Embed-Mistral/0c1a0b0589177079acc552433cad51d7c9132379/MassiveIntentClassification.json @@ -0,0 +1,137 @@ +{ + "dataset_revision": "4672e20407010da34463acc759c162ca9734bca6", + "evaluation_time": 283.396684885025, + "kg_co2_emissions": null, + "mteb_version": "1.19.2", + "scores": { + "test": [ + { + "accuracy": 0.7718560860793544, + "f1": 0.7383036343098989, + "f1_weighted": 0.7638305290060587, + "hf_subset": "ru", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.7718560860793544, + "scores_per_experiment": [ + { + "accuracy": 0.777404169468729, + "f1": 0.7370559854660887, + "f1_weighted": 0.7687330709415543 + }, + { + "accuracy": 0.7814391392064559, + "f1": 0.7523363796778161, + "f1_weighted": 0.7736351212857856 + }, + { + "accuracy": 0.7669804976462676, + "f1": 0.7365878795333431, + "f1_weighted": 0.7582574042872109 + }, + { + "accuracy": 0.7868190988567586, + "f1": 0.7451092096689428, + "f1_weighted": 0.7808337872053428 + }, + { + "accuracy": 0.7700067249495629, + "f1": 0.7299475112085703, + "f1_weighted": 0.7589373993372903 + }, + { + "accuracy": 0.7763954270342972, + "f1": 0.7463430197275412, + "f1_weighted": 0.7748130239655857 + }, + { + "accuracy": 0.7679892400806994, + "f1": 0.7255555394608271, + "f1_weighted": 0.7527355163233992 + }, + { + "accuracy": 0.7683254875588433, + "f1": 0.7375037895749078, + "f1_weighted": 0.7636723561469657 + }, + { + "accuracy": 0.7498318762609281, + "f1": 0.724030717719674, + "f1_weighted": 0.7405912844270286 + }, + { + "accuracy": 0.773369199731002, + "f1": 0.7485663110612776, + "f1_weighted": 0.7660963261404229 + } + ] + } + ], + "validation": [ + { + "accuracy": 0.7791933103787506, + "f1": 0.737426180259194, + "f1_weighted": 0.7666381598228467, + "hf_subset": "ru", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.7791933103787506, + "scores_per_experiment": [ + { + "accuracy": 0.7845548450565667, + "f1": 0.7440596664845416, + "f1_weighted": 0.7722615119198849 + }, + { + "accuracy": 0.7909493359567142, + "f1": 0.7488630037694928, + "f1_weighted": 0.7792529414980506 + }, + { + "accuracy": 0.778160354156419, + "f1": 0.7355764453222771, + "f1_weighted": 0.7661055930142987 + }, + { + "accuracy": 0.7860304968027545, + "f1": 0.7481907763117869, + "f1_weighted": 0.7732067379415821 + }, + { + "accuracy": 0.7840629611411707, + "f1": 0.7368718426066329, + "f1_weighted": 0.7715342421121536 + }, + { + "accuracy": 0.7884899163797344, + "f1": 0.7413749574303281, + "f1_weighted": 0.781955359702065 + }, + { + "accuracy": 0.7791441219872111, + "f1": 0.7255138457468365, + "f1_weighted": 0.7608562119849925 + }, + { + "accuracy": 0.764387604525332, + "f1": 0.7282226448768303, + "f1_weighted": 0.7523528285468242 + }, + { + "accuracy": 0.7540580423020167, + "f1": 0.7219137680621117, + "f1_weighted": 0.7369730678516829 + }, + { + "accuracy": 0.7820954254795868, + "f1": 0.743674851981103, + "f1_weighted": 0.7718831036569318 + } + ] + } + ] + }, + "task_name": "MassiveIntentClassification" +} \ No newline at end of file diff --git a/results/Linq-AI-Research__Linq-Embed-Mistral/0c1a0b0589177079acc552433cad51d7c9132379/MassiveScenarioClassification.json b/results/Linq-AI-Research__Linq-Embed-Mistral/0c1a0b0589177079acc552433cad51d7c9132379/MassiveScenarioClassification.json new file mode 100644 index 000000000..c6a62bf07 --- /dev/null +++ b/results/Linq-AI-Research__Linq-Embed-Mistral/0c1a0b0589177079acc552433cad51d7c9132379/MassiveScenarioClassification.json @@ -0,0 +1,137 @@ +{ + "dataset_revision": "fad2c6e8459f9e1c45d9315f4953d921437d70f8", + "evaluation_time": 163.20068669319153, + "kg_co2_emissions": null, + "mteb_version": "1.19.2", + "scores": { + "test": [ + { + "accuracy": 0.810827168796234, + "f1": 0.798338347795483, + "f1_weighted": 0.8095658628852563, + "hf_subset": "ru", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.810827168796234, + "scores_per_experiment": [ + { + "accuracy": 0.777404169468729, + "f1": 0.7764458084541596, + "f1_weighted": 0.7789823842436719 + }, + { + "accuracy": 0.8211163416274377, + "f1": 0.8059252193067358, + "f1_weighted": 0.8198936392116051 + }, + { + "accuracy": 0.8096839273705447, + "f1": 0.7934281117869947, + "f1_weighted": 0.8061563658944314 + }, + { + "accuracy": 0.8117014122394082, + "f1": 0.7999772855119596, + "f1_weighted": 0.8081322904521516 + }, + { + "accuracy": 0.8137188971082717, + "f1": 0.7966062489779197, + "f1_weighted": 0.8102946211751321 + }, + { + "accuracy": 0.8016139878950908, + "f1": 0.7887023509412651, + "f1_weighted": 0.8006372110789317 + }, + { + "accuracy": 0.7834566240753195, + "f1": 0.7734782430043259, + "f1_weighted": 0.7839118804557944 + }, + { + "accuracy": 0.8298587760591796, + "f1": 0.8140581252823026, + "f1_weighted": 0.8297610437053686 + }, + { + "accuracy": 0.8187626092804304, + "f1": 0.8094344108862249, + "f1_weighted": 0.8183519113289708 + }, + { + "accuracy": 0.8409549428379287, + "f1": 0.825327673802942, + "f1_weighted": 0.8395372813065047 + } + ] + } + ], + "validation": [ + { + "accuracy": 0.8085095917363502, + "f1": 0.7964674514148566, + "f1_weighted": 0.8053496393398817, + "hf_subset": "ru", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.8085095917363502, + "scores_per_experiment": [ + { + "accuracy": 0.7717658632562715, + "f1": 0.7728815767268206, + "f1_weighted": 0.7716027757503767 + }, + { + "accuracy": 0.8278406296114117, + "f1": 0.8150676047717574, + "f1_weighted": 0.8273558192499716 + }, + { + "accuracy": 0.8219380226266602, + "f1": 0.8040427720329218, + "f1_weighted": 0.8168604372973677 + }, + { + "accuracy": 0.793408755533694, + "f1": 0.7818631400791693, + "f1_weighted": 0.7879270643964785 + }, + { + "accuracy": 0.8268568617806198, + "f1": 0.8102574477986622, + "f1_weighted": 0.8218692755446254 + }, + { + "accuracy": 0.793408755533694, + "f1": 0.7814652947140692, + "f1_weighted": 0.7903261840308395 + }, + { + "accuracy": 0.7835710772257747, + "f1": 0.7747881129243078, + "f1_weighted": 0.7810255825997351 + }, + { + "accuracy": 0.8214461387112642, + "f1": 0.8022043687171687, + "f1_weighted": 0.8175692203966691 + }, + { + "accuracy": 0.8130841121495327, + "f1": 0.8032040913237358, + "f1_weighted": 0.8097764589856774 + }, + { + "accuracy": 0.8317757009345794, + "f1": 0.8189001050599537, + "f1_weighted": 0.8291835751470754 + } + ] + } + ] + }, + "task_name": "MassiveScenarioClassification" +} \ No newline at end of file diff --git a/results/Linq-AI-Research__Linq-Embed-Mistral/0c1a0b0589177079acc552433cad51d7c9132379/RUParaPhraserSTS.json b/results/Linq-AI-Research__Linq-Embed-Mistral/0c1a0b0589177079acc552433cad51d7c9132379/RUParaPhraserSTS.json new file mode 100644 index 000000000..e3a60cef0 --- /dev/null +++ b/results/Linq-AI-Research__Linq-Embed-Mistral/0c1a0b0589177079acc552433cad51d7c9132379/RUParaPhraserSTS.json @@ -0,0 +1,26 @@ +{ + "dataset_revision": "43265056790b8f7c59e0139acb4be0a8dad2c8f4", + "evaluation_time": 62.158459186553955, + "kg_co2_emissions": null, + "mteb_version": "1.19.2", + "scores": { + "test": [ + { + "cosine_pearson": 0.7059765768999549, + "cosine_spearman": 0.7659510997489586, + "euclidean_pearson": 0.7387695038870142, + "euclidean_spearman": 0.7659510997489586, + "hf_subset": "default", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.7659510997489586, + "manhattan_pearson": 0.7379692734636423, + "manhattan_spearman": 0.7646436745448438, + "pearson": 0.7059765768999549, + "spearman": 0.7659510997489586 + } + ] + }, + "task_name": "RUParaPhraserSTS" +} \ No newline at end of file diff --git a/results/Linq-AI-Research__Linq-Embed-Mistral/0c1a0b0589177079acc552433cad51d7c9132379/RiaNewsRetrieval.json b/results/Linq-AI-Research__Linq-Embed-Mistral/0c1a0b0589177079acc552433cad51d7c9132379/RiaNewsRetrieval.json new file mode 100644 index 000000000..5684cb73f --- /dev/null +++ b/results/Linq-AI-Research__Linq-Embed-Mistral/0c1a0b0589177079acc552433cad51d7c9132379/RiaNewsRetrieval.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "82374b0bbacda6114f39ff9c5b925fa1512ca5d7", + "evaluation_time": 30584.269805431366, + "kg_co2_emissions": null, + "mteb_version": "1.19.2", + "scores": { + "test": [ + { + "hf_subset": "default", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.80886, + "map_at_1": 0.6941, + "map_at_10": 0.77445, + "map_at_100": 0.77691, + "map_at_1000": 0.777, + "map_at_20": 0.77607, + "map_at_3": 0.76068, + "map_at_5": 0.76983, + "mrr_at_1": 0.6941, + "mrr_at_10": 0.7744712301587301, + "mrr_at_100": 0.7769313873062079, + "mrr_at_1000": 0.777016473830474, + "mrr_at_20": 0.7760916159090308, + "mrr_at_3": 0.7607, + "mrr_at_5": 0.76985, + "nauc_map_at_1000_diff1": 0.7524368763676522, + "nauc_map_at_1000_max": 0.30489404350159643, + "nauc_map_at_1000_std": -0.12173710700029534, + "nauc_map_at_100_diff1": 0.7524095722659204, + "nauc_map_at_100_max": 0.3049201274795752, + "nauc_map_at_100_std": -0.12165217719554552, + "nauc_map_at_10_diff1": 0.7515432465910883, + "nauc_map_at_10_max": 0.3058362487837298, + "nauc_map_at_10_std": -0.12164760618766078, + "nauc_map_at_1_diff1": 0.7789078982457494, + "nauc_map_at_1_max": 0.2706203646198131, + "nauc_map_at_1_std": -0.14688670214175997, + "nauc_map_at_20_diff1": 0.7521595880045785, + "nauc_map_at_20_max": 0.305056919697089, + "nauc_map_at_20_std": -0.12172684593312268, + "nauc_map_at_3_diff1": 0.751705241992875, + "nauc_map_at_3_max": 0.3024373040887015, + "nauc_map_at_3_std": -0.13120679403756552, + "nauc_map_at_5_diff1": 0.7517186702719679, + "nauc_map_at_5_max": 0.30650054909380936, + "nauc_map_at_5_std": -0.12479801076691001, + "nauc_mrr_at_1000_diff1": 0.7524187687700795, + "nauc_mrr_at_1000_max": 0.30506274903256125, + "nauc_mrr_at_1000_std": -0.12159960670326139, + "nauc_mrr_at_100_diff1": 0.7523914704471029, + "nauc_mrr_at_100_max": 0.30508876247908034, + "nauc_mrr_at_100_std": -0.12151472972878329, + "nauc_mrr_at_10_diff1": 0.7515253041861752, + "nauc_mrr_at_10_max": 0.30600288203216514, + "nauc_mrr_at_10_std": -0.12151184385866856, + "nauc_mrr_at_1_diff1": 0.7789078982457494, + "nauc_mrr_at_1_max": 0.2706203646198131, + "nauc_mrr_at_1_std": -0.14688670214175997, + "nauc_mrr_at_20_diff1": 0.7521415443742253, + "nauc_mrr_at_20_max": 0.3052248530298201, + "nauc_mrr_at_20_std": -0.12158998391941463, + "nauc_mrr_at_3_diff1": 0.7516883380515945, + "nauc_mrr_at_3_max": 0.30259416402581896, + "nauc_mrr_at_3_std": -0.13107945665030166, + "nauc_mrr_at_5_diff1": 0.7517011109476598, + "nauc_mrr_at_5_max": 0.30666378683010703, + "nauc_mrr_at_5_std": -0.1246652759737654, + "nauc_ndcg_at_1000_diff1": 0.7454155698784701, + "nauc_ndcg_at_1000_max": 0.3167074773280948, + "nauc_ndcg_at_1000_std": -0.10644231730080203, + "nauc_ndcg_at_100_diff1": 0.7445400245666669, + "nauc_ndcg_at_100_max": 0.318229372307377, + "nauc_ndcg_at_100_std": -0.10291732944558782, + "nauc_ndcg_at_10_diff1": 0.740338113433323, + "nauc_ndcg_at_10_max": 0.32131023126931196, + "nauc_ndcg_at_10_std": -0.10261671974567818, + "nauc_ndcg_at_1_diff1": 0.7789078982457494, + "nauc_ndcg_at_1_max": 0.2706203646198131, + "nauc_ndcg_at_1_std": -0.14688670214175997, + "nauc_ndcg_at_20_diff1": 0.7427169384125643, + "nauc_ndcg_at_20_max": 0.31851522952242967, + "nauc_ndcg_at_20_std": -0.10268501571075737, + "nauc_ndcg_at_3_diff1": 0.7421912234028455, + "nauc_ndcg_at_3_max": 0.3136631207405142, + "nauc_ndcg_at_3_std": -0.12522608228198107, + "nauc_ndcg_at_5_diff1": 0.7412357947767397, + "nauc_ndcg_at_5_max": 0.3225989068228711, + "nauc_ndcg_at_5_std": -0.11163201707880163, + "nauc_precision_at_1000_diff1": 0.5585011782490836, + "nauc_precision_at_1000_max": 0.7151786640582178, + "nauc_precision_at_1000_std": 0.40173106279549387, + "nauc_precision_at_100_diff1": 0.6318997295887969, + "nauc_precision_at_100_max": 0.5503868213952257, + "nauc_precision_at_100_std": 0.2553339517625119, + "nauc_precision_at_10_diff1": 0.6648183863709409, + "nauc_precision_at_10_max": 0.425487134744609, + "nauc_precision_at_10_std": 0.03738872598219465, + "nauc_precision_at_1_diff1": 0.7789078982457494, + "nauc_precision_at_1_max": 0.2706203646198131, + "nauc_precision_at_1_std": -0.14688670214175997, + "nauc_precision_at_20_diff1": 0.6631303264405549, + "nauc_precision_at_20_max": 0.4337885897458778, + "nauc_precision_at_20_std": 0.08488331638387737, + "nauc_precision_at_3_diff1": 0.7044708358425475, + "nauc_precision_at_3_max": 0.35821858678558155, + "nauc_precision_at_3_std": -0.1012694471845676, + "nauc_precision_at_5_diff1": 0.6887706593246806, + "nauc_precision_at_5_max": 0.4048496581214323, + "nauc_precision_at_5_std": -0.04189111019851313, + "nauc_recall_at_1000_diff1": 0.5585011782490785, + "nauc_recall_at_1000_max": 0.7151786640582173, + "nauc_recall_at_1000_std": 0.40173106279548276, + "nauc_recall_at_100_diff1": 0.6318997295888049, + "nauc_recall_at_100_max": 0.5503868213952233, + "nauc_recall_at_100_std": 0.2553339517625272, + "nauc_recall_at_10_diff1": 0.6648183863709409, + "nauc_recall_at_10_max": 0.42548713474460725, + "nauc_recall_at_10_std": 0.03738872598219724, + "nauc_recall_at_1_diff1": 0.7789078982457494, + "nauc_recall_at_1_max": 0.2706203646198131, + "nauc_recall_at_1_std": -0.14688670214175997, + "nauc_recall_at_20_diff1": 0.6631303264405556, + "nauc_recall_at_20_max": 0.43378858974587653, + "nauc_recall_at_20_std": 0.08488331638387889, + "nauc_recall_at_3_diff1": 0.704470835842548, + "nauc_recall_at_3_max": 0.358218586785582, + "nauc_recall_at_3_std": -0.10126944718456825, + "nauc_recall_at_5_diff1": 0.6887706593246788, + "nauc_recall_at_5_max": 0.40484965812143325, + "nauc_recall_at_5_std": -0.041891110198511626, + "ndcg_at_1": 0.6941, + "ndcg_at_10": 0.80886, + "ndcg_at_100": 0.82061, + "ndcg_at_1000": 0.82291, + "ndcg_at_20": 0.81463, + "ndcg_at_3": 0.78129, + "ndcg_at_5": 0.79778, + "precision_at_1": 0.6941, + "precision_at_10": 0.09146, + "precision_at_100": 0.00969, + "precision_at_1000": 0.00099, + "precision_at_20": 0.04686, + "precision_at_3": 0.2802, + "precision_at_5": 0.17612, + "recall_at_1": 0.6941, + "recall_at_10": 0.9146, + "recall_at_100": 0.9692, + "recall_at_1000": 0.9874, + "recall_at_20": 0.9372, + "recall_at_3": 0.8406, + "recall_at_5": 0.8806 + } + ] + }, + "task_name": "RiaNewsRetrieval" +} \ No newline at end of file diff --git a/results/Linq-AI-Research__Linq-Embed-Mistral/0c1a0b0589177079acc552433cad51d7c9132379/RuBQReranking.json b/results/Linq-AI-Research__Linq-Embed-Mistral/0c1a0b0589177079acc552433cad51d7c9132379/RuBQReranking.json new file mode 100644 index 000000000..d23ccf2de --- /dev/null +++ b/results/Linq-AI-Research__Linq-Embed-Mistral/0c1a0b0589177079acc552433cad51d7c9132379/RuBQReranking.json @@ -0,0 +1,26 @@ +{ + "dataset_revision": "2e96b8f098fa4b0950fc58eacadeb31c0d0c7fa2", + "evaluation_time": 993.5698163509369, + "kg_co2_emissions": null, + "mteb_version": "1.19.2", + "scores": { + "test": [ + { + "hf_subset": "default", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.7684186176600346, + "map": 0.7684186176600346, + "mrr": 0.8147935791552813, + "nAUC_map_diff1": 0.4071429520606149, + "nAUC_map_max": 0.13090909805545545, + "nAUC_map_std": 0.05514605787438006, + "nAUC_mrr_diff1": 0.45335906899569517, + "nAUC_mrr_max": 0.16119825867592008, + "nAUC_mrr_std": 0.06374161710572034 + } + ] + }, + "task_name": "RuBQReranking" +} \ No newline at end of file diff --git a/results/Linq-AI-Research__Linq-Embed-Mistral/0c1a0b0589177079acc552433cad51d7c9132379/RuBQRetrieval.json b/results/Linq-AI-Research__Linq-Embed-Mistral/0c1a0b0589177079acc552433cad51d7c9132379/RuBQRetrieval.json new file mode 100644 index 000000000..a4e090710 --- /dev/null +++ b/results/Linq-AI-Research__Linq-Embed-Mistral/0c1a0b0589177079acc552433cad51d7c9132379/RuBQRetrieval.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "e19b6ffa60b3bc248e0b41f4cc37c26a55c2a67b", + "evaluation_time": 1489.2378797531128, + "kg_co2_emissions": null, + "mteb_version": "1.19.2", + "scores": { + "test": [ + { + "hf_subset": "default", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.73196, + "map_at_1": 0.43062, + "map_at_10": 0.65221, + "map_at_100": 0.66189, + "map_at_1000": 0.66206, + "map_at_20": 0.65967, + "map_at_3": 0.59588, + "map_at_5": 0.62941, + "mrr_at_1": 0.6004728132387707, + "mrr_at_10": 0.7248562328792825, + "mrr_at_100": 0.7278665737915188, + "mrr_at_1000": 0.7278913871248642, + "mrr_at_20": 0.7273136485302735, + "mrr_at_3": 0.7035066981875492, + "mrr_at_5": 0.7176615445232466, + "nauc_map_at_1000_diff1": 0.3774406767256758, + "nauc_map_at_1000_max": 0.16909408121878675, + "nauc_map_at_1000_std": -0.18725006179918327, + "nauc_map_at_100_diff1": 0.37752094582786544, + "nauc_map_at_100_max": 0.16937474228085467, + "nauc_map_at_100_std": -0.1869064242615371, + "nauc_map_at_10_diff1": 0.37312520236368063, + "nauc_map_at_10_max": 0.1666374350178296, + "nauc_map_at_10_std": -0.19104185041738386, + "nauc_map_at_1_diff1": 0.4016313359721605, + "nauc_map_at_1_max": 0.07765651716636006, + "nauc_map_at_1_std": -0.17661546820910443, + "nauc_map_at_20_diff1": 0.3760261613793132, + "nauc_map_at_20_max": 0.17060877720573955, + "nauc_map_at_20_std": -0.18645850985112256, + "nauc_map_at_3_diff1": 0.3651097011942123, + "nauc_map_at_3_max": 0.1450827810851466, + "nauc_map_at_3_std": -0.19487417276983246, + "nauc_map_at_5_diff1": 0.36789612602317723, + "nauc_map_at_5_max": 0.1568607331216473, + "nauc_map_at_5_std": -0.2019489888479521, + "nauc_mrr_at_1000_diff1": 0.4423162968045083, + "nauc_mrr_at_1000_max": 0.17436343278678965, + "nauc_mrr_at_1000_std": -0.1961249545341672, + "nauc_mrr_at_100_diff1": 0.44231927069409177, + "nauc_mrr_at_100_max": 0.17441685944613178, + "nauc_mrr_at_100_std": -0.19606543214389724, + "nauc_mrr_at_10_diff1": 0.4409776761189798, + "nauc_mrr_at_10_max": 0.17648103537542026, + "nauc_mrr_at_10_std": -0.19556927228961893, + "nauc_mrr_at_1_diff1": 0.4552167903373739, + "nauc_mrr_at_1_max": 0.13758339560794378, + "nauc_mrr_at_1_std": -0.20023316577398823, + "nauc_mrr_at_20_diff1": 0.44222223321225934, + "nauc_mrr_at_20_max": 0.1754204834194535, + "nauc_mrr_at_20_std": -0.19569308693886597, + "nauc_mrr_at_3_diff1": 0.43681974853879485, + "nauc_mrr_at_3_max": 0.17265871428527627, + "nauc_mrr_at_3_std": -0.1986247677353473, + "nauc_mrr_at_5_diff1": 0.43912254508006443, + "nauc_mrr_at_5_max": 0.17702461621687704, + "nauc_mrr_at_5_std": -0.2007660092945449, + "nauc_ndcg_at_1000_diff1": 0.39473326236043155, + "nauc_ndcg_at_1000_max": 0.1875697369335963, + "nauc_ndcg_at_1000_std": -0.17921031106070007, + "nauc_ndcg_at_100_diff1": 0.3951781207409726, + "nauc_ndcg_at_100_max": 0.19315448603779048, + "nauc_ndcg_at_100_std": -0.17238256774305602, + "nauc_ndcg_at_10_diff1": 0.3803735959770975, + "nauc_ndcg_at_10_max": 0.19393780591399828, + "nauc_ndcg_at_10_std": -0.18025374354796594, + "nauc_ndcg_at_1_diff1": 0.4552167903373739, + "nauc_ndcg_at_1_max": 0.13758339560794378, + "nauc_ndcg_at_1_std": -0.20023316577398823, + "nauc_ndcg_at_20_diff1": 0.38843197056910395, + "nauc_ndcg_at_20_max": 0.20313494977980875, + "nauc_ndcg_at_20_std": -0.1681924276068553, + "nauc_ndcg_at_3_diff1": 0.36527120064858287, + "nauc_ndcg_at_3_max": 0.15848439877946896, + "nauc_ndcg_at_3_std": -0.19855178499219883, + "nauc_ndcg_at_5_diff1": 0.37045691281178134, + "nauc_ndcg_at_5_max": 0.17597511841978786, + "nauc_ndcg_at_5_std": -0.20652890218089184, + "nauc_precision_at_1000_diff1": -0.10965998196581718, + "nauc_precision_at_1000_max": 0.06915695863761862, + "nauc_precision_at_1000_std": 0.07720080938111781, + "nauc_precision_at_100_diff1": -0.0952064504767698, + "nauc_precision_at_100_max": 0.0972469451549985, + "nauc_precision_at_100_std": 0.09619799811564997, + "nauc_precision_at_10_diff1": -0.044131113002656104, + "nauc_precision_at_10_max": 0.14551461841926633, + "nauc_precision_at_10_std": 0.042473822426186955, + "nauc_precision_at_1_diff1": 0.4552167903373739, + "nauc_precision_at_1_max": 0.13758339560794378, + "nauc_precision_at_1_std": -0.20023316577398823, + "nauc_precision_at_20_diff1": -0.08074019944664297, + "nauc_precision_at_20_max": 0.1388685330868178, + "nauc_precision_at_20_std": 0.09568943019659794, + "nauc_precision_at_3_diff1": 0.08380439184824519, + "nauc_precision_at_3_max": 0.1724995307797711, + "nauc_precision_at_3_std": -0.08028904204535553, + "nauc_precision_at_5_diff1": 0.01685788048291993, + "nauc_precision_at_5_max": 0.15756229959551807, + "nauc_precision_at_5_std": -0.04699072244285311, + "nauc_recall_at_1000_diff1": 0.5492209526994482, + "nauc_recall_at_1000_max": 0.8236335770352717, + "nauc_recall_at_1000_std": 0.7923145052611901, + "nauc_recall_at_100_diff1": 0.3939252274876361, + "nauc_recall_at_100_max": 0.6324362399810495, + "nauc_recall_at_100_std": 0.4288606235832715, + "nauc_recall_at_10_diff1": 0.2506694722587436, + "nauc_recall_at_10_max": 0.2869559573468239, + "nauc_recall_at_10_std": -0.08698251447566006, + "nauc_recall_at_1_diff1": 0.4016313359721605, + "nauc_recall_at_1_max": 0.07765651716636006, + "nauc_recall_at_1_std": -0.17661546820910443, + "nauc_recall_at_20_diff1": 0.2588284524073533, + "nauc_recall_at_20_max": 0.468182261808439, + "nauc_recall_at_20_std": 0.0935255695141888, + "nauc_recall_at_3_diff1": 0.27869822150038476, + "nauc_recall_at_3_max": 0.1515277568808875, + "nauc_recall_at_3_std": -0.18087795496147846, + "nauc_recall_at_5_diff1": 0.2608385875535053, + "nauc_recall_at_5_max": 0.1951749787372531, + "nauc_recall_at_5_std": -0.2006315169151524, + "ndcg_at_1": 0.60047, + "ndcg_at_10": 0.73196, + "ndcg_at_100": 0.75901, + "ndcg_at_1000": 0.76138, + "ndcg_at_20": 0.74973, + "ndcg_at_3": 0.65601, + "ndcg_at_5": 0.6948, + "precision_at_1": 0.60047, + "precision_at_10": 0.1445, + "precision_at_100": 0.01644, + "precision_at_1000": 0.00168, + "precision_at_20": 0.0781, + "precision_at_3": 0.36072, + "precision_at_5": 0.25083, + "recall_at_1": 0.43062, + "recall_at_10": 0.88595, + "recall_at_100": 0.98309, + "recall_at_1000": 0.99773, + "recall_at_20": 0.9434, + "recall_at_3": 0.69969, + "recall_at_5": 0.79151 + } + ] + }, + "task_name": "RuBQRetrieval" +} \ No newline at end of file diff --git a/results/Linq-AI-Research__Linq-Embed-Mistral/0c1a0b0589177079acc552433cad51d7c9132379/RuReviewsClassification.json b/results/Linq-AI-Research__Linq-Embed-Mistral/0c1a0b0589177079acc552433cad51d7c9132379/RuReviewsClassification.json new file mode 100644 index 000000000..82bccc19f --- /dev/null +++ b/results/Linq-AI-Research__Linq-Embed-Mistral/0c1a0b0589177079acc552433cad51d7c9132379/RuReviewsClassification.json @@ -0,0 +1,73 @@ +{ + "dataset_revision": "f6d2c31f4dc6b88f468552750bfec05b4b41b05a", + "evaluation_time": 46.564120054244995, + "kg_co2_emissions": null, + "mteb_version": "1.19.2", + "scores": { + "test": [ + { + "accuracy": 0.72998046875, + "f1": 0.7261584743941671, + "f1_weighted": 0.7261583239060692, + "hf_subset": "default", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.72998046875, + "scores_per_experiment": [ + { + "accuracy": 0.71875, + "f1": 0.7166671262093689, + "f1_weighted": 0.7166614308719557 + }, + { + "accuracy": 0.73828125, + "f1": 0.7371807782196362, + "f1_weighted": 0.7371898000955073 + }, + { + "accuracy": 0.7265625, + "f1": 0.7176770947083518, + "f1_weighted": 0.7176719880196252 + }, + { + "accuracy": 0.7451171875, + "f1": 0.7469502366857782, + "f1_weighted": 0.746955462271585 + }, + { + "accuracy": 0.73486328125, + "f1": 0.7386980198303584, + "f1_weighted": 0.7386998046373836 + }, + { + "accuracy": 0.71826171875, + "f1": 0.7200027671780287, + "f1_weighted": 0.720005654131105 + }, + { + "accuracy": 0.72265625, + "f1": 0.7164769535839769, + "f1_weighted": 0.7164779325804977 + }, + { + "accuracy": 0.73974609375, + "f1": 0.7333907760781768, + "f1_weighted": 0.7333886974153621 + }, + { + "accuracy": 0.7099609375, + "f1": 0.6895952285573329, + "f1_weighted": 0.6895771089471955 + }, + { + "accuracy": 0.74560546875, + "f1": 0.7449457628906625, + "f1_weighted": 0.7449553600904751 + } + ] + } + ] + }, + "task_name": "RuReviewsClassification" +} \ No newline at end of file diff --git a/results/Linq-AI-Research__Linq-Embed-Mistral/0c1a0b0589177079acc552433cad51d7c9132379/RuSTSBenchmarkSTS.json b/results/Linq-AI-Research__Linq-Embed-Mistral/0c1a0b0589177079acc552433cad51d7c9132379/RuSTSBenchmarkSTS.json new file mode 100644 index 000000000..6f424917a --- /dev/null +++ b/results/Linq-AI-Research__Linq-Embed-Mistral/0c1a0b0589177079acc552433cad51d7c9132379/RuSTSBenchmarkSTS.json @@ -0,0 +1,26 @@ +{ + "dataset_revision": "7cf24f325c6da6195df55bef3d86b5e0616f3018", + "evaluation_time": 40.92873549461365, + "kg_co2_emissions": null, + "mteb_version": "1.19.2", + "scores": { + "test": [ + { + "cosine_pearson": 0.8480730455694943, + "cosine_spearman": 0.8410057482873593, + "euclidean_pearson": 0.8289786818197947, + "euclidean_spearman": 0.8410060965663902, + "hf_subset": "default", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.8410057482873593, + "manhattan_pearson": 0.8299621028521468, + "manhattan_spearman": 0.8429410856091291, + "pearson": 0.8480730455694943, + "spearman": 0.8410057482873593 + } + ] + }, + "task_name": "RuSTSBenchmarkSTS" +} \ No newline at end of file diff --git a/results/Linq-AI-Research__Linq-Embed-Mistral/0c1a0b0589177079acc552433cad51d7c9132379/RuSciBenchGRNTIClassification.json b/results/Linq-AI-Research__Linq-Embed-Mistral/0c1a0b0589177079acc552433cad51d7c9132379/RuSciBenchGRNTIClassification.json new file mode 100644 index 000000000..68a7c935d --- /dev/null +++ b/results/Linq-AI-Research__Linq-Embed-Mistral/0c1a0b0589177079acc552433cad51d7c9132379/RuSciBenchGRNTIClassification.json @@ -0,0 +1,73 @@ +{ + "dataset_revision": "673a610d6d3dd91a547a0d57ae1b56f37ebbf6a1", + "evaluation_time": 179.43042969703674, + "kg_co2_emissions": null, + "mteb_version": "1.19.2", + "scores": { + "test": [ + { + "accuracy": 0.684033203125, + "f1": 0.6764476442056325, + "f1_weighted": 0.6765378933766231, + "hf_subset": "default", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.684033203125, + "scores_per_experiment": [ + { + "accuracy": 0.69677734375, + "f1": 0.6917824502995791, + "f1_weighted": 0.6918739065040733 + }, + { + "accuracy": 0.68310546875, + "f1": 0.6747354811201706, + "f1_weighted": 0.6748125172269545 + }, + { + "accuracy": 0.68798828125, + "f1": 0.6789781784254548, + "f1_weighted": 0.6790847107896488 + }, + { + "accuracy": 0.68505859375, + "f1": 0.6754797897097325, + "f1_weighted": 0.6755684564054388 + }, + { + "accuracy": 0.6767578125, + "f1": 0.6699359386109345, + "f1_weighted": 0.6699908126574913 + }, + { + "accuracy": 0.67431640625, + "f1": 0.6689604876875322, + "f1_weighted": 0.6690461361985246 + }, + { + "accuracy": 0.689453125, + "f1": 0.6797231768075653, + "f1_weighted": 0.6798222613760152 + }, + { + "accuracy": 0.6806640625, + "f1": 0.6717963507366671, + "f1_weighted": 0.6718908727374925 + }, + { + "accuracy": 0.689453125, + "f1": 0.6825941359105433, + "f1_weighted": 0.6827444853509625 + }, + { + "accuracy": 0.6767578125, + "f1": 0.6704904527481444, + "f1_weighted": 0.6705447745196293 + } + ] + } + ] + }, + "task_name": "RuSciBenchGRNTIClassification" +} \ No newline at end of file diff --git a/results/Linq-AI-Research__Linq-Embed-Mistral/0c1a0b0589177079acc552433cad51d7c9132379/RuSciBenchGRNTIClusteringP2P.json b/results/Linq-AI-Research__Linq-Embed-Mistral/0c1a0b0589177079acc552433cad51d7c9132379/RuSciBenchGRNTIClusteringP2P.json new file mode 100644 index 000000000..901827457 --- /dev/null +++ b/results/Linq-AI-Research__Linq-Embed-Mistral/0c1a0b0589177079acc552433cad51d7c9132379/RuSciBenchGRNTIClusteringP2P.json @@ -0,0 +1,34 @@ +{ + "dataset_revision": "673a610d6d3dd91a547a0d57ae1b56f37ebbf6a1", + "evaluation_time": 149.89241099357605, + "kg_co2_emissions": null, + "mteb_version": "1.19.2", + "scores": { + "test": [ + { + "hf_subset": "default", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.6150316250650828, + "v_measure": 0.6150316250650828, + "v_measure_std": 0.009838139666195452, + "v_measures": { + "Level 0": [ + 0.6127955266452347, + 0.6143133081411107, + 0.6057551699733295, + 0.6248137407360614, + 0.6376357785665048, + 0.6141463410663988, + 0.6084134565263348, + 0.6004258320066429, + 0.6182291128122653, + 0.6137879841769446 + ] + } + } + ] + }, + "task_name": "RuSciBenchGRNTIClusteringP2P" +} \ No newline at end of file diff --git a/results/Linq-AI-Research__Linq-Embed-Mistral/0c1a0b0589177079acc552433cad51d7c9132379/RuSciBenchOECDClassification.json b/results/Linq-AI-Research__Linq-Embed-Mistral/0c1a0b0589177079acc552433cad51d7c9132379/RuSciBenchOECDClassification.json new file mode 100644 index 000000000..764e0f8b8 --- /dev/null +++ b/results/Linq-AI-Research__Linq-Embed-Mistral/0c1a0b0589177079acc552433cad51d7c9132379/RuSciBenchOECDClassification.json @@ -0,0 +1,73 @@ +{ + "dataset_revision": "26c88e99dcaba32bb45d0e1bfc21902337f6d471", + "evaluation_time": 178.73000073432922, + "kg_co2_emissions": null, + "mteb_version": "1.19.2", + "scores": { + "test": [ + { + "accuracy": 0.54130859375, + "f1": 0.5205796790821494, + "f1_weighted": 0.5205476324883288, + "hf_subset": "default", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.54130859375, + "scores_per_experiment": [ + { + "accuracy": 0.53369140625, + "f1": 0.5132340561394073, + "f1_weighted": 0.5133349767697196 + }, + { + "accuracy": 0.54443359375, + "f1": 0.5262391150048592, + "f1_weighted": 0.5261688008503416 + }, + { + "accuracy": 0.56201171875, + "f1": 0.5478358702314526, + "f1_weighted": 0.5478694504196542 + }, + { + "accuracy": 0.5234375, + "f1": 0.48776291857879484, + "f1_weighted": 0.4876760681796472 + }, + { + "accuracy": 0.54052734375, + "f1": 0.5174264708087518, + "f1_weighted": 0.5173831616502681 + }, + { + "accuracy": 0.56982421875, + "f1": 0.5534670916633719, + "f1_weighted": 0.5534358460820241 + }, + { + "accuracy": 0.53125, + "f1": 0.503413670569662, + "f1_weighted": 0.5034113968598974 + }, + { + "accuracy": 0.55419921875, + "f1": 0.542396796181948, + "f1_weighted": 0.5423867352987461 + }, + { + "accuracy": 0.5146484375, + "f1": 0.4901281619593602, + "f1_weighted": 0.48994831602259353 + }, + { + "accuracy": 0.5390625, + "f1": 0.5238926396838856, + "f1_weighted": 0.523861572750396 + } + ] + } + ] + }, + "task_name": "RuSciBenchOECDClassification" +} \ No newline at end of file diff --git a/results/Linq-AI-Research__Linq-Embed-Mistral/0c1a0b0589177079acc552433cad51d7c9132379/RuSciBenchOECDClusteringP2P.json b/results/Linq-AI-Research__Linq-Embed-Mistral/0c1a0b0589177079acc552433cad51d7c9132379/RuSciBenchOECDClusteringP2P.json new file mode 100644 index 000000000..27c3d73c3 --- /dev/null +++ b/results/Linq-AI-Research__Linq-Embed-Mistral/0c1a0b0589177079acc552433cad51d7c9132379/RuSciBenchOECDClusteringP2P.json @@ -0,0 +1,34 @@ +{ + "dataset_revision": "26c88e99dcaba32bb45d0e1bfc21902337f6d471", + "evaluation_time": 152.50048232078552, + "kg_co2_emissions": null, + "mteb_version": "1.19.2", + "scores": { + "test": [ + { + "hf_subset": "default", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.5237218824510818, + "v_measure": 0.5237218824510818, + "v_measure_std": 0.007177553133058739, + "v_measures": { + "Level 0": [ + 0.5188599209773046, + 0.5178910623566413, + 0.5344025520064645, + 0.5325652589799993, + 0.510607536299601, + 0.5172531819466448, + 0.5249596966092206, + 0.5237918934551288, + 0.5268171866449851, + 0.5300705352348278 + ] + } + } + ] + }, + "task_name": "RuSciBenchOECDClusteringP2P" +} \ No newline at end of file diff --git a/results/Linq-AI-Research__Linq-Embed-Mistral/0c1a0b0589177079acc552433cad51d7c9132379/STS22.json b/results/Linq-AI-Research__Linq-Embed-Mistral/0c1a0b0589177079acc552433cad51d7c9132379/STS22.json new file mode 100644 index 000000000..f2f0208de --- /dev/null +++ b/results/Linq-AI-Research__Linq-Embed-Mistral/0c1a0b0589177079acc552433cad51d7c9132379/STS22.json @@ -0,0 +1,26 @@ +{ + "dataset_revision": "de9d86b3b84231dc21f76c7b7af1f28e2f57f6e3", + "evaluation_time": 23.70423126220703, + "kg_co2_emissions": null, + "mteb_version": "1.19.2", + "scores": { + "test": [ + { + "cosine_pearson": 0.5799588318581563, + "cosine_spearman": 0.6391902886384432, + "euclidean_pearson": 0.5507912840414484, + "euclidean_spearman": 0.6391902886384432, + "hf_subset": "ru", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.6391902886384432, + "manhattan_pearson": 0.5537723793471323, + "manhattan_spearman": 0.6415798526116022, + "pearson": 0.5799588318581563, + "spearman": 0.6391902886384432 + } + ] + }, + "task_name": "STS22" +} \ No newline at end of file diff --git a/results/Linq-AI-Research__Linq-Embed-Mistral/0c1a0b0589177079acc552433cad51d7c9132379/SensitiveTopicsClassification.json b/results/Linq-AI-Research__Linq-Embed-Mistral/0c1a0b0589177079acc552433cad51d7c9132379/SensitiveTopicsClassification.json new file mode 100644 index 000000000..c5f2e41aa --- /dev/null +++ b/results/Linq-AI-Research__Linq-Embed-Mistral/0c1a0b0589177079acc552433cad51d7c9132379/SensitiveTopicsClassification.json @@ -0,0 +1,73 @@ +{ + "dataset_revision": "416b34a802308eac30e4192afc0ff99bb8dcc7f2", + "evaluation_time": 74.08395957946777, + "kg_co2_emissions": null, + "mteb_version": "1.19.2", + "scores": { + "test": [ + { + "accuracy": 0.33876953125, + "f1": 0.4320477109468386, + "hf_subset": "default", + "languages": [ + "rus-Cyrl" + ], + "lrap": 0.5515333387586783, + "main_score": 0.33876953125, + "scores_per_experiment": [ + { + "accuracy": 0.3203125, + "f1": 0.3797034278379044, + "lrap": 0.5184461805555506 + }, + { + "accuracy": 0.32763671875, + "f1": 0.4567120053125303, + "lrap": 0.5623508029513884 + }, + { + "accuracy": 0.328125, + "f1": 0.4064684057422513, + "lrap": 0.5253906249999954 + }, + { + "accuracy": 0.3427734375, + "f1": 0.4023477721230191, + "lrap": 0.5292697482638842 + }, + { + "accuracy": 0.33154296875, + "f1": 0.4418000621051738, + "lrap": 0.5447455512152749 + }, + { + "accuracy": 0.35498046875, + "f1": 0.453143723880101, + "lrap": 0.5694851345486107 + }, + { + "accuracy": 0.361328125, + "f1": 0.47656632928185083, + "lrap": 0.5803765190972222 + }, + { + "accuracy": 0.33203125, + "f1": 0.42460658479537217, + "lrap": 0.5488213433159701 + }, + { + "accuracy": 0.34814453125, + "f1": 0.4286615666188703, + "lrap": 0.5553385416666645 + }, + { + "accuracy": 0.3408203125, + "f1": 0.4504672317713125, + "lrap": 0.5811089409722222 + } + ] + } + ] + }, + "task_name": "SensitiveTopicsClassification" +} \ No newline at end of file diff --git a/results/Linq-AI-Research__Linq-Embed-Mistral/0c1a0b0589177079acc552433cad51d7c9132379/TERRa.json b/results/Linq-AI-Research__Linq-Embed-Mistral/0c1a0b0589177079acc552433cad51d7c9132379/TERRa.json new file mode 100644 index 000000000..4357c7f62 --- /dev/null +++ b/results/Linq-AI-Research__Linq-Embed-Mistral/0c1a0b0589177079acc552433cad51d7c9132379/TERRa.json @@ -0,0 +1,58 @@ +{ + "dataset_revision": "7b58f24536063837d644aab9a023c62199b2a612", + "evaluation_time": 10.34049105644226, + "kg_co2_emissions": null, + "mteb_version": "1.19.2", + "scores": { + "dev": [ + { + "cosine_accuracy": 0.5830618892508144, + "cosine_accuracy_threshold": 0.739809513092041, + "cosine_ap": 0.6000116897438131, + "cosine_f1": 0.6728110599078341, + "cosine_f1_threshold": 0.6015908718109131, + "cosine_precision": 0.5195729537366548, + "cosine_recall": 0.954248366013072, + "dot_accuracy": 0.5830618892508144, + "dot_accuracy_threshold": 0.7398096323013306, + "dot_ap": 0.6000116897438131, + "dot_f1": 0.6728110599078341, + "dot_f1_threshold": 0.6015908718109131, + "dot_precision": 0.5195729537366548, + "dot_recall": 0.954248366013072, + "euclidean_accuracy": 0.5830618892508144, + "euclidean_accuracy_threshold": 0.7213739156723022, + "euclidean_ap": 0.6000116897438131, + "euclidean_f1": 0.6728110599078341, + "euclidean_f1_threshold": 0.8926453590393066, + "euclidean_precision": 0.5195729537366548, + "euclidean_recall": 0.954248366013072, + "hf_subset": "default", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.6000116897438131, + "manhattan_accuracy": 0.5895765472312704, + "manhattan_accuracy_threshold": 39.278533935546875, + "manhattan_ap": 0.5965698123425456, + "manhattan_f1": 0.6757369614512472, + "manhattan_f1_threshold": 44.55304718017578, + "manhattan_precision": 0.5173611111111112, + "manhattan_recall": 0.9738562091503268, + "max_accuracy": 0.5895765472312704, + "max_ap": 0.6000116897438131, + "max_f1": 0.6757369614512472, + "max_precision": 0.5195729537366548, + "max_recall": 0.9738562091503268, + "similarity_accuracy": 0.5830618892508144, + "similarity_accuracy_threshold": 0.739809513092041, + "similarity_ap": 0.6000116897438131, + "similarity_f1": 0.6728110599078341, + "similarity_f1_threshold": 0.6015908718109131, + "similarity_precision": 0.5195729537366548, + "similarity_recall": 0.954248366013072 + } + ] + }, + "task_name": "TERRa" +} \ No newline at end of file diff --git a/results/Linq-AI-Research__Linq-Embed-Mistral/0c1a0b0589177079acc552433cad51d7c9132379/model_meta.json b/results/Linq-AI-Research__Linq-Embed-Mistral/0c1a0b0589177079acc552433cad51d7c9132379/model_meta.json new file mode 100644 index 000000000..5c3f58349 --- /dev/null +++ b/results/Linq-AI-Research__Linq-Embed-Mistral/0c1a0b0589177079acc552433cad51d7c9132379/model_meta.json @@ -0,0 +1 @@ +{"name": "Linq-AI-Research/Linq-Embed-Mistral", "revision": "0c1a0b0589177079acc552433cad51d7c9132379", "release_date": "2024-05-29", "languages": ["eng_Latn"], "n_parameters": 7110000000, "memory_usage": null, "max_tokens": 32768, "embed_dim": 4096, "license": "cc-by-nc-4.0", "open_weights": true, "public_training_data": null, "public_training_code": null, "framework": ["Sentence Transformers", "PyTorch"], "reference": "https://huggingface.co/Linq-AI-Research/Linq-Embed-Mistral", "similarity_fn_name": "cosine", "use_instructions": true, "zero_shot_benchmarks": null, "loader": "instruct_wrapper"} \ No newline at end of file diff --git a/results/McGill-NLP__LLM2Vec-Llama-2-7b-chat-hf-mntp-supervised/2c055a5d77126c0d3dc6cd8ffa30e2908f4f45f8/model_meta.json b/results/McGill-NLP__LLM2Vec-Llama-2-7b-chat-hf-mntp-supervised/2c055a5d77126c0d3dc6cd8ffa30e2908f4f45f8/model_meta.json new file mode 100644 index 000000000..3d9cee675 --- /dev/null +++ b/results/McGill-NLP__LLM2Vec-Llama-2-7b-chat-hf-mntp-supervised/2c055a5d77126c0d3dc6cd8ffa30e2908f4f45f8/model_meta.json @@ -0,0 +1 @@ +{"name": "McGill-NLP/LLM2Vec-Llama-2-7b-chat-hf-mntp-supervised", "revision": "2c055a5d77126c0d3dc6cd8ffa30e2908f4f45f8", "release_date": "2024-04-09", "languages": ["eng_Latn"], "n_parameters": 7111000000, "memory_usage": null, "max_tokens": 32768, "embed_dim": 4096, "license": "mit", "open_weights": true, "public_training_data": null, "public_training_code": null, "framework": ["LLM2Vec", "PyTorch"], "reference": "https://huggingface.co/McGill-NLP/LLM2Vec-Llama-2-7b-chat-hf-mntp-supervised", "similarity_fn_name": "cosine", "use_instructions": true, "zero_shot_benchmarks": null, "loader": "loader_inner"} \ No newline at end of file diff --git a/results/McGill-NLP__LLM2Vec-Meta-Llama-3-8B-Instruct-mntp-supervised/baa8ebf04a1c2500e61288e7dad65e8ae42601a7/model_meta.json b/results/McGill-NLP__LLM2Vec-Meta-Llama-3-8B-Instruct-mntp-supervised/baa8ebf04a1c2500e61288e7dad65e8ae42601a7/model_meta.json index e00d08a6b..e3d0903d3 100644 --- a/results/McGill-NLP__LLM2Vec-Meta-Llama-3-8B-Instruct-mntp-supervised/baa8ebf04a1c2500e61288e7dad65e8ae42601a7/model_meta.json +++ b/results/McGill-NLP__LLM2Vec-Meta-Llama-3-8B-Instruct-mntp-supervised/baa8ebf04a1c2500e61288e7dad65e8ae42601a7/model_meta.json @@ -1 +1,25 @@ -{"name": "McGill-NLP/LLM2Vec-Meta-Llama-3-8B-Instruct-mntp-supervised", "revision": "baa8ebf04a1c2500e61288e7dad65e8ae42601a7", "release_date": "2024-04-09", "languages": ["eng_Latn"], "n_parameters": 7505000000, "memory_usage": null, "max_tokens": 8192, "embed_dim": 4096, "license": "mit", "open_weights": true, "public_training_data": null, "public_training_code": null, "framework": ["LLM2Vec", "PyTorch"], "reference": "https://huggingface.co/McGill-NLP/LLM2Vec-Meta-Llama-3-8B-Instruct-mntp-supervised", "similarity_fn_name": "cosine", "use_instructions": true, "zero_shot_benchmarks": null, "loader": "loader_inner"} \ No newline at end of file +{ + "name": "McGill-NLP/LLM2Vec-Meta-Llama-3-8B-Instruct-mntp-supervised", + "revision": "baa8ebf04a1c2500e61288e7dad65e8ae42601a7", + "release_date": "2024-04-09", + "languages": [ + "eng_Latn" + ], + "n_parameters": 7505000000, + "memory_usage": null, + "max_tokens": 8192, + "embed_dim": 4096, + "license": "mit", + "open_weights": true, + "public_training_data": null, + "public_training_code": null, + "framework": [ + "LLM2Vec", + "PyTorch" + ], + "reference": "https://huggingface.co/McGill-NLP/LLM2Vec-Meta-Llama-3-8B-Instruct-mntp-supervised", + "similarity_fn_name": "cosine", + "use_instructions": true, + "zero_shot_benchmarks": null, + "loader": "loader_inner" +} \ No newline at end of file diff --git a/results/McGill-NLP__LLM2Vec-Mistral-7B-Instruct-v2-mntp-supervised/0ae69bdd5816105778b971c3138e8f8a18eaa3ae/model_meta.json b/results/McGill-NLP__LLM2Vec-Mistral-7B-Instruct-v2-mntp-supervised/0ae69bdd5816105778b971c3138e8f8a18eaa3ae/model_meta.json new file mode 100644 index 000000000..e6d275e19 --- /dev/null +++ b/results/McGill-NLP__LLM2Vec-Mistral-7B-Instruct-v2-mntp-supervised/0ae69bdd5816105778b971c3138e8f8a18eaa3ae/model_meta.json @@ -0,0 +1 @@ +{"name": "McGill-NLP/LLM2Vec-Mistral-7B-Instruct-v2-mntp-supervised", "revision": "0ae69bdd5816105778b971c3138e8f8a18eaa3ae", "release_date": "2024-04-09", "languages": ["eng_Latn"], "n_parameters": 7111000000, "memory_usage": null, "max_tokens": 32768, "embed_dim": 4096, "license": "mit", "open_weights": true, "public_training_data": null, "public_training_code": null, "framework": ["LLM2Vec", "PyTorch"], "reference": "https://huggingface.co/McGill-NLP/LLM2Vec-Mistral-7B-Instruct-v2-mntp-supervised", "similarity_fn_name": "cosine", "use_instructions": true, "zero_shot_benchmarks": null, "loader": "loader_inner"} \ No newline at end of file diff --git a/results/Salesforce__SFR-Embedding-2_R/91762139d94ed4371a9fa31db5551272e0b83818/GeoreviewClassification.json b/results/Salesforce__SFR-Embedding-2_R/91762139d94ed4371a9fa31db5551272e0b83818/GeoreviewClassification.json new file mode 100644 index 000000000..c50b087cd --- /dev/null +++ b/results/Salesforce__SFR-Embedding-2_R/91762139d94ed4371a9fa31db5551272e0b83818/GeoreviewClassification.json @@ -0,0 +1,73 @@ +{ + "dataset_revision": "3765c0d1de6b7d264bc459433c45e5a75513839c", + "evaluation_time": 79.21273803710938, + "kg_co2_emissions": null, + "mteb_version": "1.19.0", + "scores": { + "test": [ + { + "accuracy": 0.533544921875, + "f1": 0.482455104022538, + "f1_weighted": 0.48244616980650645, + "hf_subset": "default", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.533544921875, + "scores_per_experiment": [ + { + "accuracy": 0.5107421875, + "f1": 0.4442618032221092, + "f1_weighted": 0.44422136316818306 + }, + { + "accuracy": 0.5322265625, + "f1": 0.4854010057545167, + "f1_weighted": 0.48535031812746554 + }, + { + "accuracy": 0.55126953125, + "f1": 0.5071800898884422, + "f1_weighted": 0.5071828753363121 + }, + { + "accuracy": 0.53515625, + "f1": 0.45573295910851186, + "f1_weighted": 0.45566820050993173 + }, + { + "accuracy": 0.56689453125, + "f1": 0.5313465045910781, + "f1_weighted": 0.5313601876733801 + }, + { + "accuracy": 0.52392578125, + "f1": 0.4819794571508723, + "f1_weighted": 0.4820353424925155 + }, + { + "accuracy": 0.5419921875, + "f1": 0.5090516430592439, + "f1_weighted": 0.5090725896547796 + }, + { + "accuracy": 0.5107421875, + "f1": 0.4509418876950907, + "f1_weighted": 0.45091751424174836 + }, + { + "accuracy": 0.53271484375, + "f1": 0.48725104749886566, + "f1_weighted": 0.48725968909307976 + }, + { + "accuracy": 0.52978515625, + "f1": 0.47140464225664935, + "f1_weighted": 0.47139361776766886 + } + ] + } + ] + }, + "task_name": "GeoreviewClassification" +} \ No newline at end of file diff --git a/results/Salesforce__SFR-Embedding-2_R/91762139d94ed4371a9fa31db5551272e0b83818/GeoreviewClusteringP2P.json b/results/Salesforce__SFR-Embedding-2_R/91762139d94ed4371a9fa31db5551272e0b83818/GeoreviewClusteringP2P.json new file mode 100644 index 000000000..8c60818ed --- /dev/null +++ b/results/Salesforce__SFR-Embedding-2_R/91762139d94ed4371a9fa31db5551272e0b83818/GeoreviewClusteringP2P.json @@ -0,0 +1,34 @@ +{ + "dataset_revision": "97a313c8fc85b47f13f33e7e9a95c1ad888c7fec", + "evaluation_time": 118.22258162498474, + "kg_co2_emissions": null, + "mteb_version": "1.19.0", + "scores": { + "test": [ + { + "hf_subset": "default", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.751073042812265, + "v_measure": 0.751073042812265, + "v_measure_std": 0.004099166121044085, + "v_measures": { + "Level 0": [ + 0.7482804774977784, + 0.7476197077402589, + 0.7531949690137879, + 0.7562633432015669, + 0.7475548188046933, + 0.7551506409354132, + 0.7539137869472315, + 0.7540612570467135, + 0.7519866169102637, + 0.7427048100249412 + ] + } + } + ] + }, + "task_name": "GeoreviewClusteringP2P" +} \ No newline at end of file diff --git a/results/Salesforce__SFR-Embedding-2_R/91762139d94ed4371a9fa31db5551272e0b83818/HeadlineClassification.json b/results/Salesforce__SFR-Embedding-2_R/91762139d94ed4371a9fa31db5551272e0b83818/HeadlineClassification.json new file mode 100644 index 000000000..f899fb730 --- /dev/null +++ b/results/Salesforce__SFR-Embedding-2_R/91762139d94ed4371a9fa31db5551272e0b83818/HeadlineClassification.json @@ -0,0 +1,73 @@ +{ + "dataset_revision": "2fe05ee6b5832cda29f2ef7aaad7b7fe6a3609eb", + "evaluation_time": 49.94564986228943, + "kg_co2_emissions": null, + "mteb_version": "1.19.0", + "scores": { + "test": [ + { + "accuracy": 0.829150390625, + "f1": 0.8301474429623485, + "f1_weighted": 0.8301176121917424, + "hf_subset": "default", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.829150390625, + "scores_per_experiment": [ + { + "accuracy": 0.8203125, + "f1": 0.8219774558601484, + "f1_weighted": 0.8219294434415277 + }, + { + "accuracy": 0.845703125, + "f1": 0.8458538620328437, + "f1_weighted": 0.8458425897527416 + }, + { + "accuracy": 0.83203125, + "f1": 0.8328098121300466, + "f1_weighted": 0.8327854181168117 + }, + { + "accuracy": 0.8193359375, + "f1": 0.8215900475896291, + "f1_weighted": 0.8215559149269707 + }, + { + "accuracy": 0.826171875, + "f1": 0.8281227428274619, + "f1_weighted": 0.8280891983775999 + }, + { + "accuracy": 0.80859375, + "f1": 0.8095440312444571, + "f1_weighted": 0.8094952713901122 + }, + { + "accuracy": 0.8076171875, + "f1": 0.8083011987022081, + "f1_weighted": 0.8082528806657918 + }, + { + "accuracy": 0.8466796875, + "f1": 0.847333024499196, + "f1_weighted": 0.8473117933361806 + }, + { + "accuracy": 0.84326171875, + "f1": 0.8444555387051612, + "f1_weighted": 0.8444515107887093 + }, + { + "accuracy": 0.841796875, + "f1": 0.841486716032334, + "f1_weighted": 0.8414621011209772 + } + ] + } + ] + }, + "task_name": "HeadlineClassification" +} \ No newline at end of file diff --git a/results/Salesforce__SFR-Embedding-2_R/91762139d94ed4371a9fa31db5551272e0b83818/InappropriatenessClassification.json b/results/Salesforce__SFR-Embedding-2_R/91762139d94ed4371a9fa31db5551272e0b83818/InappropriatenessClassification.json new file mode 100644 index 000000000..a70322c38 --- /dev/null +++ b/results/Salesforce__SFR-Embedding-2_R/91762139d94ed4371a9fa31db5551272e0b83818/InappropriatenessClassification.json @@ -0,0 +1,95 @@ +{ + "dataset_revision": "601651fdc45ef243751676e62dd7a19f491c0285", + "evaluation_time": 43.15772271156311, + "kg_co2_emissions": null, + "mteb_version": "1.19.0", + "scores": { + "test": [ + { + "accuracy": 0.68759765625, + "ap": 0.6527360851284097, + "ap_weighted": 0.6527360851284097, + "f1": 0.6710900406230234, + "f1_weighted": 0.6710900406230234, + "hf_subset": "default", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.68759765625, + "scores_per_experiment": [ + { + "accuracy": 0.69384765625, + "ap": 0.6586874874598716, + "ap_weighted": 0.6586874874598716, + "f1": 0.6816424829732368, + "f1_weighted": 0.6816424829732368 + }, + { + "accuracy": 0.7197265625, + "ap": 0.6794949383802817, + "ap_weighted": 0.6794949383802817, + "f1": 0.712979538580387, + "f1_weighted": 0.712979538580387 + }, + { + "accuracy": 0.666015625, + "ap": 0.6440958130411255, + "ap_weighted": 0.6440958130411255, + "f1": 0.6388174687891586, + "f1_weighted": 0.6388174687891586 + }, + { + "accuracy": 0.716796875, + "ap": 0.6682602611940298, + "ap_weighted": 0.6682602611940298, + "f1": 0.7134907127613181, + "f1_weighted": 0.7134907127613181 + }, + { + "accuracy": 0.65673828125, + "ap": 0.6315541176664905, + "ap_weighted": 0.6315541176664905, + "f1": 0.6299527360479584, + "f1_weighted": 0.6299527360479584 + }, + { + "accuracy": 0.64404296875, + "ap": 0.6255387082808563, + "ap_weighted": 0.6255387082808563, + "f1": 0.6072288174051444, + "f1_weighted": 0.6072288174051444 + }, + { + "accuracy": 0.66552734375, + "ap": 0.6399060033732179, + "ap_weighted": 0.6399060033732179, + "f1": 0.64122688905853, + "f1_weighted": 0.64122688905853 + }, + { + "accuracy": 0.7392578125, + "ap": 0.6812026079963235, + "ap_weighted": 0.6812026079963235, + "f1": 0.7389351462837064, + "f1_weighted": 0.7389351462837064 + }, + { + "accuracy": 0.66015625, + "ap": 0.6324000871513944, + "ap_weighted": 0.6324000871513944, + "f1": 0.6365442575130934, + "f1_weighted": 0.6365442575130934 + }, + { + "accuracy": 0.7138671875, + "ap": 0.6662208267405063, + "ap_weighted": 0.6662208267405063, + "f1": 0.7100823568177008, + "f1_weighted": 0.7100823568177008 + } + ] + } + ] + }, + "task_name": "InappropriatenessClassification" +} \ No newline at end of file diff --git a/results/Salesforce__SFR-Embedding-2_R/91762139d94ed4371a9fa31db5551272e0b83818/KinopoiskClassification.json b/results/Salesforce__SFR-Embedding-2_R/91762139d94ed4371a9fa31db5551272e0b83818/KinopoiskClassification.json new file mode 100644 index 000000000..5aec744ab --- /dev/null +++ b/results/Salesforce__SFR-Embedding-2_R/91762139d94ed4371a9fa31db5551272e0b83818/KinopoiskClassification.json @@ -0,0 +1,73 @@ +{ + "dataset_revision": "5911f26666ac11af46cb9c6849d0dc80a378af24", + "evaluation_time": 83.00787210464478, + "kg_co2_emissions": null, + "mteb_version": "1.19.0", + "scores": { + "test": [ + { + "accuracy": 0.6661333333333334, + "f1": 0.5785594344242517, + "f1_weighted": 0.5785594344242517, + "hf_subset": "default", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.6661333333333334, + "scores_per_experiment": [ + { + "accuracy": 0.6633333333333333, + "f1": 0.5735622069196933, + "f1_weighted": 0.5735622069196932 + }, + { + "accuracy": 0.656, + "f1": 0.5494984633882773, + "f1_weighted": 0.5494984633882775 + }, + { + "accuracy": 0.6646666666666666, + "f1": 0.5750185971259335, + "f1_weighted": 0.5750185971259336 + }, + { + "accuracy": 0.664, + "f1": 0.5684387945257511, + "f1_weighted": 0.5684387945257511 + }, + { + "accuracy": 0.6633333333333333, + "f1": 0.579991742614034, + "f1_weighted": 0.5799917426140339 + }, + { + "accuracy": 0.6653333333333333, + "f1": 0.5784729738682056, + "f1_weighted": 0.5784729738682056 + }, + { + "accuracy": 0.6693333333333333, + "f1": 0.5868568262495439, + "f1_weighted": 0.586856826249544 + }, + { + "accuracy": 0.672, + "f1": 0.5920985890280074, + "f1_weighted": 0.5920985890280075 + }, + { + "accuracy": 0.6673333333333333, + "f1": 0.5819192259790215, + "f1_weighted": 0.5819192259790215 + }, + { + "accuracy": 0.676, + "f1": 0.5997369245440489, + "f1_weighted": 0.5997369245440489 + } + ] + } + ] + }, + "task_name": "KinopoiskClassification" +} \ No newline at end of file diff --git a/results/Salesforce__SFR-Embedding-2_R/91762139d94ed4371a9fa31db5551272e0b83818/MIRACLReranking.json b/results/Salesforce__SFR-Embedding-2_R/91762139d94ed4371a9fa31db5551272e0b83818/MIRACLReranking.json new file mode 100644 index 000000000..ff6343b2f --- /dev/null +++ b/results/Salesforce__SFR-Embedding-2_R/91762139d94ed4371a9fa31db5551272e0b83818/MIRACLReranking.json @@ -0,0 +1,130 @@ +{ + "dataset_revision": "6d1962c527217f8927fca80f890f14f36b2802af", + "evaluation_time": 3438.133997440338, + "kg_co2_emissions": null, + "mteb_version": "1.19.0", + "scores": { + "dev": [ + { + "MAP@1(MIRACL)": 0.35325, + "MAP@10(MIRACL)": 0.55289, + "MAP@100(MIRACL)": 0.57035, + "MAP@1000(MIRACL)": 0.57035, + "MAP@20(MIRACL)": 0.56531, + "MAP@3(MIRACL)": 0.48576, + "MAP@5(MIRACL)": 0.52345, + "NDCG@1(MIRACL)": 0.583, + "NDCG@10(MIRACL)": 0.62287, + "NDCG@100(MIRACL)": 0.65625, + "NDCG@1000(MIRACL)": 0.65625, + "NDCG@20(MIRACL)": 0.64339, + "NDCG@3(MIRACL)": 0.57547, + "NDCG@5(MIRACL)": 0.59178, + "P@1(MIRACL)": 0.583, + "P@10(MIRACL)": 0.15718, + "P@100(MIRACL)": 0.01953, + "P@1000(MIRACL)": 0.00195, + "P@20(MIRACL)": 0.08881, + "P@3(MIRACL)": 0.35124, + "P@5(MIRACL)": 0.2563, + "Recall@1(MIRACL)": 0.35325, + "Recall@10(MIRACL)": 0.70335, + "Recall@100(MIRACL)": 0.79952, + "Recall@1000(MIRACL)": 0.79952, + "Recall@20(MIRACL)": 0.75708, + "Recall@3(MIRACL)": 0.54503, + "Recall@5(MIRACL)": 0.62102, + "hf_subset": "ru", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.62287, + "nAUC_MAP@1000_diff1(MIRACL)": 0.19033683765049947, + "nAUC_MAP@1000_max(MIRACL)": 0.4025918932784632, + "nAUC_MAP@1000_std(MIRACL)": 0.20670774095395894, + "nAUC_MAP@100_diff1(MIRACL)": 0.19033683765049947, + "nAUC_MAP@100_max(MIRACL)": 0.4025918932784632, + "nAUC_MAP@100_std(MIRACL)": 0.20670774095395894, + "nAUC_MAP@10_diff1(MIRACL)": 0.19754738919104753, + "nAUC_MAP@10_max(MIRACL)": 0.37447967729883574, + "nAUC_MAP@10_std(MIRACL)": 0.190613603392509, + "nAUC_MAP@1_diff1(MIRACL)": 0.36536589258851754, + "nAUC_MAP@1_max(MIRACL)": 0.1832772037595031, + "nAUC_MAP@1_std(MIRACL)": 0.0552561122066995, + "nAUC_MAP@20_diff1(MIRACL)": 0.19144148781632353, + "nAUC_MAP@20_max(MIRACL)": 0.3932100851070387, + "nAUC_MAP@20_std(MIRACL)": 0.2038249981541636, + "nAUC_MAP@3_diff1(MIRACL)": 0.2545532405332736, + "nAUC_MAP@3_max(MIRACL)": 0.30248812683438586, + "nAUC_MAP@3_std(MIRACL)": 0.13361135477960967, + "nAUC_MAP@5_diff1(MIRACL)": 0.22412082060696048, + "nAUC_MAP@5_max(MIRACL)": 0.34348907322503214, + "nAUC_MAP@5_std(MIRACL)": 0.16139203676683989, + "nAUC_NDCG@1000_diff1(MIRACL)": 0.12410142437519599, + "nAUC_NDCG@1000_max(MIRACL)": 0.5046836916941171, + "nAUC_NDCG@1000_std(MIRACL)": 0.26928714233448886, + "nAUC_NDCG@100_diff1(MIRACL)": 0.12410142437519599, + "nAUC_NDCG@100_max(MIRACL)": 0.5046836916941171, + "nAUC_NDCG@100_std(MIRACL)": 0.26928714233448886, + "nAUC_NDCG@10_diff1(MIRACL)": 0.14242030865633748, + "nAUC_NDCG@10_max(MIRACL)": 0.447106391365235, + "nAUC_NDCG@10_std(MIRACL)": 0.24026257228617268, + "nAUC_NDCG@1_diff1(MIRACL)": 0.29168589398972683, + "nAUC_NDCG@1_max(MIRACL)": 0.4706858026176125, + "nAUC_NDCG@1_std(MIRACL)": 0.22078691481552767, + "nAUC_NDCG@20_diff1(MIRACL)": 0.12954781036067392, + "nAUC_NDCG@20_max(MIRACL)": 0.4812462687787574, + "nAUC_NDCG@20_std(MIRACL)": 0.26314524267322487, + "nAUC_NDCG@3_diff1(MIRACL)": 0.1941909978149461, + "nAUC_NDCG@3_max(MIRACL)": 0.42110012352269816, + "nAUC_NDCG@3_std(MIRACL)": 0.20230675310034152, + "nAUC_NDCG@5_diff1(MIRACL)": 0.17128759299805707, + "nAUC_NDCG@5_max(MIRACL)": 0.4218800900178248, + "nAUC_NDCG@5_std(MIRACL)": 0.21077750996767308, + "nAUC_P@1000_diff1(MIRACL)": -0.19953822572131555, + "nAUC_P@1000_max(MIRACL)": 0.3472899142415525, + "nAUC_P@1000_std(MIRACL)": 0.2114155717457209, + "nAUC_P@100_diff1(MIRACL)": -0.19953822572131508, + "nAUC_P@100_max(MIRACL)": 0.3472899142415532, + "nAUC_P@100_std(MIRACL)": 0.21141557174572123, + "nAUC_P@10_diff1(MIRACL)": -0.1996760792995945, + "nAUC_P@10_max(MIRACL)": 0.36729810893194187, + "nAUC_P@10_std(MIRACL)": 0.24771788452751695, + "nAUC_P@1_diff1(MIRACL)": 0.29168589398972683, + "nAUC_P@1_max(MIRACL)": 0.4706858026176125, + "nAUC_P@1_std(MIRACL)": 0.22078691481552767, + "nAUC_P@20_diff1(MIRACL)": -0.20626242230024544, + "nAUC_P@20_max(MIRACL)": 0.35940163474132575, + "nAUC_P@20_std(MIRACL)": 0.2446488742910146, + "nAUC_P@3_diff1(MIRACL)": -0.08287573836099785, + "nAUC_P@3_max(MIRACL)": 0.43997895282972077, + "nAUC_P@3_std(MIRACL)": 0.23940380624793628, + "nAUC_P@5_diff1(MIRACL)": -0.15570474790109645, + "nAUC_P@5_max(MIRACL)": 0.4101375327451394, + "nAUC_P@5_std(MIRACL)": 0.24536731581802354, + "nAUC_Recall@1000_diff1(MIRACL)": -0.11710896955467137, + "nAUC_Recall@1000_max(MIRACL)": 0.7060247024291173, + "nAUC_Recall@1000_std(MIRACL)": 0.4106223599324258, + "nAUC_Recall@100_diff1(MIRACL)": -0.11710896955467137, + "nAUC_Recall@100_max(MIRACL)": 0.7060247024291173, + "nAUC_Recall@100_std(MIRACL)": 0.4106223599324258, + "nAUC_Recall@10_diff1(MIRACL)": -0.006424416230081386, + "nAUC_Recall@10_max(MIRACL)": 0.45255205763813616, + "nAUC_Recall@10_std(MIRACL)": 0.27418840102613357, + "nAUC_Recall@1_diff1(MIRACL)": 0.36536589258851754, + "nAUC_Recall@1_max(MIRACL)": 0.1832772037595031, + "nAUC_Recall@1_std(MIRACL)": 0.0552561122066995, + "nAUC_Recall@20_diff1(MIRACL)": -0.06584412932921656, + "nAUC_Recall@20_max(MIRACL)": 0.5739398238769253, + "nAUC_Recall@20_std(MIRACL)": 0.36035316912935816, + "nAUC_Recall@3_diff1(MIRACL)": 0.16715333328448234, + "nAUC_Recall@3_max(MIRACL)": 0.3076506302055386, + "nAUC_Recall@3_std(MIRACL)": 0.14691555140422602, + "nAUC_Recall@5_diff1(MIRACL)": 0.09160277356421442, + "nAUC_Recall@5_max(MIRACL)": 0.37265172447561135, + "nAUC_Recall@5_std(MIRACL)": 0.19404735396909473 + } + ] + }, + "task_name": "MIRACLReranking" +} \ No newline at end of file diff --git a/results/Salesforce__SFR-Embedding-2_R/91762139d94ed4371a9fa31db5551272e0b83818/MassiveScenarioClassification.json b/results/Salesforce__SFR-Embedding-2_R/91762139d94ed4371a9fa31db5551272e0b83818/MassiveScenarioClassification.json new file mode 100644 index 000000000..3df0a09e7 --- /dev/null +++ b/results/Salesforce__SFR-Embedding-2_R/91762139d94ed4371a9fa31db5551272e0b83818/MassiveScenarioClassification.json @@ -0,0 +1,137 @@ +{ + "dataset_revision": "fad2c6e8459f9e1c45d9315f4953d921437d70f8", + "evaluation_time": 153.84353232383728, + "kg_co2_emissions": null, + "mteb_version": "1.19.0", + "scores": { + "test": [ + { + "accuracy": 0.8866173503698723, + "f1": 0.876931518341441, + "f1_weighted": 0.8836447835208376, + "hf_subset": "ru", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.8866173503698723, + "scores_per_experiment": [ + { + "accuracy": 0.8893745796906524, + "f1": 0.8806938477376733, + "f1_weighted": 0.8867524728166489 + }, + { + "accuracy": 0.8890383322125084, + "f1": 0.8795699554734846, + "f1_weighted": 0.8870009618129333 + }, + { + "accuracy": 0.8897108271687962, + "f1": 0.8771245188947406, + "f1_weighted": 0.8847515182498809 + }, + { + "accuracy": 0.8883658372562205, + "f1": 0.8775038622826093, + "f1_weighted": 0.8836300373379549 + }, + { + "accuracy": 0.8883658372562205, + "f1": 0.8766751475965635, + "f1_weighted": 0.8842288608916484 + }, + { + "accuracy": 0.8843308675184937, + "f1": 0.8750495259305986, + "f1_weighted": 0.8825856529574927 + }, + { + "accuracy": 0.8651647612642905, + "f1": 0.8583911895377763, + "f1_weighted": 0.8633455973308025 + }, + { + "accuracy": 0.8876933422999328, + "f1": 0.8783105618314875, + "f1_weighted": 0.8854289437665281 + }, + { + "accuracy": 0.8873570948217888, + "f1": 0.8779402589647946, + "f1_weighted": 0.8843006643270798 + }, + { + "accuracy": 0.8967720242098184, + "f1": 0.8880563151646819, + "f1_weighted": 0.8944231257174076 + } + ] + } + ], + "validation": [ + { + "accuracy": 0.887063453025086, + "f1": 0.875692375325753, + "f1_weighted": 0.8839357197993657, + "hf_subset": "ru", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.887063453025086, + "scores_per_experiment": [ + { + "accuracy": 0.8903098868666994, + "f1": 0.8812939993632483, + "f1_weighted": 0.888240395077318 + }, + { + "accuracy": 0.8898180029513035, + "f1": 0.8789138841066256, + "f1_weighted": 0.8880508154603761 + }, + { + "accuracy": 0.8868666994589277, + "f1": 0.8722701612342092, + "f1_weighted": 0.8812424561538283 + }, + { + "accuracy": 0.8908017707820954, + "f1": 0.8786594783059305, + "f1_weighted": 0.8860854192528625 + }, + { + "accuracy": 0.8888342351205115, + "f1": 0.8757361660188668, + "f1_weighted": 0.8850271288305297 + }, + { + "accuracy": 0.8868666994589277, + "f1": 0.8766830940180063, + "f1_weighted": 0.8849564506901489 + }, + { + "accuracy": 0.8647319232661091, + "f1": 0.8558049265589309, + "f1_weighted": 0.8622079726331933 + }, + { + "accuracy": 0.8893261190359075, + "f1": 0.8778794492077554, + "f1_weighted": 0.8863612540774493 + }, + { + "accuracy": 0.8893261190359075, + "f1": 0.8776864752873723, + "f1_weighted": 0.885773367830472 + }, + { + "accuracy": 0.8937530742744713, + "f1": 0.8819961191565838, + "f1_weighted": 0.8914119379874779 + } + ] + } + ] + }, + "task_name": "MassiveScenarioClassification" +} \ No newline at end of file diff --git a/results/Salesforce__SFR-Embedding-2_R/91762139d94ed4371a9fa31db5551272e0b83818/RUParaPhraserSTS.json b/results/Salesforce__SFR-Embedding-2_R/91762139d94ed4371a9fa31db5551272e0b83818/RUParaPhraserSTS.json new file mode 100644 index 000000000..e05de9ccf --- /dev/null +++ b/results/Salesforce__SFR-Embedding-2_R/91762139d94ed4371a9fa31db5551272e0b83818/RUParaPhraserSTS.json @@ -0,0 +1,26 @@ +{ + "dataset_revision": "43265056790b8f7c59e0139acb4be0a8dad2c8f4", + "evaluation_time": 63.38218975067139, + "kg_co2_emissions": null, + "mteb_version": "1.19.0", + "scores": { + "test": [ + { + "cosine_pearson": 0.6690354199670319, + "cosine_spearman": 0.7229462776172987, + "euclidean_pearson": 0.6931935518056518, + "euclidean_spearman": 0.7229462776172987, + "hf_subset": "default", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.7229462776172987, + "manhattan_pearson": 0.6911717924234932, + "manhattan_spearman": 0.7208219838805556, + "pearson": 0.6690354199670319, + "spearman": 0.7229462776172987 + } + ] + }, + "task_name": "RUParaPhraserSTS" +} \ No newline at end of file diff --git a/results/Salesforce__SFR-Embedding-2_R/91762139d94ed4371a9fa31db5551272e0b83818/RiaNewsRetrieval.json b/results/Salesforce__SFR-Embedding-2_R/91762139d94ed4371a9fa31db5551272e0b83818/RiaNewsRetrieval.json new file mode 100644 index 000000000..255101b30 --- /dev/null +++ b/results/Salesforce__SFR-Embedding-2_R/91762139d94ed4371a9fa31db5551272e0b83818/RiaNewsRetrieval.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "82374b0bbacda6114f39ff9c5b925fa1512ca5d7", + "evaluation_time": 30463.34087920189, + "kg_co2_emissions": null, + "mteb_version": "1.19.0", + "scores": { + "test": [ + { + "hf_subset": "default", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.81857, + "map_at_1": 0.7193, + "map_at_10": 0.78913, + "map_at_100": 0.79141, + "map_at_1000": 0.7915, + "map_at_20": 0.79058, + "map_at_3": 0.77712, + "map_at_5": 0.78518, + "mrr_at_1": 0.7195, + "mrr_at_10": 0.7892325, + "mrr_at_100": 0.7915053314612488, + "mrr_at_1000": 0.7915994475508613, + "mrr_at_20": 0.7906776665263701, + "mrr_at_3": 0.7772166666666667, + "mrr_at_5": 0.7852766666666666, + "nauc_map_at_1000_diff1": 0.7802080134075241, + "nauc_map_at_1000_max": 0.34648986145260174, + "nauc_map_at_1000_std": -0.07728668363833921, + "nauc_map_at_100_diff1": 0.7801964469546097, + "nauc_map_at_100_max": 0.3465713034209455, + "nauc_map_at_100_std": -0.07715856916925486, + "nauc_map_at_10_diff1": 0.7794016480523882, + "nauc_map_at_10_max": 0.34588000680031716, + "nauc_map_at_10_std": -0.07805970214189473, + "nauc_map_at_1_diff1": 0.8053517715327518, + "nauc_map_at_1_max": 0.308098786294874, + "nauc_map_at_1_std": -0.10448613393917475, + "nauc_map_at_20_diff1": 0.7798845820812842, + "nauc_map_at_20_max": 0.3464180097862182, + "nauc_map_at_20_std": -0.07710484429517651, + "nauc_map_at_3_diff1": 0.7796343011640257, + "nauc_map_at_3_max": 0.34538905955636756, + "nauc_map_at_3_std": -0.0883054790916272, + "nauc_map_at_5_diff1": 0.7788549877086532, + "nauc_map_at_5_max": 0.34660286516181643, + "nauc_map_at_5_std": -0.08031425674342171, + "nauc_mrr_at_1000_diff1": 0.779975669205385, + "nauc_mrr_at_1000_max": 0.34685477634271455, + "nauc_mrr_at_1000_std": -0.07724122519974827, + "nauc_mrr_at_100_diff1": 0.7799642146431082, + "nauc_mrr_at_100_max": 0.34693606689013023, + "nauc_mrr_at_100_std": -0.07711308426704075, + "nauc_mrr_at_10_diff1": 0.7791718885026365, + "nauc_mrr_at_10_max": 0.34624014997757757, + "nauc_mrr_at_10_std": -0.07801510275117948, + "nauc_mrr_at_1_diff1": 0.8050103489299117, + "nauc_mrr_at_1_max": 0.3086356652795064, + "nauc_mrr_at_1_std": -0.10443246620762446, + "nauc_mrr_at_20_diff1": 0.7796532485513028, + "nauc_mrr_at_20_max": 0.3467811155295355, + "nauc_mrr_at_20_std": -0.07705953824212754, + "nauc_mrr_at_3_diff1": 0.7794167214383998, + "nauc_mrr_at_3_max": 0.34573006570786896, + "nauc_mrr_at_3_std": -0.0882669125895829, + "nauc_mrr_at_5_diff1": 0.7786292517773432, + "nauc_mrr_at_5_max": 0.34695665146086274, + "nauc_mrr_at_5_std": -0.08027131809119366, + "nauc_ndcg_at_1000_diff1": 0.7732945386428853, + "nauc_ndcg_at_1000_max": 0.3620434674870255, + "nauc_ndcg_at_1000_std": -0.057906423190322055, + "nauc_ndcg_at_100_diff1": 0.7726776264325685, + "nauc_ndcg_at_100_max": 0.3650276292492198, + "nauc_ndcg_at_100_std": -0.05261408813510495, + "nauc_ndcg_at_10_diff1": 0.7688024862376803, + "nauc_ndcg_at_10_max": 0.3602630987472381, + "nauc_ndcg_at_10_std": -0.05795408482216004, + "nauc_ndcg_at_1_diff1": 0.8053517715327518, + "nauc_ndcg_at_1_max": 0.308098786294874, + "nauc_ndcg_at_1_std": -0.10448613393917475, + "nauc_ndcg_at_20_diff1": 0.7705155531096696, + "nauc_ndcg_at_20_max": 0.3631933730717487, + "nauc_ndcg_at_20_std": -0.05293254424022341, + "nauc_ndcg_at_3_diff1": 0.7699669906590991, + "nauc_ndcg_at_3_max": 0.3583983944138455, + "nauc_ndcg_at_3_std": -0.08118895934445464, + "nauc_ndcg_at_5_diff1": 0.7676225512206584, + "nauc_ndcg_at_5_max": 0.36140908679730643, + "nauc_ndcg_at_5_std": -0.06476672173119424, + "nauc_precision_at_1000_diff1": 0.6534449787044917, + "nauc_precision_at_1000_max": 0.741892478916698, + "nauc_precision_at_1000_std": 0.38418290667425414, + "nauc_precision_at_100_diff1": 0.6963411081058111, + "nauc_precision_at_100_max": 0.6317478670419902, + "nauc_precision_at_100_std": 0.316379786968022, + "nauc_precision_at_10_diff1": 0.7061831684741201, + "nauc_precision_at_10_max": 0.44500712835445005, + "nauc_precision_at_10_std": 0.07304539086033111, + "nauc_precision_at_1_diff1": 0.8053517715327518, + "nauc_precision_at_1_max": 0.308098786294874, + "nauc_precision_at_1_std": -0.10448613393917475, + "nauc_precision_at_20_diff1": 0.7034695066145277, + "nauc_precision_at_20_max": 0.49657684856120604, + "nauc_precision_at_20_std": 0.1572424349277631, + "nauc_precision_at_3_diff1": 0.7324277694998957, + "nauc_precision_at_3_max": 0.40847135192020184, + "nauc_precision_at_3_std": -0.053099631416378384, + "nauc_precision_at_5_diff1": 0.7140321635823325, + "nauc_precision_at_5_max": 0.43097022488119435, + "nauc_precision_at_5_std": 0.013109127745837537, + "nauc_recall_at_1000_diff1": 0.6534449787044968, + "nauc_recall_at_1000_max": 0.7418924789166966, + "nauc_recall_at_1000_std": 0.3841829066742637, + "nauc_recall_at_100_diff1": 0.6963411081058161, + "nauc_recall_at_100_max": 0.631747867041982, + "nauc_recall_at_100_std": 0.31637978696802105, + "nauc_recall_at_10_diff1": 0.7061831684741228, + "nauc_recall_at_10_max": 0.44500712835445205, + "nauc_recall_at_10_std": 0.07304539086033145, + "nauc_recall_at_1_diff1": 0.8053517715327518, + "nauc_recall_at_1_max": 0.308098786294874, + "nauc_recall_at_1_std": -0.10448613393917475, + "nauc_recall_at_20_diff1": 0.7034695066145273, + "nauc_recall_at_20_max": 0.4965768485612081, + "nauc_recall_at_20_std": 0.15724243492776627, + "nauc_recall_at_3_diff1": 0.7324277694998951, + "nauc_recall_at_3_max": 0.4084713519202018, + "nauc_recall_at_3_std": -0.05309963141637799, + "nauc_recall_at_5_diff1": 0.71403216358233, + "nauc_recall_at_5_max": 0.4309702248811922, + "nauc_recall_at_5_std": 0.013109127745838339, + "ndcg_at_1": 0.7193, + "ndcg_at_10": 0.81857, + "ndcg_at_100": 0.82945, + "ndcg_at_1000": 0.83218, + "ndcg_at_20": 0.82377, + "ndcg_at_3": 0.7947, + "ndcg_at_5": 0.80911, + "precision_at_1": 0.7193, + "precision_at_10": 0.09089, + "precision_at_100": 0.00959, + "precision_at_1000": 0.00098, + "precision_at_20": 0.04647, + "precision_at_3": 0.28173, + "precision_at_5": 0.17598, + "recall_at_1": 0.7193, + "recall_at_10": 0.9089, + "recall_at_100": 0.9593, + "recall_at_1000": 0.9813, + "recall_at_20": 0.9293, + "recall_at_3": 0.8452, + "recall_at_5": 0.8799 + } + ] + }, + "task_name": "RiaNewsRetrieval" +} \ No newline at end of file diff --git a/results/Salesforce__SFR-Embedding-2_R/91762139d94ed4371a9fa31db5551272e0b83818/RuBQRetrieval.json b/results/Salesforce__SFR-Embedding-2_R/91762139d94ed4371a9fa31db5551272e0b83818/RuBQRetrieval.json new file mode 100644 index 000000000..8dc7d096a --- /dev/null +++ b/results/Salesforce__SFR-Embedding-2_R/91762139d94ed4371a9fa31db5551272e0b83818/RuBQRetrieval.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "e19b6ffa60b3bc248e0b41f4cc37c26a55c2a67b", + "evaluation_time": 1504.726597070694, + "kg_co2_emissions": null, + "mteb_version": "1.19.0", + "scores": { + "test": [ + { + "hf_subset": "default", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.72538, + "map_at_1": 0.43154, + "map_at_10": 0.64642, + "map_at_100": 0.65507, + "map_at_1000": 0.65527, + "map_at_20": 0.65252, + "map_at_3": 0.59233, + "map_at_5": 0.62573, + "mrr_at_1": 0.6122931442080378, + "mrr_at_10": 0.7294551390296071, + "mrr_at_100": 0.731862924523419, + "mrr_at_1000": 0.7319194870195322, + "mrr_at_20": 0.7312166919110494, + "mrr_at_3": 0.7093183609141056, + "mrr_at_5": 0.723295902285264, + "nauc_map_at_1000_diff1": 0.3729893668405061, + "nauc_map_at_1000_max": 0.25418981402947627, + "nauc_map_at_1000_std": -0.08727256206567983, + "nauc_map_at_100_diff1": 0.3727736969430449, + "nauc_map_at_100_max": 0.2542426016933015, + "nauc_map_at_100_std": -0.08696875145532915, + "nauc_map_at_10_diff1": 0.3701775217787662, + "nauc_map_at_10_max": 0.2521519640793399, + "nauc_map_at_10_std": -0.08985132328903539, + "nauc_map_at_1_diff1": 0.41557762703564133, + "nauc_map_at_1_max": 0.17353335980468668, + "nauc_map_at_1_std": -0.09380326109245093, + "nauc_map_at_20_diff1": 0.3712940790134307, + "nauc_map_at_20_max": 0.25429744743759114, + "nauc_map_at_20_std": -0.08729937221168688, + "nauc_map_at_3_diff1": 0.3715842950171908, + "nauc_map_at_3_max": 0.22167045184623838, + "nauc_map_at_3_std": -0.10126206405552991, + "nauc_map_at_5_diff1": 0.36874417999629716, + "nauc_map_at_5_max": 0.23054351156830694, + "nauc_map_at_5_std": -0.10388988238804509, + "nauc_mrr_at_1000_diff1": 0.4540154550119733, + "nauc_mrr_at_1000_max": 0.2943905116122545, + "nauc_mrr_at_1000_std": -0.10168750260478021, + "nauc_mrr_at_100_diff1": 0.4539754039472757, + "nauc_mrr_at_100_max": 0.29446649075532194, + "nauc_mrr_at_100_std": -0.10154209880639944, + "nauc_mrr_at_10_diff1": 0.4519569942515509, + "nauc_mrr_at_10_max": 0.29457323081140524, + "nauc_mrr_at_10_std": -0.1012296253273361, + "nauc_mrr_at_1_diff1": 0.48474111582728396, + "nauc_mrr_at_1_max": 0.260913666047689, + "nauc_mrr_at_1_std": -0.11795876056462178, + "nauc_mrr_at_20_diff1": 0.45310495125669303, + "nauc_mrr_at_20_max": 0.29479902986250595, + "nauc_mrr_at_20_std": -0.10145581399720663, + "nauc_mrr_at_3_diff1": 0.44523758307091565, + "nauc_mrr_at_3_max": 0.2932592410677799, + "nauc_mrr_at_3_std": -0.1070611549896983, + "nauc_mrr_at_5_diff1": 0.45138111971698286, + "nauc_mrr_at_5_max": 0.29497224405064637, + "nauc_mrr_at_5_std": -0.10526795116753886, + "nauc_ndcg_at_1000_diff1": 0.3892333117768948, + "nauc_ndcg_at_1000_max": 0.28362969406930116, + "nauc_ndcg_at_1000_std": -0.07288459708615291, + "nauc_ndcg_at_100_diff1": 0.3851526661968429, + "nauc_ndcg_at_100_max": 0.2870522423232289, + "nauc_ndcg_at_100_std": -0.06367019092737335, + "nauc_ndcg_at_10_diff1": 0.3739003379920412, + "nauc_ndcg_at_10_max": 0.2847051827223434, + "nauc_ndcg_at_10_std": -0.07181866438224627, + "nauc_ndcg_at_1_diff1": 0.48474111582728396, + "nauc_ndcg_at_1_max": 0.260913666047689, + "nauc_ndcg_at_1_std": -0.11795876056462178, + "nauc_ndcg_at_20_diff1": 0.37797380908008454, + "nauc_ndcg_at_20_max": 0.29169242579888377, + "nauc_ndcg_at_20_std": -0.06437918495102345, + "nauc_ndcg_at_3_diff1": 0.3674207391766007, + "nauc_ndcg_at_3_max": 0.24795224001605004, + "nauc_ndcg_at_3_std": -0.09841614719733105, + "nauc_ndcg_at_5_diff1": 0.37037128611985937, + "nauc_ndcg_at_5_max": 0.25131220171851304, + "nauc_ndcg_at_5_std": -0.10013049076878043, + "nauc_precision_at_1000_diff1": -0.10707120055201903, + "nauc_precision_at_1000_max": 0.08273250374450333, + "nauc_precision_at_1000_std": 0.05136200166901931, + "nauc_precision_at_100_diff1": -0.10057611772138593, + "nauc_precision_at_100_max": 0.10949947222308364, + "nauc_precision_at_100_std": 0.07808266012156236, + "nauc_precision_at_10_diff1": -0.049683841795602195, + "nauc_precision_at_10_max": 0.17461443463893594, + "nauc_precision_at_10_std": 0.05055154395611409, + "nauc_precision_at_1_diff1": 0.48474111582728396, + "nauc_precision_at_1_max": 0.260913666047689, + "nauc_precision_at_1_std": -0.11795876056462178, + "nauc_precision_at_20_diff1": -0.07780130436816086, + "nauc_precision_at_20_max": 0.1564432945077386, + "nauc_precision_at_20_std": 0.074164072686806, + "nauc_precision_at_3_diff1": 0.08978496762378062, + "nauc_precision_at_3_max": 0.18634459815706172, + "nauc_precision_at_3_std": -0.04359206468412119, + "nauc_precision_at_5_diff1": 0.009059642647878688, + "nauc_precision_at_5_max": 0.15797026063186959, + "nauc_precision_at_5_std": -0.022032813447347678, + "nauc_recall_at_1000_diff1": 0.35815607618411677, + "nauc_recall_at_1000_max": 0.5081006447029666, + "nauc_recall_at_1000_std": 0.5723758900114141, + "nauc_recall_at_100_diff1": 0.19182791358098286, + "nauc_recall_at_100_max": 0.4569565421731788, + "nauc_recall_at_100_std": 0.470862560147177, + "nauc_recall_at_10_diff1": 0.22868516411707165, + "nauc_recall_at_10_max": 0.3229869685941732, + "nauc_recall_at_10_std": 0.047235054990806845, + "nauc_recall_at_1_diff1": 0.41557762703564133, + "nauc_recall_at_1_max": 0.17353335980468668, + "nauc_recall_at_1_std": -0.09380326109245093, + "nauc_recall_at_20_diff1": 0.209040277384093, + "nauc_recall_at_20_max": 0.40465880390895986, + "nauc_recall_at_20_std": 0.15286039196509446, + "nauc_recall_at_3_diff1": 0.2754606513803307, + "nauc_recall_at_3_max": 0.2143072436387688, + "nauc_recall_at_3_std": -0.07583628106687351, + "nauc_recall_at_5_diff1": 0.2551275755143425, + "nauc_recall_at_5_max": 0.2177333453840513, + "nauc_recall_at_5_std": -0.07231736731114545, + "ndcg_at_1": 0.61229, + "ndcg_at_10": 0.72538, + "ndcg_at_100": 0.75117, + "ndcg_at_1000": 0.75493, + "ndcg_at_20": 0.73953, + "ndcg_at_3": 0.65436, + "ndcg_at_5": 0.6924, + "precision_at_1": 0.61229, + "precision_at_10": 0.14226, + "precision_at_100": 0.01616, + "precision_at_1000": 0.00166, + "precision_at_20": 0.07574, + "precision_at_3": 0.35973, + "precision_at_5": 0.25059, + "recall_at_1": 0.43154, + "recall_at_10": 0.86817, + "recall_at_100": 0.96579, + "recall_at_1000": 0.99136, + "recall_at_20": 0.9139, + "recall_at_3": 0.69286, + "recall_at_5": 0.78449 + } + ] + }, + "task_name": "RuBQRetrieval" +} \ No newline at end of file diff --git a/results/Salesforce__SFR-Embedding-2_R/91762139d94ed4371a9fa31db5551272e0b83818/RuReviewsClassification.json b/results/Salesforce__SFR-Embedding-2_R/91762139d94ed4371a9fa31db5551272e0b83818/RuReviewsClassification.json new file mode 100644 index 000000000..7a94f72ac --- /dev/null +++ b/results/Salesforce__SFR-Embedding-2_R/91762139d94ed4371a9fa31db5551272e0b83818/RuReviewsClassification.json @@ -0,0 +1,73 @@ +{ + "dataset_revision": "f6d2c31f4dc6b88f468552750bfec05b4b41b05a", + "evaluation_time": 45.315327167510986, + "kg_co2_emissions": null, + "mteb_version": "1.19.0", + "scores": { + "test": [ + { + "accuracy": 0.7078125, + "f1": 0.6842300015008821, + "f1_weighted": 0.684211946877564, + "hf_subset": "default", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.7078125, + "scores_per_experiment": [ + { + "accuracy": 0.7041015625, + "f1": 0.6800482718004425, + "f1_weighted": 0.6800277749054044 + }, + { + "accuracy": 0.728515625, + "f1": 0.7268009355394365, + "f1_weighted": 0.7268214438087428 + }, + { + "accuracy": 0.6884765625, + "f1": 0.6432228351819543, + "f1_weighted": 0.6431849842230231 + }, + { + "accuracy": 0.73779296875, + "f1": 0.7259682908608175, + "f1_weighted": 0.7259572742194869 + }, + { + "accuracy": 0.7041015625, + "f1": 0.6798591951772112, + "f1_weighted": 0.6798384462777975 + }, + { + "accuracy": 0.70556640625, + "f1": 0.6857723462384913, + "f1_weighted": 0.6857582952342391 + }, + { + "accuracy": 0.70751953125, + "f1": 0.6875706117527599, + "f1_weighted": 0.6875538304228002 + }, + { + "accuracy": 0.71142578125, + "f1": 0.6849664097505497, + "f1_weighted": 0.6849431925007095 + }, + { + "accuracy": 0.6767578125, + "f1": 0.6333303441413106, + "f1_weighted": 0.6332887660612323 + }, + { + "accuracy": 0.7138671875, + "f1": 0.6947607745658472, + "f1_weighted": 0.6947454611222037 + } + ] + } + ] + }, + "task_name": "RuReviewsClassification" +} \ No newline at end of file diff --git a/results/Salesforce__SFR-Embedding-2_R/91762139d94ed4371a9fa31db5551272e0b83818/RuSTSBenchmarkSTS.json b/results/Salesforce__SFR-Embedding-2_R/91762139d94ed4371a9fa31db5551272e0b83818/RuSTSBenchmarkSTS.json new file mode 100644 index 000000000..2fad43f94 --- /dev/null +++ b/results/Salesforce__SFR-Embedding-2_R/91762139d94ed4371a9fa31db5551272e0b83818/RuSTSBenchmarkSTS.json @@ -0,0 +1,26 @@ +{ + "dataset_revision": "7cf24f325c6da6195df55bef3d86b5e0616f3018", + "evaluation_time": 41.896517515182495, + "kg_co2_emissions": null, + "mteb_version": "1.19.0", + "scores": { + "test": [ + { + "cosine_pearson": 0.7997540949040447, + "cosine_spearman": 0.7909469888326499, + "euclidean_pearson": 0.7780968455932049, + "euclidean_spearman": 0.7909469888326499, + "hf_subset": "default", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.7909469888326499, + "manhattan_pearson": 0.7791473295911902, + "manhattan_spearman": 0.7926550313373727, + "pearson": 0.7997540949040447, + "spearman": 0.7909469888326499 + } + ] + }, + "task_name": "RuSTSBenchmarkSTS" +} \ No newline at end of file diff --git a/results/Salesforce__SFR-Embedding-2_R/91762139d94ed4371a9fa31db5551272e0b83818/RuSciBenchGRNTIClassification.json b/results/Salesforce__SFR-Embedding-2_R/91762139d94ed4371a9fa31db5551272e0b83818/RuSciBenchGRNTIClassification.json new file mode 100644 index 000000000..b294a7bb4 --- /dev/null +++ b/results/Salesforce__SFR-Embedding-2_R/91762139d94ed4371a9fa31db5551272e0b83818/RuSciBenchGRNTIClassification.json @@ -0,0 +1,73 @@ +{ + "dataset_revision": "673a610d6d3dd91a547a0d57ae1b56f37ebbf6a1", + "evaluation_time": 180.28583979606628, + "kg_co2_emissions": null, + "mteb_version": "1.19.0", + "scores": { + "test": [ + { + "accuracy": 0.672607421875, + "f1": 0.6548581749172834, + "f1_weighted": 0.6549445185338045, + "hf_subset": "default", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.672607421875, + "scores_per_experiment": [ + { + "accuracy": 0.68603515625, + "f1": 0.6753213094989184, + "f1_weighted": 0.6753967689200607 + }, + { + "accuracy": 0.66748046875, + "f1": 0.6512496124273753, + "f1_weighted": 0.6512580098953011 + }, + { + "accuracy": 0.66650390625, + "f1": 0.6431846812276119, + "f1_weighted": 0.643329868574312 + }, + { + "accuracy": 0.68212890625, + "f1": 0.6649372687261403, + "f1_weighted": 0.6650084611483312 + }, + { + "accuracy": 0.67529296875, + "f1": 0.6585741060315871, + "f1_weighted": 0.6586237071731367 + }, + { + "accuracy": 0.66015625, + "f1": 0.6426204935157264, + "f1_weighted": 0.6427308456429128 + }, + { + "accuracy": 0.66845703125, + "f1": 0.6488069093408658, + "f1_weighted": 0.6489148054785124 + }, + { + "accuracy": 0.6640625, + "f1": 0.6454136962081495, + "f1_weighted": 0.6454945228323586 + }, + { + "accuracy": 0.69482421875, + "f1": 0.6798153295785568, + "f1_weighted": 0.6799141963481644 + }, + { + "accuracy": 0.6611328125, + "f1": 0.6386583426179027, + "f1_weighted": 0.6387739993249557 + } + ] + } + ] + }, + "task_name": "RuSciBenchGRNTIClassification" +} \ No newline at end of file diff --git a/results/Salesforce__SFR-Embedding-2_R/91762139d94ed4371a9fa31db5551272e0b83818/RuSciBenchGRNTIClusteringP2P.json b/results/Salesforce__SFR-Embedding-2_R/91762139d94ed4371a9fa31db5551272e0b83818/RuSciBenchGRNTIClusteringP2P.json new file mode 100644 index 000000000..21bb9fa7f --- /dev/null +++ b/results/Salesforce__SFR-Embedding-2_R/91762139d94ed4371a9fa31db5551272e0b83818/RuSciBenchGRNTIClusteringP2P.json @@ -0,0 +1,34 @@ +{ + "dataset_revision": "673a610d6d3dd91a547a0d57ae1b56f37ebbf6a1", + "evaluation_time": 152.27210760116577, + "kg_co2_emissions": null, + "mteb_version": "1.19.0", + "scores": { + "test": [ + { + "hf_subset": "default", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.6452678466288365, + "v_measure": 0.6452678466288365, + "v_measure_std": 0.006006782312477266, + "v_measures": { + "Level 0": [ + 0.6426174810877145, + 0.6463699896740169, + 0.636398141920836, + 0.6546617204386257, + 0.6425896427910388, + 0.6382174562209246, + 0.6490209467468757, + 0.6402922881159449, + 0.6482359338029867, + 0.6542748654894008 + ] + } + } + ] + }, + "task_name": "RuSciBenchGRNTIClusteringP2P" +} \ No newline at end of file diff --git a/results/Salesforce__SFR-Embedding-2_R/91762139d94ed4371a9fa31db5551272e0b83818/RuSciBenchOECDClassification.json b/results/Salesforce__SFR-Embedding-2_R/91762139d94ed4371a9fa31db5551272e0b83818/RuSciBenchOECDClassification.json new file mode 100644 index 000000000..04b1c2545 --- /dev/null +++ b/results/Salesforce__SFR-Embedding-2_R/91762139d94ed4371a9fa31db5551272e0b83818/RuSciBenchOECDClassification.json @@ -0,0 +1,73 @@ +{ + "dataset_revision": "26c88e99dcaba32bb45d0e1bfc21902337f6d471", + "evaluation_time": 183.4744803905487, + "kg_co2_emissions": null, + "mteb_version": "1.19.0", + "scores": { + "test": [ + { + "accuracy": 0.53955078125, + "f1": 0.5093428046209144, + "f1_weighted": 0.5092471924330896, + "hf_subset": "default", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.53955078125, + "scores_per_experiment": [ + { + "accuracy": 0.5419921875, + "f1": 0.5132067625786738, + "f1_weighted": 0.5132041276081083 + }, + { + "accuracy": 0.546875, + "f1": 0.5238977644084195, + "f1_weighted": 0.5237147840960017 + }, + { + "accuracy": 0.55859375, + "f1": 0.5362610872968139, + "f1_weighted": 0.5361795056374034 + }, + { + "accuracy": 0.52783203125, + "f1": 0.47765958391822266, + "f1_weighted": 0.47759043324435757 + }, + { + "accuracy": 0.53076171875, + "f1": 0.49856605780076974, + "f1_weighted": 0.49858501105901465 + }, + { + "accuracy": 0.54541015625, + "f1": 0.5189262961698865, + "f1_weighted": 0.5188121788236764 + }, + { + "accuracy": 0.53271484375, + "f1": 0.4921206143103161, + "f1_weighted": 0.4920644260088097 + }, + { + "accuracy": 0.56591796875, + "f1": 0.5466397691460307, + "f1_weighted": 0.5465398801074838 + }, + { + "accuracy": 0.5126953125, + "f1": 0.47959453490626086, + "f1_weighted": 0.479405163123415 + }, + { + "accuracy": 0.53271484375, + "f1": 0.5065555756737501, + "f1_weighted": 0.5063764146226256 + } + ] + } + ] + }, + "task_name": "RuSciBenchOECDClassification" +} \ No newline at end of file diff --git a/results/Salesforce__SFR-Embedding-2_R/91762139d94ed4371a9fa31db5551272e0b83818/RuSciBenchOECDClusteringP2P.json b/results/Salesforce__SFR-Embedding-2_R/91762139d94ed4371a9fa31db5551272e0b83818/RuSciBenchOECDClusteringP2P.json new file mode 100644 index 000000000..97925e11b --- /dev/null +++ b/results/Salesforce__SFR-Embedding-2_R/91762139d94ed4371a9fa31db5551272e0b83818/RuSciBenchOECDClusteringP2P.json @@ -0,0 +1,34 @@ +{ + "dataset_revision": "26c88e99dcaba32bb45d0e1bfc21902337f6d471", + "evaluation_time": 213.92391967773438, + "kg_co2_emissions": null, + "mteb_version": "1.19.0", + "scores": { + "test": [ + { + "hf_subset": "default", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.5496534815728105, + "v_measure": 0.5496534815728105, + "v_measure_std": 0.006578462330063986, + "v_measures": { + "Level 0": [ + 0.5485009365336291, + 0.5463687935701449, + 0.5477391873308559, + 0.5625625965455736, + 0.5540859726686398, + 0.5509262926037056, + 0.544022172024568, + 0.538986538654372, + 0.5455299114149839, + 0.5578124143816323 + ] + } + } + ] + }, + "task_name": "RuSciBenchOECDClusteringP2P" +} \ No newline at end of file diff --git a/results/Salesforce__SFR-Embedding-2_R/91762139d94ed4371a9fa31db5551272e0b83818/STS22.json b/results/Salesforce__SFR-Embedding-2_R/91762139d94ed4371a9fa31db5551272e0b83818/STS22.json new file mode 100644 index 000000000..0cce272f3 --- /dev/null +++ b/results/Salesforce__SFR-Embedding-2_R/91762139d94ed4371a9fa31db5551272e0b83818/STS22.json @@ -0,0 +1,26 @@ +{ + "dataset_revision": "de9d86b3b84231dc21f76c7b7af1f28e2f57f6e3", + "evaluation_time": 23.540674209594727, + "kg_co2_emissions": null, + "mteb_version": "1.19.0", + "scores": { + "test": [ + { + "cosine_pearson": 0.5334440647452878, + "cosine_spearman": 0.5857862685405976, + "euclidean_pearson": 0.4936175659021995, + "euclidean_spearman": 0.5857862685405976, + "hf_subset": "ru", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.5857862685405976, + "manhattan_pearson": 0.503070861612905, + "manhattan_spearman": 0.5975888662864769, + "pearson": 0.5334440647452878, + "spearman": 0.5857862685405976 + } + ] + }, + "task_name": "STS22" +} \ No newline at end of file diff --git a/results/Salesforce__SFR-Embedding-2_R/91762139d94ed4371a9fa31db5551272e0b83818/SensitiveTopicsClassification.json b/results/Salesforce__SFR-Embedding-2_R/91762139d94ed4371a9fa31db5551272e0b83818/SensitiveTopicsClassification.json new file mode 100644 index 000000000..ab73c8d41 --- /dev/null +++ b/results/Salesforce__SFR-Embedding-2_R/91762139d94ed4371a9fa31db5551272e0b83818/SensitiveTopicsClassification.json @@ -0,0 +1,73 @@ +{ + "dataset_revision": "416b34a802308eac30e4192afc0ff99bb8dcc7f2", + "evaluation_time": 79.22559475898743, + "kg_co2_emissions": null, + "mteb_version": "1.19.0", + "scores": { + "test": [ + { + "accuracy": 0.344775390625, + "f1": 0.42289916898591856, + "hf_subset": "default", + "languages": [ + "rus-Cyrl" + ], + "lrap": 0.5547159830729146, + "main_score": 0.344775390625, + "scores_per_experiment": [ + { + "accuracy": 0.3349609375, + "f1": 0.3994228449597384, + "lrap": 0.5196261935763838 + }, + { + "accuracy": 0.36328125, + "f1": 0.46065008806387603, + "lrap": 0.582845052083333 + }, + { + "accuracy": 0.353515625, + "f1": 0.41626242437674466, + "lrap": 0.5438503689236083 + }, + { + "accuracy": 0.3505859375, + "f1": 0.42705508807692105, + "lrap": 0.563307020399304 + }, + { + "accuracy": 0.3388671875, + "f1": 0.41967571130141296, + "lrap": 0.5415717230902747 + }, + { + "accuracy": 0.35205078125, + "f1": 0.41061135946889127, + "lrap": 0.5502658420138872 + }, + { + "accuracy": 0.33544921875, + "f1": 0.4202541760234866, + "lrap": 0.5579427083333314 + }, + { + "accuracy": 0.328125, + "f1": 0.4261175851902919, + "lrap": 0.5501234266493032 + }, + { + "accuracy": 0.3447265625, + "f1": 0.4011444836318907, + "lrap": 0.5631239149305535 + }, + { + "accuracy": 0.34619140625, + "f1": 0.4477979287659324, + "lrap": 0.5745035807291663 + } + ] + } + ] + }, + "task_name": "SensitiveTopicsClassification" +} \ No newline at end of file diff --git a/results/Salesforce__SFR-Embedding-2_R/91762139d94ed4371a9fa31db5551272e0b83818/model_meta.json b/results/Salesforce__SFR-Embedding-2_R/91762139d94ed4371a9fa31db5551272e0b83818/model_meta.json index 91b0070ff..eb6547e3e 100644 --- a/results/Salesforce__SFR-Embedding-2_R/91762139d94ed4371a9fa31db5551272e0b83818/model_meta.json +++ b/results/Salesforce__SFR-Embedding-2_R/91762139d94ed4371a9fa31db5551272e0b83818/model_meta.json @@ -1 +1,25 @@ -{"name": "Salesforce/SFR-Embedding-2_R", "revision": "91762139d94ed4371a9fa31db5551272e0b83818", "release_date": "2024-06-14", "languages": ["eng_Latn"], "n_parameters": 7110000000, "memory_usage": null, "max_tokens": 32768, "embed_dim": 4096, "license": "cc-by-nc-4.0", "open_weights": true, "public_training_data": null, "public_training_code": null, "framework": ["Sentence Transformers", "PyTorch"], "reference": "https://huggingface.co/Salesforce/SFR-Embedding-2_R", "similarity_fn_name": "cosine", "use_instructions": true, "zero_shot_benchmarks": null, "loader": "instruct_wrapper"} \ No newline at end of file +{ + "name": "Salesforce/SFR-Embedding-2_R", + "revision": "91762139d94ed4371a9fa31db5551272e0b83818", + "release_date": "2024-06-14", + "languages": [ + "eng_Latn" + ], + "n_parameters": 7110000000, + "memory_usage": null, + "max_tokens": 32768, + "embed_dim": 4096, + "license": "cc-by-nc-4.0", + "open_weights": true, + "public_training_data": null, + "public_training_code": null, + "framework": [ + "Sentence Transformers", + "PyTorch" + ], + "reference": "https://huggingface.co/Salesforce/SFR-Embedding-2_R", + "similarity_fn_name": "cosine", + "use_instructions": true, + "zero_shot_benchmarks": null, + "loader": "instruct_wrapper" +} \ No newline at end of file diff --git a/results/WhereIsAI__UAE-Large-V1/369c368f70f16a613f19f5598d4f12d9f44235d4/CEDRClassification.json b/results/WhereIsAI__UAE-Large-V1/369c368f70f16a613f19f5598d4f12d9f44235d4/CEDRClassification.json new file mode 100644 index 000000000..ea36d051c --- /dev/null +++ b/results/WhereIsAI__UAE-Large-V1/369c368f70f16a613f19f5598d4f12d9f44235d4/CEDRClassification.json @@ -0,0 +1,73 @@ +{ + "dataset_revision": "c0ba03d058e3e1b2f3fd20518875a4563dd12db4", + "evaluation_time": 21.70494055747986, + "kg_co2_emissions": null, + "mteb_version": "1.19.2", + "scores": { + "test": [ + { + "accuracy": 0.3427736450584485, + "f1": 0.14186054435137987, + "hf_subset": "default", + "languages": [ + "rus-Cyrl" + ], + "lrap": 0.5749282678002249, + "main_score": 0.3427736450584485, + "scores_per_experiment": [ + { + "accuracy": 0.34112646121147716, + "f1": 0.12571850776690446, + "lrap": 0.5748140276301928 + }, + { + "accuracy": 0.3416578108395324, + "f1": 0.09298022529512855, + "lrap": 0.5507438894792902 + }, + { + "accuracy": 0.3804463336875664, + "f1": 0.18928982003731074, + "lrap": 0.5942614240170148 + }, + { + "accuracy": 0.3294367693942614, + "f1": 0.16889808971709327, + "lrap": 0.5778958554729138 + }, + { + "accuracy": 0.3475026567481403, + "f1": 0.1518084191057472, + "lrap": 0.571014877789598 + }, + { + "accuracy": 0.3432518597236982, + "f1": 0.1701042547431902, + "lrap": 0.6005844845908729 + }, + { + "accuracy": 0.29861849096705634, + "f1": 0.14257899805047552, + "lrap": 0.5769394261424138 + }, + { + "accuracy": 0.3660998937300744, + "f1": 0.09751215301440189, + "lrap": 0.5539851222104268 + }, + { + "accuracy": 0.3416578108395324, + "f1": 0.13642244969270736, + "lrap": 0.5717853347502783 + }, + { + "accuracy": 0.3379383634431456, + "f1": 0.14329252609083945, + "lrap": 0.5772582359192475 + } + ] + } + ] + }, + "task_name": "CEDRClassification" +} \ No newline at end of file diff --git a/results/WhereIsAI__UAE-Large-V1/369c368f70f16a613f19f5598d4f12d9f44235d4/GeoreviewClassification.json b/results/WhereIsAI__UAE-Large-V1/369c368f70f16a613f19f5598d4f12d9f44235d4/GeoreviewClassification.json new file mode 100644 index 000000000..d013b548d --- /dev/null +++ b/results/WhereIsAI__UAE-Large-V1/369c368f70f16a613f19f5598d4f12d9f44235d4/GeoreviewClassification.json @@ -0,0 +1,73 @@ +{ + "dataset_revision": "3765c0d1de6b7d264bc459433c45e5a75513839c", + "evaluation_time": 47.42616581916809, + "kg_co2_emissions": null, + "mteb_version": "1.19.2", + "scores": { + "test": [ + { + "accuracy": 0.272900390625, + "f1": 0.27243039844640304, + "f1_weighted": 0.2724432612624273, + "hf_subset": "default", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.272900390625, + "scores_per_experiment": [ + { + "accuracy": 0.2685546875, + "f1": 0.26784229289523537, + "f1_weighted": 0.267881544016771 + }, + { + "accuracy": 0.30419921875, + "f1": 0.30729570464732825, + "f1_weighted": 0.3072991913428375 + }, + { + "accuracy": 0.27783203125, + "f1": 0.2797065372293043, + "f1_weighted": 0.2797218460288182 + }, + { + "accuracy": 0.2705078125, + "f1": 0.2728448518338992, + "f1_weighted": 0.27287161752032746 + }, + { + "accuracy": 0.27783203125, + "f1": 0.2729560746365959, + "f1_weighted": 0.27295406197356376 + }, + { + "accuracy": 0.27294921875, + "f1": 0.2706960914825669, + "f1_weighted": 0.2707201600217324 + }, + { + "accuracy": 0.26318359375, + "f1": 0.26422404943654637, + "f1_weighted": 0.2642532121667203 + }, + { + "accuracy": 0.26025390625, + "f1": 0.25982675336228966, + "f1_weighted": 0.25981613387630453 + }, + { + "accuracy": 0.24658203125, + "f1": 0.24363960048370004, + "f1_weighted": 0.24365144526204 + }, + { + "accuracy": 0.287109375, + "f1": 0.2852720284565645, + "f1_weighted": 0.2852634004151577 + } + ] + } + ] + }, + "task_name": "GeoreviewClassification" +} \ No newline at end of file diff --git a/results/WhereIsAI__UAE-Large-V1/369c368f70f16a613f19f5598d4f12d9f44235d4/GeoreviewClusteringP2P.json b/results/WhereIsAI__UAE-Large-V1/369c368f70f16a613f19f5598d4f12d9f44235d4/GeoreviewClusteringP2P.json new file mode 100644 index 000000000..e767f6216 --- /dev/null +++ b/results/WhereIsAI__UAE-Large-V1/369c368f70f16a613f19f5598d4f12d9f44235d4/GeoreviewClusteringP2P.json @@ -0,0 +1,34 @@ +{ + "dataset_revision": "97a313c8fc85b47f13f33e7e9a95c1ad888c7fec", + "evaluation_time": 104.35629892349243, + "kg_co2_emissions": null, + "mteb_version": "1.19.2", + "scores": { + "test": [ + { + "hf_subset": "default", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.2169654653410979, + "v_measure": 0.2169654653410979, + "v_measure_std": 0.008979457180302637, + "v_measures": { + "Level 0": [ + 0.22311270310081796, + 0.22313089381431747, + 0.21672366079693853, + 0.22089073665702816, + 0.2173574376957309, + 0.22678765988968605, + 0.21355139066092016, + 0.20818433272575446, + 0.2246452348785096, + 0.19527060319127582 + ] + } + } + ] + }, + "task_name": "GeoreviewClusteringP2P" +} \ No newline at end of file diff --git a/results/WhereIsAI__UAE-Large-V1/369c368f70f16a613f19f5598d4f12d9f44235d4/HeadlineClassification.json b/results/WhereIsAI__UAE-Large-V1/369c368f70f16a613f19f5598d4f12d9f44235d4/HeadlineClassification.json new file mode 100644 index 000000000..0af89b26a --- /dev/null +++ b/results/WhereIsAI__UAE-Large-V1/369c368f70f16a613f19f5598d4f12d9f44235d4/HeadlineClassification.json @@ -0,0 +1,73 @@ +{ + "dataset_revision": "2fe05ee6b5832cda29f2ef7aaad7b7fe6a3609eb", + "evaluation_time": 19.920687437057495, + "kg_co2_emissions": null, + "mteb_version": "1.19.2", + "scores": { + "test": [ + { + "accuracy": 0.312548828125, + "f1": 0.31162522456678954, + "f1_weighted": 0.3116251004027454, + "hf_subset": "default", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.312548828125, + "scores_per_experiment": [ + { + "accuracy": 0.3408203125, + "f1": 0.34011810034513673, + "f1_weighted": 0.34013942696644883 + }, + { + "accuracy": 0.314453125, + "f1": 0.3129200452717575, + "f1_weighted": 0.3129229358070194 + }, + { + "accuracy": 0.291015625, + "f1": 0.2932475616287081, + "f1_weighted": 0.29322736986189263 + }, + { + "accuracy": 0.365234375, + "f1": 0.36686485999034774, + "f1_weighted": 0.366889130608955 + }, + { + "accuracy": 0.3251953125, + "f1": 0.32556364679918726, + "f1_weighted": 0.3255708710753665 + }, + { + "accuracy": 0.330078125, + "f1": 0.326576399787136, + "f1_weighted": 0.3265865602884081 + }, + { + "accuracy": 0.2763671875, + "f1": 0.2725002292794683, + "f1_weighted": 0.2724907028428623 + }, + { + "accuracy": 0.27001953125, + "f1": 0.26537884807533846, + "f1_weighted": 0.2653885343751161 + }, + { + "accuracy": 0.29248046875, + "f1": 0.2951232518752592, + "f1_weighted": 0.29509607832535295 + }, + { + "accuracy": 0.31982421875, + "f1": 0.3179593026155565, + "f1_weighted": 0.3179393938760328 + } + ] + } + ] + }, + "task_name": "HeadlineClassification" +} \ No newline at end of file diff --git a/results/WhereIsAI__UAE-Large-V1/369c368f70f16a613f19f5598d4f12d9f44235d4/InappropriatenessClassification.json b/results/WhereIsAI__UAE-Large-V1/369c368f70f16a613f19f5598d4f12d9f44235d4/InappropriatenessClassification.json new file mode 100644 index 000000000..96a0269c2 --- /dev/null +++ b/results/WhereIsAI__UAE-Large-V1/369c368f70f16a613f19f5598d4f12d9f44235d4/InappropriatenessClassification.json @@ -0,0 +1,95 @@ +{ + "dataset_revision": "601651fdc45ef243751676e62dd7a19f491c0285", + "evaluation_time": 20.27226948738098, + "kg_co2_emissions": null, + "mteb_version": "1.19.2", + "scores": { + "test": [ + { + "accuracy": 0.517578125, + "ap": 0.5095041560064141, + "ap_weighted": 0.5095041560064141, + "f1": 0.5148976391154665, + "f1_weighted": 0.5148976391154665, + "hf_subset": "default", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.517578125, + "scores_per_experiment": [ + { + "accuracy": 0.5107421875, + "ap": 0.5055121009844868, + "ap_weighted": 0.5055121009844868, + "f1": 0.5066730645516464, + "f1_weighted": 0.5066730645516464 + }, + { + "accuracy": 0.53173828125, + "ap": 0.5167606653792134, + "ap_weighted": 0.5167606653792134, + "f1": 0.5297550767786833, + "f1_weighted": 0.5297550767786833 + }, + { + "accuracy": 0.51025390625, + "ap": 0.5052219804997793, + "ap_weighted": 0.5052219804997793, + "f1": 0.508862685577236, + "f1_weighted": 0.508862685577236 + }, + { + "accuracy": 0.482421875, + "ap": 0.4916218547077922, + "ap_weighted": 0.4916218547077922, + "f1": 0.47433621290003264, + "f1_weighted": 0.47433621290003264 + }, + { + "accuracy": 0.5400390625, + "ap": 0.5215118963068182, + "ap_weighted": 0.5215118963068182, + "f1": 0.5394047741831975, + "f1_weighted": 0.5394047741831975 + }, + { + "accuracy": 0.53857421875, + "ap": 0.5208062336428215, + "ap_weighted": 0.5208062336428215, + "f1": 0.538525698145123, + "f1_weighted": 0.538525698145123 + }, + { + "accuracy": 0.54443359375, + "ap": 0.524436037836169, + "ap_weighted": 0.524436037836169, + "f1": 0.5430424473309443, + "f1_weighted": 0.5430424473309443 + }, + { + "accuracy": 0.49267578125, + "ap": 0.49639332114026236, + "ap_weighted": 0.49639332114026236, + "f1": 0.4925440264808745, + "f1_weighted": 0.4925440264808745 + }, + { + "accuracy": 0.51513671875, + "ap": 0.5077700955234308, + "ap_weighted": 0.5077700955234308, + "f1": 0.5128928668691585, + "f1_weighted": 0.5128928668691585 + }, + { + "accuracy": 0.509765625, + "ap": 0.5050073740433674, + "ap_weighted": 0.5050073740433674, + "f1": 0.5029395383377684, + "f1_weighted": 0.5029395383377684 + } + ] + } + ] + }, + "task_name": "InappropriatenessClassification" +} \ No newline at end of file diff --git a/results/WhereIsAI__UAE-Large-V1/369c368f70f16a613f19f5598d4f12d9f44235d4/KinopoiskClassification.json b/results/WhereIsAI__UAE-Large-V1/369c368f70f16a613f19f5598d4f12d9f44235d4/KinopoiskClassification.json new file mode 100644 index 000000000..5779c989d --- /dev/null +++ b/results/WhereIsAI__UAE-Large-V1/369c368f70f16a613f19f5598d4f12d9f44235d4/KinopoiskClassification.json @@ -0,0 +1,73 @@ +{ + "dataset_revision": "5911f26666ac11af46cb9c6849d0dc80a378af24", + "evaluation_time": 48.36592936515808, + "kg_co2_emissions": null, + "mteb_version": "1.19.2", + "scores": { + "test": [ + { + "accuracy": 0.35453333333333337, + "f1": 0.35161533925418653, + "f1_weighted": 0.35161533925418653, + "hf_subset": "default", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.35453333333333337, + "scores_per_experiment": [ + { + "accuracy": 0.31466666666666665, + "f1": 0.3137970760931348, + "f1_weighted": 0.31379707609313484 + }, + { + "accuracy": 0.3586666666666667, + "f1": 0.35714382696189234, + "f1_weighted": 0.35714382696189234 + }, + { + "accuracy": 0.36733333333333335, + "f1": 0.36395798108383887, + "f1_weighted": 0.3639579810838389 + }, + { + "accuracy": 0.37266666666666665, + "f1": 0.3715984556072967, + "f1_weighted": 0.3715984556072967 + }, + { + "accuracy": 0.382, + "f1": 0.381425805706358, + "f1_weighted": 0.381425805706358 + }, + { + "accuracy": 0.3473333333333333, + "f1": 0.34008463348084855, + "f1_weighted": 0.34008463348084855 + }, + { + "accuracy": 0.3506666666666667, + "f1": 0.34743742077192313, + "f1_weighted": 0.34743742077192324 + }, + { + "accuracy": 0.36733333333333335, + "f1": 0.36438095411140153, + "f1_weighted": 0.3643809541114015 + }, + { + "accuracy": 0.344, + "f1": 0.3408678780508911, + "f1_weighted": 0.3408678780508911 + }, + { + "accuracy": 0.3406666666666667, + "f1": 0.33545936067428056, + "f1_weighted": 0.33545936067428056 + } + ] + } + ] + }, + "task_name": "KinopoiskClassification" +} \ No newline at end of file diff --git a/results/WhereIsAI__UAE-Large-V1/369c368f70f16a613f19f5598d4f12d9f44235d4/MIRACLReranking.json b/results/WhereIsAI__UAE-Large-V1/369c368f70f16a613f19f5598d4f12d9f44235d4/MIRACLReranking.json new file mode 100644 index 000000000..61bb96344 --- /dev/null +++ b/results/WhereIsAI__UAE-Large-V1/369c368f70f16a613f19f5598d4f12d9f44235d4/MIRACLReranking.json @@ -0,0 +1,130 @@ +{ + "dataset_revision": "6d1962c527217f8927fca80f890f14f36b2802af", + "evaluation_time": 2190.397878885269, + "kg_co2_emissions": null, + "mteb_version": "1.19.2", + "scores": { + "dev": [ + { + "MAP@1(MIRACL)": 0.10775, + "MAP@10(MIRACL)": 0.18689, + "MAP@100(MIRACL)": 0.21904, + "MAP@1000(MIRACL)": 0.21904, + "MAP@20(MIRACL)": 0.20097, + "MAP@3(MIRACL)": 0.14911, + "MAP@5(MIRACL)": 0.16576, + "NDCG@1(MIRACL)": 0.18123, + "NDCG@10(MIRACL)": 0.25179, + "NDCG@100(MIRACL)": 0.37058, + "NDCG@1000(MIRACL)": 0.37058, + "NDCG@20(MIRACL)": 0.29144, + "NDCG@3(MIRACL)": 0.19393, + "NDCG@5(MIRACL)": 0.21143, + "P@1(MIRACL)": 0.18123, + "P@10(MIRACL)": 0.07562, + "P@100(MIRACL)": 0.01953, + "P@1000(MIRACL)": 0.00195, + "P@20(MIRACL)": 0.05401, + "P@3(MIRACL)": 0.12056, + "P@5(MIRACL)": 0.098, + "Recall@1(MIRACL)": 0.10775, + "Recall@10(MIRACL)": 0.35319, + "Recall@100(MIRACL)": 0.79952, + "Recall@1000(MIRACL)": 0.79952, + "Recall@20(MIRACL)": 0.4747, + "Recall@3(MIRACL)": 0.19176, + "Recall@5(MIRACL)": 0.24545, + "hf_subset": "ru", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.25179, + "nAUC_MAP@1000_diff1(MIRACL)": 0.23538738533131684, + "nAUC_MAP@1000_max(MIRACL)": 0.14082934104816378, + "nAUC_MAP@1000_std(MIRACL)": 0.047002079186633156, + "nAUC_MAP@100_diff1(MIRACL)": 0.23538738533131684, + "nAUC_MAP@100_max(MIRACL)": 0.14082934104816378, + "nAUC_MAP@100_std(MIRACL)": 0.047002079186633156, + "nAUC_MAP@10_diff1(MIRACL)": 0.25390058840989105, + "nAUC_MAP@10_max(MIRACL)": 0.1259853531463474, + "nAUC_MAP@10_std(MIRACL)": 0.03855391177966912, + "nAUC_MAP@1_diff1(MIRACL)": 0.346807409310064, + "nAUC_MAP@1_max(MIRACL)": 0.10837170975571755, + "nAUC_MAP@1_std(MIRACL)": -0.010592015192321466, + "nAUC_MAP@20_diff1(MIRACL)": 0.24432027657682487, + "nAUC_MAP@20_max(MIRACL)": 0.13336061481738615, + "nAUC_MAP@20_std(MIRACL)": 0.04566914055715195, + "nAUC_MAP@3_diff1(MIRACL)": 0.2820692290535767, + "nAUC_MAP@3_max(MIRACL)": 0.11425586689886244, + "nAUC_MAP@3_std(MIRACL)": 0.015101537609068386, + "nAUC_MAP@5_diff1(MIRACL)": 0.27205554900087325, + "nAUC_MAP@5_max(MIRACL)": 0.11895201370686258, + "nAUC_MAP@5_std(MIRACL)": 0.024442792915727492, + "nAUC_NDCG@1000_diff1(MIRACL)": 0.15247107108575006, + "nAUC_NDCG@1000_max(MIRACL)": 0.17655363456869763, + "nAUC_NDCG@1000_std(MIRACL)": 0.07142183233554175, + "nAUC_NDCG@100_diff1(MIRACL)": 0.15247107108575006, + "nAUC_NDCG@100_max(MIRACL)": 0.17655363456869763, + "nAUC_NDCG@100_std(MIRACL)": 0.07142183233554175, + "nAUC_NDCG@10_diff1(MIRACL)": 0.20567073853343834, + "nAUC_NDCG@10_max(MIRACL)": 0.13343542784915202, + "nAUC_NDCG@10_std(MIRACL)": 0.05415841638349995, + "nAUC_NDCG@1_diff1(MIRACL)": 0.2794189516412678, + "nAUC_NDCG@1_max(MIRACL)": 0.13338645967503324, + "nAUC_NDCG@1_std(MIRACL)": 0.04122624776790232, + "nAUC_NDCG@20_diff1(MIRACL)": 0.18733284842298017, + "nAUC_NDCG@20_max(MIRACL)": 0.14836752264584926, + "nAUC_NDCG@20_std(MIRACL)": 0.06993705335869221, + "nAUC_NDCG@3_diff1(MIRACL)": 0.24017667734614448, + "nAUC_NDCG@3_max(MIRACL)": 0.12781942068174074, + "nAUC_NDCG@3_std(MIRACL)": 0.03107655360058877, + "nAUC_NDCG@5_diff1(MIRACL)": 0.2319527631256309, + "nAUC_NDCG@5_max(MIRACL)": 0.12843525770163072, + "nAUC_NDCG@5_std(MIRACL)": 0.03400590628174582, + "nAUC_P@1000_diff1(MIRACL)": -0.09365640710949047, + "nAUC_P@1000_max(MIRACL)": 0.13569984980483787, + "nAUC_P@1000_std(MIRACL)": 0.04899419557725115, + "nAUC_P@100_diff1(MIRACL)": -0.09365640710948986, + "nAUC_P@100_max(MIRACL)": 0.1356998498048383, + "nAUC_P@100_std(MIRACL)": 0.04899419557725159, + "nAUC_P@10_diff1(MIRACL)": 0.04080854724871233, + "nAUC_P@10_max(MIRACL)": 0.1443846377713033, + "nAUC_P@10_std(MIRACL)": 0.11310450584719269, + "nAUC_P@1_diff1(MIRACL)": 0.2794189516412678, + "nAUC_P@1_max(MIRACL)": 0.13338645967503324, + "nAUC_P@1_std(MIRACL)": 0.04122624776790232, + "nAUC_P@20_diff1(MIRACL)": -0.01337747826101748, + "nAUC_P@20_max(MIRACL)": 0.16097004663794012, + "nAUC_P@20_std(MIRACL)": 0.115777875196793, + "nAUC_P@3_diff1(MIRACL)": 0.15825663319888217, + "nAUC_P@3_max(MIRACL)": 0.15463573384209442, + "nAUC_P@3_std(MIRACL)": 0.08179021037863447, + "nAUC_P@5_diff1(MIRACL)": 0.11577872314265653, + "nAUC_P@5_max(MIRACL)": 0.15190377432963748, + "nAUC_P@5_std(MIRACL)": 0.09040874720408776, + "nAUC_Recall@1000_diff1(MIRACL)": -0.12898795660364237, + "nAUC_Recall@1000_max(MIRACL)": 0.33072876448786254, + "nAUC_Recall@1000_std(MIRACL)": 0.18449554907692, + "nAUC_Recall@100_diff1(MIRACL)": -0.12898795660364237, + "nAUC_Recall@100_max(MIRACL)": 0.33072876448786254, + "nAUC_Recall@100_std(MIRACL)": 0.18449554907692, + "nAUC_Recall@10_diff1(MIRACL)": 0.1260157106173291, + "nAUC_Recall@10_max(MIRACL)": 0.11814747248207881, + "nAUC_Recall@10_std(MIRACL)": 0.07354602101279308, + "nAUC_Recall@1_diff1(MIRACL)": 0.346807409310064, + "nAUC_Recall@1_max(MIRACL)": 0.10837170975571755, + "nAUC_Recall@1_std(MIRACL)": -0.010592015192321466, + "nAUC_Recall@20_diff1(MIRACL)": 0.08299194226552467, + "nAUC_Recall@20_max(MIRACL)": 0.1451349726429474, + "nAUC_Recall@20_std(MIRACL)": 0.11547280235793832, + "nAUC_Recall@3_diff1(MIRACL)": 0.21346563301293073, + "nAUC_Recall@3_max(MIRACL)": 0.11032498284718112, + "nAUC_Recall@3_std(MIRACL)": 0.017744540224794728, + "nAUC_Recall@5_diff1(MIRACL)": 0.18526725862374696, + "nAUC_Recall@5_max(MIRACL)": 0.11551428785352265, + "nAUC_Recall@5_std(MIRACL)": 0.03332656874091986 + } + ] + }, + "task_name": "MIRACLReranking" +} \ No newline at end of file diff --git a/results/WhereIsAI__UAE-Large-V1/369c368f70f16a613f19f5598d4f12d9f44235d4/MIRACLRetrieval.json b/results/WhereIsAI__UAE-Large-V1/369c368f70f16a613f19f5598d4f12d9f44235d4/MIRACLRetrieval.json new file mode 100644 index 000000000..b3446eaf4 --- /dev/null +++ b/results/WhereIsAI__UAE-Large-V1/369c368f70f16a613f19f5598d4f12d9f44235d4/MIRACLRetrieval.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "main", + "evaluation_time": 86561.4031059742, + "kg_co2_emissions": null, + "mteb_version": "1.19.4", + "scores": { + "dev": [ + { + "hf_subset": "ru", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.1038, + "map_at_1": 0.03858, + "map_at_10": 0.071, + "map_at_100": 0.08075, + "map_at_1000": 0.08191, + "map_at_20": 0.07577, + "map_at_3": 0.05841, + "map_at_5": 0.06428, + "mrr_at_1": 0.08226837060702875, + "mrr_at_10": 0.13392508494345554, + "mrr_at_100": 0.14284592878788827, + "mrr_at_1000": 0.14353778178007612, + "mrr_at_20": 0.13914323084610378, + "mrr_at_3": 0.11701277955271565, + "mrr_at_5": 0.12679712460063897, + "nauc_map_at_1000_diff1": 0.2549397898558356, + "nauc_map_at_1000_max": 0.31572329928531917, + "nauc_map_at_1000_std": 0.20641785695827758, + "nauc_map_at_100_diff1": 0.25588903805989494, + "nauc_map_at_100_max": 0.31304230014150136, + "nauc_map_at_100_std": 0.20280245482581574, + "nauc_map_at_10_diff1": 0.2683965055844689, + "nauc_map_at_10_max": 0.28593255145637475, + "nauc_map_at_10_std": 0.16109905843218433, + "nauc_map_at_1_diff1": 0.36879739207499834, + "nauc_map_at_1_max": 0.2732785619648901, + "nauc_map_at_1_std": 0.07342254191231093, + "nauc_map_at_20_diff1": 0.26235729735617325, + "nauc_map_at_20_max": 0.292755366907038, + "nauc_map_at_20_std": 0.1775156667084958, + "nauc_map_at_3_diff1": 0.2834311597033785, + "nauc_map_at_3_max": 0.2594960830845603, + "nauc_map_at_3_std": 0.1153196698800567, + "nauc_map_at_5_diff1": 0.27543031846789595, + "nauc_map_at_5_max": 0.26599156884440905, + "nauc_map_at_5_std": 0.1277878958183748, + "nauc_mrr_at_1000_diff1": 0.2370580168541847, + "nauc_mrr_at_1000_max": 0.2730060791758593, + "nauc_mrr_at_1000_std": 0.17733857524662813, + "nauc_mrr_at_100_diff1": 0.23669978422697888, + "nauc_mrr_at_100_max": 0.27270631029675496, + "nauc_mrr_at_100_std": 0.17708819340278287, + "nauc_mrr_at_10_diff1": 0.24146281351339963, + "nauc_mrr_at_10_max": 0.2651234100647721, + "nauc_mrr_at_10_std": 0.16216782530171006, + "nauc_mrr_at_1_diff1": 0.3024594170878796, + "nauc_mrr_at_1_max": 0.24084083309730395, + "nauc_mrr_at_1_std": 0.09870127875555809, + "nauc_mrr_at_20_diff1": 0.23759091245509195, + "nauc_mrr_at_20_max": 0.2679838384317707, + "nauc_mrr_at_20_std": 0.16968297115366945, + "nauc_mrr_at_3_diff1": 0.2534538336094808, + "nauc_mrr_at_3_max": 0.25029576329103276, + "nauc_mrr_at_3_std": 0.13846870314998896, + "nauc_mrr_at_5_diff1": 0.2443881265100262, + "nauc_mrr_at_5_max": 0.24860333570330365, + "nauc_mrr_at_5_std": 0.14284263650555806, + "nauc_ndcg_at_1000_diff1": 0.20571482127635268, + "nauc_ndcg_at_1000_max": 0.39856778281624816, + "nauc_ndcg_at_1000_std": 0.3484459175681837, + "nauc_ndcg_at_100_diff1": 0.2100404901044162, + "nauc_ndcg_at_100_max": 0.37662346555698484, + "nauc_ndcg_at_100_std": 0.3193895380836668, + "nauc_ndcg_at_10_diff1": 0.23815861687280232, + "nauc_ndcg_at_10_max": 0.2969283859908895, + "nauc_ndcg_at_10_std": 0.20290992921700127, + "nauc_ndcg_at_1_diff1": 0.3024594170878796, + "nauc_ndcg_at_1_max": 0.24084083309730395, + "nauc_ndcg_at_1_std": 0.09870127875555809, + "nauc_ndcg_at_20_diff1": 0.22647473694371006, + "nauc_ndcg_at_20_max": 0.30998582245792955, + "nauc_ndcg_at_20_std": 0.2333802711661617, + "nauc_ndcg_at_3_diff1": 0.2567424178962605, + "nauc_ndcg_at_3_max": 0.25646487631982273, + "nauc_ndcg_at_3_std": 0.13624240788647418, + "nauc_ndcg_at_5_diff1": 0.2461575770561425, + "nauc_ndcg_at_5_max": 0.2607055241975762, + "nauc_ndcg_at_5_std": 0.14897109953452312, + "nauc_precision_at_1000_diff1": 0.06017507739607772, + "nauc_precision_at_1000_max": 0.39282070092050714, + "nauc_precision_at_1000_std": 0.40744063314037543, + "nauc_precision_at_100_diff1": 0.08464414039587322, + "nauc_precision_at_100_max": 0.43297670013859224, + "nauc_precision_at_100_std": 0.44292742697295456, + "nauc_precision_at_10_diff1": 0.18115325240729918, + "nauc_precision_at_10_max": 0.33458663989235976, + "nauc_precision_at_10_std": 0.28747414404946564, + "nauc_precision_at_1_diff1": 0.3024594170878796, + "nauc_precision_at_1_max": 0.24084083309730395, + "nauc_precision_at_1_std": 0.09870127875555809, + "nauc_precision_at_20_diff1": 0.1436798768761113, + "nauc_precision_at_20_max": 0.3492740668905507, + "nauc_precision_at_20_std": 0.3348070790170325, + "nauc_precision_at_3_diff1": 0.20736372619443005, + "nauc_precision_at_3_max": 0.26591890239412463, + "nauc_precision_at_3_std": 0.18040529093895402, + "nauc_precision_at_5_diff1": 0.19694730648081546, + "nauc_precision_at_5_max": 0.2753183593004797, + "nauc_precision_at_5_std": 0.20140207024469375, + "nauc_recall_at_1000_diff1": 0.12002547719107197, + "nauc_recall_at_1000_max": 0.5026954310742026, + "nauc_recall_at_1000_std": 0.5505702646819981, + "nauc_recall_at_100_diff1": 0.13352166476468807, + "nauc_recall_at_100_max": 0.4387063154731406, + "nauc_recall_at_100_std": 0.463211323706859, + "nauc_recall_at_10_diff1": 0.18839721938134393, + "nauc_recall_at_10_max": 0.30592643867506236, + "nauc_recall_at_10_std": 0.25645862608918313, + "nauc_recall_at_1_diff1": 0.36879739207499834, + "nauc_recall_at_1_max": 0.2732785619648901, + "nauc_recall_at_1_std": 0.07342254191231093, + "nauc_recall_at_20_diff1": 0.16655538745848308, + "nauc_recall_at_20_max": 0.31066280713752653, + "nauc_recall_at_20_std": 0.29290539133247157, + "nauc_recall_at_3_diff1": 0.22143187935586245, + "nauc_recall_at_3_max": 0.237647240629027, + "nauc_recall_at_3_std": 0.14304934980311962, + "nauc_recall_at_5_diff1": 0.20100708483699167, + "nauc_recall_at_5_max": 0.24476528159552152, + "nauc_recall_at_5_std": 0.16262591046386768, + "ndcg_at_1": 0.08227, + "ndcg_at_10": 0.1038, + "ndcg_at_100": 0.15109, + "ndcg_at_1000": 0.1777, + "ndcg_at_20": 0.1204, + "ndcg_at_3": 0.08408, + "ndcg_at_5": 0.08997, + "precision_at_1": 0.08227, + "precision_at_10": 0.03299, + "precision_at_100": 0.00824, + "precision_at_1000": 0.00131, + "precision_at_20": 0.02324, + "precision_at_3": 0.05751, + "precision_at_5": 0.04601, + "recall_at_1": 0.03858, + "recall_at_10": 0.13985, + "recall_at_100": 0.31816, + "recall_at_1000": 0.47873, + "recall_at_20": 0.19116, + "recall_at_3": 0.0804, + "recall_at_5": 0.10205 + } + ] + }, + "task_name": "MIRACLRetrieval" +} \ No newline at end of file diff --git a/results/WhereIsAI__UAE-Large-V1/369c368f70f16a613f19f5598d4f12d9f44235d4/MassiveScenarioClassification.json b/results/WhereIsAI__UAE-Large-V1/369c368f70f16a613f19f5598d4f12d9f44235d4/MassiveScenarioClassification.json new file mode 100644 index 000000000..48157f924 --- /dev/null +++ b/results/WhereIsAI__UAE-Large-V1/369c368f70f16a613f19f5598d4f12d9f44235d4/MassiveScenarioClassification.json @@ -0,0 +1,137 @@ +{ + "dataset_revision": "fad2c6e8459f9e1c45d9315f4953d921437d70f8", + "evaluation_time": 56.624640226364136, + "kg_co2_emissions": null, + "mteb_version": "1.19.2", + "scores": { + "test": [ + { + "accuracy": 0.4236381977135172, + "f1": 0.41705520686157993, + "f1_weighted": 0.4239215483469339, + "hf_subset": "ru", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.4236381977135172, + "scores_per_experiment": [ + { + "accuracy": 0.4465366509751177, + "f1": 0.44604062816498835, + "f1_weighted": 0.44192562325734114 + }, + { + "accuracy": 0.4021519838601211, + "f1": 0.3956447035898876, + "f1_weighted": 0.4040488052126315 + }, + { + "accuracy": 0.43611297915265634, + "f1": 0.4287192477495157, + "f1_weighted": 0.43449850735068657 + }, + { + "accuracy": 0.4317417619367855, + "f1": 0.41765344750787653, + "f1_weighted": 0.4308987241780749 + }, + { + "accuracy": 0.425689307330195, + "f1": 0.42702320699254454, + "f1_weighted": 0.42916422735077003 + }, + { + "accuracy": 0.3930733019502354, + "f1": 0.38778148219910313, + "f1_weighted": 0.3948560736620025 + }, + { + "accuracy": 0.4388029589778077, + "f1": 0.42938969575568336, + "f1_weighted": 0.4474275880746285 + }, + { + "accuracy": 0.43611297915265634, + "f1": 0.42325118123837774, + "f1_weighted": 0.4292598799017616 + }, + { + "accuracy": 0.3910558170813719, + "f1": 0.39060039069226166, + "f1_weighted": 0.3906572520499222 + }, + { + "accuracy": 0.4351042367182246, + "f1": 0.42444808472556, + "f1_weighted": 0.43647880243151993 + } + ] + } + ], + "validation": [ + { + "accuracy": 0.4260206591244466, + "f1": 0.42441332041151014, + "f1_weighted": 0.4252546986873379, + "hf_subset": "ru", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.4260206591244466, + "scores_per_experiment": [ + { + "accuracy": 0.4722085587801279, + "f1": 0.47139336935500853, + "f1_weighted": 0.47114934527797875 + }, + { + "accuracy": 0.4023610427939006, + "f1": 0.40234265697536836, + "f1_weighted": 0.4027359424741011 + }, + { + "accuracy": 0.4461387112641417, + "f1": 0.44373925764175776, + "f1_weighted": 0.44536220780945274 + }, + { + "accuracy": 0.41515002459419575, + "f1": 0.41251021251837117, + "f1_weighted": 0.4115498101948993 + }, + { + "accuracy": 0.42744712247909494, + "f1": 0.44114912484945934, + "f1_weighted": 0.42983072530625344 + }, + { + "accuracy": 0.3890801770782095, + "f1": 0.39434172930845546, + "f1_weighted": 0.3894287047307646 + }, + { + "accuracy": 0.4441711756025578, + "f1": 0.4371024119354954, + "f1_weighted": 0.45109991642265485 + }, + { + "accuracy": 0.43138219380226267, + "f1": 0.42465071529354464, + "f1_weighted": 0.4181113765540303 + }, + { + "accuracy": 0.3890801770782095, + "f1": 0.39073919283106817, + "f1_weighted": 0.3899731286015203 + }, + { + "accuracy": 0.44318740777176585, + "f1": 0.42616453340657245, + "f1_weighted": 0.44330582950172365 + } + ] + } + ] + }, + "task_name": "MassiveScenarioClassification" +} \ No newline at end of file diff --git a/results/WhereIsAI__UAE-Large-V1/369c368f70f16a613f19f5598d4f12d9f44235d4/RUParaPhraserSTS.json b/results/WhereIsAI__UAE-Large-V1/369c368f70f16a613f19f5598d4f12d9f44235d4/RUParaPhraserSTS.json new file mode 100644 index 000000000..4915ec2db --- /dev/null +++ b/results/WhereIsAI__UAE-Large-V1/369c368f70f16a613f19f5598d4f12d9f44235d4/RUParaPhraserSTS.json @@ -0,0 +1,26 @@ +{ + "dataset_revision": "43265056790b8f7c59e0139acb4be0a8dad2c8f4", + "evaluation_time": 21.202986478805542, + "kg_co2_emissions": null, + "mteb_version": "1.19.2", + "scores": { + "test": [ + { + "cosine_pearson": 0.4696855704802466, + "cosine_spearman": 0.4733669918568403, + "euclidean_pearson": 0.48491498371927355, + "euclidean_spearman": 0.47228027531781247, + "hf_subset": "default", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.4733669918568403, + "manhattan_pearson": 0.48481099894041924, + "manhattan_spearman": 0.47227718471403635, + "pearson": 0.4696855704802466, + "spearman": 0.4733669918568403 + } + ] + }, + "task_name": "RUParaPhraserSTS" +} \ No newline at end of file diff --git a/results/WhereIsAI__UAE-Large-V1/369c368f70f16a613f19f5598d4f12d9f44235d4/RiaNewsRetrieval.json b/results/WhereIsAI__UAE-Large-V1/369c368f70f16a613f19f5598d4f12d9f44235d4/RiaNewsRetrieval.json new file mode 100644 index 000000000..7edfc7faa --- /dev/null +++ b/results/WhereIsAI__UAE-Large-V1/369c368f70f16a613f19f5598d4f12d9f44235d4/RiaNewsRetrieval.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "82374b0bbacda6114f39ff9c5b925fa1512ca5d7", + "evaluation_time": 16204.289098262787, + "kg_co2_emissions": null, + "mteb_version": "1.19.2", + "scores": { + "test": [ + { + "hf_subset": "default", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.25023, + "map_at_1": 0.1725, + "map_at_10": 0.22237, + "map_at_100": 0.22879, + "map_at_1000": 0.22944, + "map_at_20": 0.226, + "map_at_3": 0.2073, + "map_at_5": 0.21537, + "mrr_at_1": 0.173, + "mrr_at_10": 0.2226395634920635, + "mrr_at_100": 0.22906312298988568, + "mrr_at_1000": 0.2297114661040529, + "mrr_at_20": 0.22627118371179905, + "mrr_at_3": 0.20756666666666665, + "mrr_at_5": 0.21565666666666666, + "nauc_map_at_1000_diff1": 0.4667566179444467, + "nauc_map_at_1000_max": 0.32916719324640653, + "nauc_map_at_1000_std": 0.16069581891155724, + "nauc_map_at_100_diff1": 0.46675610070093165, + "nauc_map_at_100_max": 0.32908453063755794, + "nauc_map_at_100_std": 0.16055350340645755, + "nauc_map_at_10_diff1": 0.4705517348754555, + "nauc_map_at_10_max": 0.32893796180649487, + "nauc_map_at_10_std": 0.15659187293597085, + "nauc_map_at_1_diff1": 0.5472807458877814, + "nauc_map_at_1_max": 0.33080395063071405, + "nauc_map_at_1_std": 0.12576730440615885, + "nauc_map_at_20_diff1": 0.4680068880052351, + "nauc_map_at_20_max": 0.32873162544691104, + "nauc_map_at_20_std": 0.15847313933350834, + "nauc_map_at_3_diff1": 0.484067051670243, + "nauc_map_at_3_max": 0.32876777956916464, + "nauc_map_at_3_std": 0.14551141366614823, + "nauc_map_at_5_diff1": 0.4759253600999169, + "nauc_map_at_5_max": 0.32838997708214007, + "nauc_map_at_5_std": 0.1514566995912955, + "nauc_mrr_at_1000_diff1": 0.46539831058578457, + "nauc_mrr_at_1000_max": 0.3291055238406423, + "nauc_mrr_at_1000_std": 0.16056316111678426, + "nauc_mrr_at_100_diff1": 0.4653989607672544, + "nauc_mrr_at_100_max": 0.3290230318945373, + "nauc_mrr_at_100_std": 0.16042114947296168, + "nauc_mrr_at_10_diff1": 0.469191254331448, + "nauc_mrr_at_10_max": 0.32887965042585043, + "nauc_mrr_at_10_std": 0.15646678661268895, + "nauc_mrr_at_1_diff1": 0.5441221003959906, + "nauc_mrr_at_1_max": 0.3306864619720654, + "nauc_mrr_at_1_std": 0.12569542557675512, + "nauc_mrr_at_20_diff1": 0.46664997611861037, + "nauc_mrr_at_20_max": 0.3286707506515306, + "nauc_mrr_at_20_std": 0.15834323638139008, + "nauc_mrr_at_3_diff1": 0.48266633648918833, + "nauc_mrr_at_3_max": 0.32869874770034524, + "nauc_mrr_at_3_std": 0.14539584749310863, + "nauc_mrr_at_5_diff1": 0.4745368595506971, + "nauc_mrr_at_5_max": 0.3283444566217237, + "nauc_mrr_at_5_std": 0.151301491457963, + "nauc_ndcg_at_1000_diff1": 0.4241241602934732, + "nauc_ndcg_at_1000_max": 0.331996211422215, + "nauc_ndcg_at_1000_std": 0.19772154268629752, + "nauc_ndcg_at_100_diff1": 0.4243419633594812, + "nauc_ndcg_at_100_max": 0.32934347634267547, + "nauc_ndcg_at_100_std": 0.19349578401984968, + "nauc_ndcg_at_10_diff1": 0.44126609946166423, + "nauc_ndcg_at_10_max": 0.32880952022331156, + "nauc_ndcg_at_10_std": 0.17278759815975006, + "nauc_ndcg_at_1_diff1": 0.5472807458877814, + "nauc_ndcg_at_1_max": 0.33080395063071405, + "nauc_ndcg_at_1_std": 0.12576730440615885, + "nauc_ndcg_at_20_diff1": 0.43251754709847956, + "nauc_ndcg_at_20_max": 0.3275587416275954, + "nauc_ndcg_at_20_std": 0.17874611630485496, + "nauc_ndcg_at_3_diff1": 0.4664562295266938, + "nauc_ndcg_at_3_max": 0.3282517672441085, + "nauc_ndcg_at_3_std": 0.15130471142216048, + "nauc_ndcg_at_5_diff1": 0.45289738933869256, + "nauc_ndcg_at_5_max": 0.32770658303931216, + "nauc_ndcg_at_5_std": 0.16145071293082452, + "nauc_precision_at_1000_diff1": 0.2631169202066752, + "nauc_precision_at_1000_max": 0.3542409053281427, + "nauc_precision_at_1000_std": 0.3719760299711744, + "nauc_precision_at_100_diff1": 0.2969369799477999, + "nauc_precision_at_100_max": 0.33056458868154226, + "nauc_precision_at_100_std": 0.3070584569616868, + "nauc_precision_at_10_diff1": 0.3660567393068115, + "nauc_precision_at_10_max": 0.3287006543417761, + "nauc_precision_at_10_std": 0.2163075742969608, + "nauc_precision_at_1_diff1": 0.5472807458877814, + "nauc_precision_at_1_max": 0.33080395063071405, + "nauc_precision_at_1_std": 0.12576730440615885, + "nauc_precision_at_20_diff1": 0.33798551210738814, + "nauc_precision_at_20_max": 0.323489993608473, + "nauc_precision_at_20_std": 0.23464595527481943, + "nauc_precision_at_3_diff1": 0.4209889700832435, + "nauc_precision_at_3_max": 0.32693433566431507, + "nauc_precision_at_3_std": 0.1663462628289668, + "nauc_precision_at_5_diff1": 0.3943694747513904, + "nauc_precision_at_5_max": 0.3260235470059195, + "nauc_precision_at_5_std": 0.18748781663984374, + "nauc_recall_at_1000_diff1": 0.2631169202066761, + "nauc_recall_at_1000_max": 0.3542409053281432, + "nauc_recall_at_1000_std": 0.37197602997117524, + "nauc_recall_at_100_diff1": 0.29693697994780005, + "nauc_recall_at_100_max": 0.3305645886815423, + "nauc_recall_at_100_std": 0.307058456961687, + "nauc_recall_at_10_diff1": 0.3660567393068117, + "nauc_recall_at_10_max": 0.3287006543417766, + "nauc_recall_at_10_std": 0.21630757429696099, + "nauc_recall_at_1_diff1": 0.5472807458877814, + "nauc_recall_at_1_max": 0.33080395063071405, + "nauc_recall_at_1_std": 0.12576730440615885, + "nauc_recall_at_20_diff1": 0.3379855121073879, + "nauc_recall_at_20_max": 0.3234899936084729, + "nauc_recall_at_20_std": 0.23464595527481943, + "nauc_recall_at_3_diff1": 0.4209889700832433, + "nauc_recall_at_3_max": 0.32693433566431523, + "nauc_recall_at_3_std": 0.16634626282896686, + "nauc_recall_at_5_diff1": 0.39436947475139045, + "nauc_recall_at_5_max": 0.3260235470059194, + "nauc_recall_at_5_std": 0.18748781663984365, + "ndcg_at_1": 0.1725, + "ndcg_at_10": 0.25023, + "ndcg_at_100": 0.28426, + "ndcg_at_1000": 0.30441, + "ndcg_at_20": 0.26342, + "ndcg_at_3": 0.21891, + "ndcg_at_5": 0.23343, + "precision_at_1": 0.1725, + "precision_at_10": 0.03394, + "precision_at_100": 0.00505, + "precision_at_1000": 0.00067, + "precision_at_20": 0.01957, + "precision_at_3": 0.0842, + "precision_at_5": 0.05756, + "recall_at_1": 0.1725, + "recall_at_10": 0.3394, + "recall_at_100": 0.5052, + "recall_at_1000": 0.6703, + "recall_at_20": 0.3915, + "recall_at_3": 0.2526, + "recall_at_5": 0.2878 + } + ] + }, + "task_name": "RiaNewsRetrieval" +} \ No newline at end of file diff --git a/results/WhereIsAI__UAE-Large-V1/369c368f70f16a613f19f5598d4f12d9f44235d4/RuBQRetrieval.json b/results/WhereIsAI__UAE-Large-V1/369c368f70f16a613f19f5598d4f12d9f44235d4/RuBQRetrieval.json new file mode 100644 index 000000000..160d77c19 --- /dev/null +++ b/results/WhereIsAI__UAE-Large-V1/369c368f70f16a613f19f5598d4f12d9f44235d4/RuBQRetrieval.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "e19b6ffa60b3bc248e0b41f4cc37c26a55c2a67b", + "evaluation_time": 877.4626908302307, + "kg_co2_emissions": null, + "mteb_version": "1.19.2", + "scores": { + "test": [ + { + "hf_subset": "default", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.19716, + "map_at_1": 0.1034, + "map_at_10": 0.15808, + "map_at_100": 0.16502, + "map_at_1000": 0.16589, + "map_at_20": 0.16147, + "map_at_3": 0.1422, + "map_at_5": 0.15039, + "mrr_at_1": 0.1607565011820331, + "mrr_at_10": 0.2171711415062479, + "mrr_at_100": 0.2239702483277161, + "mrr_at_1000": 0.22467645528873267, + "mrr_at_20": 0.2207132078535203, + "mrr_at_3": 0.2018321513002364, + "mrr_at_5": 0.2095744680851064, + "nauc_map_at_1000_diff1": 0.3873930677446431, + "nauc_map_at_1000_max": 0.3887791338077492, + "nauc_map_at_1000_std": 0.2208617402929542, + "nauc_map_at_100_diff1": 0.38757973959767217, + "nauc_map_at_100_max": 0.3887672387670951, + "nauc_map_at_100_std": 0.22055850407857655, + "nauc_map_at_10_diff1": 0.39214566162297126, + "nauc_map_at_10_max": 0.3890429849373504, + "nauc_map_at_10_std": 0.21791673147880286, + "nauc_map_at_1_diff1": 0.4401195950542, + "nauc_map_at_1_max": 0.34251559207876753, + "nauc_map_at_1_std": 0.14555540869384673, + "nauc_map_at_20_diff1": 0.3891574243240744, + "nauc_map_at_20_max": 0.3892175870561461, + "nauc_map_at_20_std": 0.21978574935333156, + "nauc_map_at_3_diff1": 0.3993137775327199, + "nauc_map_at_3_max": 0.38369480572847603, + "nauc_map_at_3_std": 0.20242014052665935, + "nauc_map_at_5_diff1": 0.3930058133965449, + "nauc_map_at_5_max": 0.38725560011061233, + "nauc_map_at_5_std": 0.2126875269607504, + "nauc_mrr_at_1000_diff1": 0.3854881840704125, + "nauc_mrr_at_1000_max": 0.38586882310158704, + "nauc_mrr_at_1000_std": 0.18903776765747046, + "nauc_mrr_at_100_diff1": 0.38539206872853826, + "nauc_mrr_at_100_max": 0.3857965465965916, + "nauc_mrr_at_100_std": 0.18887771081434981, + "nauc_mrr_at_10_diff1": 0.39043716490651104, + "nauc_mrr_at_10_max": 0.38803525471366773, + "nauc_mrr_at_10_std": 0.1871797650611585, + "nauc_mrr_at_1_diff1": 0.434369041772976, + "nauc_mrr_at_1_max": 0.3643916441023078, + "nauc_mrr_at_1_std": 0.1423039514489491, + "nauc_mrr_at_20_diff1": 0.387072007203457, + "nauc_mrr_at_20_max": 0.3868113310288484, + "nauc_mrr_at_20_std": 0.18851829350796587, + "nauc_mrr_at_3_diff1": 0.39588198968503563, + "nauc_mrr_at_3_max": 0.3892473046477677, + "nauc_mrr_at_3_std": 0.18063446707353928, + "nauc_mrr_at_5_diff1": 0.3922380745261246, + "nauc_mrr_at_5_max": 0.38994361407936473, + "nauc_mrr_at_5_std": 0.18657219684678328, + "nauc_ndcg_at_1000_diff1": 0.35356408552538, + "nauc_ndcg_at_1000_max": 0.3882428554245445, + "nauc_ndcg_at_1000_std": 0.24152162416775844, + "nauc_ndcg_at_100_diff1": 0.35624099775748447, + "nauc_ndcg_at_100_max": 0.3882139038884509, + "nauc_ndcg_at_100_std": 0.23602949786891156, + "nauc_ndcg_at_10_diff1": 0.37751545109083007, + "nauc_ndcg_at_10_max": 0.39579789936301396, + "nauc_ndcg_at_10_std": 0.22793784678603216, + "nauc_ndcg_at_1_diff1": 0.4416777136063084, + "nauc_ndcg_at_1_max": 0.36082474187235164, + "nauc_ndcg_at_1_std": 0.14097948034678678, + "nauc_ndcg_at_20_diff1": 0.368092851257554, + "nauc_ndcg_at_20_max": 0.39422757532148156, + "nauc_ndcg_at_20_std": 0.2339450770280416, + "nauc_ndcg_at_3_diff1": 0.3886930988282777, + "nauc_ndcg_at_3_max": 0.3930983769416757, + "nauc_ndcg_at_3_std": 0.2081134909869786, + "nauc_ndcg_at_5_diff1": 0.38103251341791367, + "nauc_ndcg_at_5_max": 0.39728234121124045, + "nauc_ndcg_at_5_std": 0.22178546206478658, + "nauc_precision_at_1000_diff1": 0.12039012930294533, + "nauc_precision_at_1000_max": 0.25029980767596843, + "nauc_precision_at_1000_std": 0.21896595956099646, + "nauc_precision_at_100_diff1": 0.2024979588240234, + "nauc_precision_at_100_max": 0.32005619577875216, + "nauc_precision_at_100_std": 0.24343864635536985, + "nauc_precision_at_10_diff1": 0.31422827878362003, + "nauc_precision_at_10_max": 0.41075871535109076, + "nauc_precision_at_10_std": 0.26793877928951737, + "nauc_precision_at_1_diff1": 0.4416777136063084, + "nauc_precision_at_1_max": 0.36082474187235164, + "nauc_precision_at_1_std": 0.14097948034678678, + "nauc_precision_at_20_diff1": 0.2783883764412421, + "nauc_precision_at_20_max": 0.3900150865910402, + "nauc_precision_at_20_std": 0.26908920626234184, + "nauc_precision_at_3_diff1": 0.35523078360720356, + "nauc_precision_at_3_max": 0.4224536165759014, + "nauc_precision_at_3_std": 0.2410725226379458, + "nauc_precision_at_5_diff1": 0.33008956071536477, + "nauc_precision_at_5_max": 0.42015281513856434, + "nauc_precision_at_5_std": 0.2599471512801571, + "nauc_recall_at_1000_diff1": 0.20217093371706651, + "nauc_recall_at_1000_max": 0.3134413819363736, + "nauc_recall_at_1000_std": 0.28923296814268634, + "nauc_recall_at_100_diff1": 0.23691710280372855, + "nauc_recall_at_100_max": 0.3326825027302337, + "nauc_recall_at_100_std": 0.26172899613469813, + "nauc_recall_at_10_diff1": 0.31683592925608334, + "nauc_recall_at_10_max": 0.3780011480291154, + "nauc_recall_at_10_std": 0.25316249087444853, + "nauc_recall_at_1_diff1": 0.4401195950542, + "nauc_recall_at_1_max": 0.34251559207876753, + "nauc_recall_at_1_std": 0.14555540869384673, + "nauc_recall_at_20_diff1": 0.29168630558999514, + "nauc_recall_at_20_max": 0.3711294658263492, + "nauc_recall_at_20_std": 0.2689362306712218, + "nauc_recall_at_3_diff1": 0.3539620004573367, + "nauc_recall_at_3_max": 0.39504026261188663, + "nauc_recall_at_3_std": 0.22901073034367433, + "nauc_recall_at_5_diff1": 0.33884405490563646, + "nauc_recall_at_5_max": 0.40218623144317556, + "nauc_recall_at_5_std": 0.25354221807530575, + "ndcg_at_1": 0.15957, + "ndcg_at_10": 0.19716, + "ndcg_at_100": 0.23309, + "ndcg_at_1000": 0.258, + "ndcg_at_20": 0.20893, + "ndcg_at_3": 0.16742, + "ndcg_at_5": 0.17979, + "precision_at_1": 0.15957, + "precision_at_10": 0.0409, + "precision_at_100": 0.00677, + "precision_at_1000": 0.00098, + "precision_at_20": 0.02405, + "precision_at_3": 0.09535, + "precision_at_5": 0.0669, + "recall_at_1": 0.1034, + "recall_at_10": 0.25752, + "recall_at_100": 0.41198, + "recall_at_1000": 0.59194, + "recall_at_20": 0.2976, + "recall_at_3": 0.17917, + "recall_at_5": 0.20958 + } + ] + }, + "task_name": "RuBQRetrieval" +} \ No newline at end of file diff --git a/results/WhereIsAI__UAE-Large-V1/369c368f70f16a613f19f5598d4f12d9f44235d4/RuReviewsClassification.json b/results/WhereIsAI__UAE-Large-V1/369c368f70f16a613f19f5598d4f12d9f44235d4/RuReviewsClassification.json new file mode 100644 index 000000000..c24a702df --- /dev/null +++ b/results/WhereIsAI__UAE-Large-V1/369c368f70f16a613f19f5598d4f12d9f44235d4/RuReviewsClassification.json @@ -0,0 +1,73 @@ +{ + "dataset_revision": "f6d2c31f4dc6b88f468552750bfec05b4b41b05a", + "evaluation_time": 23.670599937438965, + "kg_co2_emissions": null, + "mteb_version": "1.19.2", + "scores": { + "test": [ + { + "accuracy": 0.43466796875, + "f1": 0.43239744692493065, + "f1_weighted": 0.43238955440069826, + "hf_subset": "default", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.43466796875, + "scores_per_experiment": [ + { + "accuracy": 0.44775390625, + "f1": 0.43169102711645585, + "f1_weighted": 0.43166400530338367 + }, + { + "accuracy": 0.44189453125, + "f1": 0.44289683593006385, + "f1_weighted": 0.44289372636731783 + }, + { + "accuracy": 0.44873046875, + "f1": 0.4506152103597203, + "f1_weighted": 0.45058617602914414 + }, + { + "accuracy": 0.47509765625, + "f1": 0.47700219271440014, + "f1_weighted": 0.476984316367728 + }, + { + "accuracy": 0.46728515625, + "f1": 0.4665405429073073, + "f1_weighted": 0.4665238666678694 + }, + { + "accuracy": 0.40625, + "f1": 0.4087480757527148, + "f1_weighted": 0.4087401170795813 + }, + { + "accuracy": 0.39794921875, + "f1": 0.3857884513499344, + "f1_weighted": 0.3858147395549545 + }, + { + "accuracy": 0.4736328125, + "f1": 0.4721482908969887, + "f1_weighted": 0.4721337824921066 + }, + { + "accuracy": 0.34375, + "f1": 0.34310152050478643, + "f1_weighted": 0.3431144088180463 + }, + { + "accuracy": 0.4443359375, + "f1": 0.44544232171693493, + "f1_weighted": 0.4454404053268516 + } + ] + } + ] + }, + "task_name": "RuReviewsClassification" +} \ No newline at end of file diff --git a/results/WhereIsAI__UAE-Large-V1/369c368f70f16a613f19f5598d4f12d9f44235d4/RuSTSBenchmarkSTS.json b/results/WhereIsAI__UAE-Large-V1/369c368f70f16a613f19f5598d4f12d9f44235d4/RuSTSBenchmarkSTS.json new file mode 100644 index 000000000..94ca66138 --- /dev/null +++ b/results/WhereIsAI__UAE-Large-V1/369c368f70f16a613f19f5598d4f12d9f44235d4/RuSTSBenchmarkSTS.json @@ -0,0 +1,26 @@ +{ + "dataset_revision": "7cf24f325c6da6195df55bef3d86b5e0616f3018", + "evaluation_time": 14.815508127212524, + "kg_co2_emissions": null, + "mteb_version": "1.19.2", + "scores": { + "test": [ + { + "cosine_pearson": 0.6074677772304615, + "cosine_spearman": 0.5980140276148966, + "euclidean_pearson": 0.5935316419435811, + "euclidean_spearman": 0.5896725802017485, + "hf_subset": "default", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.5980140276148966, + "manhattan_pearson": 0.5937559587146164, + "manhattan_spearman": 0.5896524482305341, + "pearson": 0.6074677772304615, + "spearman": 0.5980140276148966 + } + ] + }, + "task_name": "RuSTSBenchmarkSTS" +} \ No newline at end of file diff --git a/results/WhereIsAI__UAE-Large-V1/369c368f70f16a613f19f5598d4f12d9f44235d4/RuSciBenchGRNTIClassification.json b/results/WhereIsAI__UAE-Large-V1/369c368f70f16a613f19f5598d4f12d9f44235d4/RuSciBenchGRNTIClassification.json new file mode 100644 index 000000000..7c8a8457d --- /dev/null +++ b/results/WhereIsAI__UAE-Large-V1/369c368f70f16a613f19f5598d4f12d9f44235d4/RuSciBenchGRNTIClassification.json @@ -0,0 +1,73 @@ +{ + "dataset_revision": "673a610d6d3dd91a547a0d57ae1b56f37ebbf6a1", + "evaluation_time": 110.86036396026611, + "kg_co2_emissions": null, + "mteb_version": "1.19.2", + "scores": { + "test": [ + { + "accuracy": 0.21435546875, + "f1": 0.2134801234706058, + "f1_weighted": 0.21355578015530763, + "hf_subset": "default", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.21435546875, + "scores_per_experiment": [ + { + "accuracy": 0.2373046875, + "f1": 0.2385830211771008, + "f1_weighted": 0.2385976490923937 + }, + { + "accuracy": 0.201171875, + "f1": 0.2010408941516249, + "f1_weighted": 0.2011426046983844 + }, + { + "accuracy": 0.1884765625, + "f1": 0.18857595964818685, + "f1_weighted": 0.1885315033945787 + }, + { + "accuracy": 0.2314453125, + "f1": 0.22895947097013197, + "f1_weighted": 0.22902451917688996 + }, + { + "accuracy": 0.201171875, + "f1": 0.20026662761338834, + "f1_weighted": 0.20029068921133922 + }, + { + "accuracy": 0.20556640625, + "f1": 0.20502353719919084, + "f1_weighted": 0.20513872779224637 + }, + { + "accuracy": 0.2275390625, + "f1": 0.22701029036231352, + "f1_weighted": 0.22719745494728247 + }, + { + "accuracy": 0.20263671875, + "f1": 0.1997286640271915, + "f1_weighted": 0.19982043714326167 + }, + { + "accuracy": 0.216796875, + "f1": 0.21466135740932474, + "f1_weighted": 0.21476023716353349 + }, + { + "accuracy": 0.2314453125, + "f1": 0.2309514121476048, + "f1_weighted": 0.23105397893316632 + } + ] + } + ] + }, + "task_name": "RuSciBenchGRNTIClassification" +} \ No newline at end of file diff --git a/results/WhereIsAI__UAE-Large-V1/369c368f70f16a613f19f5598d4f12d9f44235d4/RuSciBenchGRNTIClusteringP2P.json b/results/WhereIsAI__UAE-Large-V1/369c368f70f16a613f19f5598d4f12d9f44235d4/RuSciBenchGRNTIClusteringP2P.json new file mode 100644 index 000000000..29ef0ee26 --- /dev/null +++ b/results/WhereIsAI__UAE-Large-V1/369c368f70f16a613f19f5598d4f12d9f44235d4/RuSciBenchGRNTIClusteringP2P.json @@ -0,0 +1,34 @@ +{ + "dataset_revision": "673a610d6d3dd91a547a0d57ae1b56f37ebbf6a1", + "evaluation_time": 111.25953912734985, + "kg_co2_emissions": null, + "mteb_version": "1.19.2", + "scores": { + "test": [ + { + "hf_subset": "default", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.1925761117548493, + "v_measure": 0.1925761117548493, + "v_measure_std": 0.0068850991780520735, + "v_measures": { + "Level 0": [ + 0.19687664198989274, + 0.19517792425504651, + 0.1930207471719472, + 0.19150533945149248, + 0.19798665004386787, + 0.19611756740791741, + 0.17356687141904467, + 0.19110732511611908, + 0.19896486225626947, + 0.19143718843689522 + ] + } + } + ] + }, + "task_name": "RuSciBenchGRNTIClusteringP2P" +} \ No newline at end of file diff --git a/results/WhereIsAI__UAE-Large-V1/369c368f70f16a613f19f5598d4f12d9f44235d4/RuSciBenchOECDClassification.json b/results/WhereIsAI__UAE-Large-V1/369c368f70f16a613f19f5598d4f12d9f44235d4/RuSciBenchOECDClassification.json new file mode 100644 index 000000000..4e6c52346 --- /dev/null +++ b/results/WhereIsAI__UAE-Large-V1/369c368f70f16a613f19f5598d4f12d9f44235d4/RuSciBenchOECDClassification.json @@ -0,0 +1,73 @@ +{ + "dataset_revision": "26c88e99dcaba32bb45d0e1bfc21902337f6d471", + "evaluation_time": 109.49619007110596, + "kg_co2_emissions": null, + "mteb_version": "1.19.2", + "scores": { + "test": [ + { + "accuracy": 0.1580078125, + "f1": 0.15812693607109746, + "f1_weighted": 0.15814366627703486, + "hf_subset": "default", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.1580078125, + "scores_per_experiment": [ + { + "accuracy": 0.14208984375, + "f1": 0.14054734043992723, + "f1_weighted": 0.14046772434146537 + }, + { + "accuracy": 0.1591796875, + "f1": 0.1598796629898343, + "f1_weighted": 0.15993901774092084 + }, + { + "accuracy": 0.15625, + "f1": 0.15692787977171707, + "f1_weighted": 0.1568770666103347 + }, + { + "accuracy": 0.158203125, + "f1": 0.15845438021278008, + "f1_weighted": 0.1584237148344379 + }, + { + "accuracy": 0.1533203125, + "f1": 0.1531090381994811, + "f1_weighted": 0.15323492030189503 + }, + { + "accuracy": 0.15966796875, + "f1": 0.15907095881124939, + "f1_weighted": 0.15914830397109078 + }, + { + "accuracy": 0.17333984375, + "f1": 0.17343774310767776, + "f1_weighted": 0.1734316704293661 + }, + { + "accuracy": 0.1455078125, + "f1": 0.14521661026261698, + "f1_weighted": 0.1452602466496854 + }, + { + "accuracy": 0.16015625, + "f1": 0.16229848835523478, + "f1_weighted": 0.1623135683957477 + }, + { + "accuracy": 0.17236328125, + "f1": 0.17232725856045575, + "f1_weighted": 0.1723404294954048 + } + ] + } + ] + }, + "task_name": "RuSciBenchOECDClassification" +} \ No newline at end of file diff --git a/results/WhereIsAI__UAE-Large-V1/369c368f70f16a613f19f5598d4f12d9f44235d4/RuSciBenchOECDClusteringP2P.json b/results/WhereIsAI__UAE-Large-V1/369c368f70f16a613f19f5598d4f12d9f44235d4/RuSciBenchOECDClusteringP2P.json new file mode 100644 index 000000000..81220a4bc --- /dev/null +++ b/results/WhereIsAI__UAE-Large-V1/369c368f70f16a613f19f5598d4f12d9f44235d4/RuSciBenchOECDClusteringP2P.json @@ -0,0 +1,34 @@ +{ + "dataset_revision": "26c88e99dcaba32bb45d0e1bfc21902337f6d471", + "evaluation_time": 115.65804266929626, + "kg_co2_emissions": null, + "mteb_version": "1.19.2", + "scores": { + "test": [ + { + "hf_subset": "default", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.16507585198824953, + "v_measure": 0.16507585198824953, + "v_measure_std": 0.0053782342889032815, + "v_measures": { + "Level 0": [ + 0.16492095099795576, + 0.15494137058244264, + 0.16989426324198345, + 0.16687378917680634, + 0.16645898594665878, + 0.16710458948056867, + 0.16658144158359067, + 0.154640270613698, + 0.16910706430801395, + 0.17023579395077704 + ] + } + } + ] + }, + "task_name": "RuSciBenchOECDClusteringP2P" +} \ No newline at end of file diff --git a/results/WhereIsAI__UAE-Large-V1/369c368f70f16a613f19f5598d4f12d9f44235d4/STS22.json b/results/WhereIsAI__UAE-Large-V1/369c368f70f16a613f19f5598d4f12d9f44235d4/STS22.json new file mode 100644 index 000000000..f9a636e7b --- /dev/null +++ b/results/WhereIsAI__UAE-Large-V1/369c368f70f16a613f19f5598d4f12d9f44235d4/STS22.json @@ -0,0 +1,26 @@ +{ + "dataset_revision": "de9d86b3b84231dc21f76c7b7af1f28e2f57f6e3", + "evaluation_time": 12.66112470626831, + "kg_co2_emissions": null, + "mteb_version": "1.19.2", + "scores": { + "test": [ + { + "cosine_pearson": 0.1782435900481098, + "cosine_spearman": 0.27179526500755036, + "euclidean_pearson": 0.17048187185502803, + "euclidean_spearman": 0.26052247152024105, + "hf_subset": "ru", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.27179526500755036, + "manhattan_pearson": 0.16990930699633905, + "manhattan_spearman": 0.2633700191090123, + "pearson": 0.1782435900481098, + "spearman": 0.27179526500755036 + } + ] + }, + "task_name": "STS22" +} \ No newline at end of file diff --git a/results/WhereIsAI__UAE-Large-V1/369c368f70f16a613f19f5598d4f12d9f44235d4/SensitiveTopicsClassification.json b/results/WhereIsAI__UAE-Large-V1/369c368f70f16a613f19f5598d4f12d9f44235d4/SensitiveTopicsClassification.json new file mode 100644 index 000000000..70d6d6845 --- /dev/null +++ b/results/WhereIsAI__UAE-Large-V1/369c368f70f16a613f19f5598d4f12d9f44235d4/SensitiveTopicsClassification.json @@ -0,0 +1,73 @@ +{ + "dataset_revision": "416b34a802308eac30e4192afc0ff99bb8dcc7f2", + "evaluation_time": 31.848329067230225, + "kg_co2_emissions": null, + "mteb_version": "1.19.2", + "scores": { + "test": [ + { + "accuracy": 0.181201171875, + "f1": 0.042459985789176445, + "hf_subset": "default", + "languages": [ + "rus-Cyrl" + ], + "lrap": 0.26630859374999355, + "main_score": 0.181201171875, + "scores_per_experiment": [ + { + "accuracy": 0.17578125, + "f1": 0.013251126335653358, + "lrap": 0.25630696614582704 + }, + { + "accuracy": 0.18603515625, + "f1": 0.052759526262812016, + "lrap": 0.26745605468749356 + }, + { + "accuracy": 0.17919921875, + "f1": 0.045725585250916384, + "lrap": 0.2685275607638823 + }, + { + "accuracy": 0.177734375, + "f1": 0.030195680472199296, + "lrap": 0.2602267795138825 + }, + { + "accuracy": 0.173828125, + "f1": 0.029593701631172767, + "lrap": 0.2615017361111047 + }, + { + "accuracy": 0.1875, + "f1": 0.07398149434084218, + "lrap": 0.2761094835069379 + }, + { + "accuracy": 0.17919921875, + "f1": 0.02815435871306044, + "lrap": 0.2654622395833269 + }, + { + "accuracy": 0.177734375, + "f1": 0.03119131812149384, + "lrap": 0.26363118489582693 + }, + { + "accuracy": 0.1806640625, + "f1": 0.04897015440219323, + "lrap": 0.2655707465277715 + }, + { + "accuracy": 0.1943359375, + "f1": 0.07077691236142095, + "lrap": 0.27829318576388223 + } + ] + } + ] + }, + "task_name": "SensitiveTopicsClassification" +} \ No newline at end of file diff --git a/results/WhereIsAI__UAE-Large-V1/369c368f70f16a613f19f5598d4f12d9f44235d4/model_meta.json b/results/WhereIsAI__UAE-Large-V1/369c368f70f16a613f19f5598d4f12d9f44235d4/model_meta.json index e41f6b724..57b0faa06 100644 --- a/results/WhereIsAI__UAE-Large-V1/369c368f70f16a613f19f5598d4f12d9f44235d4/model_meta.json +++ b/results/WhereIsAI__UAE-Large-V1/369c368f70f16a613f19f5598d4f12d9f44235d4/model_meta.json @@ -1 +1,25 @@ -{"name": "WhereIsAI/UAE-Large-V1", "revision": "369c368f70f16a613f19f5598d4f12d9f44235d4", "release_date": "2023-12-04", "languages": ["eng_Latn"], "n_parameters": 335000, "memory_usage": null, "max_tokens": 512, "embed_dim": 1024, "license": "mit", "open_weights": true, "public_training_data": null, "public_training_code": null, "framework": ["Sentence Transformers", "PyTorch"], "reference": "https://huggingface.co/WhereIsAI/UAE-Large-V1", "similarity_fn_name": "cosine", "use_instructions": false, "zero_shot_benchmarks": null, "loader": "UAEWrapper"} \ No newline at end of file +{ + "name": "WhereIsAI/UAE-Large-V1", + "revision": "369c368f70f16a613f19f5598d4f12d9f44235d4", + "release_date": "2023-12-04", + "languages": [ + "eng_Latn" + ], + "n_parameters": 335000, + "memory_usage": null, + "max_tokens": 512, + "embed_dim": 1024, + "license": "mit", + "open_weights": true, + "public_training_data": null, + "public_training_code": null, + "framework": [ + "Sentence Transformers", + "PyTorch" + ], + "reference": "https://huggingface.co/WhereIsAI/UAE-Large-V1", + "similarity_fn_name": "cosine", + "use_instructions": false, + "zero_shot_benchmarks": null, + "loader": "UAEWrapper" +} \ No newline at end of file diff --git a/results/avsolatorio__GIST-Embedding-v0/bf6b2e55e92f510a570ad4d7d2da2ec8cd22590c/CEDRClassification.json b/results/avsolatorio__GIST-Embedding-v0/bf6b2e55e92f510a570ad4d7d2da2ec8cd22590c/CEDRClassification.json new file mode 100644 index 000000000..260f4671f --- /dev/null +++ b/results/avsolatorio__GIST-Embedding-v0/bf6b2e55e92f510a570ad4d7d2da2ec8cd22590c/CEDRClassification.json @@ -0,0 +1,73 @@ +{ + "dataset_revision": "c0ba03d058e3e1b2f3fd20518875a4563dd12db4", + "evaluation_time": 12.639983177185059, + "kg_co2_emissions": null, + "mteb_version": "1.19.2", + "scores": { + "test": [ + { + "accuracy": 0.3324654622741764, + "f1": 0.13282824627157988, + "hf_subset": "default", + "languages": [ + "rus-Cyrl" + ], + "lrap": 0.5652736450584611, + "main_score": 0.3324654622741764, + "scores_per_experiment": [ + { + "accuracy": 0.2842720510095643, + "f1": 0.14983694353813168, + "lrap": 0.5844314558979932 + }, + { + "accuracy": 0.3161530286928799, + "f1": 0.11137317205428661, + "lrap": 0.5546227417640935 + }, + { + "accuracy": 0.3368756641870351, + "f1": 0.16483318111459427, + "lrap": 0.592136025504794 + }, + { + "accuracy": 0.3368756641870351, + "f1": 0.12675603181626083, + "lrap": 0.5599893730074516 + }, + { + "accuracy": 0.36344314558979807, + "f1": 0.12809603019233612, + "lrap": 0.5562699256110649 + }, + { + "accuracy": 0.3528161530286929, + "f1": 0.11372304977750884, + "lrap": 0.551487778958568 + }, + { + "accuracy": 0.35015940488841657, + "f1": 0.14831477867924214, + "lrap": 0.5659936238044758 + }, + { + "accuracy": 0.3097768331562168, + "f1": 0.13385485706378522, + "lrap": 0.5619553666312561 + }, + { + "accuracy": 0.3331562167906483, + "f1": 0.13623077616684992, + "lrap": 0.5677205100956556 + }, + { + "accuracy": 0.34112646121147716, + "f1": 0.11526364231280303, + "lrap": 0.5581296493092579 + } + ] + } + ] + }, + "task_name": "CEDRClassification" +} \ No newline at end of file diff --git a/results/avsolatorio__GIST-Embedding-v0/bf6b2e55e92f510a570ad4d7d2da2ec8cd22590c/GeoreviewClassification.json b/results/avsolatorio__GIST-Embedding-v0/bf6b2e55e92f510a570ad4d7d2da2ec8cd22590c/GeoreviewClassification.json new file mode 100644 index 000000000..e0c8920dd --- /dev/null +++ b/results/avsolatorio__GIST-Embedding-v0/bf6b2e55e92f510a570ad4d7d2da2ec8cd22590c/GeoreviewClassification.json @@ -0,0 +1,73 @@ +{ + "dataset_revision": "3765c0d1de6b7d264bc459433c45e5a75513839c", + "evaluation_time": 23.458097457885742, + "kg_co2_emissions": null, + "mteb_version": "1.19.2", + "scores": { + "test": [ + { + "accuracy": 0.27890625, + "f1": 0.2770714728263111, + "f1_weighted": 0.27706653059400976, + "hf_subset": "default", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.27890625, + "scores_per_experiment": [ + { + "accuracy": 0.265625, + "f1": 0.26579310584476784, + "f1_weighted": 0.2658120517981484 + }, + { + "accuracy": 0.30224609375, + "f1": 0.30098112575752983, + "f1_weighted": 0.3009545902291218 + }, + { + "accuracy": 0.28564453125, + "f1": 0.2861199935809627, + "f1_weighted": 0.2861160884963123 + }, + { + "accuracy": 0.25927734375, + "f1": 0.26027865960193103, + "f1_weighted": 0.2602808181549225 + }, + { + "accuracy": 0.28564453125, + "f1": 0.28381388768537985, + "f1_weighted": 0.2838067190221567 + }, + { + "accuracy": 0.27490234375, + "f1": 0.27250455820694713, + "f1_weighted": 0.2725166481933218 + }, + { + "accuracy": 0.2734375, + "f1": 0.2757574424571254, + "f1_weighted": 0.27577132818067884 + }, + { + "accuracy": 0.27685546875, + "f1": 0.2672831952637146, + "f1_weighted": 0.2672755727910743 + }, + { + "accuracy": 0.27001953125, + "f1": 0.26621290034672235, + "f1_weighted": 0.2661595030621535 + }, + { + "accuracy": 0.29541015625, + "f1": 0.29196985951803034, + "f1_weighted": 0.29197198601220753 + } + ] + } + ] + }, + "task_name": "GeoreviewClassification" +} \ No newline at end of file diff --git a/results/avsolatorio__GIST-Embedding-v0/bf6b2e55e92f510a570ad4d7d2da2ec8cd22590c/GeoreviewClusteringP2P.json b/results/avsolatorio__GIST-Embedding-v0/bf6b2e55e92f510a570ad4d7d2da2ec8cd22590c/GeoreviewClusteringP2P.json new file mode 100644 index 000000000..4c23cb6bd --- /dev/null +++ b/results/avsolatorio__GIST-Embedding-v0/bf6b2e55e92f510a570ad4d7d2da2ec8cd22590c/GeoreviewClusteringP2P.json @@ -0,0 +1,34 @@ +{ + "dataset_revision": "97a313c8fc85b47f13f33e7e9a95c1ad888c7fec", + "evaluation_time": 84.74524521827698, + "kg_co2_emissions": null, + "mteb_version": "1.19.2", + "scores": { + "test": [ + { + "hf_subset": "default", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.23895393571829432, + "v_measure": 0.23895393571829432, + "v_measure_std": 0.005464111001775365, + "v_measures": { + "Level 0": [ + 0.2385086366188436, + 0.23609261641248983, + 0.24216702097073606, + 0.2466892876090826, + 0.23686844757257336, + 0.2435091492562068, + 0.23999957940843578, + 0.2254246608811317, + 0.23814338371312874, + 0.24213657474031489 + ] + } + } + ] + }, + "task_name": "GeoreviewClusteringP2P" +} \ No newline at end of file diff --git a/results/avsolatorio__GIST-Embedding-v0/bf6b2e55e92f510a570ad4d7d2da2ec8cd22590c/HeadlineClassification.json b/results/avsolatorio__GIST-Embedding-v0/bf6b2e55e92f510a570ad4d7d2da2ec8cd22590c/HeadlineClassification.json new file mode 100644 index 000000000..0edd940a5 --- /dev/null +++ b/results/avsolatorio__GIST-Embedding-v0/bf6b2e55e92f510a570ad4d7d2da2ec8cd22590c/HeadlineClassification.json @@ -0,0 +1,73 @@ +{ + "dataset_revision": "2fe05ee6b5832cda29f2ef7aaad7b7fe6a3609eb", + "evaluation_time": 12.673069715499878, + "kg_co2_emissions": null, + "mteb_version": "1.19.2", + "scores": { + "test": [ + { + "accuracy": 0.2994140625, + "f1": 0.2973469062998874, + "f1_weighted": 0.2973460884060384, + "hf_subset": "default", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.2994140625, + "scores_per_experiment": [ + { + "accuracy": 0.318359375, + "f1": 0.3152949105498351, + "f1_weighted": 0.31531487867263025 + }, + { + "accuracy": 0.27978515625, + "f1": 0.2716512429683555, + "f1_weighted": 0.2716633230676073 + }, + { + "accuracy": 0.2880859375, + "f1": 0.2903702844824818, + "f1_weighted": 0.29034839220339737 + }, + { + "accuracy": 0.3310546875, + "f1": 0.3272478394377511, + "f1_weighted": 0.3272959318979395 + }, + { + "accuracy": 0.318359375, + "f1": 0.31839111816427235, + "f1_weighted": 0.3183834261052079 + }, + { + "accuracy": 0.31591796875, + "f1": 0.31456925997194185, + "f1_weighted": 0.31458747106132073 + }, + { + "accuracy": 0.2607421875, + "f1": 0.2614621654754548, + "f1_weighted": 0.26144036050010067 + }, + { + "accuracy": 0.2490234375, + "f1": 0.25042799350533024, + "f1_weighted": 0.25040947832558746 + }, + { + "accuracy": 0.31787109375, + "f1": 0.31645871710373175, + "f1_weighted": 0.31641719012902625 + }, + { + "accuracy": 0.31494140625, + "f1": 0.3075955313397199, + "f1_weighted": 0.3076004320975664 + } + ] + } + ] + }, + "task_name": "HeadlineClassification" +} \ No newline at end of file diff --git a/results/avsolatorio__GIST-Embedding-v0/bf6b2e55e92f510a570ad4d7d2da2ec8cd22590c/InappropriatenessClassification.json b/results/avsolatorio__GIST-Embedding-v0/bf6b2e55e92f510a570ad4d7d2da2ec8cd22590c/InappropriatenessClassification.json new file mode 100644 index 000000000..cc3b9ae90 --- /dev/null +++ b/results/avsolatorio__GIST-Embedding-v0/bf6b2e55e92f510a570ad4d7d2da2ec8cd22590c/InappropriatenessClassification.json @@ -0,0 +1,95 @@ +{ + "dataset_revision": "601651fdc45ef243751676e62dd7a19f491c0285", + "evaluation_time": 12.102399587631226, + "kg_co2_emissions": null, + "mteb_version": "1.19.2", + "scores": { + "test": [ + { + "accuracy": 0.512744140625, + "ap": 0.5068530710416295, + "ap_weighted": 0.5068530710416295, + "f1": 0.5121831976581946, + "f1_weighted": 0.5121831976581946, + "hf_subset": "default", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.512744140625, + "scores_per_experiment": [ + { + "accuracy": 0.494140625, + "ap": 0.49710663094008267, + "ap_weighted": 0.49710663094008267, + "f1": 0.493762120726251, + "f1_weighted": 0.493762120726251 + }, + { + "accuracy": 0.51806640625, + "ap": 0.5093725214149746, + "ap_weighted": 0.5093725214149746, + "f1": 0.517891577026524, + "f1_weighted": 0.517891577026524 + }, + { + "accuracy": 0.5263671875, + "ap": 0.5139393411624205, + "ap_weighted": 0.5139393411624205, + "f1": 0.5256066749769557, + "f1_weighted": 0.5256066749769557 + }, + { + "accuracy": 0.48974609375, + "ap": 0.4949722782258064, + "ap_weighted": 0.4949722782258064, + "f1": 0.4892930172245723, + "f1_weighted": 0.4892930172245723 + }, + { + "accuracy": 0.54931640625, + "ap": 0.5271611966080403, + "ap_weighted": 0.5271611966080403, + "f1": 0.5492260215483002, + "f1_weighted": 0.5492260215483002 + }, + { + "accuracy": 0.525390625, + "ap": 0.5133902138157895, + "ap_weighted": 0.5133902138157895, + "f1": 0.5247701743781316, + "f1_weighted": 0.5247701743781316 + }, + { + "accuracy": 0.51513671875, + "ap": 0.5078027441308691, + "ap_weighted": 0.5078027441308691, + "f1": 0.5150755584169394, + "f1_weighted": 0.5150755584169394 + }, + { + "accuracy": 0.5009765625, + "ap": 0.5004891771788991, + "ap_weighted": 0.5004891771788991, + "f1": 0.5004577622443047, + "f1_weighted": 0.5004577622443047 + }, + { + "accuracy": 0.51220703125, + "ap": 0.5062340441884089, + "ap_weighted": 0.5062340441884089, + "f1": 0.5097495278892209, + "f1_weighted": 0.5097495278892209 + }, + { + "accuracy": 0.49609375, + "ap": 0.498062562751004, + "ap_weighted": 0.498062562751004, + "f1": 0.4959995421507469, + "f1_weighted": 0.4959995421507469 + } + ] + } + ] + }, + "task_name": "InappropriatenessClassification" +} \ No newline at end of file diff --git a/results/avsolatorio__GIST-Embedding-v0/bf6b2e55e92f510a570ad4d7d2da2ec8cd22590c/KinopoiskClassification.json b/results/avsolatorio__GIST-Embedding-v0/bf6b2e55e92f510a570ad4d7d2da2ec8cd22590c/KinopoiskClassification.json new file mode 100644 index 000000000..31e75b9c9 --- /dev/null +++ b/results/avsolatorio__GIST-Embedding-v0/bf6b2e55e92f510a570ad4d7d2da2ec8cd22590c/KinopoiskClassification.json @@ -0,0 +1,73 @@ +{ + "dataset_revision": "5911f26666ac11af46cb9c6849d0dc80a378af24", + "evaluation_time": 20.25071144104004, + "kg_co2_emissions": null, + "mteb_version": "1.19.2", + "scores": { + "test": [ + { + "accuracy": 0.3531333333333333, + "f1": 0.3471596148171109, + "f1_weighted": 0.3471596148171109, + "hf_subset": "default", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.3531333333333333, + "scores_per_experiment": [ + { + "accuracy": 0.324, + "f1": 0.3236385706303712, + "f1_weighted": 0.3236385706303712 + }, + { + "accuracy": 0.368, + "f1": 0.36181122055002607, + "f1_weighted": 0.36181122055002607 + }, + { + "accuracy": 0.36466666666666664, + "f1": 0.3495095151924568, + "f1_weighted": 0.34950951519245677 + }, + { + "accuracy": 0.374, + "f1": 0.3736896025471685, + "f1_weighted": 0.37368960254716843 + }, + { + "accuracy": 0.358, + "f1": 0.34438800276515846, + "f1_weighted": 0.34438800276515846 + }, + { + "accuracy": 0.33866666666666667, + "f1": 0.3180035383079564, + "f1_weighted": 0.3180035383079564 + }, + { + "accuracy": 0.3606666666666667, + "f1": 0.3602013557515354, + "f1_weighted": 0.3602013557515354 + }, + { + "accuracy": 0.346, + "f1": 0.34595025136507823, + "f1_weighted": 0.3459502513650782 + }, + { + "accuracy": 0.3473333333333333, + "f1": 0.34527175333669363, + "f1_weighted": 0.3452717533366936 + }, + { + "accuracy": 0.35, + "f1": 0.3491323377246638, + "f1_weighted": 0.3491323377246638 + } + ] + } + ] + }, + "task_name": "KinopoiskClassification" +} \ No newline at end of file diff --git a/results/avsolatorio__GIST-Embedding-v0/bf6b2e55e92f510a570ad4d7d2da2ec8cd22590c/MIRACLReranking.json b/results/avsolatorio__GIST-Embedding-v0/bf6b2e55e92f510a570ad4d7d2da2ec8cd22590c/MIRACLReranking.json new file mode 100644 index 000000000..cf4c96221 --- /dev/null +++ b/results/avsolatorio__GIST-Embedding-v0/bf6b2e55e92f510a570ad4d7d2da2ec8cd22590c/MIRACLReranking.json @@ -0,0 +1,130 @@ +{ + "dataset_revision": "6d1962c527217f8927fca80f890f14f36b2802af", + "evaluation_time": 876.5592341423035, + "kg_co2_emissions": null, + "mteb_version": "1.19.2", + "scores": { + "dev": [ + { + "MAP@1(MIRACL)": 0.06214, + "MAP@10(MIRACL)": 0.11602, + "MAP@100(MIRACL)": 0.14934, + "MAP@1000(MIRACL)": 0.14934, + "MAP@20(MIRACL)": 0.12894, + "MAP@3(MIRACL)": 0.08944, + "MAP@5(MIRACL)": 0.10016, + "NDCG@1(MIRACL)": 0.09864, + "NDCG@10(MIRACL)": 0.16478, + "NDCG@100(MIRACL)": 0.30575, + "NDCG@1000(MIRACL)": 0.30575, + "NDCG@20(MIRACL)": 0.20468, + "NDCG@3(MIRACL)": 0.11593, + "NDCG@5(MIRACL)": 0.13046, + "P@1(MIRACL)": 0.09864, + "P@10(MIRACL)": 0.05237, + "P@100(MIRACL)": 0.01953, + "P@1000(MIRACL)": 0.00195, + "P@20(MIRACL)": 0.04162, + "P@3(MIRACL)": 0.07324, + "P@5(MIRACL)": 0.06239, + "Recall@1(MIRACL)": 0.06214, + "Recall@10(MIRACL)": 0.24834, + "Recall@100(MIRACL)": 0.79952, + "Recall@1000(MIRACL)": 0.79952, + "Recall@20(MIRACL)": 0.37325, + "Recall@3(MIRACL)": 0.11944, + "Recall@5(MIRACL)": 0.15821, + "hf_subset": "ru", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.16478, + "nAUC_MAP@1000_diff1(MIRACL)": 0.14621611681442312, + "nAUC_MAP@1000_max(MIRACL)": 0.06822458074721523, + "nAUC_MAP@1000_std(MIRACL)": 0.05108459163491569, + "nAUC_MAP@100_diff1(MIRACL)": 0.14621611681442312, + "nAUC_MAP@100_max(MIRACL)": 0.06822458074721523, + "nAUC_MAP@100_std(MIRACL)": 0.05108459163491569, + "nAUC_MAP@10_diff1(MIRACL)": 0.1588910111309091, + "nAUC_MAP@10_max(MIRACL)": 0.05166664575581377, + "nAUC_MAP@10_std(MIRACL)": 0.04199321136072298, + "nAUC_MAP@1_diff1(MIRACL)": 0.25695769382707545, + "nAUC_MAP@1_max(MIRACL)": 0.08150630226975161, + "nAUC_MAP@1_std(MIRACL)": 0.026520810018842655, + "nAUC_MAP@20_diff1(MIRACL)": 0.15104175917913476, + "nAUC_MAP@20_max(MIRACL)": 0.057958706346398325, + "nAUC_MAP@20_std(MIRACL)": 0.048945896389972036, + "nAUC_MAP@3_diff1(MIRACL)": 0.19916549856537918, + "nAUC_MAP@3_max(MIRACL)": 0.054580443837388394, + "nAUC_MAP@3_std(MIRACL)": 0.03201636189062581, + "nAUC_MAP@5_diff1(MIRACL)": 0.1722860275431054, + "nAUC_MAP@5_max(MIRACL)": 0.04306105749485529, + "nAUC_MAP@5_std(MIRACL)": 0.027298333687719452, + "nAUC_NDCG@1000_diff1(MIRACL)": 0.09439657465279892, + "nAUC_NDCG@1000_max(MIRACL)": 0.11390619091549667, + "nAUC_NDCG@1000_std(MIRACL)": 0.08751321540693553, + "nAUC_NDCG@100_diff1(MIRACL)": 0.09439657465279892, + "nAUC_NDCG@100_max(MIRACL)": 0.11390619091549667, + "nAUC_NDCG@100_std(MIRACL)": 0.08751321540693553, + "nAUC_NDCG@10_diff1(MIRACL)": 0.12093305923250075, + "nAUC_NDCG@10_max(MIRACL)": 0.0539025069689858, + "nAUC_NDCG@10_std(MIRACL)": 0.055950265305502005, + "nAUC_NDCG@1_diff1(MIRACL)": 0.2112403330603597, + "nAUC_NDCG@1_max(MIRACL)": 0.1104236959190068, + "nAUC_NDCG@1_std(MIRACL)": 0.09082717689868323, + "nAUC_NDCG@20_diff1(MIRACL)": 0.10948721198251336, + "nAUC_NDCG@20_max(MIRACL)": 0.06994002194076654, + "nAUC_NDCG@20_std(MIRACL)": 0.07502305086665699, + "nAUC_NDCG@3_diff1(MIRACL)": 0.16971524347059957, + "nAUC_NDCG@3_max(MIRACL)": 0.06250754035215514, + "nAUC_NDCG@3_std(MIRACL)": 0.04880456752899729, + "nAUC_NDCG@5_diff1(MIRACL)": 0.13824289922334296, + "nAUC_NDCG@5_max(MIRACL)": 0.041382700888776665, + "nAUC_NDCG@5_std(MIRACL)": 0.03393492430975926, + "nAUC_P@1000_diff1(MIRACL)": -0.040330076193292944, + "nAUC_P@1000_max(MIRACL)": 0.09998988399521574, + "nAUC_P@1000_std(MIRACL)": 0.04160477662305493, + "nAUC_P@100_diff1(MIRACL)": -0.04033007619329272, + "nAUC_P@100_max(MIRACL)": 0.0999898839952161, + "nAUC_P@100_std(MIRACL)": 0.04160477662305544, + "nAUC_P@10_diff1(MIRACL)": 0.012524113746635443, + "nAUC_P@10_max(MIRACL)": 0.06878528985883052, + "nAUC_P@10_std(MIRACL)": 0.07864202241240201, + "nAUC_P@1_diff1(MIRACL)": 0.2112403330603597, + "nAUC_P@1_max(MIRACL)": 0.1104236959190068, + "nAUC_P@1_std(MIRACL)": 0.09082717689868323, + "nAUC_P@20_diff1(MIRACL)": 0.00031835737236572746, + "nAUC_P@20_max(MIRACL)": 0.08528319658058431, + "nAUC_P@20_std(MIRACL)": 0.07987301340595462, + "nAUC_P@3_diff1(MIRACL)": 0.11480929366447717, + "nAUC_P@3_max(MIRACL)": 0.06622009095674788, + "nAUC_P@3_std(MIRACL)": 0.0846548962491291, + "nAUC_P@5_diff1(MIRACL)": 0.041095658040191316, + "nAUC_P@5_max(MIRACL)": 0.03818686652622348, + "nAUC_P@5_std(MIRACL)": 0.05766702850367041, + "nAUC_Recall@1000_diff1(MIRACL)": -0.028356523888922475, + "nAUC_Recall@1000_max(MIRACL)": 0.2887594990207768, + "nAUC_Recall@1000_std(MIRACL)": 0.21266430754706794, + "nAUC_Recall@100_diff1(MIRACL)": -0.028356523888922475, + "nAUC_Recall@100_max(MIRACL)": 0.2887594990207768, + "nAUC_Recall@100_std(MIRACL)": 0.21266430754706794, + "nAUC_Recall@10_diff1(MIRACL)": 0.06721721895887127, + "nAUC_Recall@10_max(MIRACL)": 0.03206037445096784, + "nAUC_Recall@10_std(MIRACL)": 0.060877937254556855, + "nAUC_Recall@1_diff1(MIRACL)": 0.25695769382707545, + "nAUC_Recall@1_max(MIRACL)": 0.08150630226975161, + "nAUC_Recall@1_std(MIRACL)": 0.026520810018842655, + "nAUC_Recall@20_diff1(MIRACL)": 0.04378157150822391, + "nAUC_Recall@20_max(MIRACL)": 0.06422168550195774, + "nAUC_Recall@20_std(MIRACL)": 0.10325295495403795, + "nAUC_Recall@3_diff1(MIRACL)": 0.15801997149893643, + "nAUC_Recall@3_max(MIRACL)": 0.037340877941018515, + "nAUC_Recall@3_std(MIRACL)": 0.03128835819488035, + "nAUC_Recall@5_diff1(MIRACL)": 0.09962750713182308, + "nAUC_Recall@5_max(MIRACL)": 0.0043879601064250855, + "nAUC_Recall@5_std(MIRACL)": 0.012612183875219526 + } + ] + }, + "task_name": "MIRACLReranking" +} \ No newline at end of file diff --git a/results/avsolatorio__GIST-Embedding-v0/bf6b2e55e92f510a570ad4d7d2da2ec8cd22590c/MIRACLRetrieval.json b/results/avsolatorio__GIST-Embedding-v0/bf6b2e55e92f510a570ad4d7d2da2ec8cd22590c/MIRACLRetrieval.json new file mode 100644 index 000000000..8eb2d0d27 --- /dev/null +++ b/results/avsolatorio__GIST-Embedding-v0/bf6b2e55e92f510a570ad4d7d2da2ec8cd22590c/MIRACLRetrieval.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "main", + "evaluation_time": 181928.02845215797, + "kg_co2_emissions": null, + "mteb_version": "1.19.4", + "scores": { + "dev": [ + { + "hf_subset": "ru", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.03117, + "map_at_1": 0.01365, + "map_at_10": 0.02151, + "map_at_100": 0.02521, + "map_at_1000": 0.0259, + "map_at_20": 0.02295, + "map_at_3": 0.0174, + "map_at_5": 0.01931, + "mrr_at_1": 0.027955271565495207, + "mrr_at_10": 0.0406958390384908, + "mrr_at_100": 0.045008468919462766, + "mrr_at_1000": 0.04561702203979863, + "mrr_at_20": 0.042707331574184854, + "mrr_at_3": 0.034078807241746535, + "mrr_at_5": 0.037473375931842384, + "nauc_map_at_1000_diff1": 0.257502691379278, + "nauc_map_at_1000_max": 0.322955877850376, + "nauc_map_at_1000_std": 0.3413709478159263, + "nauc_map_at_100_diff1": 0.25993997900582705, + "nauc_map_at_100_max": 0.3189478267263155, + "nauc_map_at_100_std": 0.33420606143617604, + "nauc_map_at_10_diff1": 0.2750338698914035, + "nauc_map_at_10_max": 0.2957220469488424, + "nauc_map_at_10_std": 0.28662601613814465, + "nauc_map_at_1_diff1": 0.3928851727009956, + "nauc_map_at_1_max": 0.35901211973452524, + "nauc_map_at_1_std": 0.22127646065504292, + "nauc_map_at_20_diff1": 0.2659836013531062, + "nauc_map_at_20_max": 0.30922839040155314, + "nauc_map_at_20_std": 0.307385904868114, + "nauc_map_at_3_diff1": 0.3215598038302512, + "nauc_map_at_3_max": 0.3022437033168311, + "nauc_map_at_3_std": 0.23854594646540575, + "nauc_map_at_5_diff1": 0.308387557968314, + "nauc_map_at_5_max": 0.3084194344235418, + "nauc_map_at_5_std": 0.2666484875862237, + "nauc_mrr_at_1000_diff1": 0.22042405823585595, + "nauc_mrr_at_1000_max": 0.30155236624596243, + "nauc_mrr_at_1000_std": 0.30037201356942916, + "nauc_mrr_at_100_diff1": 0.22064580953108756, + "nauc_mrr_at_100_max": 0.300599467228226, + "nauc_mrr_at_100_std": 0.2989367848860514, + "nauc_mrr_at_10_diff1": 0.2288199422931353, + "nauc_mrr_at_10_max": 0.2939149994690223, + "nauc_mrr_at_10_std": 0.2758949681163856, + "nauc_mrr_at_1_diff1": 0.28322909654149087, + "nauc_mrr_at_1_max": 0.2862922759153856, + "nauc_mrr_at_1_std": 0.2561526348065721, + "nauc_mrr_at_20_diff1": 0.22400696988912736, + "nauc_mrr_at_20_max": 0.29951928960106206, + "nauc_mrr_at_20_std": 0.2889097659817532, + "nauc_mrr_at_3_diff1": 0.25801682970341855, + "nauc_mrr_at_3_max": 0.29813600916861116, + "nauc_mrr_at_3_std": 0.2587298840829686, + "nauc_mrr_at_5_diff1": 0.2515365425269771, + "nauc_mrr_at_5_max": 0.3007342826947947, + "nauc_mrr_at_5_std": 0.26852408217741275, + "nauc_ndcg_at_1000_diff1": 0.17962869765959785, + "nauc_ndcg_at_1000_max": 0.35752463836132503, + "nauc_ndcg_at_1000_std": 0.44745313877787307, + "nauc_ndcg_at_100_diff1": 0.18160221536352214, + "nauc_ndcg_at_100_max": 0.3305677975204959, + "nauc_ndcg_at_100_std": 0.4166912539454801, + "nauc_ndcg_at_10_diff1": 0.22246909315402685, + "nauc_ndcg_at_10_max": 0.2801068164742353, + "nauc_ndcg_at_10_std": 0.30295178903206327, + "nauc_ndcg_at_1_diff1": 0.28322909654149087, + "nauc_ndcg_at_1_max": 0.2862922759153856, + "nauc_ndcg_at_1_std": 0.2561526348065721, + "nauc_ndcg_at_20_diff1": 0.20822497996180536, + "nauc_ndcg_at_20_max": 0.3108624598455599, + "nauc_ndcg_at_20_std": 0.34891655328796845, + "nauc_ndcg_at_3_diff1": 0.2763028800150276, + "nauc_ndcg_at_3_max": 0.2944143327651395, + "nauc_ndcg_at_3_std": 0.24893756403162082, + "nauc_ndcg_at_5_diff1": 0.2717640842472917, + "nauc_ndcg_at_5_max": 0.30652637091714025, + "nauc_ndcg_at_5_std": 0.28169803869576104, + "nauc_precision_at_1000_diff1": 0.11967137452581932, + "nauc_precision_at_1000_max": 0.37814282112243275, + "nauc_precision_at_1000_std": 0.5051357946940825, + "nauc_precision_at_100_diff1": 0.11167031465094225, + "nauc_precision_at_100_max": 0.36316992142955007, + "nauc_precision_at_100_std": 0.5177843064866507, + "nauc_precision_at_10_diff1": 0.1426453781968446, + "nauc_precision_at_10_max": 0.28614677032220354, + "nauc_precision_at_10_std": 0.3694722616152845, + "nauc_precision_at_1_diff1": 0.28322909654149087, + "nauc_precision_at_1_max": 0.2862922759153856, + "nauc_precision_at_1_std": 0.2561526348065721, + "nauc_precision_at_20_diff1": 0.12398389153988969, + "nauc_precision_at_20_max": 0.339611849927438, + "nauc_precision_at_20_std": 0.44705194471677245, + "nauc_precision_at_3_diff1": 0.20781005436489638, + "nauc_precision_at_3_max": 0.2590100386991451, + "nauc_precision_at_3_std": 0.265680747931602, + "nauc_precision_at_5_diff1": 0.18817468504931864, + "nauc_precision_at_5_max": 0.28456165327552657, + "nauc_precision_at_5_std": 0.32401374547922207, + "nauc_recall_at_1000_diff1": 0.12891955437558916, + "nauc_recall_at_1000_max": 0.3708013992490127, + "nauc_recall_at_1000_std": 0.5002972167425704, + "nauc_recall_at_100_diff1": 0.11084949959275703, + "nauc_recall_at_100_max": 0.32978103366269124, + "nauc_recall_at_100_std": 0.48385712115418206, + "nauc_recall_at_10_diff1": 0.16499073907768158, + "nauc_recall_at_10_max": 0.24522086975745808, + "nauc_recall_at_10_std": 0.32990675664043567, + "nauc_recall_at_1_diff1": 0.3928851727009956, + "nauc_recall_at_1_max": 0.35901211973452524, + "nauc_recall_at_1_std": 0.22127646065504292, + "nauc_recall_at_20_diff1": 0.16047959615792132, + "nauc_recall_at_20_max": 0.31708670177821113, + "nauc_recall_at_20_std": 0.4099586505077514, + "nauc_recall_at_3_diff1": 0.27726456101192193, + "nauc_recall_at_3_max": 0.2702318788898111, + "nauc_recall_at_3_std": 0.24775019954602154, + "nauc_recall_at_5_diff1": 0.2665514256394964, + "nauc_recall_at_5_max": 0.29809289773517994, + "nauc_recall_at_5_std": 0.30379209351782327, + "ndcg_at_1": 0.02796, + "ndcg_at_10": 0.03117, + "ndcg_at_100": 0.05247, + "ndcg_at_1000": 0.07009, + "ndcg_at_20": 0.03674, + "ndcg_at_3": 0.02403, + "ndcg_at_5": 0.02625, + "precision_at_1": 0.02796, + "precision_at_10": 0.00982, + "precision_at_100": 0.00312, + "precision_at_1000": 0.0006, + "precision_at_20": 0.00731, + "precision_at_3": 0.01544, + "precision_at_5": 0.0131, + "recall_at_1": 0.01365, + "recall_at_10": 0.04178, + "recall_at_100": 0.12788, + "recall_at_1000": 0.23871, + "recall_at_20": 0.05874, + "recall_at_3": 0.0211, + "recall_at_5": 0.02848 + } + ] + }, + "task_name": "MIRACLRetrieval" +} \ No newline at end of file diff --git a/results/avsolatorio__GIST-Embedding-v0/bf6b2e55e92f510a570ad4d7d2da2ec8cd22590c/MassiveIntentClassification.json b/results/avsolatorio__GIST-Embedding-v0/bf6b2e55e92f510a570ad4d7d2da2ec8cd22590c/MassiveIntentClassification.json new file mode 100644 index 000000000..8edf73bf9 --- /dev/null +++ b/results/avsolatorio__GIST-Embedding-v0/bf6b2e55e92f510a570ad4d7d2da2ec8cd22590c/MassiveIntentClassification.json @@ -0,0 +1,137 @@ +{ + "dataset_revision": "4672e20407010da34463acc759c162ca9734bca6", + "evaluation_time": 56.10311555862427, + "kg_co2_emissions": null, + "mteb_version": "1.19.2", + "scores": { + "test": [ + { + "accuracy": 0.31281102891728313, + "f1": 0.2818318390855832, + "f1_weighted": 0.2883537570593143, + "hf_subset": "ru", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.31281102891728313, + "scores_per_experiment": [ + { + "accuracy": 0.3238063214525891, + "f1": 0.3016677606458697, + "f1_weighted": 0.294376122124558 + }, + { + "accuracy": 0.312373907195696, + "f1": 0.2815534109757773, + "f1_weighted": 0.28958248160296896 + }, + { + "accuracy": 0.3069939475453934, + "f1": 0.2900615759603937, + "f1_weighted": 0.28879043197553805 + }, + { + "accuracy": 0.30497646267652995, + "f1": 0.2640428099021538, + "f1_weighted": 0.28412731116719364 + }, + { + "accuracy": 0.3406186953597848, + "f1": 0.295183195352672, + "f1_weighted": 0.31604625004386205 + }, + { + "accuracy": 0.2915265635507734, + "f1": 0.25587550204153986, + "f1_weighted": 0.27534843547064186 + }, + { + "accuracy": 0.31002017484868866, + "f1": 0.2863754962632543, + "f1_weighted": 0.2859610128278995 + }, + { + "accuracy": 0.3073301950235373, + "f1": 0.2713466784530624, + "f1_weighted": 0.27772235105704474 + }, + { + "accuracy": 0.2965702757229321, + "f1": 0.27179307168919714, + "f1_weighted": 0.26429254424396653 + }, + { + "accuracy": 0.3338937457969065, + "f1": 0.30041888957191176, + "f1_weighted": 0.30729063007946944 + } + ] + } + ], + "validation": [ + { + "accuracy": 0.3122970978848992, + "f1": 0.2767086067308245, + "f1_weighted": 0.29046449249745737, + "hf_subset": "ru", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.3122970978848992, + "scores_per_experiment": [ + { + "accuracy": 0.30545991146089524, + "f1": 0.2793609033189319, + "f1_weighted": 0.2785324900738541 + }, + { + "accuracy": 0.3034923757993114, + "f1": 0.26847102018667274, + "f1_weighted": 0.2801336556934657 + }, + { + "accuracy": 0.3157894736842105, + "f1": 0.2899203435341625, + "f1_weighted": 0.3053547570327551 + }, + { + "accuracy": 0.30644367929168714, + "f1": 0.2618600980299359, + "f1_weighted": 0.2828577127510715 + }, + { + "accuracy": 0.33448106246925724, + "f1": 0.2813645133665386, + "f1_weighted": 0.31142120822248376 + }, + { + "accuracy": 0.2985735366453517, + "f1": 0.26918388107809854, + "f1_weighted": 0.2842832098289532 + }, + { + "accuracy": 0.32267584849975406, + "f1": 0.29020604439146963, + "f1_weighted": 0.3022471383931498 + }, + { + "accuracy": 0.30644367929168714, + "f1": 0.26468556529101467, + "f1_weighted": 0.28394335459496545 + }, + { + "accuracy": 0.2951303492375799, + "f1": 0.2745526838876486, + "f1_weighted": 0.2673737181399241 + }, + { + "accuracy": 0.33448106246925724, + "f1": 0.2874810142237714, + "f1_weighted": 0.3084976802439507 + } + ] + } + ] + }, + "task_name": "MassiveIntentClassification" +} \ No newline at end of file diff --git a/results/avsolatorio__GIST-Embedding-v0/bf6b2e55e92f510a570ad4d7d2da2ec8cd22590c/MassiveScenarioClassification.json b/results/avsolatorio__GIST-Embedding-v0/bf6b2e55e92f510a570ad4d7d2da2ec8cd22590c/MassiveScenarioClassification.json new file mode 100644 index 000000000..91f899fa1 --- /dev/null +++ b/results/avsolatorio__GIST-Embedding-v0/bf6b2e55e92f510a570ad4d7d2da2ec8cd22590c/MassiveScenarioClassification.json @@ -0,0 +1,137 @@ +{ + "dataset_revision": "fad2c6e8459f9e1c45d9315f4953d921437d70f8", + "evaluation_time": 32.806586265563965, + "kg_co2_emissions": null, + "mteb_version": "1.19.2", + "scores": { + "test": [ + { + "accuracy": 0.3642905178211163, + "f1": 0.3392313193902316, + "f1_weighted": 0.3457909204677908, + "hf_subset": "ru", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.3642905178211163, + "scores_per_experiment": [ + { + "accuracy": 0.36583725622057833, + "f1": 0.34697182901572, + "f1_weighted": 0.3444778893143892 + }, + { + "accuracy": 0.34667114996637527, + "f1": 0.3218448766016233, + "f1_weighted": 0.3235188627294629 + }, + { + "accuracy": 0.3762609280430397, + "f1": 0.3447620687696501, + "f1_weighted": 0.35923586608789043 + }, + { + "accuracy": 0.37491593813046403, + "f1": 0.34340425800102575, + "f1_weighted": 0.3629645842512604 + }, + { + "accuracy": 0.3624747814391392, + "f1": 0.33789469966393226, + "f1_weighted": 0.3410742050031575 + }, + { + "accuracy": 0.34700739744451914, + "f1": 0.3187169018129359, + "f1_weighted": 0.3224815754888547 + }, + { + "accuracy": 0.36617350369872226, + "f1": 0.34003207469873936, + "f1_weighted": 0.3507067460546692 + }, + { + "accuracy": 0.3695359784801614, + "f1": 0.35260055838847126, + "f1_weighted": 0.34979385561417387 + }, + { + "accuracy": 0.34667114996637527, + "f1": 0.32329452984220336, + "f1_weighted": 0.32982637460853054 + }, + { + "accuracy": 0.3873570948217888, + "f1": 0.3627913971080149, + "f1_weighted": 0.3738292455255192 + } + ] + } + ], + "validation": [ + { + "accuracy": 0.361829808165273, + "f1": 0.34162649825507574, + "f1_weighted": 0.3405604655066674, + "hf_subset": "ru", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.361829808165273, + "scores_per_experiment": [ + { + "accuracy": 0.35809149040826366, + "f1": 0.34427396665739296, + "f1_weighted": 0.33587068047918206 + }, + { + "accuracy": 0.33841613379242497, + "f1": 0.32088229779804217, + "f1_weighted": 0.3109422049945239 + }, + { + "accuracy": 0.38465322183964584, + "f1": 0.35765341742457113, + "f1_weighted": 0.36589417320851175 + }, + { + "accuracy": 0.3753074274471225, + "f1": 0.34409702944396664, + "f1_weighted": 0.3580173340783647 + }, + { + "accuracy": 0.35809149040826366, + "f1": 0.3401300534414439, + "f1_weighted": 0.3299072352960808 + }, + { + "accuracy": 0.3497294638465322, + "f1": 0.3217344696595402, + "f1_weighted": 0.33504191455611426 + }, + { + "accuracy": 0.3630103295622233, + "f1": 0.3445320691979703, + "f1_weighted": 0.34178713227306284 + }, + { + "accuracy": 0.3630103295622233, + "f1": 0.34704929995226985, + "f1_weighted": 0.33742996897777866 + }, + { + "accuracy": 0.3364485981308411, + "f1": 0.32239065609098055, + "f1_weighted": 0.31777840934389723 + }, + { + "accuracy": 0.3915395966551894, + "f1": 0.3735217228845796, + "f1_weighted": 0.372935601859158 + } + ] + } + ] + }, + "task_name": "MassiveScenarioClassification" +} \ No newline at end of file diff --git a/results/avsolatorio__GIST-Embedding-v0/bf6b2e55e92f510a570ad4d7d2da2ec8cd22590c/RUParaPhraserSTS.json b/results/avsolatorio__GIST-Embedding-v0/bf6b2e55e92f510a570ad4d7d2da2ec8cd22590c/RUParaPhraserSTS.json new file mode 100644 index 000000000..ad21046e6 --- /dev/null +++ b/results/avsolatorio__GIST-Embedding-v0/bf6b2e55e92f510a570ad4d7d2da2ec8cd22590c/RUParaPhraserSTS.json @@ -0,0 +1,26 @@ +{ + "dataset_revision": "43265056790b8f7c59e0139acb4be0a8dad2c8f4", + "evaluation_time": 11.849887371063232, + "kg_co2_emissions": null, + "mteb_version": "1.19.2", + "scores": { + "test": [ + { + "cosine_pearson": 0.49222202561401174, + "cosine_spearman": 0.4966047857163402, + "euclidean_pearson": 0.5066525524774866, + "euclidean_spearman": 0.4966047857163402, + "hf_subset": "default", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.4966047857163402, + "manhattan_pearson": 0.5044463373060302, + "manhattan_spearman": 0.49384537924609784, + "pearson": 0.49222202561401174, + "spearman": 0.4966047857163402 + } + ] + }, + "task_name": "RUParaPhraserSTS" +} \ No newline at end of file diff --git a/results/avsolatorio__GIST-Embedding-v0/bf6b2e55e92f510a570ad4d7d2da2ec8cd22590c/RiaNewsRetrieval.json b/results/avsolatorio__GIST-Embedding-v0/bf6b2e55e92f510a570ad4d7d2da2ec8cd22590c/RiaNewsRetrieval.json new file mode 100644 index 000000000..b1ed2377c --- /dev/null +++ b/results/avsolatorio__GIST-Embedding-v0/bf6b2e55e92f510a570ad4d7d2da2ec8cd22590c/RiaNewsRetrieval.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "82374b0bbacda6114f39ff9c5b925fa1512ca5d7", + "evaluation_time": 5114.905699014664, + "kg_co2_emissions": null, + "mteb_version": "1.19.2", + "scores": { + "test": [ + { + "hf_subset": "default", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.13748, + "map_at_1": 0.0873, + "map_at_10": 0.11895, + "map_at_100": 0.12412, + "map_at_1000": 0.12474, + "map_at_20": 0.12168, + "map_at_3": 0.10823, + "map_at_5": 0.11415, + "mrr_at_1": 0.0871, + "mrr_at_10": 0.1188381746031746, + "mrr_at_100": 0.12400476381534634, + "mrr_at_1000": 0.12463376096734083, + "mrr_at_20": 0.12157594398162896, + "mrr_at_3": 0.10811666666666668, + "mrr_at_5": 0.11403666666666666, + "nauc_map_at_1000_diff1": 0.3365453027327383, + "nauc_map_at_1000_max": 0.22432012444825095, + "nauc_map_at_1000_std": 0.20160469840455508, + "nauc_map_at_100_diff1": 0.3369829424957315, + "nauc_map_at_100_max": 0.22446475092037768, + "nauc_map_at_100_std": 0.20148995083179488, + "nauc_map_at_10_diff1": 0.3430999571779901, + "nauc_map_at_10_max": 0.22536301982306617, + "nauc_map_at_10_std": 0.19736049143697582, + "nauc_map_at_1_diff1": 0.426410353105769, + "nauc_map_at_1_max": 0.24818274496714637, + "nauc_map_at_1_std": 0.18031417435874744, + "nauc_map_at_20_diff1": 0.3397852362439211, + "nauc_map_at_20_max": 0.22559015178813868, + "nauc_map_at_20_std": 0.20006700553014092, + "nauc_map_at_3_diff1": 0.3686667441825634, + "nauc_map_at_3_max": 0.2357942235188482, + "nauc_map_at_3_std": 0.1954133041693481, + "nauc_map_at_5_diff1": 0.35470626732086524, + "nauc_map_at_5_max": 0.22990501200143412, + "nauc_map_at_5_std": 0.1963598299033195, + "nauc_mrr_at_1000_diff1": 0.3374237609250784, + "nauc_mrr_at_1000_max": 0.22420884492308718, + "nauc_mrr_at_1000_std": 0.20126567965165532, + "nauc_mrr_at_100_diff1": 0.33786209867310224, + "nauc_mrr_at_100_max": 0.22435359375620972, + "nauc_mrr_at_100_std": 0.201150756984768, + "nauc_mrr_at_10_diff1": 0.3439953687486192, + "nauc_mrr_at_10_max": 0.22524922276992607, + "nauc_mrr_at_10_std": 0.197015651920589, + "nauc_mrr_at_1_diff1": 0.42870917132157227, + "nauc_mrr_at_1_max": 0.2478407943769626, + "nauc_mrr_at_1_std": 0.17959643486556823, + "nauc_mrr_at_20_diff1": 0.3406553342383306, + "nauc_mrr_at_20_max": 0.22549094042841814, + "nauc_mrr_at_20_std": 0.1997522580695578, + "nauc_mrr_at_3_diff1": 0.3696718065225796, + "nauc_mrr_at_3_max": 0.23569635196755176, + "nauc_mrr_at_3_std": 0.1950390125000601, + "nauc_mrr_at_5_diff1": 0.3556342609204071, + "nauc_mrr_at_5_max": 0.2297942182692602, + "nauc_mrr_at_5_std": 0.19600926305905167, + "nauc_ndcg_at_1000_diff1": 0.2732941701598263, + "nauc_ndcg_at_1000_max": 0.20710370570901807, + "nauc_ndcg_at_1000_std": 0.2224577423128696, + "nauc_ndcg_at_100_diff1": 0.2830153432084584, + "nauc_ndcg_at_100_max": 0.20928846057957504, + "nauc_ndcg_at_100_std": 0.22135646846113446, + "nauc_ndcg_at_10_diff1": 0.30716610128998345, + "nauc_ndcg_at_10_max": 0.21359423817153847, + "nauc_ndcg_at_10_std": 0.20271588041919456, + "nauc_ndcg_at_1_diff1": 0.426410353105769, + "nauc_ndcg_at_1_max": 0.24818274496714637, + "nauc_ndcg_at_1_std": 0.18031417435874744, + "nauc_ndcg_at_20_diff1": 0.2977664286935497, + "nauc_ndcg_at_20_max": 0.21463866765134215, + "nauc_ndcg_at_20_std": 0.21173454093227914, + "nauc_ndcg_at_3_diff1": 0.3534080745228557, + "nauc_ndcg_at_3_max": 0.23314770228073184, + "nauc_ndcg_at_3_std": 0.19967651056819646, + "nauc_ndcg_at_5_diff1": 0.3308609680822211, + "nauc_ndcg_at_5_max": 0.22322279678498724, + "nauc_ndcg_at_5_std": 0.20084595876671754, + "nauc_precision_at_1000_diff1": 0.11861780708871764, + "nauc_precision_at_1000_max": 0.16754766416680686, + "nauc_precision_at_1000_std": 0.27452146705529257, + "nauc_precision_at_100_diff1": 0.16891607389848493, + "nauc_precision_at_100_max": 0.17745105988317728, + "nauc_precision_at_100_std": 0.2709996391836234, + "nauc_precision_at_10_diff1": 0.22873247080290277, + "nauc_precision_at_10_max": 0.1870777498626921, + "nauc_precision_at_10_std": 0.21373359081680038, + "nauc_precision_at_1_diff1": 0.426410353105769, + "nauc_precision_at_1_max": 0.24818274496714637, + "nauc_precision_at_1_std": 0.18031417435874744, + "nauc_precision_at_20_diff1": 0.20869457467610028, + "nauc_precision_at_20_max": 0.19205811318834354, + "nauc_precision_at_20_std": 0.23895144642330982, + "nauc_precision_at_3_diff1": 0.3164041200987156, + "nauc_precision_at_3_max": 0.226896869863769, + "nauc_precision_at_3_std": 0.21008814770204617, + "nauc_precision_at_5_diff1": 0.275644460797996, + "nauc_precision_at_5_max": 0.2073949384611652, + "nauc_precision_at_5_std": 0.21086701142857248, + "nauc_recall_at_1000_diff1": 0.11861780708871777, + "nauc_recall_at_1000_max": 0.16754766416680728, + "nauc_recall_at_1000_std": 0.27452146705529284, + "nauc_recall_at_100_diff1": 0.16891607389848484, + "nauc_recall_at_100_max": 0.17745105988317705, + "nauc_recall_at_100_std": 0.27099963918362313, + "nauc_recall_at_10_diff1": 0.22873247080290288, + "nauc_recall_at_10_max": 0.1870777498626922, + "nauc_recall_at_10_std": 0.21373359081680032, + "nauc_recall_at_1_diff1": 0.426410353105769, + "nauc_recall_at_1_max": 0.24818274496714637, + "nauc_recall_at_1_std": 0.18031417435874744, + "nauc_recall_at_20_diff1": 0.2086945746761003, + "nauc_recall_at_20_max": 0.19205811318834334, + "nauc_recall_at_20_std": 0.23895144642330993, + "nauc_recall_at_3_diff1": 0.3164041200987157, + "nauc_recall_at_3_max": 0.226896869863769, + "nauc_recall_at_3_std": 0.210088147702046, + "nauc_recall_at_5_diff1": 0.2756444607979961, + "nauc_recall_at_5_max": 0.20739493846116516, + "nauc_recall_at_5_std": 0.21086701142857253, + "ndcg_at_1": 0.0873, + "ndcg_at_10": 0.13748, + "ndcg_at_100": 0.16575, + "ndcg_at_1000": 0.1855, + "ndcg_at_20": 0.14752, + "ndcg_at_3": 0.11518, + "ndcg_at_5": 0.12582, + "precision_at_1": 0.0873, + "precision_at_10": 0.01972, + "precision_at_100": 0.00337, + "precision_at_1000": 0.0005, + "precision_at_20": 0.01186, + "precision_at_3": 0.0451, + "precision_at_5": 0.03222, + "recall_at_1": 0.0873, + "recall_at_10": 0.1972, + "recall_at_100": 0.3367, + "recall_at_1000": 0.4988, + "recall_at_20": 0.2371, + "recall_at_3": 0.1353, + "recall_at_5": 0.1611 + } + ] + }, + "task_name": "RiaNewsRetrieval" +} \ No newline at end of file diff --git a/results/avsolatorio__GIST-Embedding-v0/bf6b2e55e92f510a570ad4d7d2da2ec8cd22590c/RuBQReranking.json b/results/avsolatorio__GIST-Embedding-v0/bf6b2e55e92f510a570ad4d7d2da2ec8cd22590c/RuBQReranking.json new file mode 100644 index 000000000..0ed951579 --- /dev/null +++ b/results/avsolatorio__GIST-Embedding-v0/bf6b2e55e92f510a570ad4d7d2da2ec8cd22590c/RuBQReranking.json @@ -0,0 +1,26 @@ +{ + "dataset_revision": "2e96b8f098fa4b0950fc58eacadeb31c0d0c7fa2", + "evaluation_time": 206.9095892906189, + "kg_co2_emissions": null, + "mteb_version": "1.19.2", + "scores": { + "test": [ + { + "hf_subset": "default", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.36595660133238134, + "map": 0.36595660133238134, + "mrr": 0.4062975243826307, + "nAUC_map_diff1": 0.15116356452689286, + "nAUC_map_max": 0.14089754512122496, + "nAUC_map_std": 0.1053770755205493, + "nAUC_mrr_diff1": 0.14312377595592476, + "nAUC_mrr_max": 0.13256306985702837, + "nAUC_mrr_std": 0.08265205636921726 + } + ] + }, + "task_name": "RuBQReranking" +} \ No newline at end of file diff --git a/results/avsolatorio__GIST-Embedding-v0/bf6b2e55e92f510a570ad4d7d2da2ec8cd22590c/RuBQRetrieval.json b/results/avsolatorio__GIST-Embedding-v0/bf6b2e55e92f510a570ad4d7d2da2ec8cd22590c/RuBQRetrieval.json new file mode 100644 index 000000000..3385ed28b --- /dev/null +++ b/results/avsolatorio__GIST-Embedding-v0/bf6b2e55e92f510a570ad4d7d2da2ec8cd22590c/RuBQRetrieval.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "e19b6ffa60b3bc248e0b41f4cc37c26a55c2a67b", + "evaluation_time": 297.71597170829773, + "kg_co2_emissions": null, + "mteb_version": "1.19.2", + "scores": { + "test": [ + { + "hf_subset": "default", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.07999, + "map_at_1": 0.03716, + "map_at_10": 0.06185, + "map_at_100": 0.06608, + "map_at_1000": 0.06681, + "map_at_20": 0.06384, + "map_at_3": 0.05371, + "map_at_5": 0.05842, + "mrr_at_1": 0.057328605200945626, + "mrr_at_10": 0.085477644564524, + "mrr_at_100": 0.09020629648496871, + "mrr_at_1000": 0.09094217386915024, + "mrr_at_20": 0.08763817847444859, + "mrr_at_3": 0.07584712371946416, + "mrr_at_5": 0.08110717100078801, + "nauc_map_at_1000_diff1": 0.385489746405905, + "nauc_map_at_1000_max": 0.3296334745691113, + "nauc_map_at_1000_std": 0.19647834123970387, + "nauc_map_at_100_diff1": 0.3861931471087145, + "nauc_map_at_100_max": 0.3298381487895035, + "nauc_map_at_100_std": 0.1954826175292499, + "nauc_map_at_10_diff1": 0.40049890283226375, + "nauc_map_at_10_max": 0.33717817360374125, + "nauc_map_at_10_std": 0.19170896057960446, + "nauc_map_at_1_diff1": 0.4930118797970527, + "nauc_map_at_1_max": 0.3526916980145528, + "nauc_map_at_1_std": 0.1394006235139239, + "nauc_map_at_20_diff1": 0.39396519600011914, + "nauc_map_at_20_max": 0.3328257001970114, + "nauc_map_at_20_std": 0.1933185833257439, + "nauc_map_at_3_diff1": 0.4336920452473483, + "nauc_map_at_3_max": 0.349061919823475, + "nauc_map_at_3_std": 0.18017588227147693, + "nauc_map_at_5_diff1": 0.4115642079021201, + "nauc_map_at_5_max": 0.3445903221387509, + "nauc_map_at_5_std": 0.18830913047946313, + "nauc_mrr_at_1000_diff1": 0.3580796003390215, + "nauc_mrr_at_1000_max": 0.3127453620033407, + "nauc_mrr_at_1000_std": 0.14819121211532785, + "nauc_mrr_at_100_diff1": 0.3586220032456663, + "nauc_mrr_at_100_max": 0.3130115511512007, + "nauc_mrr_at_100_std": 0.14774541972207142, + "nauc_mrr_at_10_diff1": 0.3693543691462704, + "nauc_mrr_at_10_max": 0.31941681707638725, + "nauc_mrr_at_10_std": 0.1441519411394712, + "nauc_mrr_at_1_diff1": 0.4482080061615412, + "nauc_mrr_at_1_max": 0.3499727835165633, + "nauc_mrr_at_1_std": 0.09793478087090934, + "nauc_mrr_at_20_diff1": 0.3641668319943668, + "nauc_mrr_at_20_max": 0.31554563176821865, + "nauc_mrr_at_20_std": 0.1458956474613942, + "nauc_mrr_at_3_diff1": 0.3956100475786266, + "nauc_mrr_at_3_max": 0.3305009084233761, + "nauc_mrr_at_3_std": 0.13659832408347147, + "nauc_mrr_at_5_diff1": 0.3802200489372644, + "nauc_mrr_at_5_max": 0.3286325382111682, + "nauc_mrr_at_5_std": 0.14253346470715011, + "nauc_ndcg_at_1000_diff1": 0.3017527338102732, + "nauc_ndcg_at_1000_max": 0.29226257175202053, + "nauc_ndcg_at_1000_std": 0.22643933971884164, + "nauc_ndcg_at_100_diff1": 0.3113186458875538, + "nauc_ndcg_at_100_max": 0.290669863507816, + "nauc_ndcg_at_100_std": 0.204010269785058, + "nauc_ndcg_at_10_diff1": 0.35824866875783423, + "nauc_ndcg_at_10_max": 0.3174532931330254, + "nauc_ndcg_at_10_std": 0.19142462727534412, + "nauc_ndcg_at_1_diff1": 0.46973626944114294, + "nauc_ndcg_at_1_max": 0.3434095386443362, + "nauc_ndcg_at_1_std": 0.09739805150358201, + "nauc_ndcg_at_20_diff1": 0.3424729485054116, + "nauc_ndcg_at_20_max": 0.30549287527389574, + "nauc_ndcg_at_20_std": 0.19407651902773512, + "nauc_ndcg_at_3_diff1": 0.40637175168930995, + "nauc_ndcg_at_3_max": 0.3368921666456892, + "nauc_ndcg_at_3_std": 0.17592743694397248, + "nauc_ndcg_at_5_diff1": 0.37849031631011265, + "nauc_ndcg_at_5_max": 0.333637786007359, + "nauc_ndcg_at_5_std": 0.1879507427853588, + "nauc_precision_at_1000_diff1": 0.1357525435841633, + "nauc_precision_at_1000_max": 0.20472288918742598, + "nauc_precision_at_1000_std": 0.28808536589579836, + "nauc_precision_at_100_diff1": 0.17884531297482986, + "nauc_precision_at_100_max": 0.23187190566036614, + "nauc_precision_at_100_std": 0.24732437924645476, + "nauc_precision_at_10_diff1": 0.27897811412042406, + "nauc_precision_at_10_max": 0.29204675695413224, + "nauc_precision_at_10_std": 0.20704809131671434, + "nauc_precision_at_1_diff1": 0.46973626944114294, + "nauc_precision_at_1_max": 0.3434095386443362, + "nauc_precision_at_1_std": 0.09739805150358201, + "nauc_precision_at_20_diff1": 0.24362972785428538, + "nauc_precision_at_20_max": 0.2569664430053101, + "nauc_precision_at_20_std": 0.20878227094943638, + "nauc_precision_at_3_diff1": 0.3784416289930288, + "nauc_precision_at_3_max": 0.3441964001137959, + "nauc_precision_at_3_std": 0.17944565232251747, + "nauc_precision_at_5_diff1": 0.3283099970139272, + "nauc_precision_at_5_max": 0.3323594818715414, + "nauc_precision_at_5_std": 0.20390917270859446, + "nauc_recall_at_1000_diff1": 0.1659588914627122, + "nauc_recall_at_1000_max": 0.2250435363438709, + "nauc_recall_at_1000_std": 0.2817909020954406, + "nauc_recall_at_100_diff1": 0.19191846400250734, + "nauc_recall_at_100_max": 0.2169979679241474, + "nauc_recall_at_100_std": 0.22092592192042665, + "nauc_recall_at_10_diff1": 0.29002543834771116, + "nauc_recall_at_10_max": 0.2826176286204075, + "nauc_recall_at_10_std": 0.2131146374348301, + "nauc_recall_at_1_diff1": 0.4930118797970527, + "nauc_recall_at_1_max": 0.3526916980145528, + "nauc_recall_at_1_std": 0.1394006235139239, + "nauc_recall_at_20_diff1": 0.25867644735177964, + "nauc_recall_at_20_max": 0.2557532503092326, + "nauc_recall_at_20_std": 0.20958654974542046, + "nauc_recall_at_3_diff1": 0.3823287561853335, + "nauc_recall_at_3_max": 0.33625086528725173, + "nauc_recall_at_3_std": 0.20544913076435908, + "nauc_recall_at_5_diff1": 0.3251144344658206, + "nauc_recall_at_5_max": 0.31806232252343114, + "nauc_recall_at_5_std": 0.21336111017181394, + "ndcg_at_1": 0.05615, + "ndcg_at_10": 0.07999, + "ndcg_at_100": 0.10322, + "ndcg_at_1000": 0.12686, + "ndcg_at_20": 0.0872, + "ndcg_at_3": 0.06398, + "ndcg_at_5": 0.07168, + "precision_at_1": 0.05615, + "precision_at_10": 0.01791, + "precision_at_100": 0.00346, + "precision_at_1000": 0.00064, + "precision_at_20": 0.01099, + "precision_at_3": 0.03664, + "precision_at_5": 0.02801, + "recall_at_1": 0.03716, + "recall_at_10": 0.11119, + "recall_at_100": 0.21249, + "recall_at_1000": 0.38479, + "recall_at_20": 0.13655, + "recall_at_3": 0.07068, + "recall_at_5": 0.08884 + } + ] + }, + "task_name": "RuBQRetrieval" +} \ No newline at end of file diff --git a/results/avsolatorio__GIST-Embedding-v0/bf6b2e55e92f510a570ad4d7d2da2ec8cd22590c/RuReviewsClassification.json b/results/avsolatorio__GIST-Embedding-v0/bf6b2e55e92f510a570ad4d7d2da2ec8cd22590c/RuReviewsClassification.json new file mode 100644 index 000000000..cbff630a1 --- /dev/null +++ b/results/avsolatorio__GIST-Embedding-v0/bf6b2e55e92f510a570ad4d7d2da2ec8cd22590c/RuReviewsClassification.json @@ -0,0 +1,73 @@ +{ + "dataset_revision": "f6d2c31f4dc6b88f468552750bfec05b4b41b05a", + "evaluation_time": 12.88510012626648, + "kg_co2_emissions": null, + "mteb_version": "1.19.2", + "scores": { + "test": [ + { + "accuracy": 0.433203125, + "f1": 0.42942779195142045, + "f1_weighted": 0.4294193156114181, + "hf_subset": "default", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.433203125, + "scores_per_experiment": [ + { + "accuracy": 0.4375, + "f1": 0.4272511964667314, + "f1_weighted": 0.42721406551458674 + }, + { + "accuracy": 0.4228515625, + "f1": 0.41845386080546293, + "f1_weighted": 0.4184614850555495 + }, + { + "accuracy": 0.42431640625, + "f1": 0.4257651170178969, + "f1_weighted": 0.4257495011185535 + }, + { + "accuracy": 0.48388671875, + "f1": 0.4826082393426141, + "f1_weighted": 0.48261915594091376 + }, + { + "accuracy": 0.4658203125, + "f1": 0.45974963367644134, + "f1_weighted": 0.4597299801772599 + }, + { + "accuracy": 0.44189453125, + "f1": 0.44274134898443324, + "f1_weighted": 0.4427510424927441 + }, + { + "accuracy": 0.40625, + "f1": 0.3990142048570459, + "f1_weighted": 0.3990132964436126 + }, + { + "accuracy": 0.4619140625, + "f1": 0.4575037114476396, + "f1_weighted": 0.4574967678800208 + }, + { + "accuracy": 0.34033203125, + "f1": 0.335288262599644, + "f1_weighted": 0.33528327876297914 + }, + { + "accuracy": 0.447265625, + "f1": 0.44590234431629544, + "f1_weighted": 0.4458745827279605 + } + ] + } + ] + }, + "task_name": "RuReviewsClassification" +} \ No newline at end of file diff --git a/results/avsolatorio__GIST-Embedding-v0/bf6b2e55e92f510a570ad4d7d2da2ec8cd22590c/RuSTSBenchmarkSTS.json b/results/avsolatorio__GIST-Embedding-v0/bf6b2e55e92f510a570ad4d7d2da2ec8cd22590c/RuSTSBenchmarkSTS.json new file mode 100644 index 000000000..34ef83f91 --- /dev/null +++ b/results/avsolatorio__GIST-Embedding-v0/bf6b2e55e92f510a570ad4d7d2da2ec8cd22590c/RuSTSBenchmarkSTS.json @@ -0,0 +1,26 @@ +{ + "dataset_revision": "7cf24f325c6da6195df55bef3d86b5e0616f3018", + "evaluation_time": 7.820411443710327, + "kg_co2_emissions": null, + "mteb_version": "1.19.2", + "scores": { + "test": [ + { + "cosine_pearson": 0.620005311119234, + "cosine_spearman": 0.608145515890056, + "euclidean_pearson": 0.6110197972605675, + "euclidean_spearman": 0.608145515890056, + "hf_subset": "default", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.608145515890056, + "manhattan_pearson": 0.6106799540057721, + "manhattan_spearman": 0.6081736626149064, + "pearson": 0.620005311119234, + "spearman": 0.608145515890056 + } + ] + }, + "task_name": "RuSTSBenchmarkSTS" +} \ No newline at end of file diff --git a/results/avsolatorio__GIST-Embedding-v0/bf6b2e55e92f510a570ad4d7d2da2ec8cd22590c/RuSciBenchGRNTIClassification.json b/results/avsolatorio__GIST-Embedding-v0/bf6b2e55e92f510a570ad4d7d2da2ec8cd22590c/RuSciBenchGRNTIClassification.json new file mode 100644 index 000000000..5aec7f434 --- /dev/null +++ b/results/avsolatorio__GIST-Embedding-v0/bf6b2e55e92f510a570ad4d7d2da2ec8cd22590c/RuSciBenchGRNTIClassification.json @@ -0,0 +1,73 @@ +{ + "dataset_revision": "673a610d6d3dd91a547a0d57ae1b56f37ebbf6a1", + "evaluation_time": 39.11285328865051, + "kg_co2_emissions": null, + "mteb_version": "1.19.2", + "scores": { + "test": [ + { + "accuracy": 0.1751953125, + "f1": 0.15470480582547086, + "f1_weighted": 0.15485655128662157, + "hf_subset": "default", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.1751953125, + "scores_per_experiment": [ + { + "accuracy": 0.19091796875, + "f1": 0.17556648610678302, + "f1_weighted": 0.17565333422907453 + }, + { + "accuracy": 0.1611328125, + "f1": 0.13950817329613918, + "f1_weighted": 0.13970893906714935 + }, + { + "accuracy": 0.18603515625, + "f1": 0.16985314267459298, + "f1_weighted": 0.1700372435332325 + }, + { + "accuracy": 0.16796875, + "f1": 0.14589560451524197, + "f1_weighted": 0.14601024705909785 + }, + { + "accuracy": 0.18115234375, + "f1": 0.15738826581794088, + "f1_weighted": 0.15746694012655463 + }, + { + "accuracy": 0.1826171875, + "f1": 0.16128988760053353, + "f1_weighted": 0.1614557860927248 + }, + { + "accuracy": 0.177734375, + "f1": 0.15756932287941092, + "f1_weighted": 0.15771383501999764 + }, + { + "accuracy": 0.15625, + "f1": 0.13644648225350395, + "f1_weighted": 0.13662814823148975 + }, + { + "accuracy": 0.16845703125, + "f1": 0.1471633524578529, + "f1_weighted": 0.14736259338023985 + }, + { + "accuracy": 0.1796875, + "f1": 0.15636734065270927, + "f1_weighted": 0.15652844612665462 + } + ] + } + ] + }, + "task_name": "RuSciBenchGRNTIClassification" +} \ No newline at end of file diff --git a/results/avsolatorio__GIST-Embedding-v0/bf6b2e55e92f510a570ad4d7d2da2ec8cd22590c/RuSciBenchGRNTIClusteringP2P.json b/results/avsolatorio__GIST-Embedding-v0/bf6b2e55e92f510a570ad4d7d2da2ec8cd22590c/RuSciBenchGRNTIClusteringP2P.json new file mode 100644 index 000000000..4541bf377 --- /dev/null +++ b/results/avsolatorio__GIST-Embedding-v0/bf6b2e55e92f510a570ad4d7d2da2ec8cd22590c/RuSciBenchGRNTIClusteringP2P.json @@ -0,0 +1,34 @@ +{ + "dataset_revision": "673a610d6d3dd91a547a0d57ae1b56f37ebbf6a1", + "evaluation_time": 83.11611366271973, + "kg_co2_emissions": null, + "mteb_version": "1.19.2", + "scores": { + "test": [ + { + "hf_subset": "default", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.1697255436034443, + "v_measure": 0.1697255436034443, + "v_measure_std": 0.006384382554195144, + "v_measures": { + "Level 0": [ + 0.170418121516057, + 0.1814863027046673, + 0.1603819618398447, + 0.17749638659852143, + 0.16786487010216047, + 0.16749409548563196, + 0.16776503659734338, + 0.17400716257905355, + 0.1602204892817019, + 0.17012100932946128 + ] + } + } + ] + }, + "task_name": "RuSciBenchGRNTIClusteringP2P" +} \ No newline at end of file diff --git a/results/avsolatorio__GIST-Embedding-v0/bf6b2e55e92f510a570ad4d7d2da2ec8cd22590c/RuSciBenchOECDClassification.json b/results/avsolatorio__GIST-Embedding-v0/bf6b2e55e92f510a570ad4d7d2da2ec8cd22590c/RuSciBenchOECDClassification.json new file mode 100644 index 000000000..38eb929d1 --- /dev/null +++ b/results/avsolatorio__GIST-Embedding-v0/bf6b2e55e92f510a570ad4d7d2da2ec8cd22590c/RuSciBenchOECDClassification.json @@ -0,0 +1,73 @@ +{ + "dataset_revision": "26c88e99dcaba32bb45d0e1bfc21902337f6d471", + "evaluation_time": 39.18341279029846, + "kg_co2_emissions": null, + "mteb_version": "1.19.2", + "scores": { + "test": [ + { + "accuracy": 0.1361328125, + "f1": 0.12138635311919925, + "f1_weighted": 0.1213946301139565, + "hf_subset": "default", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.1361328125, + "scores_per_experiment": [ + { + "accuracy": 0.11376953125, + "f1": 0.10384873828009245, + "f1_weighted": 0.10368945308940578 + }, + { + "accuracy": 0.130859375, + "f1": 0.1171301494850572, + "f1_weighted": 0.11714481706320679 + }, + { + "accuracy": 0.1435546875, + "f1": 0.13115140314121648, + "f1_weighted": 0.13109340980740067 + }, + { + "accuracy": 0.13525390625, + "f1": 0.12000741333594586, + "f1_weighted": 0.12012510344012213 + }, + { + "accuracy": 0.14501953125, + "f1": 0.13102371690204104, + "f1_weighted": 0.1311118496539423 + }, + { + "accuracy": 0.12255859375, + "f1": 0.11384830939829704, + "f1_weighted": 0.11386320802390429 + }, + { + "accuracy": 0.14892578125, + "f1": 0.13062224491737132, + "f1_weighted": 0.13064133287192917 + }, + { + "accuracy": 0.13330078125, + "f1": 0.12021381834248897, + "f1_weighted": 0.12028738442106686 + }, + { + "accuracy": 0.1572265625, + "f1": 0.137454456004924, + "f1_weighted": 0.1374292708533484 + }, + { + "accuracy": 0.130859375, + "f1": 0.10856328138455806, + "f1_weighted": 0.10856047191523874 + } + ] + } + ] + }, + "task_name": "RuSciBenchOECDClassification" +} \ No newline at end of file diff --git a/results/avsolatorio__GIST-Embedding-v0/bf6b2e55e92f510a570ad4d7d2da2ec8cd22590c/RuSciBenchOECDClusteringP2P.json b/results/avsolatorio__GIST-Embedding-v0/bf6b2e55e92f510a570ad4d7d2da2ec8cd22590c/RuSciBenchOECDClusteringP2P.json new file mode 100644 index 000000000..effdd2e54 --- /dev/null +++ b/results/avsolatorio__GIST-Embedding-v0/bf6b2e55e92f510a570ad4d7d2da2ec8cd22590c/RuSciBenchOECDClusteringP2P.json @@ -0,0 +1,34 @@ +{ + "dataset_revision": "26c88e99dcaba32bb45d0e1bfc21902337f6d471", + "evaluation_time": 80.97057127952576, + "kg_co2_emissions": null, + "mteb_version": "1.19.2", + "scores": { + "test": [ + { + "hf_subset": "default", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.15197409749505192, + "v_measure": 0.15197409749505192, + "v_measure_std": 0.005125044018204567, + "v_measures": { + "Level 0": [ + 0.15446211379820207, + 0.15091125472844874, + 0.1544462487828715, + 0.14969303930586197, + 0.14812110962945332, + 0.16016604891077144, + 0.1411919070856263, + 0.15505404411009932, + 0.14867598407359042, + 0.15701922452559422 + ] + } + } + ] + }, + "task_name": "RuSciBenchOECDClusteringP2P" +} \ No newline at end of file diff --git a/results/avsolatorio__GIST-Embedding-v0/bf6b2e55e92f510a570ad4d7d2da2ec8cd22590c/STS22.json b/results/avsolatorio__GIST-Embedding-v0/bf6b2e55e92f510a570ad4d7d2da2ec8cd22590c/STS22.json new file mode 100644 index 000000000..ba8041de1 --- /dev/null +++ b/results/avsolatorio__GIST-Embedding-v0/bf6b2e55e92f510a570ad4d7d2da2ec8cd22590c/STS22.json @@ -0,0 +1,26 @@ +{ + "dataset_revision": "de9d86b3b84231dc21f76c7b7af1f28e2f57f6e3", + "evaluation_time": 3.96528959274292, + "kg_co2_emissions": null, + "mteb_version": "1.19.2", + "scores": { + "test": [ + { + "cosine_pearson": 0.14125424726002145, + "cosine_spearman": 0.21730991271130268, + "euclidean_pearson": 0.1428789553990079, + "euclidean_spearman": 0.21728199591847644, + "hf_subset": "ru", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.21730991271130268, + "manhattan_pearson": 0.14145785419916376, + "manhattan_spearman": 0.2155311578060724, + "pearson": 0.14125424726002145, + "spearman": 0.21730991271130268 + } + ] + }, + "task_name": "STS22" +} \ No newline at end of file diff --git a/results/avsolatorio__GIST-Embedding-v0/bf6b2e55e92f510a570ad4d7d2da2ec8cd22590c/SensitiveTopicsClassification.json b/results/avsolatorio__GIST-Embedding-v0/bf6b2e55e92f510a570ad4d7d2da2ec8cd22590c/SensitiveTopicsClassification.json new file mode 100644 index 000000000..ba3cd1cf7 --- /dev/null +++ b/results/avsolatorio__GIST-Embedding-v0/bf6b2e55e92f510a570ad4d7d2da2ec8cd22590c/SensitiveTopicsClassification.json @@ -0,0 +1,73 @@ +{ + "dataset_revision": "416b34a802308eac30e4192afc0ff99bb8dcc7f2", + "evaluation_time": 17.91078782081604, + "kg_co2_emissions": null, + "mteb_version": "1.19.2", + "scores": { + "test": [ + { + "accuracy": 0.17685546875, + "f1": 0.02740114755619024, + "hf_subset": "default", + "languages": [ + "rus-Cyrl" + ], + "lrap": 0.26103515624999357, + "main_score": 0.17685546875, + "scores_per_experiment": [ + { + "accuracy": 0.17919921875, + "f1": 0.018571123919533306, + "lrap": 0.26182725694443804 + }, + { + "accuracy": 0.1767578125, + "f1": 0.02077858191843575, + "lrap": 0.2588433159722158 + }, + { + "accuracy": 0.17578125, + "f1": 0.019848758963124078, + "lrap": 0.2601725260416603 + }, + { + "accuracy": 0.1796875, + "f1": 0.02200653945745962, + "lrap": 0.2617323133680492 + }, + { + "accuracy": 0.1767578125, + "f1": 0.03666502917167592, + "lrap": 0.2633463541666603 + }, + { + "accuracy": 0.18017578125, + "f1": 0.02006115626358645, + "lrap": 0.2594401041666602 + }, + { + "accuracy": 0.173828125, + "f1": 0.022767120866524874, + "lrap": 0.25821940104166025 + }, + { + "accuracy": 0.17333984375, + "f1": 0.02192596996979524, + "lrap": 0.25873480902777135 + }, + { + "accuracy": 0.1748046875, + "f1": 0.027438844052591314, + "lrap": 0.25697157118054914 + }, + { + "accuracy": 0.17822265625, + "f1": 0.0639483509791759, + "lrap": 0.2710639105902712 + } + ] + } + ] + }, + "task_name": "SensitiveTopicsClassification" +} \ No newline at end of file diff --git a/results/avsolatorio__GIST-Embedding-v0/bf6b2e55e92f510a570ad4d7d2da2ec8cd22590c/TERRa.json b/results/avsolatorio__GIST-Embedding-v0/bf6b2e55e92f510a570ad4d7d2da2ec8cd22590c/TERRa.json new file mode 100644 index 000000000..cbabfad32 --- /dev/null +++ b/results/avsolatorio__GIST-Embedding-v0/bf6b2e55e92f510a570ad4d7d2da2ec8cd22590c/TERRa.json @@ -0,0 +1,58 @@ +{ + "dataset_revision": "7b58f24536063837d644aab9a023c62199b2a612", + "evaluation_time": 2.207289934158325, + "kg_co2_emissions": null, + "mteb_version": "1.19.2", + "scores": { + "dev": [ + { + "cosine_accuracy": 0.5179153094462541, + "cosine_accuracy_threshold": 0.8462473154067993, + "cosine_ap": 0.4824973995221279, + "cosine_f1": 0.6681222707423581, + "cosine_f1_threshold": 0.7648097276687622, + "cosine_precision": 0.5016393442622951, + "cosine_recall": 1.0, + "dot_accuracy": 0.5179153094462541, + "dot_accuracy_threshold": 0.8462472558021545, + "dot_ap": 0.4824973995221279, + "dot_f1": 0.6681222707423581, + "dot_f1_threshold": 0.764809787273407, + "dot_precision": 0.5016393442622951, + "dot_recall": 1.0, + "euclidean_accuracy": 0.5179153094462541, + "euclidean_accuracy_threshold": 0.5545315742492676, + "euclidean_ap": 0.4824973995221279, + "euclidean_f1": 0.6681222707423581, + "euclidean_f1_threshold": 0.685809850692749, + "euclidean_precision": 0.5016393442622951, + "euclidean_recall": 1.0, + "hf_subset": "default", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.4829227722891509, + "manhattan_accuracy": 0.50814332247557, + "manhattan_accuracy_threshold": 12.291492462158203, + "manhattan_ap": 0.4829227722891509, + "manhattan_f1": 0.6681222707423581, + "manhattan_f1_threshold": 15.240278244018555, + "manhattan_precision": 0.5016393442622951, + "manhattan_recall": 1.0, + "max_accuracy": 0.5179153094462541, + "max_ap": 0.4829227722891509, + "max_f1": 0.6681222707423581, + "max_precision": 0.5016393442622951, + "max_recall": 1.0, + "similarity_accuracy": 0.5179153094462541, + "similarity_accuracy_threshold": 0.8462473154067993, + "similarity_ap": 0.4824973995221279, + "similarity_f1": 0.6681222707423581, + "similarity_f1_threshold": 0.7648097276687622, + "similarity_precision": 0.5016393442622951, + "similarity_recall": 1.0 + } + ] + }, + "task_name": "TERRa" +} \ No newline at end of file diff --git a/results/avsolatorio__GIST-Embedding-v0/bf6b2e55e92f510a570ad4d7d2da2ec8cd22590c/model_meta.json b/results/avsolatorio__GIST-Embedding-v0/bf6b2e55e92f510a570ad4d7d2da2ec8cd22590c/model_meta.json new file mode 100644 index 000000000..fccdc3638 --- /dev/null +++ b/results/avsolatorio__GIST-Embedding-v0/bf6b2e55e92f510a570ad4d7d2da2ec8cd22590c/model_meta.json @@ -0,0 +1,22 @@ +{ + "name": "avsolatorio/GIST-Embedding-v0", + "revision": "bf6b2e55e92f510a570ad4d7d2da2ec8cd22590c", + "release_date": null, + "languages": [], + "n_parameters": null, + "memory_usage": null, + "max_tokens": null, + "embed_dim": null, + "license": null, + "open_weights": null, + "public_training_data": null, + "public_training_code": null, + "framework": [ + "Sentence Transformers" + ], + "reference": null, + "similarity_fn_name": null, + "use_instructions": null, + "zero_shot_benchmarks": null, + "loader": null +} \ No newline at end of file diff --git a/results/avsolatorio__GIST-large-Embedding-v0/7831200e2f7819b994490c091cf3258a2b821f0c/CEDRClassification.json b/results/avsolatorio__GIST-large-Embedding-v0/7831200e2f7819b994490c091cf3258a2b821f0c/CEDRClassification.json new file mode 100644 index 000000000..5f408e0cb --- /dev/null +++ b/results/avsolatorio__GIST-large-Embedding-v0/7831200e2f7819b994490c091cf3258a2b821f0c/CEDRClassification.json @@ -0,0 +1,73 @@ +{ + "dataset_revision": "c0ba03d058e3e1b2f3fd20518875a4563dd12db4", + "evaluation_time": 19.967292308807373, + "kg_co2_emissions": null, + "mteb_version": "1.19.2", + "scores": { + "test": [ + { + "accuracy": 0.34006376195536664, + "f1": 0.1374323350893255, + "hf_subset": "default", + "languages": [ + "rus-Cyrl" + ], + "lrap": 0.5735494155154215, + "main_score": 0.34006376195536664, + "scores_per_experiment": [ + { + "accuracy": 0.3368756641870351, + "f1": 0.11743078531165567, + "lrap": 0.5643464399575046 + }, + { + "accuracy": 0.3804463336875664, + "f1": 0.1344065828190694, + "lrap": 0.571466524973445 + }, + { + "accuracy": 0.35015940488841657, + "f1": 0.15494928954931442, + "lrap": 0.5802337938363568 + }, + { + "accuracy": 0.32199787460148777, + "f1": 0.14956085255748536, + "lrap": 0.5770988310308306 + }, + { + "accuracy": 0.34006376195536664, + "f1": 0.1472136345065937, + "lrap": 0.5782146652497467 + }, + { + "accuracy": 0.3575982996811902, + "f1": 0.1265290716471919, + "lrap": 0.5625398512221171 + }, + { + "accuracy": 0.30765143464399575, + "f1": 0.11635784996367235, + "lrap": 0.567747077577058 + }, + { + "accuracy": 0.33581296493092455, + "f1": 0.14975312144086977, + "lrap": 0.5833156216790771 + }, + { + "accuracy": 0.3198724760892667, + "f1": 0.1381538027757735, + "lrap": 0.5725823591923609 + }, + { + "accuracy": 0.35015940488841657, + "f1": 0.13996836032162938, + "lrap": 0.5779489904357188 + } + ] + } + ] + }, + "task_name": "CEDRClassification" +} \ No newline at end of file diff --git a/results/avsolatorio__GIST-large-Embedding-v0/7831200e2f7819b994490c091cf3258a2b821f0c/GeoreviewClassification.json b/results/avsolatorio__GIST-large-Embedding-v0/7831200e2f7819b994490c091cf3258a2b821f0c/GeoreviewClassification.json new file mode 100644 index 000000000..eaced2453 --- /dev/null +++ b/results/avsolatorio__GIST-large-Embedding-v0/7831200e2f7819b994490c091cf3258a2b821f0c/GeoreviewClassification.json @@ -0,0 +1,73 @@ +{ + "dataset_revision": "3765c0d1de6b7d264bc459433c45e5a75513839c", + "evaluation_time": 48.09922194480896, + "kg_co2_emissions": null, + "mteb_version": "1.19.2", + "scores": { + "test": [ + { + "accuracy": 0.27705078125, + "f1": 0.2755638750366433, + "f1_weighted": 0.2755742750163824, + "hf_subset": "default", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.27705078125, + "scores_per_experiment": [ + { + "accuracy": 0.28369140625, + "f1": 0.28392140120139053, + "f1_weighted": 0.28395051395930826 + }, + { + "accuracy": 0.31298828125, + "f1": 0.31434143072639414, + "f1_weighted": 0.3143440543582186 + }, + { + "accuracy": 0.2744140625, + "f1": 0.2734264097213739, + "f1_weighted": 0.27343387447541034 + }, + { + "accuracy": 0.27001953125, + "f1": 0.2656662767369736, + "f1_weighted": 0.26569681867826905 + }, + { + "accuracy": 0.28271484375, + "f1": 0.2794194362492434, + "f1_weighted": 0.27942163692401545 + }, + { + "accuracy": 0.26513671875, + "f1": 0.26197248892470965, + "f1_weighted": 0.26198804465451364 + }, + { + "accuracy": 0.2724609375, + "f1": 0.27332888522180465, + "f1_weighted": 0.2733487631894003 + }, + { + "accuracy": 0.26123046875, + "f1": 0.25990176309499735, + "f1_weighted": 0.2598855591683728 + }, + { + "accuracy": 0.2578125, + "f1": 0.25794710962117073, + "f1_weighted": 0.25796351383715327 + }, + { + "accuracy": 0.2900390625, + "f1": 0.2857135488683749, + "f1_weighted": 0.2857099709191617 + } + ] + } + ] + }, + "task_name": "GeoreviewClassification" +} \ No newline at end of file diff --git a/results/avsolatorio__GIST-large-Embedding-v0/7831200e2f7819b994490c091cf3258a2b821f0c/GeoreviewClusteringP2P.json b/results/avsolatorio__GIST-large-Embedding-v0/7831200e2f7819b994490c091cf3258a2b821f0c/GeoreviewClusteringP2P.json new file mode 100644 index 000000000..c7557ad65 --- /dev/null +++ b/results/avsolatorio__GIST-large-Embedding-v0/7831200e2f7819b994490c091cf3258a2b821f0c/GeoreviewClusteringP2P.json @@ -0,0 +1,34 @@ +{ + "dataset_revision": "97a313c8fc85b47f13f33e7e9a95c1ad888c7fec", + "evaluation_time": 97.63377928733826, + "kg_co2_emissions": null, + "mteb_version": "1.19.2", + "scores": { + "test": [ + { + "hf_subset": "default", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.27120048222122684, + "v_measure": 0.27120048222122684, + "v_measure_std": 0.00891409978378047, + "v_measures": { + "Level 0": [ + 0.26430312654667787, + 0.261392299088465, + 0.2716794998652693, + 0.2809457755332931, + 0.27339043545857344, + 0.27277484416168757, + 0.26068219221822264, + 0.28867993296914324, + 0.27692662626382963, + 0.26123009010710657 + ] + } + } + ] + }, + "task_name": "GeoreviewClusteringP2P" +} \ No newline at end of file diff --git a/results/avsolatorio__GIST-large-Embedding-v0/7831200e2f7819b994490c091cf3258a2b821f0c/HeadlineClassification.json b/results/avsolatorio__GIST-large-Embedding-v0/7831200e2f7819b994490c091cf3258a2b821f0c/HeadlineClassification.json new file mode 100644 index 000000000..380553124 --- /dev/null +++ b/results/avsolatorio__GIST-large-Embedding-v0/7831200e2f7819b994490c091cf3258a2b821f0c/HeadlineClassification.json @@ -0,0 +1,73 @@ +{ + "dataset_revision": "2fe05ee6b5832cda29f2ef7aaad7b7fe6a3609eb", + "evaluation_time": 19.925139904022217, + "kg_co2_emissions": null, + "mteb_version": "1.19.2", + "scores": { + "test": [ + { + "accuracy": 0.3240234375, + "f1": 0.32197825767624877, + "f1_weighted": 0.32197731279413755, + "hf_subset": "default", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.3240234375, + "scores_per_experiment": [ + { + "accuracy": 0.36083984375, + "f1": 0.3597120692650362, + "f1_weighted": 0.35972051978529124 + }, + { + "accuracy": 0.3203125, + "f1": 0.3145493186634998, + "f1_weighted": 0.31457626395039334 + }, + { + "accuracy": 0.291015625, + "f1": 0.28961827529097955, + "f1_weighted": 0.28959672879245196 + }, + { + "accuracy": 0.37646484375, + "f1": 0.37728929959055674, + "f1_weighted": 0.37730862464824344 + }, + { + "accuracy": 0.326171875, + "f1": 0.3257719814465465, + "f1_weighted": 0.32575669138974717 + }, + { + "accuracy": 0.34716796875, + "f1": 0.34609946187540985, + "f1_weighted": 0.346106122778941 + }, + { + "accuracy": 0.29443359375, + "f1": 0.29240861478408914, + "f1_weighted": 0.29240595325696744 + }, + { + "accuracy": 0.2841796875, + "f1": 0.27772390015370224, + "f1_weighted": 0.2777602352385998 + }, + { + "accuracy": 0.32373046875, + "f1": 0.3218609212863998, + "f1_weighted": 0.3218217473858124 + }, + { + "accuracy": 0.31591796875, + "f1": 0.314748734406268, + "f1_weighted": 0.314720240714928 + } + ] + } + ] + }, + "task_name": "HeadlineClassification" +} \ No newline at end of file diff --git a/results/avsolatorio__GIST-large-Embedding-v0/7831200e2f7819b994490c091cf3258a2b821f0c/InappropriatenessClassification.json b/results/avsolatorio__GIST-large-Embedding-v0/7831200e2f7819b994490c091cf3258a2b821f0c/InappropriatenessClassification.json new file mode 100644 index 000000000..967621a74 --- /dev/null +++ b/results/avsolatorio__GIST-large-Embedding-v0/7831200e2f7819b994490c091cf3258a2b821f0c/InappropriatenessClassification.json @@ -0,0 +1,95 @@ +{ + "dataset_revision": "601651fdc45ef243751676e62dd7a19f491c0285", + "evaluation_time": 20.388413429260254, + "kg_co2_emissions": null, + "mteb_version": "1.19.2", + "scores": { + "test": [ + { + "accuracy": 0.515087890625, + "ap": 0.5080279752180195, + "ap_weighted": 0.5080279752180195, + "f1": 0.5123686946331695, + "f1_weighted": 0.5123686946331695, + "hf_subset": "default", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.515087890625, + "scores_per_experiment": [ + { + "accuracy": 0.51904296875, + "ap": 0.5099136046396515, + "ap_weighted": 0.5099136046396515, + "f1": 0.5183621332855821, + "f1_weighted": 0.5183621332855821 + }, + { + "accuracy": 0.51171875, + "ap": 0.5059791577938672, + "ap_weighted": 0.5059791577938672, + "f1": 0.5090852782153716, + "f1_weighted": 0.5090852782153716 + }, + { + "accuracy": 0.52783203125, + "ap": 0.5148184193686007, + "ap_weighted": 0.5148184193686007, + "f1": 0.5254532467155922, + "f1_weighted": 0.5254532467155922 + }, + { + "accuracy": 0.48828125, + "ap": 0.4942989864864865, + "ap_weighted": 0.4942989864864865, + "f1": 0.48601468266740233, + "f1_weighted": 0.48601468266740233 + }, + { + "accuracy": 0.54638671875, + "ap": 0.5253058897111698, + "ap_weighted": 0.5253058897111698, + "f1": 0.5463476732993271, + "f1_weighted": 0.5463476732993271 + }, + { + "accuracy": 0.52587890625, + "ap": 0.5136218322450248, + "ap_weighted": 0.5136218322450248, + "f1": 0.5258380955582849, + "f1_weighted": 0.5258380955582849 + }, + { + "accuracy": 0.513671875, + "ap": 0.507080078125, + "ap_weighted": 0.507080078125, + "f1": 0.5069001794665511, + "f1_weighted": 0.5069001794665511 + }, + { + "accuracy": 0.5009765625, + "ap": 0.500489277742347, + "ap_weighted": 0.500489277742347, + "f1": 0.5007461177072242, + "f1_weighted": 0.5007461177072242 + }, + { + "accuracy": 0.51611328125, + "ap": 0.5082829122340426, + "ap_weighted": 0.5082829122340426, + "f1": 0.5134684069506842, + "f1_weighted": 0.5134684069506842 + }, + { + "accuracy": 0.5009765625, + "ap": 0.5004895938340054, + "ap_weighted": 0.5004895938340054, + "f1": 0.49147113246567464, + "f1_weighted": 0.49147113246567464 + } + ] + } + ] + }, + "task_name": "InappropriatenessClassification" +} \ No newline at end of file diff --git a/results/avsolatorio__GIST-large-Embedding-v0/7831200e2f7819b994490c091cf3258a2b821f0c/KinopoiskClassification.json b/results/avsolatorio__GIST-large-Embedding-v0/7831200e2f7819b994490c091cf3258a2b821f0c/KinopoiskClassification.json new file mode 100644 index 000000000..40754add0 --- /dev/null +++ b/results/avsolatorio__GIST-large-Embedding-v0/7831200e2f7819b994490c091cf3258a2b821f0c/KinopoiskClassification.json @@ -0,0 +1,73 @@ +{ + "dataset_revision": "5911f26666ac11af46cb9c6849d0dc80a378af24", + "evaluation_time": 48.28503751754761, + "kg_co2_emissions": null, + "mteb_version": "1.19.2", + "scores": { + "test": [ + { + "accuracy": 0.3589333333333334, + "f1": 0.35035047328246594, + "f1_weighted": 0.35035047328246594, + "hf_subset": "default", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.3589333333333334, + "scores_per_experiment": [ + { + "accuracy": 0.3406666666666667, + "f1": 0.3376549542705298, + "f1_weighted": 0.3376549542705297 + }, + { + "accuracy": 0.35733333333333334, + "f1": 0.3478576711183961, + "f1_weighted": 0.3478576711183961 + }, + { + "accuracy": 0.38066666666666665, + "f1": 0.3629145451332783, + "f1_weighted": 0.36291454513327837 + }, + { + "accuracy": 0.36, + "f1": 0.35888202804545216, + "f1_weighted": 0.35888202804545216 + }, + { + "accuracy": 0.37466666666666665, + "f1": 0.3721199758326496, + "f1_weighted": 0.3721199758326496 + }, + { + "accuracy": 0.3526666666666667, + "f1": 0.3410041530569057, + "f1_weighted": 0.3410041530569057 + }, + { + "accuracy": 0.3446666666666667, + "f1": 0.32117861502731, + "f1_weighted": 0.32117861502731 + }, + { + "accuracy": 0.366, + "f1": 0.35977302073407347, + "f1_weighted": 0.3597730207340735 + }, + { + "accuracy": 0.3453333333333333, + "f1": 0.34428716108688534, + "f1_weighted": 0.34428716108688534 + }, + { + "accuracy": 0.36733333333333335, + "f1": 0.35783260851917875, + "f1_weighted": 0.35783260851917875 + } + ] + } + ] + }, + "task_name": "KinopoiskClassification" +} \ No newline at end of file diff --git a/results/avsolatorio__GIST-large-Embedding-v0/7831200e2f7819b994490c091cf3258a2b821f0c/MIRACLReranking.json b/results/avsolatorio__GIST-large-Embedding-v0/7831200e2f7819b994490c091cf3258a2b821f0c/MIRACLReranking.json new file mode 100644 index 000000000..9dce5e733 --- /dev/null +++ b/results/avsolatorio__GIST-large-Embedding-v0/7831200e2f7819b994490c091cf3258a2b821f0c/MIRACLReranking.json @@ -0,0 +1,130 @@ +{ + "dataset_revision": "6d1962c527217f8927fca80f890f14f36b2802af", + "evaluation_time": 2203.003876686096, + "kg_co2_emissions": null, + "mteb_version": "1.19.2", + "scores": { + "dev": [ + { + "MAP@1(MIRACL)": 0.07317, + "MAP@10(MIRACL)": 0.14059, + "MAP@100(MIRACL)": 0.17399, + "MAP@1000(MIRACL)": 0.17399, + "MAP@20(MIRACL)": 0.15478, + "MAP@3(MIRACL)": 0.10741, + "MAP@5(MIRACL)": 0.12225, + "NDCG@1(MIRACL)": 0.12029, + "NDCG@10(MIRACL)": 0.19762, + "NDCG@100(MIRACL)": 0.3293, + "NDCG@1000(MIRACL)": 0.3293, + "NDCG@20(MIRACL)": 0.23961, + "NDCG@3(MIRACL)": 0.14086, + "NDCG@5(MIRACL)": 0.1599, + "P@1(MIRACL)": 0.12029, + "P@10(MIRACL)": 0.06239, + "P@100(MIRACL)": 0.01953, + "P@1000(MIRACL)": 0.00195, + "P@20(MIRACL)": 0.04755, + "P@3(MIRACL)": 0.09008, + "P@5(MIRACL)": 0.07618, + "Recall@1(MIRACL)": 0.07317, + "Recall@10(MIRACL)": 0.29712, + "Recall@100(MIRACL)": 0.79952, + "Recall@1000(MIRACL)": 0.79952, + "Recall@20(MIRACL)": 0.42834, + "Recall@3(MIRACL)": 0.1455, + "Recall@5(MIRACL)": 0.19785, + "hf_subset": "ru", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.19762, + "nAUC_MAP@1000_diff1(MIRACL)": 0.21258217701170978, + "nAUC_MAP@1000_max(MIRACL)": 0.06044749395241681, + "nAUC_MAP@1000_std(MIRACL)": 0.011128954466212999, + "nAUC_MAP@100_diff1(MIRACL)": 0.21258217701170978, + "nAUC_MAP@100_max(MIRACL)": 0.06044749395241681, + "nAUC_MAP@100_std(MIRACL)": 0.011128954466212999, + "nAUC_MAP@10_diff1(MIRACL)": 0.23560150306460761, + "nAUC_MAP@10_max(MIRACL)": 0.03932444702726981, + "nAUC_MAP@10_std(MIRACL)": -0.0037845704211965166, + "nAUC_MAP@1_diff1(MIRACL)": 0.3650184535079333, + "nAUC_MAP@1_max(MIRACL)": -0.005867912376223306, + "nAUC_MAP@1_std(MIRACL)": -0.038560138345324776, + "nAUC_MAP@20_diff1(MIRACL)": 0.23005546972901197, + "nAUC_MAP@20_max(MIRACL)": 0.052419176010986765, + "nAUC_MAP@20_std(MIRACL)": 0.005312888603848898, + "nAUC_MAP@3_diff1(MIRACL)": 0.2672381502123203, + "nAUC_MAP@3_max(MIRACL)": 0.013611126987351334, + "nAUC_MAP@3_std(MIRACL)": -0.02745265539368297, + "nAUC_MAP@5_diff1(MIRACL)": 0.25070242946179255, + "nAUC_MAP@5_max(MIRACL)": 0.02838715447421787, + "nAUC_MAP@5_std(MIRACL)": -0.009633627774843808, + "nAUC_NDCG@1000_diff1(MIRACL)": 0.11563275931498515, + "nAUC_NDCG@1000_max(MIRACL)": 0.11160455300315476, + "nAUC_NDCG@1000_std(MIRACL)": 0.04955761315513242, + "nAUC_NDCG@100_diff1(MIRACL)": 0.11563275931498515, + "nAUC_NDCG@100_max(MIRACL)": 0.11160455300315476, + "nAUC_NDCG@100_std(MIRACL)": 0.04955761315513242, + "nAUC_NDCG@10_diff1(MIRACL)": 0.18594603663620085, + "nAUC_NDCG@10_max(MIRACL)": 0.06029444485111004, + "nAUC_NDCG@10_std(MIRACL)": 0.010536446667053313, + "nAUC_NDCG@1_diff1(MIRACL)": 0.23582152515325258, + "nAUC_NDCG@1_max(MIRACL)": 0.003101842161587907, + "nAUC_NDCG@1_std(MIRACL)": 0.02555545145569327, + "nAUC_NDCG@20_diff1(MIRACL)": 0.180827308898951, + "nAUC_NDCG@20_max(MIRACL)": 0.08295632937952964, + "nAUC_NDCG@20_std(MIRACL)": 0.030138316379924705, + "nAUC_NDCG@3_diff1(MIRACL)": 0.2105217666468589, + "nAUC_NDCG@3_max(MIRACL)": 0.03123476764411779, + "nAUC_NDCG@3_std(MIRACL)": -0.01174373483451869, + "nAUC_NDCG@5_diff1(MIRACL)": 0.20285245228848833, + "nAUC_NDCG@5_max(MIRACL)": 0.045295513507828934, + "nAUC_NDCG@5_std(MIRACL)": 0.0018352349527268853, + "nAUC_P@1000_diff1(MIRACL)": -0.13247955978318746, + "nAUC_P@1000_max(MIRACL)": 0.12726683962894347, + "nAUC_P@1000_std(MIRACL)": 0.0556717512908741, + "nAUC_P@100_diff1(MIRACL)": -0.13247955978318704, + "nAUC_P@100_max(MIRACL)": 0.12726683962894356, + "nAUC_P@100_std(MIRACL)": 0.055671751290874406, + "nAUC_P@10_diff1(MIRACL)": 0.03426423145541141, + "nAUC_P@10_max(MIRACL)": 0.12439220417852946, + "nAUC_P@10_std(MIRACL)": 0.05603931894428385, + "nAUC_P@1_diff1(MIRACL)": 0.23582152515325258, + "nAUC_P@1_max(MIRACL)": 0.003101842161587907, + "nAUC_P@1_std(MIRACL)": 0.02555545145569327, + "nAUC_P@20_diff1(MIRACL)": 0.006988620137136568, + "nAUC_P@20_max(MIRACL)": 0.15753207035019326, + "nAUC_P@20_std(MIRACL)": 0.08159100119681585, + "nAUC_P@3_diff1(MIRACL)": 0.12867854901993275, + "nAUC_P@3_max(MIRACL)": 0.07953882454076788, + "nAUC_P@3_std(MIRACL)": 0.03140574114107525, + "nAUC_P@5_diff1(MIRACL)": 0.10084591492400478, + "nAUC_P@5_max(MIRACL)": 0.10705950355548124, + "nAUC_P@5_std(MIRACL)": 0.06581105301363233, + "nAUC_Recall@1000_diff1(MIRACL)": -0.16359005729509252, + "nAUC_Recall@1000_max(MIRACL)": 0.296746725300613, + "nAUC_Recall@1000_std(MIRACL)": 0.19572817899587883, + "nAUC_Recall@100_diff1(MIRACL)": -0.16359005729509252, + "nAUC_Recall@100_max(MIRACL)": 0.296746725300613, + "nAUC_Recall@100_std(MIRACL)": 0.19572817899587883, + "nAUC_Recall@10_diff1(MIRACL)": 0.12519249581251166, + "nAUC_Recall@10_max(MIRACL)": 0.0713244321428019, + "nAUC_Recall@10_std(MIRACL)": 0.0191900133794734, + "nAUC_Recall@1_diff1(MIRACL)": 0.3650184535079333, + "nAUC_Recall@1_max(MIRACL)": -0.005867912376223306, + "nAUC_Recall@1_std(MIRACL)": -0.038560138345324776, + "nAUC_Recall@20_diff1(MIRACL)": 0.12125019557233109, + "nAUC_Recall@20_max(MIRACL)": 0.11052318579476551, + "nAUC_Recall@20_std(MIRACL)": 0.05968726703645123, + "nAUC_Recall@3_diff1(MIRACL)": 0.194530933295139, + "nAUC_Recall@3_max(MIRACL)": 0.026025498479086855, + "nAUC_Recall@3_std(MIRACL)": -0.04106490981420787, + "nAUC_Recall@5_diff1(MIRACL)": 0.1586375370170182, + "nAUC_Recall@5_max(MIRACL)": 0.052886088744921345, + "nAUC_Recall@5_std(MIRACL)": -0.0018249345370233662 + } + ] + }, + "task_name": "MIRACLReranking" +} \ No newline at end of file diff --git a/results/avsolatorio__GIST-large-Embedding-v0/7831200e2f7819b994490c091cf3258a2b821f0c/MIRACLRetrieval.json b/results/avsolatorio__GIST-large-Embedding-v0/7831200e2f7819b994490c091cf3258a2b821f0c/MIRACLRetrieval.json new file mode 100644 index 000000000..e76683fb9 --- /dev/null +++ b/results/avsolatorio__GIST-large-Embedding-v0/7831200e2f7819b994490c091cf3258a2b821f0c/MIRACLRetrieval.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "main", + "evaluation_time": 86618.13805007935, + "kg_co2_emissions": null, + "mteb_version": "1.19.4", + "scores": { + "dev": [ + { + "hf_subset": "ru", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.05468, + "map_at_1": 0.01527, + "map_at_10": 0.03545, + "map_at_100": 0.04181, + "map_at_1000": 0.04288, + "map_at_20": 0.03792, + "map_at_3": 0.02706, + "map_at_5": 0.03124, + "mrr_at_1": 0.03274760383386582, + "mrr_at_10": 0.06528379988843247, + "mrr_at_100": 0.07160423000168167, + "mrr_at_1000": 0.07247493831297569, + "mrr_at_20": 0.0683174560871175, + "mrr_at_3": 0.053381256656017034, + "mrr_at_5": 0.05901224707135251, + "nauc_map_at_1000_diff1": 0.21476615658955792, + "nauc_map_at_1000_max": 0.2784800049321351, + "nauc_map_at_1000_std": 0.22029126592874584, + "nauc_map_at_100_diff1": 0.21707619284373383, + "nauc_map_at_100_max": 0.27352108556653654, + "nauc_map_at_100_std": 0.21455417631257145, + "nauc_map_at_10_diff1": 0.22698947731927596, + "nauc_map_at_10_max": 0.2382683935524157, + "nauc_map_at_10_std": 0.16420498489588445, + "nauc_map_at_1_diff1": 0.4038418535128401, + "nauc_map_at_1_max": 0.26241694108336766, + "nauc_map_at_1_std": 0.11140621662955093, + "nauc_map_at_20_diff1": 0.22150178188772238, + "nauc_map_at_20_max": 0.24387783020060547, + "nauc_map_at_20_std": 0.17606647333314634, + "nauc_map_at_3_diff1": 0.26257102023399004, + "nauc_map_at_3_max": 0.18906566808039044, + "nauc_map_at_3_std": 0.10468374795975115, + "nauc_map_at_5_diff1": 0.23725136850957262, + "nauc_map_at_5_max": 0.22599204546386487, + "nauc_map_at_5_std": 0.13443668806054485, + "nauc_mrr_at_1000_diff1": 0.18345741381707883, + "nauc_mrr_at_1000_max": 0.224432014970286, + "nauc_mrr_at_1000_std": 0.16435188823560026, + "nauc_mrr_at_100_diff1": 0.18355991163714244, + "nauc_mrr_at_100_max": 0.22351548358907739, + "nauc_mrr_at_100_std": 0.163284186732973, + "nauc_mrr_at_10_diff1": 0.19004254452604477, + "nauc_mrr_at_10_max": 0.20869135894092258, + "nauc_mrr_at_10_std": 0.14063249718682827, + "nauc_mrr_at_1_diff1": 0.25269858824634994, + "nauc_mrr_at_1_max": 0.1477690660850008, + "nauc_mrr_at_1_std": 0.08139216645489833, + "nauc_mrr_at_20_diff1": 0.1839659050276887, + "nauc_mrr_at_20_max": 0.21105824210033194, + "nauc_mrr_at_20_std": 0.14758403937953687, + "nauc_mrr_at_3_diff1": 0.19720188807153904, + "nauc_mrr_at_3_max": 0.15838149080749375, + "nauc_mrr_at_3_std": 0.09257819898643264, + "nauc_mrr_at_5_diff1": 0.1972994416235115, + "nauc_mrr_at_5_max": 0.19320271145929374, + "nauc_mrr_at_5_std": 0.11779479308635822, + "nauc_ndcg_at_1000_diff1": 0.15575618757311804, + "nauc_ndcg_at_1000_max": 0.38547826273897773, + "nauc_ndcg_at_1000_std": 0.3648904784604195, + "nauc_ndcg_at_100_diff1": 0.16976795995801455, + "nauc_ndcg_at_100_max": 0.3547082048017675, + "nauc_ndcg_at_100_std": 0.3272659669483649, + "nauc_ndcg_at_10_diff1": 0.19465930607012205, + "nauc_ndcg_at_10_max": 0.25419599401370074, + "nauc_ndcg_at_10_std": 0.18986924881239953, + "nauc_ndcg_at_1_diff1": 0.25269858824634994, + "nauc_ndcg_at_1_max": 0.1477690660850008, + "nauc_ndcg_at_1_std": 0.08139216645489833, + "nauc_ndcg_at_20_diff1": 0.1786087179623157, + "nauc_ndcg_at_20_max": 0.2627671680060024, + "nauc_ndcg_at_20_std": 0.21323529158559176, + "nauc_ndcg_at_3_diff1": 0.2153200555964684, + "nauc_ndcg_at_3_max": 0.16961788530572286, + "nauc_ndcg_at_3_std": 0.09627638069815397, + "nauc_ndcg_at_5_diff1": 0.2039806000762679, + "nauc_ndcg_at_5_max": 0.23185889781185898, + "nauc_ndcg_at_5_std": 0.14369780660084278, + "nauc_precision_at_1000_diff1": 0.06067160568277711, + "nauc_precision_at_1000_max": 0.4205418062568635, + "nauc_precision_at_1000_std": 0.42834580444245746, + "nauc_precision_at_100_diff1": 0.09794468914445092, + "nauc_precision_at_100_max": 0.438317580498105, + "nauc_precision_at_100_std": 0.4397566528579009, + "nauc_precision_at_10_diff1": 0.15747495844749157, + "nauc_precision_at_10_max": 0.2880365581466585, + "nauc_precision_at_10_std": 0.2530175708341407, + "nauc_precision_at_1_diff1": 0.25269858824634994, + "nauc_precision_at_1_max": 0.1477690660850008, + "nauc_precision_at_1_std": 0.08139216645489833, + "nauc_precision_at_20_diff1": 0.13334639768867573, + "nauc_precision_at_20_max": 0.3002288551288548, + "nauc_precision_at_20_std": 0.2771117517789323, + "nauc_precision_at_3_diff1": 0.17967363731377575, + "nauc_precision_at_3_max": 0.15836929369218458, + "nauc_precision_at_3_std": 0.11367703912365247, + "nauc_precision_at_5_diff1": 0.16420491142345311, + "nauc_precision_at_5_max": 0.2521689578325436, + "nauc_precision_at_5_std": 0.1702284214684505, + "nauc_recall_at_1000_diff1": 0.09092332172113748, + "nauc_recall_at_1000_max": 0.4688605918171676, + "nauc_recall_at_1000_std": 0.4928285625367123, + "nauc_recall_at_100_diff1": 0.12063910429404995, + "nauc_recall_at_100_max": 0.43386962611438945, + "nauc_recall_at_100_std": 0.44666715096681586, + "nauc_recall_at_10_diff1": 0.15811062525507497, + "nauc_recall_at_10_max": 0.28537641567040223, + "nauc_recall_at_10_std": 0.2305924894998115, + "nauc_recall_at_1_diff1": 0.4038418535128401, + "nauc_recall_at_1_max": 0.26241694108336766, + "nauc_recall_at_1_std": 0.11140621662955093, + "nauc_recall_at_20_diff1": 0.12752116078910117, + "nauc_recall_at_20_max": 0.2865023766463221, + "nauc_recall_at_20_std": 0.26276809091723213, + "nauc_recall_at_3_diff1": 0.21571789296427407, + "nauc_recall_at_3_max": 0.18336066427905678, + "nauc_recall_at_3_std": 0.11102651836723056, + "nauc_recall_at_5_diff1": 0.1789920084921232, + "nauc_recall_at_5_max": 0.2657085272627233, + "nauc_recall_at_5_std": 0.1795559967449063, + "ndcg_at_1": 0.03275, + "ndcg_at_10": 0.05468, + "ndcg_at_100": 0.08862, + "ndcg_at_1000": 0.11542, + "ndcg_at_20": 0.06381, + "ndcg_at_3": 0.04035, + "ndcg_at_5": 0.04517, + "precision_at_1": 0.03275, + "precision_at_10": 0.01909, + "precision_at_100": 0.00542, + "precision_at_1000": 0.001, + "precision_at_20": 0.01338, + "precision_at_3": 0.02982, + "precision_at_5": 0.02508, + "recall_at_1": 0.01527, + "recall_at_10": 0.08007, + "recall_at_100": 0.21387, + "recall_at_1000": 0.37872, + "recall_at_20": 0.10795, + "recall_at_3": 0.04064, + "recall_at_5": 0.05575 + } + ] + }, + "task_name": "MIRACLRetrieval" +} \ No newline at end of file diff --git a/results/avsolatorio__GIST-large-Embedding-v0/7831200e2f7819b994490c091cf3258a2b821f0c/MassiveIntentClassification.json b/results/avsolatorio__GIST-large-Embedding-v0/7831200e2f7819b994490c091cf3258a2b821f0c/MassiveIntentClassification.json new file mode 100644 index 000000000..b090bda6b --- /dev/null +++ b/results/avsolatorio__GIST-large-Embedding-v0/7831200e2f7819b994490c091cf3258a2b821f0c/MassiveIntentClassification.json @@ -0,0 +1,137 @@ +{ + "dataset_revision": "4672e20407010da34463acc759c162ca9734bca6", + "evaluation_time": 94.60019040107727, + "kg_co2_emissions": null, + "mteb_version": "1.19.2", + "scores": { + "test": [ + { + "accuracy": 0.3421318090114324, + "f1": 0.31603976518794374, + "f1_weighted": 0.32170964871737595, + "hf_subset": "ru", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.3421318090114324, + "scores_per_experiment": [ + { + "accuracy": 0.34229993275050435, + "f1": 0.3232323049560755, + "f1_weighted": 0.3158059737935334 + }, + { + "accuracy": 0.33927370544720914, + "f1": 0.311309466939703, + "f1_weighted": 0.3235044694389404 + }, + { + "accuracy": 0.33053127101546736, + "f1": 0.31731342607263174, + "f1_weighted": 0.3179582246761472 + }, + { + "accuracy": 0.3439811701412239, + "f1": 0.30539400822163304, + "f1_weighted": 0.3283257579882539 + }, + { + "accuracy": 0.35406859448554134, + "f1": 0.3237682005772445, + "f1_weighted": 0.3319410711079611 + }, + { + "accuracy": 0.33187626092804307, + "f1": 0.30085113804177466, + "f1_weighted": 0.31973732704597224 + }, + { + "accuracy": 0.3523873570948218, + "f1": 0.3340003317152589, + "f1_weighted": 0.32994936176085676 + }, + { + "accuracy": 0.34734364492266306, + "f1": 0.31742733272026574, + "f1_weighted": 0.3139588620797258 + }, + { + "accuracy": 0.32447881640887694, + "f1": 0.30415549449099144, + "f1_weighted": 0.30156513168903876 + }, + { + "accuracy": 0.3550773369199731, + "f1": 0.3229459481438593, + "f1_weighted": 0.3343503075933291 + } + ] + } + ], + "validation": [ + { + "accuracy": 0.3317757009345794, + "f1": 0.30235085489219743, + "f1_weighted": 0.3142707796407248, + "hf_subset": "ru", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.3317757009345794, + "scores_per_experiment": [ + { + "accuracy": 0.3305459911460895, + "f1": 0.30437794230076715, + "f1_weighted": 0.3117445709013516 + }, + { + "accuracy": 0.3246433841613379, + "f1": 0.3002974096886172, + "f1_weighted": 0.30812844009901214 + }, + { + "accuracy": 0.3413674372848008, + "f1": 0.3115019204693708, + "f1_weighted": 0.3297562868132212 + }, + { + "accuracy": 0.33448106246925724, + "f1": 0.2995329373559814, + "f1_weighted": 0.32075360840894096 + }, + { + "accuracy": 0.3605509099852435, + "f1": 0.31890337884882075, + "f1_weighted": 0.34352027687513015 + }, + { + "accuracy": 0.3192326610919823, + "f1": 0.2910237841067648, + "f1_weighted": 0.3081223318733984 + }, + { + "accuracy": 0.3192326610919823, + "f1": 0.30457385838441037, + "f1_weighted": 0.29849871962371294 + }, + { + "accuracy": 0.33448106246925724, + "f1": 0.2934797200024305, + "f1_weighted": 0.3093487764595314 + }, + { + "accuracy": 0.3044761436301033, + "f1": 0.28085735152155133, + "f1_weighted": 0.2814839821799514 + }, + { + "accuracy": 0.3487456960157403, + "f1": 0.31896024624326014, + "f1_weighted": 0.33135080317299737 + } + ] + } + ] + }, + "task_name": "MassiveIntentClassification" +} \ No newline at end of file diff --git a/results/avsolatorio__GIST-large-Embedding-v0/7831200e2f7819b994490c091cf3258a2b821f0c/MassiveScenarioClassification.json b/results/avsolatorio__GIST-large-Embedding-v0/7831200e2f7819b994490c091cf3258a2b821f0c/MassiveScenarioClassification.json new file mode 100644 index 000000000..d4894bd89 --- /dev/null +++ b/results/avsolatorio__GIST-large-Embedding-v0/7831200e2f7819b994490c091cf3258a2b821f0c/MassiveScenarioClassification.json @@ -0,0 +1,137 @@ +{ + "dataset_revision": "fad2c6e8459f9e1c45d9315f4953d921437d70f8", + "evaluation_time": 55.61965775489807, + "kg_co2_emissions": null, + "mteb_version": "1.19.2", + "scores": { + "test": [ + { + "accuracy": 0.39038332212508414, + "f1": 0.3703014420070158, + "f1_weighted": 0.3783183530317334, + "hf_subset": "ru", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.39038332212508414, + "scores_per_experiment": [ + { + "accuracy": 0.39004707464694016, + "f1": 0.379606479019285, + "f1_weighted": 0.37548147443710184 + }, + { + "accuracy": 0.3765971755211836, + "f1": 0.35876153301428837, + "f1_weighted": 0.36086600233697647 + }, + { + "accuracy": 0.4098856758574311, + "f1": 0.3843082091919536, + "f1_weighted": 0.4000994374389898 + }, + { + "accuracy": 0.40282447881640887, + "f1": 0.3827737864828574, + "f1_weighted": 0.3989130503544746 + }, + { + "accuracy": 0.39509078681909887, + "f1": 0.3668522493066833, + "f1_weighted": 0.37501108897551866 + }, + { + "accuracy": 0.3776059179556153, + "f1": 0.352398836620617, + "f1_weighted": 0.3648601955988587 + }, + { + "accuracy": 0.4092131809011432, + "f1": 0.38520284513671654, + "f1_weighted": 0.40008912654655554 + }, + { + "accuracy": 0.3964357767316745, + "f1": 0.3789334514347173, + "f1_weighted": 0.38467250025139865 + }, + { + "accuracy": 0.34162743779421656, + "f1": 0.3281271119383458, + "f1_weighted": 0.3273770360488625 + }, + { + "accuracy": 0.40450571620712844, + "f1": 0.38604991792469384, + "f1_weighted": 0.3958136183285971 + } + ] + } + ], + "validation": [ + { + "accuracy": 0.38607968519429414, + "f1": 0.37169781298763793, + "f1_weighted": 0.37306210698187925, + "hf_subset": "ru", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.38607968519429414, + "scores_per_experiment": [ + { + "accuracy": 0.39350713231677326, + "f1": 0.3874975212697251, + "f1_weighted": 0.3857069230537102 + }, + { + "accuracy": 0.3743236596163306, + "f1": 0.36217626897423044, + "f1_weighted": 0.3617426074906449 + }, + { + "accuracy": 0.4171175602557796, + "f1": 0.3988719202101778, + "f1_weighted": 0.40603406148637894 + }, + { + "accuracy": 0.38317757009345793, + "f1": 0.3700452902080645, + "f1_weighted": 0.373892431479232 + }, + { + "accuracy": 0.3989178553861289, + "f1": 0.3824191495985928, + "f1_weighted": 0.37206341511830976 + }, + { + "accuracy": 0.36645351696999506, + "f1": 0.3470027032770395, + "f1_weighted": 0.35522727814418364 + }, + { + "accuracy": 0.3836694540088539, + "f1": 0.36567221351416934, + "f1_weighted": 0.3748336535557361 + }, + { + "accuracy": 0.39203148057058534, + "f1": 0.38264509947724423, + "f1_weighted": 0.3775186968671488 + }, + { + "accuracy": 0.3305459911460895, + "f1": 0.32494606817637167, + "f1_weighted": 0.31442075266779346 + }, + { + "accuracy": 0.42105263157894735, + "f1": 0.3957018951707639, + "f1_weighted": 0.40918124995565486 + } + ] + } + ] + }, + "task_name": "MassiveScenarioClassification" +} \ No newline at end of file diff --git a/results/avsolatorio__GIST-large-Embedding-v0/7831200e2f7819b994490c091cf3258a2b821f0c/RUParaPhraserSTS.json b/results/avsolatorio__GIST-large-Embedding-v0/7831200e2f7819b994490c091cf3258a2b821f0c/RUParaPhraserSTS.json new file mode 100644 index 000000000..f6ba41eac --- /dev/null +++ b/results/avsolatorio__GIST-large-Embedding-v0/7831200e2f7819b994490c091cf3258a2b821f0c/RUParaPhraserSTS.json @@ -0,0 +1,26 @@ +{ + "dataset_revision": "43265056790b8f7c59e0139acb4be0a8dad2c8f4", + "evaluation_time": 22.20662498474121, + "kg_co2_emissions": null, + "mteb_version": "1.19.2", + "scores": { + "test": [ + { + "cosine_pearson": 0.461228551294567, + "cosine_spearman": 0.4628861573829746, + "euclidean_pearson": 0.4777818874203278, + "euclidean_spearman": 0.4628861573829746, + "hf_subset": "default", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.4628861573829746, + "manhattan_pearson": 0.4780365432461127, + "manhattan_spearman": 0.46336763098498096, + "pearson": 0.461228551294567, + "spearman": 0.4628861573829746 + } + ] + }, + "task_name": "RUParaPhraserSTS" +} \ No newline at end of file diff --git a/results/avsolatorio__GIST-large-Embedding-v0/7831200e2f7819b994490c091cf3258a2b821f0c/RiaNewsRetrieval.json b/results/avsolatorio__GIST-large-Embedding-v0/7831200e2f7819b994490c091cf3258a2b821f0c/RiaNewsRetrieval.json new file mode 100644 index 000000000..cebd97db8 --- /dev/null +++ b/results/avsolatorio__GIST-large-Embedding-v0/7831200e2f7819b994490c091cf3258a2b821f0c/RiaNewsRetrieval.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "82374b0bbacda6114f39ff9c5b925fa1512ca5d7", + "evaluation_time": 16202.857517957687, + "kg_co2_emissions": null, + "mteb_version": "1.19.4", + "scores": { + "test": [ + { + "hf_subset": "default", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.18562, + "map_at_1": 0.1154, + "map_at_10": 0.15989, + "map_at_100": 0.16638, + "map_at_1000": 0.16715, + "map_at_20": 0.16336, + "map_at_3": 0.14535, + "map_at_5": 0.15348, + "mrr_at_1": 0.1154, + "mrr_at_10": 0.1599246031746032, + "mrr_at_100": 0.16641882492027854, + "mrr_at_1000": 0.16718644975788333, + "mrr_at_20": 0.1633932375880054, + "mrr_at_3": 0.14535, + "mrr_at_5": 0.15352000000000002, + "nauc_map_at_1000_diff1": 0.30712212803209266, + "nauc_map_at_1000_max": 0.1777474149063804, + "nauc_map_at_1000_std": 0.12261886137887144, + "nauc_map_at_100_diff1": 0.3070900515532286, + "nauc_map_at_100_max": 0.17763856617578389, + "nauc_map_at_100_std": 0.12232515701428281, + "nauc_map_at_10_diff1": 0.31187484698304235, + "nauc_map_at_10_max": 0.1789233059132743, + "nauc_map_at_10_std": 0.1189232950959738, + "nauc_map_at_1_diff1": 0.4048627218756255, + "nauc_map_at_1_max": 0.19124900959795157, + "nauc_map_at_1_std": 0.09480418162719592, + "nauc_map_at_20_diff1": 0.30841817375977854, + "nauc_map_at_20_max": 0.17811128171011725, + "nauc_map_at_20_std": 0.12106199035309492, + "nauc_map_at_3_diff1": 0.3282031698548156, + "nauc_map_at_3_max": 0.18177331933216026, + "nauc_map_at_3_std": 0.10800115715697496, + "nauc_map_at_5_diff1": 0.31872191263295574, + "nauc_map_at_5_max": 0.1785337503855839, + "nauc_map_at_5_std": 0.11393118916953827, + "nauc_mrr_at_1000_diff1": 0.30691413916919613, + "nauc_mrr_at_1000_max": 0.17773473399313874, + "nauc_mrr_at_1000_std": 0.12250227668923411, + "nauc_mrr_at_100_diff1": 0.30688223061075587, + "nauc_mrr_at_100_max": 0.17762596287280638, + "nauc_mrr_at_100_std": 0.12220873580322382, + "nauc_mrr_at_10_diff1": 0.31166779802369965, + "nauc_mrr_at_10_max": 0.17890712045414595, + "nauc_mrr_at_10_std": 0.118805706922945, + "nauc_mrr_at_1_diff1": 0.4048627218756255, + "nauc_mrr_at_1_max": 0.19105908615657777, + "nauc_mrr_at_1_std": 0.09434398251925177, + "nauc_mrr_at_20_diff1": 0.3082093744266332, + "nauc_mrr_at_20_max": 0.17809788663162296, + "nauc_mrr_at_20_std": 0.12094498800902703, + "nauc_mrr_at_3_diff1": 0.32788361375404396, + "nauc_mrr_at_3_max": 0.18185462711627678, + "nauc_mrr_at_3_std": 0.10792335757674186, + "nauc_mrr_at_5_diff1": 0.3185106679494191, + "nauc_mrr_at_5_max": 0.17851843768300235, + "nauc_mrr_at_5_std": 0.11381627967334536, + "nauc_ndcg_at_1000_diff1": 0.26118401706027067, + "nauc_ndcg_at_1000_max": 0.1729019258013486, + "nauc_ndcg_at_1000_std": 0.15472791875900815, + "nauc_ndcg_at_100_diff1": 0.2601260030909401, + "nauc_ndcg_at_100_max": 0.17009763240585227, + "nauc_ndcg_at_100_std": 0.14770418762388077, + "nauc_ndcg_at_10_diff1": 0.27844054631319287, + "nauc_ndcg_at_10_max": 0.17465948443438367, + "nauc_ndcg_at_10_std": 0.1324677210648628, + "nauc_ndcg_at_1_diff1": 0.4048627218756255, + "nauc_ndcg_at_1_max": 0.19124900959795157, + "nauc_ndcg_at_1_std": 0.09480418162719592, + "nauc_ndcg_at_20_diff1": 0.26809455616235744, + "nauc_ndcg_at_20_max": 0.17190530003904866, + "nauc_ndcg_at_20_std": 0.13905280632228684, + "nauc_ndcg_at_3_diff1": 0.30763236862706184, + "nauc_ndcg_at_3_max": 0.17942678827712372, + "nauc_ndcg_at_3_std": 0.11244020677687531, + "nauc_ndcg_at_5_diff1": 0.2927442437587806, + "nauc_ndcg_at_5_max": 0.1740492469984622, + "nauc_ndcg_at_5_std": 0.12219783424236874, + "nauc_precision_at_1000_diff1": 0.13322176257707177, + "nauc_precision_at_1000_max": 0.16842619124292021, + "nauc_precision_at_1000_std": 0.2783526804513925, + "nauc_precision_at_100_diff1": 0.14745741213163976, + "nauc_precision_at_100_max": 0.15230811896739624, + "nauc_precision_at_100_std": 0.21697563203339731, + "nauc_precision_at_10_diff1": 0.20273739712359182, + "nauc_precision_at_10_max": 0.16507151131946762, + "nauc_precision_at_10_std": 0.16460097088319137, + "nauc_precision_at_1_diff1": 0.4048627218756255, + "nauc_precision_at_1_max": 0.19124900959795157, + "nauc_precision_at_1_std": 0.09480418162719592, + "nauc_precision_at_20_diff1": 0.17674798393736868, + "nauc_precision_at_20_max": 0.15710052746844316, + "nauc_precision_at_20_std": 0.18212629306616887, + "nauc_precision_at_3_diff1": 0.257915523035279, + "nauc_precision_at_3_max": 0.17380697417064306, + "nauc_precision_at_3_std": 0.12340417627408787, + "nauc_precision_at_5_diff1": 0.23272107581859303, + "nauc_precision_at_5_max": 0.16339692324133623, + "nauc_precision_at_5_std": 0.14206980816780343, + "nauc_recall_at_1000_diff1": 0.13322176257707286, + "nauc_recall_at_1000_max": 0.16842619124292108, + "nauc_recall_at_1000_std": 0.27835268045139333, + "nauc_recall_at_100_diff1": 0.14745741213163968, + "nauc_recall_at_100_max": 0.15230811896739624, + "nauc_recall_at_100_std": 0.21697563203339762, + "nauc_recall_at_10_diff1": 0.2027373971235918, + "nauc_recall_at_10_max": 0.1650715113194677, + "nauc_recall_at_10_std": 0.16460097088319137, + "nauc_recall_at_1_diff1": 0.4048627218756255, + "nauc_recall_at_1_max": 0.19124900959795157, + "nauc_recall_at_1_std": 0.09480418162719592, + "nauc_recall_at_20_diff1": 0.1767479839373689, + "nauc_recall_at_20_max": 0.15710052746844325, + "nauc_recall_at_20_std": 0.182126293066169, + "nauc_recall_at_3_diff1": 0.257915523035279, + "nauc_recall_at_3_max": 0.173806974170643, + "nauc_recall_at_3_std": 0.12340417627408787, + "nauc_recall_at_5_diff1": 0.2327210758185931, + "nauc_recall_at_5_max": 0.16339692324133623, + "nauc_recall_at_5_std": 0.1420698081678036, + "ndcg_at_1": 0.1154, + "ndcg_at_10": 0.18562, + "ndcg_at_100": 0.22103, + "ndcg_at_1000": 0.24442, + "ndcg_at_20": 0.1983, + "ndcg_at_3": 0.15541, + "ndcg_at_5": 0.17007, + "precision_at_1": 0.1154, + "precision_at_10": 0.02684, + "precision_at_100": 0.00443, + "precision_at_1000": 0.00063, + "precision_at_20": 0.01593, + "precision_at_3": 0.06153, + "precision_at_5": 0.04404, + "recall_at_1": 0.1154, + "recall_at_10": 0.2684, + "recall_at_100": 0.4429, + "recall_at_1000": 0.6337, + "recall_at_20": 0.3186, + "recall_at_3": 0.1846, + "recall_at_5": 0.2202 + } + ] + }, + "task_name": "RiaNewsRetrieval" +} \ No newline at end of file diff --git a/results/avsolatorio__GIST-large-Embedding-v0/7831200e2f7819b994490c091cf3258a2b821f0c/RuBQReranking.json b/results/avsolatorio__GIST-large-Embedding-v0/7831200e2f7819b994490c091cf3258a2b821f0c/RuBQReranking.json new file mode 100644 index 000000000..c274ac339 --- /dev/null +++ b/results/avsolatorio__GIST-large-Embedding-v0/7831200e2f7819b994490c091cf3258a2b821f0c/RuBQReranking.json @@ -0,0 +1,26 @@ +{ + "dataset_revision": "2e96b8f098fa4b0950fc58eacadeb31c0d0c7fa2", + "evaluation_time": 573.9482209682465, + "kg_co2_emissions": null, + "mteb_version": "1.19.2", + "scores": { + "test": [ + { + "hf_subset": "default", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.39896478938391794, + "map": 0.39896478938391794, + "mrr": 0.4458470213789363, + "nAUC_map_diff1": 0.1795794272028112, + "nAUC_map_max": 0.16644577568472524, + "nAUC_map_std": 0.11267357154985082, + "nAUC_mrr_diff1": 0.1710741452938078, + "nAUC_mrr_max": 0.17483082682167855, + "nAUC_mrr_std": 0.10957013596086365 + } + ] + }, + "task_name": "RuBQReranking" +} \ No newline at end of file diff --git a/results/avsolatorio__GIST-large-Embedding-v0/7831200e2f7819b994490c091cf3258a2b821f0c/RuBQRetrieval.json b/results/avsolatorio__GIST-large-Embedding-v0/7831200e2f7819b994490c091cf3258a2b821f0c/RuBQRetrieval.json new file mode 100644 index 000000000..d01ea2682 --- /dev/null +++ b/results/avsolatorio__GIST-large-Embedding-v0/7831200e2f7819b994490c091cf3258a2b821f0c/RuBQRetrieval.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "e19b6ffa60b3bc248e0b41f4cc37c26a55c2a67b", + "evaluation_time": 877.4548399448395, + "kg_co2_emissions": null, + "mteb_version": "1.19.2", + "scores": { + "test": [ + { + "hf_subset": "default", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.12771, + "map_at_1": 0.06032, + "map_at_10": 0.09868, + "map_at_100": 0.10454, + "map_at_1000": 0.10544, + "map_at_20": 0.10196, + "map_at_3": 0.0866, + "map_at_5": 0.09329, + "mrr_at_1": 0.09692671394799054, + "mrr_at_10": 0.13937647754137114, + "mrr_at_100": 0.14533795598973942, + "mrr_at_1000": 0.14615950790796958, + "mrr_at_20": 0.1427124884546305, + "mrr_at_3": 0.1260835303388495, + "mrr_at_5": 0.13293932230102443, + "nauc_map_at_1000_diff1": 0.3237048235619749, + "nauc_map_at_1000_max": 0.28892940826152974, + "nauc_map_at_1000_std": 0.16112818975153012, + "nauc_map_at_100_diff1": 0.3240472054357601, + "nauc_map_at_100_max": 0.28888858367378595, + "nauc_map_at_100_std": 0.16067140475412217, + "nauc_map_at_10_diff1": 0.33181727349790335, + "nauc_map_at_10_max": 0.2879176381234685, + "nauc_map_at_10_std": 0.1515297264074532, + "nauc_map_at_1_diff1": 0.3931869570966022, + "nauc_map_at_1_max": 0.2567742807819213, + "nauc_map_at_1_std": 0.08326603221790994, + "nauc_map_at_20_diff1": 0.3266646798337965, + "nauc_map_at_20_max": 0.28947894425368614, + "nauc_map_at_20_std": 0.15810696129330842, + "nauc_map_at_3_diff1": 0.35191810814877744, + "nauc_map_at_3_max": 0.2821271633279404, + "nauc_map_at_3_std": 0.13970542986511586, + "nauc_map_at_5_diff1": 0.3378827308232957, + "nauc_map_at_5_max": 0.28179632519357567, + "nauc_map_at_5_std": 0.1431413287683582, + "nauc_mrr_at_1000_diff1": 0.2918975476737011, + "nauc_mrr_at_1000_max": 0.26050169546568575, + "nauc_mrr_at_1000_std": 0.12955088744008147, + "nauc_mrr_at_100_diff1": 0.2920803857886148, + "nauc_mrr_at_100_max": 0.2606004895871058, + "nauc_mrr_at_100_std": 0.12946987488381195, + "nauc_mrr_at_10_diff1": 0.29543141148505647, + "nauc_mrr_at_10_max": 0.2608268745817352, + "nauc_mrr_at_10_std": 0.12436931136902667, + "nauc_mrr_at_1_diff1": 0.3472927713637943, + "nauc_mrr_at_1_max": 0.2527241540689997, + "nauc_mrr_at_1_std": 0.07674855930327977, + "nauc_mrr_at_20_diff1": 0.2929648498207863, + "nauc_mrr_at_20_max": 0.2607089225087446, + "nauc_mrr_at_20_std": 0.12740693907258488, + "nauc_mrr_at_3_diff1": 0.3140193601505446, + "nauc_mrr_at_3_max": 0.2617046458797621, + "nauc_mrr_at_3_std": 0.11774489706211855, + "nauc_mrr_at_5_diff1": 0.30250668665486136, + "nauc_mrr_at_5_max": 0.25735154777444685, + "nauc_mrr_at_5_std": 0.11776211443930405, + "nauc_ndcg_at_1000_diff1": 0.27105520761583135, + "nauc_ndcg_at_1000_max": 0.2844798522327179, + "nauc_ndcg_at_1000_std": 0.1955830531062527, + "nauc_ndcg_at_100_diff1": 0.27413078998324175, + "nauc_ndcg_at_100_max": 0.2858186695719294, + "nauc_ndcg_at_100_std": 0.19042651372121996, + "nauc_ndcg_at_10_diff1": 0.29764360456627204, + "nauc_ndcg_at_10_max": 0.28657348123015697, + "nauc_ndcg_at_10_std": 0.16245217508232604, + "nauc_ndcg_at_1_diff1": 0.35294723314896825, + "nauc_ndcg_at_1_max": 0.2481396396767849, + "nauc_ndcg_at_1_std": 0.07765195133779187, + "nauc_ndcg_at_20_diff1": 0.2861535556318614, + "nauc_ndcg_at_20_max": 0.2880977309777763, + "nauc_ndcg_at_20_std": 0.17742683313937144, + "nauc_ndcg_at_3_diff1": 0.33109898759615863, + "nauc_ndcg_at_3_max": 0.28021580076936664, + "nauc_ndcg_at_3_std": 0.14504094711057028, + "nauc_ndcg_at_5_diff1": 0.3109713024099143, + "nauc_ndcg_at_5_max": 0.27616094337329034, + "nauc_ndcg_at_5_std": 0.14742506761993263, + "nauc_precision_at_1000_diff1": 0.10821881092021673, + "nauc_precision_at_1000_max": 0.21554788566276348, + "nauc_precision_at_1000_std": 0.22673537772451774, + "nauc_precision_at_100_diff1": 0.1526129169208604, + "nauc_precision_at_100_max": 0.26567741287028745, + "nauc_precision_at_100_std": 0.23924476169374992, + "nauc_precision_at_10_diff1": 0.22913449388639118, + "nauc_precision_at_10_max": 0.30144407512136806, + "nauc_precision_at_10_std": 0.19205730717815575, + "nauc_precision_at_1_diff1": 0.35294723314896825, + "nauc_precision_at_1_max": 0.2481396396767849, + "nauc_precision_at_1_std": 0.07765195133779187, + "nauc_precision_at_20_diff1": 0.19766153313206344, + "nauc_precision_at_20_max": 0.29037558784392536, + "nauc_precision_at_20_std": 0.21785393545061907, + "nauc_precision_at_3_diff1": 0.2992100176209946, + "nauc_precision_at_3_max": 0.30223875292903735, + "nauc_precision_at_3_std": 0.16433362718253466, + "nauc_precision_at_5_diff1": 0.2615500895162633, + "nauc_precision_at_5_max": 0.2932816837557532, + "nauc_precision_at_5_std": 0.16467258297234946, + "nauc_recall_at_1000_diff1": 0.15843105276944466, + "nauc_recall_at_1000_max": 0.24967329986330022, + "nauc_recall_at_1000_std": 0.2555004453141908, + "nauc_recall_at_100_diff1": 0.17349137341417067, + "nauc_recall_at_100_max": 0.2659463122364354, + "nauc_recall_at_100_std": 0.24560826044231268, + "nauc_recall_at_10_diff1": 0.23920014363599337, + "nauc_recall_at_10_max": 0.28943564335278055, + "nauc_recall_at_10_std": 0.19376267304083789, + "nauc_recall_at_1_diff1": 0.3931869570966022, + "nauc_recall_at_1_max": 0.2567742807819213, + "nauc_recall_at_1_std": 0.08326603221790994, + "nauc_recall_at_20_diff1": 0.21270243461201227, + "nauc_recall_at_20_max": 0.2875922755684256, + "nauc_recall_at_20_std": 0.22522182725386294, + "nauc_recall_at_3_diff1": 0.30285378784045214, + "nauc_recall_at_3_max": 0.27900940738553376, + "nauc_recall_at_3_std": 0.16985866602319674, + "nauc_recall_at_5_diff1": 0.2642394031127524, + "nauc_recall_at_5_max": 0.26588010274228585, + "nauc_recall_at_5_std": 0.16734612699765036, + "ndcg_at_1": 0.09634, + "ndcg_at_10": 0.12771, + "ndcg_at_100": 0.15819, + "ndcg_at_1000": 0.18532, + "ndcg_at_20": 0.13915, + "ndcg_at_3": 0.10397, + "ndcg_at_5": 0.11478, + "precision_at_1": 0.09634, + "precision_at_10": 0.02831, + "precision_at_100": 0.00508, + "precision_at_1000": 0.00084, + "precision_at_20": 0.01746, + "precision_at_3": 0.06087, + "precision_at_5": 0.04504, + "recall_at_1": 0.06032, + "recall_at_10": 0.17478, + "recall_at_100": 0.30573, + "recall_at_1000": 0.50312, + "recall_at_20": 0.21472, + "recall_at_3": 0.11488, + "recall_at_5": 0.13956 + } + ] + }, + "task_name": "RuBQRetrieval" +} \ No newline at end of file diff --git a/results/avsolatorio__GIST-large-Embedding-v0/7831200e2f7819b994490c091cf3258a2b821f0c/RuReviewsClassification.json b/results/avsolatorio__GIST-large-Embedding-v0/7831200e2f7819b994490c091cf3258a2b821f0c/RuReviewsClassification.json new file mode 100644 index 000000000..bf88f44ca --- /dev/null +++ b/results/avsolatorio__GIST-large-Embedding-v0/7831200e2f7819b994490c091cf3258a2b821f0c/RuReviewsClassification.json @@ -0,0 +1,73 @@ +{ + "dataset_revision": "f6d2c31f4dc6b88f468552750bfec05b4b41b05a", + "evaluation_time": 24.671889781951904, + "kg_co2_emissions": null, + "mteb_version": "1.19.2", + "scores": { + "test": [ + { + "accuracy": 0.44287109375, + "f1": 0.4396589951053059, + "f1_weighted": 0.4396480900208874, + "hf_subset": "default", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.44287109375, + "scores_per_experiment": [ + { + "accuracy": 0.44482421875, + "f1": 0.42799813436706097, + "f1_weighted": 0.42797023544090473 + }, + { + "accuracy": 0.43505859375, + "f1": 0.4371193733639081, + "f1_weighted": 0.4371183898148416 + }, + { + "accuracy": 0.4423828125, + "f1": 0.4441858498395601, + "f1_weighted": 0.4441546377993899 + }, + { + "accuracy": 0.4921875, + "f1": 0.49390341318316794, + "f1_weighted": 0.4938942138594361 + }, + { + "accuracy": 0.47119140625, + "f1": 0.47138519636332954, + "f1_weighted": 0.4713767933562708 + }, + { + "accuracy": 0.44287109375, + "f1": 0.4454299642359869, + "f1_weighted": 0.4454246694117571 + }, + { + "accuracy": 0.4052734375, + "f1": 0.38638542518598706, + "f1_weighted": 0.3864028695215703 + }, + { + "accuracy": 0.47021484375, + "f1": 0.46807112422156666, + "f1_weighted": 0.46805049269834686 + }, + { + "accuracy": 0.36572265625, + "f1": 0.36326678911303106, + "f1_weighted": 0.36327297811358605 + }, + { + "accuracy": 0.458984375, + "f1": 0.4588446811794607, + "f1_weighted": 0.45881562019277056 + } + ] + } + ] + }, + "task_name": "RuReviewsClassification" +} \ No newline at end of file diff --git a/results/avsolatorio__GIST-large-Embedding-v0/7831200e2f7819b994490c091cf3258a2b821f0c/RuSTSBenchmarkSTS.json b/results/avsolatorio__GIST-large-Embedding-v0/7831200e2f7819b994490c091cf3258a2b821f0c/RuSTSBenchmarkSTS.json new file mode 100644 index 000000000..4240ae957 --- /dev/null +++ b/results/avsolatorio__GIST-large-Embedding-v0/7831200e2f7819b994490c091cf3258a2b821f0c/RuSTSBenchmarkSTS.json @@ -0,0 +1,26 @@ +{ + "dataset_revision": "7cf24f325c6da6195df55bef3d86b5e0616f3018", + "evaluation_time": 14.980468273162842, + "kg_co2_emissions": null, + "mteb_version": "1.19.2", + "scores": { + "test": [ + { + "cosine_pearson": 0.6060604921362047, + "cosine_spearman": 0.5983011438414694, + "euclidean_pearson": 0.5993328276832193, + "euclidean_spearman": 0.5982995230581083, + "hf_subset": "default", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.5983011438414694, + "manhattan_pearson": 0.5990771459184501, + "manhattan_spearman": 0.5975760256356963, + "pearson": 0.6060604921362047, + "spearman": 0.5983011438414694 + } + ] + }, + "task_name": "RuSTSBenchmarkSTS" +} \ No newline at end of file diff --git a/results/avsolatorio__GIST-large-Embedding-v0/7831200e2f7819b994490c091cf3258a2b821f0c/RuSciBenchGRNTIClassification.json b/results/avsolatorio__GIST-large-Embedding-v0/7831200e2f7819b994490c091cf3258a2b821f0c/RuSciBenchGRNTIClassification.json new file mode 100644 index 000000000..ad5577957 --- /dev/null +++ b/results/avsolatorio__GIST-large-Embedding-v0/7831200e2f7819b994490c091cf3258a2b821f0c/RuSciBenchGRNTIClassification.json @@ -0,0 +1,73 @@ +{ + "dataset_revision": "673a610d6d3dd91a547a0d57ae1b56f37ebbf6a1", + "evaluation_time": 112.26027941703796, + "kg_co2_emissions": null, + "mteb_version": "1.19.2", + "scores": { + "test": [ + { + "accuracy": 0.23056640625, + "f1": 0.2129782988383518, + "f1_weighted": 0.21309959999392353, + "hf_subset": "default", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.23056640625, + "scores_per_experiment": [ + { + "accuracy": 0.23828125, + "f1": 0.21246686332845233, + "f1_weighted": 0.21255793899731384 + }, + { + "accuracy": 0.2158203125, + "f1": 0.20024498556149323, + "f1_weighted": 0.2003999174412629 + }, + { + "accuracy": 0.236328125, + "f1": 0.22392982696483385, + "f1_weighted": 0.22397486393458874 + }, + { + "accuracy": 0.228515625, + "f1": 0.21084753254873106, + "f1_weighted": 0.21097697269637622 + }, + { + "accuracy": 0.21826171875, + "f1": 0.20315842024130595, + "f1_weighted": 0.20315882397350302 + }, + { + "accuracy": 0.24755859375, + "f1": 0.2275344044777491, + "f1_weighted": 0.22770180587062597 + }, + { + "accuracy": 0.24267578125, + "f1": 0.23246335913712227, + "f1_weighted": 0.2326756076858877 + }, + { + "accuracy": 0.21728515625, + "f1": 0.1918371787144316, + "f1_weighted": 0.19200978968149282 + }, + { + "accuracy": 0.22998046875, + "f1": 0.2113321464488836, + "f1_weighted": 0.21144513818648314 + }, + { + "accuracy": 0.23095703125, + "f1": 0.21596827096051463, + "f1_weighted": 0.21609514147170106 + } + ] + } + ] + }, + "task_name": "RuSciBenchGRNTIClassification" +} \ No newline at end of file diff --git a/results/avsolatorio__GIST-large-Embedding-v0/7831200e2f7819b994490c091cf3258a2b821f0c/RuSciBenchGRNTIClusteringP2P.json b/results/avsolatorio__GIST-large-Embedding-v0/7831200e2f7819b994490c091cf3258a2b821f0c/RuSciBenchGRNTIClusteringP2P.json new file mode 100644 index 000000000..134eb1827 --- /dev/null +++ b/results/avsolatorio__GIST-large-Embedding-v0/7831200e2f7819b994490c091cf3258a2b821f0c/RuSciBenchGRNTIClusteringP2P.json @@ -0,0 +1,34 @@ +{ + "dataset_revision": "673a610d6d3dd91a547a0d57ae1b56f37ebbf6a1", + "evaluation_time": 130.6393928527832, + "kg_co2_emissions": null, + "mteb_version": "1.19.2", + "scores": { + "test": [ + { + "hf_subset": "default", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.21974286349707817, + "v_measure": 0.21974286349707817, + "v_measure_std": 0.006873066513163634, + "v_measures": { + "Level 0": [ + 0.21962870092249542, + 0.20932110742761484, + 0.2213395368894164, + 0.20804719664170995, + 0.22346292712254562, + 0.220044225675708, + 0.23325825527847238, + 0.21808851098514678, + 0.21949219293683292, + 0.22474598109083932 + ] + } + } + ] + }, + "task_name": "RuSciBenchGRNTIClusteringP2P" +} \ No newline at end of file diff --git a/results/avsolatorio__GIST-large-Embedding-v0/7831200e2f7819b994490c091cf3258a2b821f0c/RuSciBenchOECDClassification.json b/results/avsolatorio__GIST-large-Embedding-v0/7831200e2f7819b994490c091cf3258a2b821f0c/RuSciBenchOECDClassification.json new file mode 100644 index 000000000..fd28f5c9a --- /dev/null +++ b/results/avsolatorio__GIST-large-Embedding-v0/7831200e2f7819b994490c091cf3258a2b821f0c/RuSciBenchOECDClassification.json @@ -0,0 +1,73 @@ +{ + "dataset_revision": "26c88e99dcaba32bb45d0e1bfc21902337f6d471", + "evaluation_time": 110.10194039344788, + "kg_co2_emissions": null, + "mteb_version": "1.19.2", + "scores": { + "test": [ + { + "accuracy": 0.173388671875, + "f1": 0.15780357191122157, + "f1_weighted": 0.1577723326863361, + "hf_subset": "default", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.173388671875, + "scores_per_experiment": [ + { + "accuracy": 0.15087890625, + "f1": 0.13938960019253333, + "f1_weighted": 0.1392754080650646 + }, + { + "accuracy": 0.17333984375, + "f1": 0.15415543416401556, + "f1_weighted": 0.15414445830195755 + }, + { + "accuracy": 0.17626953125, + "f1": 0.1634976044964715, + "f1_weighted": 0.1634760033937302 + }, + { + "accuracy": 0.171875, + "f1": 0.15479382883918819, + "f1_weighted": 0.15477962581392646 + }, + { + "accuracy": 0.1708984375, + "f1": 0.15274322309986174, + "f1_weighted": 0.15273547095442466 + }, + { + "accuracy": 0.169921875, + "f1": 0.15749615517060495, + "f1_weighted": 0.15753190697484726 + }, + { + "accuracy": 0.19970703125, + "f1": 0.18213337127300805, + "f1_weighted": 0.182097772293072 + }, + { + "accuracy": 0.154296875, + "f1": 0.1376280428062301, + "f1_weighted": 0.1375861010968735 + }, + { + "accuracy": 0.17822265625, + "f1": 0.16269850675927322, + "f1_weighted": 0.16270702160059436 + }, + { + "accuracy": 0.1884765625, + "f1": 0.17349995231102913, + "f1_weighted": 0.1733895583688702 + } + ] + } + ] + }, + "task_name": "RuSciBenchOECDClassification" +} \ No newline at end of file diff --git a/results/avsolatorio__GIST-large-Embedding-v0/7831200e2f7819b994490c091cf3258a2b821f0c/RuSciBenchOECDClusteringP2P.json b/results/avsolatorio__GIST-large-Embedding-v0/7831200e2f7819b994490c091cf3258a2b821f0c/RuSciBenchOECDClusteringP2P.json new file mode 100644 index 000000000..05333c421 --- /dev/null +++ b/results/avsolatorio__GIST-large-Embedding-v0/7831200e2f7819b994490c091cf3258a2b821f0c/RuSciBenchOECDClusteringP2P.json @@ -0,0 +1,34 @@ +{ + "dataset_revision": "26c88e99dcaba32bb45d0e1bfc21902337f6d471", + "evaluation_time": 133.3897943496704, + "kg_co2_emissions": null, + "mteb_version": "1.19.2", + "scores": { + "test": [ + { + "hf_subset": "default", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.18134022105541847, + "v_measure": 0.18134022105541847, + "v_measure_std": 0.006705393599580357, + "v_measures": { + "Level 0": [ + 0.18368240348637763, + 0.18561029455706718, + 0.18945500378129068, + 0.1830393001478907, + 0.18070707348190015, + 0.16759204897649463, + 0.18089332300517236, + 0.175971570344788, + 0.17511318002054665, + 0.19133801275265683 + ] + } + } + ] + }, + "task_name": "RuSciBenchOECDClusteringP2P" +} \ No newline at end of file diff --git a/results/avsolatorio__GIST-large-Embedding-v0/7831200e2f7819b994490c091cf3258a2b821f0c/STS22.json b/results/avsolatorio__GIST-large-Embedding-v0/7831200e2f7819b994490c091cf3258a2b821f0c/STS22.json new file mode 100644 index 000000000..9c430408a --- /dev/null +++ b/results/avsolatorio__GIST-large-Embedding-v0/7831200e2f7819b994490c091cf3258a2b821f0c/STS22.json @@ -0,0 +1,26 @@ +{ + "dataset_revision": "de9d86b3b84231dc21f76c7b7af1f28e2f57f6e3", + "evaluation_time": 12.66511869430542, + "kg_co2_emissions": null, + "mteb_version": "1.19.2", + "scores": { + "test": [ + { + "cosine_pearson": 0.21334682911263944, + "cosine_spearman": 0.29723680885391623, + "euclidean_pearson": 0.21315736448056602, + "euclidean_spearman": 0.29718099259820363, + "hf_subset": "ru", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.29723680885391623, + "manhattan_pearson": 0.21237088712409571, + "manhattan_spearman": 0.29882841471021937, + "pearson": 0.21334682911263944, + "spearman": 0.29723680885391623 + } + ] + }, + "task_name": "STS22" +} \ No newline at end of file diff --git a/results/avsolatorio__GIST-large-Embedding-v0/7831200e2f7819b994490c091cf3258a2b821f0c/SensitiveTopicsClassification.json b/results/avsolatorio__GIST-large-Embedding-v0/7831200e2f7819b994490c091cf3258a2b821f0c/SensitiveTopicsClassification.json new file mode 100644 index 000000000..c933be558 --- /dev/null +++ b/results/avsolatorio__GIST-large-Embedding-v0/7831200e2f7819b994490c091cf3258a2b821f0c/SensitiveTopicsClassification.json @@ -0,0 +1,73 @@ +{ + "dataset_revision": "416b34a802308eac30e4192afc0ff99bb8dcc7f2", + "evaluation_time": 31.852798461914062, + "kg_co2_emissions": null, + "mteb_version": "1.19.2", + "scores": { + "test": [ + { + "accuracy": 0.181103515625, + "f1": 0.04999320231184619, + "hf_subset": "default", + "languages": [ + "rus-Cyrl" + ], + "lrap": 0.2678276909722157, + "main_score": 0.181103515625, + "scores_per_experiment": [ + { + "accuracy": 0.18017578125, + "f1": 0.03508583830500519, + "lrap": 0.26197645399304914 + }, + { + "accuracy": 0.1845703125, + "f1": 0.036302779200526845, + "lrap": 0.2627495659722158 + }, + { + "accuracy": 0.1787109375, + "f1": 0.04450644213690832, + "lrap": 0.26586914062499367 + }, + { + "accuracy": 0.1748046875, + "f1": 0.055332004640639, + "lrap": 0.2701009114583267 + }, + { + "accuracy": 0.1806640625, + "f1": 0.04776141903517473, + "lrap": 0.27041286892360444 + }, + { + "accuracy": 0.18505859375, + "f1": 0.06278394047910132, + "lrap": 0.2752821180555491 + }, + { + "accuracy": 0.1845703125, + "f1": 0.0447174632764852, + "lrap": 0.26654730902777135 + }, + { + "accuracy": 0.18359375, + "f1": 0.06747731218068012, + "lrap": 0.27073838975693787 + }, + { + "accuracy": 0.18115234375, + "f1": 0.04560074068821829, + "lrap": 0.2676323784722156 + }, + { + "accuracy": 0.177734375, + "f1": 0.060364083175722914, + "lrap": 0.2669677734374934 + } + ] + } + ] + }, + "task_name": "SensitiveTopicsClassification" +} \ No newline at end of file diff --git a/results/avsolatorio__GIST-large-Embedding-v0/7831200e2f7819b994490c091cf3258a2b821f0c/TERRa.json b/results/avsolatorio__GIST-large-Embedding-v0/7831200e2f7819b994490c091cf3258a2b821f0c/TERRa.json new file mode 100644 index 000000000..f7d194bf7 --- /dev/null +++ b/results/avsolatorio__GIST-large-Embedding-v0/7831200e2f7819b994490c091cf3258a2b821f0c/TERRa.json @@ -0,0 +1,58 @@ +{ + "dataset_revision": "7b58f24536063837d644aab9a023c62199b2a612", + "evaluation_time": 5.162427186965942, + "kg_co2_emissions": null, + "mteb_version": "1.19.2", + "scores": { + "dev": [ + { + "cosine_accuracy": 0.50814332247557, + "cosine_accuracy_threshold": 0.9259141683578491, + "cosine_ap": 0.47941452935025874, + "cosine_f1": 0.6695842450765863, + "cosine_f1_threshold": 0.5899693965911865, + "cosine_precision": 0.5032894736842105, + "cosine_recall": 1.0, + "dot_accuracy": 0.50814332247557, + "dot_accuracy_threshold": 0.9259141087532043, + "dot_ap": 0.47941452935025874, + "dot_f1": 0.6695842450765863, + "dot_f1_threshold": 0.5899693965911865, + "dot_precision": 0.5032894736842105, + "dot_recall": 1.0, + "euclidean_accuracy": 0.50814332247557, + "euclidean_accuracy_threshold": 0.3841145634651184, + "euclidean_ap": 0.47941452935025874, + "euclidean_f1": 0.6695842450765863, + "euclidean_f1_threshold": 0.9055453538894653, + "euclidean_precision": 0.5032894736842105, + "euclidean_recall": 1.0, + "hf_subset": "default", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.479840337550861, + "manhattan_accuracy": 0.50814332247557, + "manhattan_accuracy_threshold": 9.819066047668457, + "manhattan_ap": 0.479840337550861, + "manhattan_f1": 0.6681222707423581, + "manhattan_f1_threshold": 23.54391860961914, + "manhattan_precision": 0.5016393442622951, + "manhattan_recall": 1.0, + "max_accuracy": 0.50814332247557, + "max_ap": 0.479840337550861, + "max_f1": 0.6695842450765863, + "max_precision": 0.5032894736842105, + "max_recall": 1.0, + "similarity_accuracy": 0.50814332247557, + "similarity_accuracy_threshold": 0.9259141683578491, + "similarity_ap": 0.47941452935025874, + "similarity_f1": 0.6695842450765863, + "similarity_f1_threshold": 0.5899693965911865, + "similarity_precision": 0.5032894736842105, + "similarity_recall": 1.0 + } + ] + }, + "task_name": "TERRa" +} \ No newline at end of file diff --git a/results/avsolatorio__GIST-large-Embedding-v0/7831200e2f7819b994490c091cf3258a2b821f0c/model_meta.json b/results/avsolatorio__GIST-large-Embedding-v0/7831200e2f7819b994490c091cf3258a2b821f0c/model_meta.json new file mode 100644 index 000000000..bc57df69f --- /dev/null +++ b/results/avsolatorio__GIST-large-Embedding-v0/7831200e2f7819b994490c091cf3258a2b821f0c/model_meta.json @@ -0,0 +1,22 @@ +{ + "name": "avsolatorio/GIST-large-Embedding-v0", + "revision": "7831200e2f7819b994490c091cf3258a2b821f0c", + "release_date": null, + "languages": [], + "n_parameters": null, + "memory_usage": null, + "max_tokens": null, + "embed_dim": null, + "license": null, + "open_weights": null, + "public_training_data": null, + "public_training_code": null, + "framework": [ + "Sentence Transformers" + ], + "reference": null, + "similarity_fn_name": null, + "use_instructions": null, + "zero_shot_benchmarks": null, + "loader": null +} \ No newline at end of file diff --git a/results/cointegrated__rubert-tiny2/dad72b8f77c5eef6995dd3e4691b758ba56b90c3/STS22.json b/results/cointegrated__rubert-tiny2/dad72b8f77c5eef6995dd3e4691b758ba56b90c3/STS22.json index 81e45702e..517c41154 100644 --- a/results/cointegrated__rubert-tiny2/dad72b8f77c5eef6995dd3e4691b758ba56b90c3/STS22.json +++ b/results/cointegrated__rubert-tiny2/dad72b8f77c5eef6995dd3e4691b758ba56b90c3/STS22.json @@ -6,25 +6,19 @@ "scores": { "test": [ { - "cosine_pearson": 0.33049852937052177, - "cosine_spearman": 0.502346800640991, - "euclidean_pearson": 0.38102300750999274, - "euclidean_spearman": 0.502346800640991, + "cosine_pearson": 0.4879372497014252, + "cosine_spearman": 0.5438394520252663, + "euclidean_pearson": 0.4972486629967743, + "euclidean_spearman": 0.5438394520252663, "hf_subset": "ru", "languages": [ "rus-Cyrl" ], - "main_score": 0.502346800640991, - "manhattan_pearson": 0.39210595416564953, - "manhattan_spearman": 0.5013868254141902, - "pearson": [ - 0.3304985656320087, - 2.5049225717439663e-08 - ], - "spearman": [ - 0.502402210520363, - 9.747218577470791e-19 - ] + "main_score": 0.5438394520252663, + "manhattan_pearson": 0.4933960837139526, + "manhattan_spearman": 0.5428316521837694, + "pearson": 0.4879372497014252, + "spearman": 0.5438394520252663 } ] }, diff --git a/results/cointegrated__rubert-tiny2/dad72b8f77c5eef6995dd3e4691b758ba56b90c3/model_meta.json b/results/cointegrated__rubert-tiny2/dad72b8f77c5eef6995dd3e4691b758ba56b90c3/model_meta.json index 0542821ce..cf6e79e67 100644 --- a/results/cointegrated__rubert-tiny2/dad72b8f77c5eef6995dd3e4691b758ba56b90c3/model_meta.json +++ b/results/cointegrated__rubert-tiny2/dad72b8f77c5eef6995dd3e4691b758ba56b90c3/model_meta.json @@ -1 +1,17 @@ -{"name": "cointegrated/rubert-tiny2", "revision": "dad72b8f77c5eef6995dd3e4691b758ba56b90c3", "release_date": "2021-10-28", "languages": ["rus_Cyrl"], "n_parameters": null, "memory_usage": null, "max_tokens": null, "embed_dim": null, "license": null, "open_source": true, "similarity_fn_name": null, "framework": [], "loader": null} \ No newline at end of file +{ + "name": "cointegrated/rubert-tiny2", + "revision": "dad72b8f77c5eef6995dd3e4691b758ba56b90c3", + "release_date": "2021-10-28", + "languages": [ + "rus_Cyrl" + ], + "n_parameters": null, + "memory_usage": null, + "max_tokens": null, + "embed_dim": null, + "license": null, + "open_source": true, + "similarity_fn_name": null, + "framework": [], + "loader": null +} \ No newline at end of file diff --git a/results/deepvk__USER-base/436a489a2087d61aa670b3496a9915f84e46c861/CEDRClassification.json b/results/deepvk__USER-base/436a489a2087d61aa670b3496a9915f84e46c861/CEDRClassification.json index 66dee1b14..5b721e677 100644 --- a/results/deepvk__USER-base/436a489a2087d61aa670b3496a9915f84e46c861/CEDRClassification.json +++ b/results/deepvk__USER-base/436a489a2087d61aa670b3496a9915f84e46c861/CEDRClassification.json @@ -6,64 +6,64 @@ "scores": { "test": [ { - "accuracy": 0.4647183846971307, - "f1": 0.41661640522782273, + "accuracy": 0.4678002125398512, + "f1": 0.4200825473545164, "hf_subset": "default", "languages": [ "rus-Cyrl" ], - "lrap": 0.7163575982996911, - "main_score": 0.4647183846971307, + "lrap": 0.7179729011689793, + "main_score": 0.4678002125398512, "scores_per_experiment": [ { - "accuracy": 0.46174282678002126, - "f1": 0.4028223709438977, - "lrap": 0.6903294367694054 + "accuracy": 0.46014877789585545, + "f1": 0.3964940852129505, + "lrap": 0.6904888416578212 }, { - "accuracy": 0.4861849096705632, - "f1": 0.4164608064875185, - "lrap": 0.7157013815090432 + "accuracy": 0.5069075451647184, + "f1": 0.4389331623379797, + "lrap": 0.7341392136025605 }, { - "accuracy": 0.48831030818278426, - "f1": 0.39170685254728466, - "lrap": 0.6967056323060676 + "accuracy": 0.4957492029755579, + "f1": 0.4112816642259675, + "lrap": 0.7014877789585656 }, { "accuracy": 0.4596174282678002, - "f1": 0.41703517637204063, - "lrap": 0.7064293304994783 + "f1": 0.4211298302862515, + "lrap": 0.7111052072263649 }, { - "accuracy": 0.5015940488841658, - "f1": 0.4729685640622082, - "lrap": 0.761211477151975 + "accuracy": 0.4814027630180659, + "f1": 0.46449910226946667, + "lrap": 0.7492029755579264 }, { - "accuracy": 0.4383634431455898, - "f1": 0.35533874669314136, - "lrap": 0.6825717321997982 + "accuracy": 0.42826780021253985, + "f1": 0.34600557707076623, + "lrap": 0.6755047821466638 }, { - "accuracy": 0.47130712008501596, - "f1": 0.43686904568434565, - "lrap": 0.7294367693942717 + "accuracy": 0.4782146652497343, + "f1": 0.4446492117458275, + "lrap": 0.7310839532412431 }, { - "accuracy": 0.44208289054197664, - "f1": 0.426660068512298, - "lrap": 0.7373538788522942 + "accuracy": 0.46014877789585545, + "f1": 0.4280212993700722, + "lrap": 0.7434112646121241 }, { - "accuracy": 0.44155154091392135, - "f1": 0.4032543335768275, - "lrap": 0.7121147715196701 + "accuracy": 0.45111583421891605, + "f1": 0.41142942052671383, + "lrap": 0.7145058448459187 }, { "accuracy": 0.45642933049946866, - "f1": 0.44304808739866486, - "lrap": 0.7317215727949081 + "f1": 0.4383821204991687, + "lrap": 0.7287991498406042 } ] } diff --git a/results/deepvk__USER-base/436a489a2087d61aa670b3496a9915f84e46c861/GeoreviewClassification.json b/results/deepvk__USER-base/436a489a2087d61aa670b3496a9915f84e46c861/GeoreviewClassification.json index 90c7d60ff..7f3bade97 100644 --- a/results/deepvk__USER-base/436a489a2087d61aa670b3496a9915f84e46c861/GeoreviewClassification.json +++ b/results/deepvk__USER-base/436a489a2087d61aa670b3496a9915f84e46c861/GeoreviewClassification.json @@ -6,64 +6,64 @@ "scores": { "test": [ { - "accuracy": 0.472314453125, - "f1": 0.4595720657178293, - "f1_weighted": 0.4595554526400026, + "accuracy": 0.468798828125, + "f1": 0.45670435152629035, + "f1_weighted": 0.45668717978375445, "hf_subset": "default", "languages": [ "rus-Cyrl" ], - "main_score": 0.472314453125, + "main_score": 0.468798828125, "scores_per_experiment": [ { - "accuracy": 0.4775390625, - "f1": 0.45653453510888997, - "f1_weighted": 0.45655862744498843 + "accuracy": 0.48095703125, + "f1": 0.46220408237172883, + "f1_weighted": 0.46223020671600723 }, { - "accuracy": 0.4638671875, - "f1": 0.446591101335643, - "f1_weighted": 0.44656107657838207 + "accuracy": 0.45947265625, + "f1": 0.4446673676848203, + "f1_weighted": 0.44464775125859957 }, { - "accuracy": 0.49560546875, - "f1": 0.4663249472948614, - "f1_weighted": 0.46631224765441315 + "accuracy": 0.48974609375, + "f1": 0.4596965599536279, + "f1_weighted": 0.4596803541245564 }, { - "accuracy": 0.47705078125, - "f1": 0.47511322227615144, - "f1_weighted": 0.47510685085093607 + "accuracy": 0.46240234375, + "f1": 0.4615325072757604, + "f1_weighted": 0.4615359080145784 }, { - "accuracy": 0.47705078125, - "f1": 0.4617632694723204, - "f1_weighted": 0.46173718949674625 + "accuracy": 0.478515625, + "f1": 0.46385983852499335, + "f1_weighted": 0.46383473061432734 }, { - "accuracy": 0.44482421875, - "f1": 0.4400302252790114, - "f1_weighted": 0.43999337281579765 + "accuracy": 0.43505859375, + "f1": 0.4307864437711714, + "f1_weighted": 0.43074265735104694 }, { - "accuracy": 0.4873046875, - "f1": 0.47802438219053867, - "f1_weighted": 0.47796190147491 + "accuracy": 0.49072265625, + "f1": 0.4779555536932104, + "f1_weighted": 0.4778831059410943 }, { - "accuracy": 0.49267578125, - "f1": 0.4834813864303464, - "f1_weighted": 0.4834691104096889 + "accuracy": 0.4912109375, + "f1": 0.4816540686974607, + "f1_weighted": 0.4816330634711452 }, { - "accuracy": 0.41455078125, - "f1": 0.41328809468581706, - "f1_weighted": 0.41329468257171975 + "accuracy": 0.41162109375, + "f1": 0.41129372590991087, + "f1_weighted": 0.4113019230519703 }, { - "accuracy": 0.49267578125, - "f1": 0.47456949310471347, - "f1_weighted": 0.4745594671024429 + "accuracy": 0.48828125, + "f1": 0.4733933673802196, + "f1_weighted": 0.47338209729421865 } ] } diff --git a/results/deepvk__USER-base/436a489a2087d61aa670b3496a9915f84e46c861/GeoreviewClusteringP2P.json b/results/deepvk__USER-base/436a489a2087d61aa670b3496a9915f84e46c861/GeoreviewClusteringP2P.json index 6e3b1f73d..775198147 100644 --- a/results/deepvk__USER-base/436a489a2087d61aa670b3496a9915f84e46c861/GeoreviewClusteringP2P.json +++ b/results/deepvk__USER-base/436a489a2087d61aa670b3496a9915f84e46c861/GeoreviewClusteringP2P.json @@ -10,21 +10,21 @@ "languages": [ "rus-Cyrl" ], - "main_score": 0.6416417664157499, - "v_measure": 0.6416417664157499, - "v_measure_std": 0.01124409305992548, + "main_score": 0.6341332979158375, + "v_measure": 0.6341332979158375, + "v_measure_std": 0.007655399079626395, "v_measures": { "Level 0": [ - 0.6600012800088786, - 0.6413319756371793, - 0.6597410605277495, - 0.6451029131522003, - 0.6397238199781189, - 0.6392598962148115, - 0.6294939396527088, - 0.6251488475822357, - 0.6465648380399792, - 0.6300490933636365 + 0.6402882916715936, + 0.6323255537857142, + 0.6376825054180131, + 0.6451157454090838, + 0.639495148527725, + 0.6198927620661097, + 0.6274034880399882, + 0.6264747215478271, + 0.6420968699047286, + 0.6305578927875919 ] } } diff --git a/results/deepvk__USER-base/436a489a2087d61aa670b3496a9915f84e46c861/HeadlineClassification.json b/results/deepvk__USER-base/436a489a2087d61aa670b3496a9915f84e46c861/HeadlineClassification.json index 392fcabef..dc97b7b9d 100644 --- a/results/deepvk__USER-base/436a489a2087d61aa670b3496a9915f84e46c861/HeadlineClassification.json +++ b/results/deepvk__USER-base/436a489a2087d61aa670b3496a9915f84e46c861/HeadlineClassification.json @@ -6,64 +6,64 @@ "scores": { "test": [ { - "accuracy": 0.748828125, - "f1": 0.7482956081813782, - "f1_weighted": 0.7483000274006628, + "accuracy": 0.75, + "f1": 0.7494035396594927, + "f1_weighted": 0.7494071035932528, "hf_subset": "default", "languages": [ "rus-Cyrl" ], - "main_score": 0.748828125, + "main_score": 0.75, "scores_per_experiment": [ { - "accuracy": 0.73974609375, - "f1": 0.7399064327868778, - "f1_weighted": 0.7399043376126357 + "accuracy": 0.7294921875, + "f1": 0.7295068936284249, + "f1_weighted": 0.7295070556308487 }, { - "accuracy": 0.75, - "f1": 0.7476940694289936, - "f1_weighted": 0.7477062976724325 + "accuracy": 0.751953125, + "f1": 0.7503201301890599, + "f1_weighted": 0.7503294755321126 }, { - "accuracy": 0.7451171875, - "f1": 0.7458974618793807, - "f1_weighted": 0.7459037633660177 + "accuracy": 0.74853515625, + "f1": 0.7490998081100568, + "f1_weighted": 0.7490964374369221 }, { "accuracy": 0.77880859375, - "f1": 0.7794440720408637, - "f1_weighted": 0.7794284411888823 + "f1": 0.7791334102686097, + "f1_weighted": 0.7791249824724787 }, { "accuracy": 0.74267578125, - "f1": 0.7442234828536972, - "f1_weighted": 0.7442405208128124 + "f1": 0.7441428165593451, + "f1_weighted": 0.7441566998122844 }, { - "accuracy": 0.755859375, - "f1": 0.7567706049036321, - "f1_weighted": 0.7567696538509523 + "accuracy": 0.763671875, + "f1": 0.7640342937658052, + "f1_weighted": 0.7640317808414093 }, { - "accuracy": 0.73193359375, - "f1": 0.7308353577928991, - "f1_weighted": 0.730832243625688 + "accuracy": 0.7392578125, + "f1": 0.7389984695119841, + "f1_weighted": 0.7389931468717391 }, { - "accuracy": 0.76123046875, - "f1": 0.7590644130832839, - "f1_weighted": 0.7590704699873088 + "accuracy": 0.7607421875, + "f1": 0.75877243199269, + "f1_weighted": 0.7587793785250465 }, { - "accuracy": 0.72265625, - "f1": 0.7201851608948804, - "f1_weighted": 0.7202050809066333 + "accuracy": 0.7265625, + "f1": 0.7232154389696487, + "f1_weighted": 0.7232374006868612 }, { - "accuracy": 0.76025390625, - "f1": 0.758935026149273, - "f1_weighted": 0.7589394649832648 + "accuracy": 0.75830078125, + "f1": 0.7568117035993024, + "f1_weighted": 0.7568146781228258 } ] } diff --git a/results/deepvk__USER-base/436a489a2087d61aa670b3496a9915f84e46c861/InappropriatenessClassification.json b/results/deepvk__USER-base/436a489a2087d61aa670b3496a9915f84e46c861/InappropriatenessClassification.json index 34ef6f924..25e38ed51 100644 --- a/results/deepvk__USER-base/436a489a2087d61aa670b3496a9915f84e46c861/InappropriatenessClassification.json +++ b/results/deepvk__USER-base/436a489a2087d61aa670b3496a9915f84e46c861/InappropriatenessClassification.json @@ -6,86 +6,86 @@ "scores": { "test": [ { - "accuracy": 0.619384765625, - "ap": 0.576739144098376, - "ap_weighted": 0.576739144098376, - "f1": 0.6179898369031541, - "f1_weighted": 0.6179898369031541, + "accuracy": 0.618310546875, + "ap": 0.5762117098775502, + "ap_weighted": 0.5762117098775502, + "f1": 0.6165747295848639, + "f1_weighted": 0.6165747295848639, "hf_subset": "default", "languages": [ "rus-Cyrl" ], - "main_score": 0.619384765625, + "main_score": 0.618310546875, "scores_per_experiment": [ { - "accuracy": 0.6337890625, - "ap": 0.5852972436621486, - "ap_weighted": 0.5852972436621486, - "f1": 0.6337205974932754, - "f1_weighted": 0.6337205974932754 + "accuracy": 0.6328125, + "ap": 0.5845412901606426, + "ap_weighted": 0.5845412901606426, + "f1": 0.632743852419924, + "f1_weighted": 0.632743852419924 }, { - "accuracy": 0.6708984375, - "ap": 0.6140959874880267, - "ap_weighted": 0.6140959874880267, - "f1": 0.6708670489357886, - "f1_weighted": 0.6708670489357886 + "accuracy": 0.67724609375, + "ap": 0.6207610748626373, + "ap_weighted": 0.6207610748626373, + "f1": 0.6772053817860997, + "f1_weighted": 0.6772053817860997 }, { - "accuracy": 0.6494140625, - "ap": 0.5981295226050205, - "ap_weighted": 0.5981295226050205, - "f1": 0.6492213740458015, - "f1_weighted": 0.6492213740458015 + "accuracy": 0.65234375, + "ap": 0.600672519329897, + "ap_weighted": 0.600672519329897, + "f1": 0.6521018812861038, + "f1_weighted": 0.6521018812861038 }, { - "accuracy": 0.58349609375, - "ap": 0.5507052815244667, - "ap_weighted": 0.5507052815244667, - "f1": 0.5783155010832112, - "f1_weighted": 0.5783155010832112 + "accuracy": 0.57568359375, + "ap": 0.545257066806258, + "ap_weighted": 0.545257066806258, + "f1": 0.5701194319344254, + "f1_weighted": 0.5701194319344254 }, { - "accuracy": 0.61962890625, - "ap": 0.5790714058393561, - "ap_weighted": 0.5790714058393561, - "f1": 0.6132509602716032, - "f1_weighted": 0.6132509602716032 + "accuracy": 0.62353515625, + "ap": 0.5834419155253121, + "ap_weighted": 0.5834419155253121, + "f1": 0.6151103170144022, + "f1_weighted": 0.6151103170144022 }, { - "accuracy": 0.53076171875, - "ap": 0.5164352599632753, - "ap_weighted": 0.5164352599632753, - "f1": 0.5295250448272755, - "f1_weighted": 0.5295250448272755 + "accuracy": 0.521484375, + "ap": 0.5112593168763676, + "ap_weighted": 0.5112593168763676, + "f1": 0.5200999281718444, + "f1_weighted": 0.5200999281718444 }, { - "accuracy": 0.67333984375, - "ap": 0.6165706237852284, - "ap_weighted": 0.6165706237852284, - "f1": 0.6733378966921371, - "f1_weighted": 0.6733378966921371 + "accuracy": 0.6728515625, + "ap": 0.6160718341206395, + "ap_weighted": 0.6160718341206395, + "f1": 0.6728465705348288, + "f1_weighted": 0.6728465705348288 }, { - "accuracy": 0.6533203125, - "ap": 0.600305823305501, - "ap_weighted": 0.600305823305501, - "f1": 0.6533173368988343, - "f1_weighted": 0.6533173368988343 + "accuracy": 0.646484375, + "ap": 0.5925503212873462, + "ap_weighted": 0.5925503212873462, + "f1": 0.6453856066092236, + "f1_weighted": 0.6453856066092236 }, { - "accuracy": 0.59228515625, - "ap": 0.5553321958969968, - "ap_weighted": 0.5553321958969968, - "f1": 0.591737633750402, - "f1_weighted": 0.591737633750402 + "accuracy": 0.58642578125, + "ap": 0.5510576923076924, + "ap_weighted": 0.5510576923076924, + "f1": 0.5861888979778396, + "f1_weighted": 0.5861888979778396 }, { - "accuracy": 0.5869140625, - "ap": 0.5514480969137396, - "ap_weighted": 0.5514480969137396, - "f1": 0.5866049750332127, - "f1_weighted": 0.5866049750332127 + "accuracy": 0.59423828125, + "ap": 0.5565040674987101, + "ap_weighted": 0.5565040674987101, + "f1": 0.593945428113948, + "f1_weighted": 0.593945428113948 } ] } diff --git a/results/deepvk__USER-base/436a489a2087d61aa670b3496a9915f84e46c861/KinopoiskClassification.json b/results/deepvk__USER-base/436a489a2087d61aa670b3496a9915f84e46c861/KinopoiskClassification.json index 94a016574..7d8a8c952 100644 --- a/results/deepvk__USER-base/436a489a2087d61aa670b3496a9915f84e46c861/KinopoiskClassification.json +++ b/results/deepvk__USER-base/436a489a2087d61aa670b3496a9915f84e46c861/KinopoiskClassification.json @@ -6,64 +6,64 @@ "scores": { "test": [ { - "accuracy": 0.5569333333333333, - "f1": 0.5368981323248281, - "f1_weighted": 0.536898132324828, + "accuracy": 0.5603333333333333, + "f1": 0.539841577478658, + "f1_weighted": 0.539841577478658, "hf_subset": "default", "languages": [ "rus-Cyrl" ], - "main_score": 0.5569333333333333, + "main_score": 0.5603333333333333, "scores_per_experiment": [ { - "accuracy": 0.482, - "f1": 0.47305670358234514, - "f1_weighted": 0.4730567035823451 + "accuracy": 0.47733333333333333, + "f1": 0.46901057852540334, + "f1_weighted": 0.4690105785254033 }, { - "accuracy": 0.5513333333333333, - "f1": 0.5260416872357171, - "f1_weighted": 0.5260416872357171 + "accuracy": 0.5566666666666666, + "f1": 0.5326792314102361, + "f1_weighted": 0.5326792314102361 }, { - "accuracy": 0.5873333333333334, - "f1": 0.5615033363329621, - "f1_weighted": 0.5615033363329621 + "accuracy": 0.5946666666666667, + "f1": 0.5687958919550601, + "f1_weighted": 0.5687958919550602 }, { - "accuracy": 0.5486666666666666, - "f1": 0.5346727257131088, - "f1_weighted": 0.5346727257131088 + "accuracy": 0.5513333333333333, + "f1": 0.5364587257495759, + "f1_weighted": 0.5364587257495759 }, { - "accuracy": 0.5573333333333333, - "f1": 0.5440471773244905, - "f1_weighted": 0.5440471773244905 + "accuracy": 0.5606666666666666, + "f1": 0.547204056748021, + "f1_weighted": 0.547204056748021 }, { - "accuracy": 0.58, - "f1": 0.5632743458516963, - "f1_weighted": 0.5632743458516962 + "accuracy": 0.5866666666666667, + "f1": 0.5691277694727549, + "f1_weighted": 0.5691277694727549 }, { - "accuracy": 0.5873333333333334, - "f1": 0.5538601811946463, - "f1_weighted": 0.5538601811946463 + "accuracy": 0.582, + "f1": 0.5470107077766914, + "f1_weighted": 0.5470107077766914 }, { - "accuracy": 0.5586666666666666, - "f1": 0.5331567991337282, - "f1_weighted": 0.5331567991337282 + "accuracy": 0.5666666666666667, + "f1": 0.5390757610612683, + "f1_weighted": 0.5390757610612683 }, { - "accuracy": 0.566, - "f1": 0.5391817247080405, - "f1_weighted": 0.5391817247080405 + "accuracy": 0.5726666666666667, + "f1": 0.5446687235639353, + "f1_weighted": 0.5446687235639353 }, { - "accuracy": 0.5506666666666666, - "f1": 0.5401866421715457, - "f1_weighted": 0.5401866421715458 + "accuracy": 0.5546666666666666, + "f1": 0.5443843285236342, + "f1_weighted": 0.5443843285236343 } ] } diff --git a/results/deepvk__USER-base/436a489a2087d61aa670b3496a9915f84e46c861/MIRACLRetrieval.json b/results/deepvk__USER-base/436a489a2087d61aa670b3496a9915f84e46c861/MIRACLRetrieval.json index 7b3eccd9c..7fd2c7051 100644 --- a/results/deepvk__USER-base/436a489a2087d61aa670b3496a9915f84e46c861/MIRACLRetrieval.json +++ b/results/deepvk__USER-base/436a489a2087d61aa670b3496a9915f84e46c861/MIRACLRetrieval.json @@ -10,147 +10,147 @@ "languages": [ "rus-Cyrl" ], - "main_score": 0.35216, - "map_at_1": 0.14044, - "map_at_10": 0.26309, - "map_at_100": 0.28981, - "map_at_1000": 0.29178, - "map_at_20": 0.27807, - "map_at_3": 0.2155, - "map_at_5": 0.23917, - "mrr_at_1": 0.28594249201277955, - "mrr_at_10": 0.4109466124042799, - "mrr_at_100": 0.42077830360362506, - "mrr_at_1000": 0.42113716816178787, - "mrr_at_20": 0.4176780722978009, - "mrr_at_3": 0.3772630457933969, - "mrr_at_5": 0.3970713525026616, - "nauc_map_at_1000_diff1": 0.16481568994209758, - "nauc_map_at_1000_max": 0.13778244917939705, - "nauc_map_at_1000_std": 0.04034157349782001, - "nauc_map_at_100_diff1": 0.16517153022323655, - "nauc_map_at_100_max": 0.1360329155622734, - "nauc_map_at_100_std": 0.03872677269446993, - "nauc_map_at_10_diff1": 0.16937910311866264, - "nauc_map_at_10_max": 0.11622704479010856, - "nauc_map_at_10_std": 0.007703975112991193, - "nauc_map_at_1_diff1": 0.2228846096271204, - "nauc_map_at_1_max": 0.07765431366600557, - "nauc_map_at_1_std": -0.04089387584581535, - "nauc_map_at_20_diff1": 0.1650518990050143, - "nauc_map_at_20_max": 0.12767733816688695, - "nauc_map_at_20_std": 0.022903038006970706, - "nauc_map_at_3_diff1": 0.18715860335275827, - "nauc_map_at_3_max": 0.0919729239336384, - "nauc_map_at_3_std": -0.018578092608548556, - "nauc_map_at_5_diff1": 0.17824264455272681, - "nauc_map_at_5_max": 0.1050961042878537, - "nauc_map_at_5_std": -0.007053798354989882, - "nauc_mrr_at_1000_diff1": 0.1435798304083784, - "nauc_mrr_at_1000_max": 0.19479990493636898, - "nauc_mrr_at_1000_std": 0.1072567331574904, - "nauc_mrr_at_100_diff1": 0.14348547139617326, - "nauc_mrr_at_100_max": 0.19493692705901222, - "nauc_mrr_at_100_std": 0.10767814905190118, - "nauc_mrr_at_10_diff1": 0.14313944343725465, - "nauc_mrr_at_10_max": 0.19297074786708043, - "nauc_mrr_at_10_std": 0.10470901572340778, - "nauc_mrr_at_1_diff1": 0.1717974919463759, - "nauc_mrr_at_1_max": 0.18036177614568435, - "nauc_mrr_at_1_std": 0.06062602208644375, - "nauc_mrr_at_20_diff1": 0.14294385157334125, - "nauc_mrr_at_20_max": 0.19434834549819424, - "nauc_mrr_at_20_std": 0.10758721954050914, - "nauc_mrr_at_3_diff1": 0.1491852013469599, - "nauc_mrr_at_3_max": 0.18367271730534854, - "nauc_mrr_at_3_std": 0.08855199931404897, - "nauc_mrr_at_5_diff1": 0.14161939109263946, - "nauc_mrr_at_5_max": 0.1882628841049982, - "nauc_mrr_at_5_std": 0.10194120096163446, - "nauc_ndcg_at_1000_diff1": 0.1432075438376385, - "nauc_ndcg_at_1000_max": 0.20755126685248662, - "nauc_ndcg_at_1000_std": 0.14053295605990943, - "nauc_ndcg_at_100_diff1": 0.1450681578686777, - "nauc_ndcg_at_100_max": 0.19762138854219458, - "nauc_ndcg_at_100_std": 0.13850065768744688, - "nauc_ndcg_at_10_diff1": 0.14811269776863384, - "nauc_ndcg_at_10_max": 0.14858935275235824, - "nauc_ndcg_at_10_std": 0.055353893746812356, - "nauc_ndcg_at_1_diff1": 0.1717974919463759, - "nauc_ndcg_at_1_max": 0.18036177614568435, - "nauc_ndcg_at_1_std": 0.06062602208644375, - "nauc_ndcg_at_20_diff1": 0.14051177320452046, - "nauc_ndcg_at_20_max": 0.17032992847697223, - "nauc_ndcg_at_20_std": 0.08648598936236548, - "nauc_ndcg_at_3_diff1": 0.15848490503685106, - "nauc_ndcg_at_3_max": 0.1420363788575197, - "nauc_ndcg_at_3_std": 0.04378165071699611, - "nauc_ndcg_at_5_diff1": 0.15441586061122994, - "nauc_ndcg_at_5_max": 0.13943370801564334, - "nauc_ndcg_at_5_std": 0.04299078112088833, - "nauc_precision_at_1000_diff1": -0.0693850001257654, - "nauc_precision_at_1000_max": 0.23895457301265866, - "nauc_precision_at_1000_std": 0.3104748618015318, - "nauc_precision_at_100_diff1": -0.04124129473256874, - "nauc_precision_at_100_max": 0.2650630128799022, - "nauc_precision_at_100_std": 0.34597349056547744, - "nauc_precision_at_10_diff1": 0.011975278358837714, - "nauc_precision_at_10_max": 0.22743813089548828, - "nauc_precision_at_10_std": 0.1993056614622371, - "nauc_precision_at_1_diff1": 0.1717974919463759, - "nauc_precision_at_1_max": 0.18036177614568435, - "nauc_precision_at_1_std": 0.06062602208644375, - "nauc_precision_at_20_diff1": -0.01325046000697272, - "nauc_precision_at_20_max": 0.25277266222735995, - "nauc_precision_at_20_std": 0.26257037178413756, - "nauc_precision_at_3_diff1": 0.07952419005134455, - "nauc_precision_at_3_max": 0.20262624120983022, - "nauc_precision_at_3_std": 0.13721345332727053, - "nauc_precision_at_5_diff1": 0.050353434949167254, - "nauc_precision_at_5_max": 0.22305126160186645, - "nauc_precision_at_5_std": 0.16948618118197217, - "nauc_recall_at_1000_diff1": 0.05450086317600606, - "nauc_recall_at_1000_max": 0.5131843341034735, - "nauc_recall_at_1000_std": 0.6259159892171178, - "nauc_recall_at_100_diff1": 0.08555859909425792, - "nauc_recall_at_100_max": 0.24547678339071335, - "nauc_recall_at_100_std": 0.32674693031150087, - "nauc_recall_at_10_diff1": 0.0956565022136966, - "nauc_recall_at_10_max": 0.1153704756423791, - "nauc_recall_at_10_std": 0.052319521754105446, - "nauc_recall_at_1_diff1": 0.2228846096271204, - "nauc_recall_at_1_max": 0.07765431366600557, - "nauc_recall_at_1_std": -0.04089387584581535, - "nauc_recall_at_20_diff1": 0.07192642711241914, - "nauc_recall_at_20_max": 0.15817325629728196, - "nauc_recall_at_20_std": 0.11737077259493674, - "nauc_recall_at_3_diff1": 0.1566837237945779, - "nauc_recall_at_3_max": 0.07826704455497915, - "nauc_recall_at_3_std": -0.006212673105043959, - "nauc_recall_at_5_diff1": 0.1251760704584453, - "nauc_recall_at_5_max": 0.09749037650409106, - "nauc_recall_at_5_std": 0.023717947434930144, - "ndcg_at_1": 0.28594, - "ndcg_at_10": 0.35216, - "ndcg_at_100": 0.44159, - "ndcg_at_1000": 0.47062, - "ndcg_at_20": 0.39015, - "ndcg_at_3": 0.29528, - "ndcg_at_5": 0.31334, - "precision_at_1": 0.28594, - "precision_at_10": 0.11134, - "precision_at_100": 0.02098, - "precision_at_1000": 0.00263, - "precision_at_20": 0.073, - "precision_at_3": 0.20634, - "precision_at_5": 0.16102, - "recall_at_1": 0.14044, - "recall_at_10": 0.45461, - "recall_at_100": 0.76199, - "recall_at_1000": 0.92997, - "recall_at_20": 0.56236, - "recall_at_3": 0.27919, - "recall_at_5": 0.34925 + "main_score": 0.33844, + "map_at_1": 0.13068, + "map_at_10": 0.25046, + "map_at_100": 0.27732, + "map_at_1000": 0.27928, + "map_at_20": 0.26525, + "map_at_3": 0.20366, + "map_at_5": 0.22639, + "mrr_at_1": 0.26677316293929715, + "mrr_at_10": 0.39275635174197476, + "mrr_at_100": 0.40251530672653774, + "mrr_at_1000": 0.4028399017212941, + "mrr_at_20": 0.39910527122826456, + "mrr_at_3": 0.3600905218317359, + "mrr_at_5": 0.3768237486687966, + "nauc_map_at_1000_diff1": 0.15794456692584777, + "nauc_map_at_1000_max": 0.12909830197930153, + "nauc_map_at_1000_std": 0.05635667662761512, + "nauc_map_at_100_diff1": 0.1582216992910556, + "nauc_map_at_100_max": 0.12761162058075068, + "nauc_map_at_100_std": 0.05492827727156516, + "nauc_map_at_10_diff1": 0.16654193130773418, + "nauc_map_at_10_max": 0.10793214410872834, + "nauc_map_at_10_std": 0.02206722533842584, + "nauc_map_at_1_diff1": 0.2202296827172708, + "nauc_map_at_1_max": 0.05007807753465177, + "nauc_map_at_1_std": -0.0424527704620777, + "nauc_map_at_20_diff1": 0.15990837209368072, + "nauc_map_at_20_max": 0.11758875162383055, + "nauc_map_at_20_std": 0.03866042502495979, + "nauc_map_at_3_diff1": 0.17298902584280004, + "nauc_map_at_3_max": 0.0708025259548859, + "nauc_map_at_3_std": -0.014114591652724913, + "nauc_map_at_5_diff1": 0.17107369964120805, + "nauc_map_at_5_max": 0.08884024812521009, + "nauc_map_at_5_std": 0.00410687004094618, + "nauc_mrr_at_1000_diff1": 0.14678937073220286, + "nauc_mrr_at_1000_max": 0.18731025158376166, + "nauc_mrr_at_1000_std": 0.11547867492193277, + "nauc_mrr_at_100_diff1": 0.14683069202217944, + "nauc_mrr_at_100_max": 0.18748259599301134, + "nauc_mrr_at_100_std": 0.11590202211160058, + "nauc_mrr_at_10_diff1": 0.1468853659152555, + "nauc_mrr_at_10_max": 0.1863453763215933, + "nauc_mrr_at_10_std": 0.11318586039374369, + "nauc_mrr_at_1_diff1": 0.16456638080577274, + "nauc_mrr_at_1_max": 0.15939890420309868, + "nauc_mrr_at_1_std": 0.05360415569963384, + "nauc_mrr_at_20_diff1": 0.14639127381285355, + "nauc_mrr_at_20_max": 0.18687457310593394, + "nauc_mrr_at_20_std": 0.11616968872525864, + "nauc_mrr_at_3_diff1": 0.1408745131554105, + "nauc_mrr_at_3_max": 0.17556777502082033, + "nauc_mrr_at_3_std": 0.09505406488871362, + "nauc_mrr_at_5_diff1": 0.14367856471781695, + "nauc_mrr_at_5_max": 0.1819697216276511, + "nauc_mrr_at_5_std": 0.10795771149299833, + "nauc_ndcg_at_1000_diff1": 0.14239762529777542, + "nauc_ndcg_at_1000_max": 0.20217762435660713, + "nauc_ndcg_at_1000_std": 0.158579589024919, + "nauc_ndcg_at_100_diff1": 0.14383079803230445, + "nauc_ndcg_at_100_max": 0.19280818193880073, + "nauc_ndcg_at_100_std": 0.15718131275984396, + "nauc_ndcg_at_10_diff1": 0.15735654960633083, + "nauc_ndcg_at_10_max": 0.1465849156932288, + "nauc_ndcg_at_10_std": 0.07319348028577521, + "nauc_ndcg_at_1_diff1": 0.16456638080577274, + "nauc_ndcg_at_1_max": 0.15939890420309868, + "nauc_ndcg_at_1_std": 0.05360415569963384, + "nauc_ndcg_at_20_diff1": 0.14606452043788754, + "nauc_ndcg_at_20_max": 0.16416062258364467, + "nauc_ndcg_at_20_std": 0.10843266302820105, + "nauc_ndcg_at_3_diff1": 0.14182846394024623, + "nauc_ndcg_at_3_max": 0.1290490230593256, + "nauc_ndcg_at_3_std": 0.049656725946823015, + "nauc_ndcg_at_5_diff1": 0.15096573190022916, + "nauc_ndcg_at_5_max": 0.12886027229806496, + "nauc_ndcg_at_5_std": 0.05444709656686989, + "nauc_precision_at_1000_diff1": -0.05791482758993109, + "nauc_precision_at_1000_max": 0.2603059102469055, + "nauc_precision_at_1000_std": 0.31337831260075083, + "nauc_precision_at_100_diff1": -0.03798256716740301, + "nauc_precision_at_100_max": 0.28308841601781953, + "nauc_precision_at_100_std": 0.35180410991445726, + "nauc_precision_at_10_diff1": 0.026288757546998522, + "nauc_precision_at_10_max": 0.25254776423338243, + "nauc_precision_at_10_std": 0.21660680710366514, + "nauc_precision_at_1_diff1": 0.16456638080577274, + "nauc_precision_at_1_max": 0.15939890420309868, + "nauc_precision_at_1_std": 0.05360415569963384, + "nauc_precision_at_20_diff1": -0.010350888375741113, + "nauc_precision_at_20_max": 0.2728525722835546, + "nauc_precision_at_20_std": 0.28808772261757315, + "nauc_precision_at_3_diff1": 0.08051771370714532, + "nauc_precision_at_3_max": 0.19430778426240003, + "nauc_precision_at_3_std": 0.1302511738508494, + "nauc_precision_at_5_diff1": 0.06692509567790808, + "nauc_precision_at_5_max": 0.22809517853781344, + "nauc_precision_at_5_std": 0.17554371476383185, + "nauc_recall_at_1000_diff1": 0.017715900641017086, + "nauc_recall_at_1000_max": 0.4628680800153984, + "nauc_recall_at_1000_std": 0.6334730091940246, + "nauc_recall_at_100_diff1": 0.0810956581211278, + "nauc_recall_at_100_max": 0.2353004394071857, + "nauc_recall_at_100_std": 0.35066919944458336, + "nauc_recall_at_10_diff1": 0.12965774216821505, + "nauc_recall_at_10_max": 0.11747723459629096, + "nauc_recall_at_10_std": 0.07454992130116075, + "nauc_recall_at_1_diff1": 0.2202296827172708, + "nauc_recall_at_1_max": 0.05007807753465177, + "nauc_recall_at_1_std": -0.0424527704620777, + "nauc_recall_at_20_diff1": 0.09819535211143614, + "nauc_recall_at_20_max": 0.14546629307158732, + "nauc_recall_at_20_std": 0.14926340396029164, + "nauc_recall_at_3_diff1": 0.13791984241632801, + "nauc_recall_at_3_max": 0.0585109217167637, + "nauc_recall_at_3_std": 0.0025290352760174973, + "nauc_recall_at_5_diff1": 0.13186776982550455, + "nauc_recall_at_5_max": 0.08328165050941126, + "nauc_recall_at_5_std": 0.0349565144303296, + "ndcg_at_1": 0.26677, + "ndcg_at_10": 0.33844, + "ndcg_at_100": 0.42957, + "ndcg_at_1000": 0.45829, + "ndcg_at_20": 0.37607, + "ndcg_at_3": 0.28204, + "ndcg_at_5": 0.29792, + "precision_at_1": 0.26677, + "precision_at_10": 0.11038, + "precision_at_100": 0.02095, + "precision_at_1000": 0.00262, + "precision_at_20": 0.07177, + "precision_at_3": 0.19995, + "precision_at_5": 0.15591, + "recall_at_1": 0.13068, + "recall_at_10": 0.44009, + "recall_at_100": 0.75878, + "recall_at_1000": 0.92524, + "recall_at_20": 0.54953, + "recall_at_3": 0.26759, + "recall_at_5": 0.33175 } ] }, diff --git a/results/deepvk__USER-base/436a489a2087d61aa670b3496a9915f84e46c861/MassiveIntentClassification.json b/results/deepvk__USER-base/436a489a2087d61aa670b3496a9915f84e46c861/MassiveIntentClassification.json index 86db9d6d3..4690207b1 100644 --- a/results/deepvk__USER-base/436a489a2087d61aa670b3496a9915f84e46c861/MassiveIntentClassification.json +++ b/results/deepvk__USER-base/436a489a2087d61aa670b3496a9915f84e46c861/MassiveIntentClassification.json @@ -6,128 +6,128 @@ "scores": { "test": [ { - "accuracy": 0.655682582380632, - "f1": 0.6338961487774227, - "f1_weighted": 0.6480597449616844, + "accuracy": 0.6592468056489577, + "f1": 0.6332930220303818, + "f1_weighted": 0.6508443105041003, "hf_subset": "ru", "languages": [ "rus-Cyrl" ], - "main_score": 0.655682582380632, + "main_score": 0.6592468056489577, "scores_per_experiment": [ { - "accuracy": 0.6704774714189644, - "f1": 0.647475414512512, - "f1_weighted": 0.6659397649485388 + "accuracy": 0.6772024209818427, + "f1": 0.6500098054988735, + "f1_weighted": 0.6712776417318309 }, { - "accuracy": 0.6819098856758574, - "f1": 0.662489895036288, - "f1_weighted": 0.6783883254671151 + "accuracy": 0.6866173503698723, + "f1": 0.6586748551381465, + "f1_weighted": 0.6803276161338214 }, { "accuracy": 0.6466039004707465, - "f1": 0.6242232867603106, - "f1_weighted": 0.6410093192677391 + "f1": 0.6234221619344356, + "f1_weighted": 0.6392532910780089 }, { - "accuracy": 0.6735036987222596, - "f1": 0.6410911362221813, - "f1_weighted": 0.6709072150514942 + "accuracy": 0.6809011432414257, + "f1": 0.6429988453339669, + "f1_weighted": 0.6766179041247198 }, { - "accuracy": 0.6607262945527909, - "f1": 0.6231508432557643, - "f1_weighted": 0.6480460006458304 + "accuracy": 0.6711499663752521, + "f1": 0.6282120171695812, + "f1_weighted": 0.6571848243839463 }, { - "accuracy": 0.6062542030934768, - "f1": 0.6011023141096451, - "f1_weighted": 0.5998313956299846 + "accuracy": 0.6116341627437795, + "f1": 0.6023394971110142, + "f1_weighted": 0.6084770533954141 }, { - "accuracy": 0.6503026227303296, - "f1": 0.632475051390601, - "f1_weighted": 0.6379971091877489 + "accuracy": 0.6449226630800269, + "f1": 0.6265072171676476, + "f1_weighted": 0.6312289797130667 }, { - "accuracy": 0.6455951580363147, - "f1": 0.621413080428444, - "f1_weighted": 0.6368622535829532 + "accuracy": 0.6476126429051782, + "f1": 0.6196337549636551, + "f1_weighted": 0.638569614574751 }, { - "accuracy": 0.6472763954270343, - "f1": 0.6290294095590201, - "f1_weighted": 0.6363984985256705 + "accuracy": 0.6489576328177539, + "f1": 0.6273829161739766, + "f1_weighted": 0.6364352110265142 }, { - "accuracy": 0.6741761936785474, - "f1": 0.6565110564994615, - "f1_weighted": 0.6652175673097682 + "accuracy": 0.6768661735036987, + "f1": 0.65374914981252, + "f1_weighted": 0.6690709688789301 } ] } ], "validation": [ { - "accuracy": 0.6546483030004919, - "f1": 0.6170084072887285, - "f1_weighted": 0.6467102103838726, + "accuracy": 0.6599114608952287, + "f1": 0.6229951814190612, + "f1_weighted": 0.6515177972982181, "hf_subset": "ru", "languages": [ "rus-Cyrl" ], - "main_score": 0.6546483030004919, + "main_score": 0.6599114608952287, "scores_per_experiment": [ { - "accuracy": 0.6655189375307428, - "f1": 0.6407137156386896, - "f1_weighted": 0.6620056024271209 + "accuracy": 0.6733890801770782, + "f1": 0.6440174924035045, + "f1_weighted": 0.669484695114143 }, { - "accuracy": 0.6886374815543532, - "f1": 0.6482929448250859, - "f1_weighted": 0.6897276180359616 + "accuracy": 0.690113133300541, + "f1": 0.6484248161401168, + "f1_weighted": 0.6880499764521711 }, { - "accuracy": 0.663059517953763, - "f1": 0.6233875880499811, - "f1_weighted": 0.6584387792587484 + "accuracy": 0.6684702410231186, + "f1": 0.6326134051706417, + "f1_weighted": 0.6607817737157313 }, { - "accuracy": 0.6660108214461387, - "f1": 0.6206756768910606, - "f1_weighted": 0.6620542885960824 + "accuracy": 0.6832267584849976, + "f1": 0.6386553370956893, + "f1_weighted": 0.6803230255098757 }, { - "accuracy": 0.6581406787998032, - "f1": 0.5963637854752606, - "f1_weighted": 0.6468825589459524 + "accuracy": 0.6596163305459911, + "f1": 0.5910124014439783, + "f1_weighted": 0.6446930950348159 }, { - "accuracy": 0.6242006886374816, - "f1": 0.5999780060723752, - "f1_weighted": 0.612971527080006 + "accuracy": 0.6207575012297097, + "f1": 0.603298155224133, + "f1_weighted": 0.6104857701116787 }, { - "accuracy": 0.6350221347761928, - "f1": 0.6009547207122825, - "f1_weighted": 0.6205747529672364 + "accuracy": 0.6345302508607968, + "f1": 0.6062149764255728, + "f1_weighted": 0.6184320065392511 }, { - "accuracy": 0.6305951795376291, - "f1": 0.5976554502118897, - "f1_weighted": 0.6185263433996023 + "accuracy": 0.6433841613379242, + "f1": 0.6090008557567135, + "f1_weighted": 0.6351164935117489 }, { - "accuracy": 0.6433841613379242, - "f1": 0.6086049008903142, - "f1_weighted": 0.632679114520136 + "accuracy": 0.6502705361534677, + "f1": 0.6218459450015268, + "f1_weighted": 0.6405272983462579 }, { - "accuracy": 0.6719134284308903, - "f1": 0.6334572841203467, - "f1_weighted": 0.6632415186078794 + "accuracy": 0.675356615838662, + "f1": 0.634868429528735, + "f1_weighted": 0.6672838386465079 } ] } diff --git a/results/deepvk__USER-base/436a489a2087d61aa670b3496a9915f84e46c861/MassiveScenarioClassification.json b/results/deepvk__USER-base/436a489a2087d61aa670b3496a9915f84e46c861/MassiveScenarioClassification.json index 21a89a205..5ac70f1cb 100644 --- a/results/deepvk__USER-base/436a489a2087d61aa670b3496a9915f84e46c861/MassiveScenarioClassification.json +++ b/results/deepvk__USER-base/436a489a2087d61aa670b3496a9915f84e46c861/MassiveScenarioClassification.json @@ -6,128 +6,128 @@ "scores": { "test": [ { - "accuracy": 0.683322125084062, - "f1": 0.6789216595117911, - "f1_weighted": 0.6789220857044962, + "accuracy": 0.6906186953597848, + "f1": 0.6866308033968759, + "f1_weighted": 0.6857808586537245, "hf_subset": "ru", "languages": [ "rus-Cyrl" ], - "main_score": 0.683322125084062, + "main_score": 0.6906186953597848, "scores_per_experiment": [ { - "accuracy": 0.6983860121049092, - "f1": 0.6968361395616279, - "f1_weighted": 0.6929963003740454 + "accuracy": 0.6943510423671823, + "f1": 0.6956218548560213, + "f1_weighted": 0.6882126592890419 }, { - "accuracy": 0.6926698049764627, - "f1": 0.688248480634289, - "f1_weighted": 0.6876800769668283 + "accuracy": 0.7088096839273705, + "f1": 0.7047826005245202, + "f1_weighted": 0.7020026346057006 }, { - "accuracy": 0.6987222595830531, - "f1": 0.6848560416757307, - "f1_weighted": 0.6951191442932766 + "accuracy": 0.7044384667114997, + "f1": 0.6945882647057474, + "f1_weighted": 0.7014520494294398 }, { - "accuracy": 0.6926698049764627, - "f1": 0.689563552303926, - "f1_weighted": 0.6894947221162891 + "accuracy": 0.6882985877605918, + "f1": 0.685481174275943, + "f1_weighted": 0.6857969859523981 }, { - "accuracy": 0.6802286482851378, - "f1": 0.6735849206101423, - "f1_weighted": 0.672357646723595 + "accuracy": 0.6906523201075991, + "f1": 0.6811043694536849, + "f1_weighted": 0.6831180352162991 }, { - "accuracy": 0.6553463349024883, - "f1": 0.6498833588673948, - "f1_weighted": 0.646846684218786 + "accuracy": 0.6647612642905179, + "f1": 0.6603349453338198, + "f1_weighted": 0.6568200721651536 }, { - "accuracy": 0.6775386684599866, - "f1": 0.671397045034112, - "f1_weighted": 0.6746504777597576 + "accuracy": 0.6893073301950235, + "f1": 0.682045802868291, + "f1_weighted": 0.6858794077890968 }, { - "accuracy": 0.6731674512441157, - "f1": 0.6745934047572872, - "f1_weighted": 0.6708795063517968 + "accuracy": 0.6761936785474109, + "f1": 0.677670424643281, + "f1_weighted": 0.6740623573484348 }, { - "accuracy": 0.691324815063887, - "f1": 0.6927110898650731, - "f1_weighted": 0.6907736149313112 + "accuracy": 0.7078009414929388, + "f1": 0.7070321509364649, + "f1_weighted": 0.7061897600267596 }, { - "accuracy": 0.6731674512441157, - "f1": 0.6675425618083283, - "f1_weighted": 0.6684226833092751 + "accuracy": 0.6815736381977135, + "f1": 0.6776464463709849, + "f1_weighted": 0.6742746247149205 } ] } ], "validation": [ { - "accuracy": 0.679390063944909, - "f1": 0.6722476179254533, - "f1_weighted": 0.6753394907636693, + "accuracy": 0.6871618298081653, + "f1": 0.6813267871979182, + "f1_weighted": 0.6829653996893705, "hf_subset": "ru", "languages": [ "rus-Cyrl" ], - "main_score": 0.679390063944909, + "main_score": 0.6871618298081653, "scores_per_experiment": [ { - "accuracy": 0.7122479094933596, - "f1": 0.7024048170048803, - "f1_weighted": 0.7107818271296087 + "accuracy": 0.7043777668470241, + "f1": 0.6979482540228524, + "f1_weighted": 0.7029182184820072 }, { - "accuracy": 0.6896212493851451, - "f1": 0.684277242929466, - "f1_weighted": 0.686303230631246 + "accuracy": 0.6984751598622725, + "f1": 0.6917065413502284, + "f1_weighted": 0.6936176411951938 }, { - "accuracy": 0.6965076242006887, - "f1": 0.6886425162146992, - "f1_weighted": 0.695663445160082 + "accuracy": 0.7151992129857354, + "f1": 0.707958668914985, + "f1_weighted": 0.7141162700925685 }, { - "accuracy": 0.6724053123462863, - "f1": 0.6714164004634391, - "f1_weighted": 0.6681062450471776 + "accuracy": 0.677816035415642, + "f1": 0.6761573175170243, + "f1_weighted": 0.6754785615031049 }, { - "accuracy": 0.6748647319232661, - "f1": 0.6664117338549709, - "f1_weighted": 0.6640226694768981 + "accuracy": 0.690113133300541, + "f1": 0.6832732302226258, + "f1_weighted": 0.6814574028127867 }, { - "accuracy": 0.6424003935071323, - "f1": 0.6350949840373314, - "f1_weighted": 0.634009695905311 + "accuracy": 0.6537137235612396, + "f1": 0.6471686484392954, + "f1_weighted": 0.6458431068892121 }, { - "accuracy": 0.6606000983767831, - "f1": 0.6471564479083354, - "f1_weighted": 0.657953708288241 + "accuracy": 0.6738809640924742, + "f1": 0.6601962857775775, + "f1_weighted": 0.6706001983981201 }, { - "accuracy": 0.6660108214461387, - "f1": 0.6605051304087408, - "f1_weighted": 0.6631458167556192 + "accuracy": 0.6669945892769307, + "f1": 0.6655781960376382, + "f1_weighted": 0.6625443441785197 }, { - "accuracy": 0.6950319724545008, - "f1": 0.694118079813324, - "f1_weighted": 0.692255566688595 + "accuracy": 0.7009345794392523, + "f1": 0.7024633875587701, + "f1_weighted": 0.6979562052486792 }, { - "accuracy": 0.6842105263157895, - "f1": 0.6724488266193469, - "f1_weighted": 0.6811527025539142 + "accuracy": 0.690113133300541, + "f1": 0.6808173421381848, + "f1_weighted": 0.6851220480935127 } ] } diff --git a/results/deepvk__USER-base/436a489a2087d61aa670b3496a9915f84e46c861/RUParaPhraserSTS.json b/results/deepvk__USER-base/436a489a2087d61aa670b3496a9915f84e46c861/RUParaPhraserSTS.json index f6389edda..0d4997727 100644 --- a/results/deepvk__USER-base/436a489a2087d61aa670b3496a9915f84e46c861/RUParaPhraserSTS.json +++ b/results/deepvk__USER-base/436a489a2087d61aa670b3496a9915f84e46c861/RUParaPhraserSTS.json @@ -6,19 +6,19 @@ "scores": { "test": [ { - "cosine_pearson": 0.6765443254605465, - "cosine_spearman": 0.7355688178876088, - "euclidean_pearson": 0.71006958615556, - "euclidean_spearman": 0.7355689990798404, + "cosine_pearson": 0.6670518427839249, + "cosine_spearman": 0.7306689608816719, + "euclidean_pearson": 0.7020320660603422, + "euclidean_spearman": 0.7306689608816719, "hf_subset": "default", "languages": [ "rus-Cyrl" ], - "main_score": 0.7355688178876088, - "manhattan_pearson": 0.7091290027333726, - "manhattan_spearman": 0.7345607381263434, - "pearson": 0.6765443254605465, - "spearman": 0.7355688178876088 + "main_score": 0.7306689608816719, + "manhattan_pearson": 0.7017125796132406, + "manhattan_spearman": 0.7304388004252601, + "pearson": 0.6670518427839249, + "spearman": 0.7306689608816719 } ] }, diff --git a/results/deepvk__USER-base/436a489a2087d61aa670b3496a9915f84e46c861/RiaNewsRetrieval.json b/results/deepvk__USER-base/436a489a2087d61aa670b3496a9915f84e46c861/RiaNewsRetrieval.json index c091e5c1f..ac1b1491a 100644 --- a/results/deepvk__USER-base/436a489a2087d61aa670b3496a9915f84e46c861/RiaNewsRetrieval.json +++ b/results/deepvk__USER-base/436a489a2087d61aa670b3496a9915f84e46c861/RiaNewsRetrieval.json @@ -10,147 +10,147 @@ "languages": [ "rus-Cyrl" ], - "main_score": 0.77829, - "map_at_1": 0.6628, - "map_at_10": 0.74297, - "map_at_100": 0.74605, - "map_at_1000": 0.74616, - "map_at_20": 0.745, - "map_at_3": 0.72798, - "map_at_5": 0.73735, - "mrr_at_1": 0.6628, - "mrr_at_10": 0.7429651984126956, - "mrr_at_100": 0.7460477460722268, - "mrr_at_1000": 0.7461577611433811, - "mrr_at_20": 0.7449991904496391, - "mrr_at_3": 0.7279833333333295, - "mrr_at_5": 0.7373533333333278, - "nauc_map_at_1000_diff1": 0.736622372105114, - "nauc_map_at_1000_max": 0.2785780089829999, - "nauc_map_at_1000_std": -0.12538403863036626, - "nauc_map_at_100_diff1": 0.736591113069233, - "nauc_map_at_100_max": 0.278629822051765, - "nauc_map_at_100_std": -0.12528993786819634, - "nauc_map_at_10_diff1": 0.735870437568321, - "nauc_map_at_10_max": 0.27767073687451005, - "nauc_map_at_10_std": -0.12669765074968925, - "nauc_map_at_1_diff1": 0.7685605026496174, - "nauc_map_at_1_max": 0.2607596681487257, - "nauc_map_at_1_std": -0.1411672866572636, - "nauc_map_at_20_diff1": 0.7361901227143981, - "nauc_map_at_20_max": 0.27832148807921264, - "nauc_map_at_20_std": -0.1256774912739158, - "nauc_map_at_3_diff1": 0.7357124992282189, - "nauc_map_at_3_max": 0.2773130064080976, - "nauc_map_at_3_std": -0.13280535901836082, - "nauc_map_at_5_diff1": 0.7359724459749499, - "nauc_map_at_5_max": 0.27765224706350766, - "nauc_map_at_5_std": -0.12820598831363197, - "nauc_mrr_at_1000_diff1": 0.736622372105114, - "nauc_mrr_at_1000_max": 0.2785780089829999, - "nauc_mrr_at_1000_std": -0.12538403863036626, - "nauc_mrr_at_100_diff1": 0.736591113069233, - "nauc_mrr_at_100_max": 0.278629822051765, - "nauc_mrr_at_100_std": -0.12528993786819634, - "nauc_mrr_at_10_diff1": 0.735870437568321, - "nauc_mrr_at_10_max": 0.27767073687451005, - "nauc_mrr_at_10_std": -0.12669765074968925, - "nauc_mrr_at_1_diff1": 0.7685605026496174, - "nauc_mrr_at_1_max": 0.2607596681487257, - "nauc_mrr_at_1_std": -0.1411672866572636, - "nauc_mrr_at_20_diff1": 0.7361901227143981, - "nauc_mrr_at_20_max": 0.27832148807921264, - "nauc_mrr_at_20_std": -0.1256774912739158, - "nauc_mrr_at_3_diff1": 0.7357124992282189, - "nauc_mrr_at_3_max": 0.2773130064080976, - "nauc_mrr_at_3_std": -0.13280535901836082, - "nauc_mrr_at_5_diff1": 0.7359724459749499, - "nauc_mrr_at_5_max": 0.27765224706350766, - "nauc_mrr_at_5_std": -0.12820598831363197, - "nauc_ndcg_at_1000_diff1": 0.7277173684457977, - "nauc_ndcg_at_1000_max": 0.2898972707524927, - "nauc_ndcg_at_1000_std": -0.11010798697822632, - "nauc_ndcg_at_100_diff1": 0.7267230572665282, - "nauc_ndcg_at_100_max": 0.2915612438336247, - "nauc_ndcg_at_100_std": -0.1063534267721977, - "nauc_ndcg_at_10_diff1": 0.722880620753823, - "nauc_ndcg_at_10_max": 0.28550851315972114, - "nauc_ndcg_at_10_std": -0.11473460727592111, - "nauc_ndcg_at_1_diff1": 0.7685605026496174, - "nauc_ndcg_at_1_max": 0.2607596681487257, - "nauc_ndcg_at_1_std": -0.1411672866572636, - "nauc_ndcg_at_20_diff1": 0.723874221154429, - "nauc_ndcg_at_20_max": 0.2885783635417774, - "nauc_ndcg_at_20_std": -0.10999364231956645, - "nauc_ndcg_at_3_diff1": 0.7241076074815131, - "nauc_ndcg_at_3_max": 0.28389069376109455, - "nauc_ndcg_at_3_std": -0.12858987660153443, - "nauc_ndcg_at_5_diff1": 0.7237464488506445, - "nauc_ndcg_at_5_max": 0.2848893748301354, - "nauc_ndcg_at_5_std": -0.1193694550404816, - "nauc_precision_at_1000_diff1": 0.5513122740845418, - "nauc_precision_at_1000_max": 0.6604196133899108, - "nauc_precision_at_1000_std": 0.34105721496518665, - "nauc_precision_at_100_diff1": 0.6132841015193903, - "nauc_precision_at_100_max": 0.5157860113742464, - "nauc_precision_at_100_std": 0.22783401239283466, - "nauc_precision_at_10_diff1": 0.6480069855804411, - "nauc_precision_at_10_max": 0.3316448142516633, - "nauc_precision_at_10_std": -0.03906716204052076, - "nauc_precision_at_1_diff1": 0.7685605026496174, - "nauc_precision_at_1_max": 0.2607596681487257, - "nauc_precision_at_1_std": -0.1411672866572636, - "nauc_precision_at_20_diff1": 0.6326144239747181, - "nauc_precision_at_20_max": 0.3745387578108166, - "nauc_precision_at_20_std": 0.02833384956546615, - "nauc_precision_at_3_diff1": 0.6813643670664915, - "nauc_precision_at_3_max": 0.3084338969066664, - "nauc_precision_at_3_std": -0.11251669503261442, - "nauc_precision_at_5_diff1": 0.669733578584559, - "nauc_precision_at_5_max": 0.3172746976905631, - "nauc_precision_at_5_std": -0.07766486585466084, - "nauc_recall_at_1000_diff1": 0.5513122740845461, - "nauc_recall_at_1000_max": 0.6604196133899095, - "nauc_recall_at_1000_std": 0.3410572149651935, - "nauc_recall_at_100_diff1": 0.613284101519397, - "nauc_recall_at_100_max": 0.5157860113742444, - "nauc_recall_at_100_std": 0.22783401239283643, - "nauc_recall_at_10_diff1": 0.6480069855804396, - "nauc_recall_at_10_max": 0.33164481425166403, - "nauc_recall_at_10_std": -0.03906716204051677, - "nauc_recall_at_1_diff1": 0.7685605026496174, - "nauc_recall_at_1_max": 0.2607596681487257, - "nauc_recall_at_1_std": -0.1411672866572636, - "nauc_recall_at_20_diff1": 0.6326144239747162, - "nauc_recall_at_20_max": 0.3745387578108153, - "nauc_recall_at_20_std": 0.02833384956546687, - "nauc_recall_at_3_diff1": 0.6813643670664928, - "nauc_recall_at_3_max": 0.3084338969066667, - "nauc_recall_at_3_std": -0.11251669503261486, - "nauc_recall_at_5_diff1": 0.6697335785845574, - "nauc_recall_at_5_max": 0.31727469769056577, - "nauc_recall_at_5_std": -0.07766486585465927, - "ndcg_at_1": 0.6628, - "ndcg_at_10": 0.77829, - "ndcg_at_100": 0.79304, - "ndcg_at_1000": 0.79604, - "ndcg_at_20": 0.78568, - "ndcg_at_3": 0.74804, - "ndcg_at_5": 0.7648, - "precision_at_1": 0.6628, - "precision_at_10": 0.08876, + "main_score": 0.77718, + "map_at_1": 0.6616, + "map_at_10": 0.74172, + "map_at_100": 0.74482, + "map_at_1000": 0.74493, + "map_at_20": 0.74374, + "map_at_3": 0.72683, + "map_at_5": 0.73617, + "mrr_at_1": 0.6616, + "mrr_at_10": 0.7417152777777778, + "mrr_at_100": 0.7448198371566934, + "mrr_at_1000": 0.7449315403240029, + "mrr_at_20": 0.7437411665664917, + "mrr_at_3": 0.7268333333333333, + "mrr_at_5": 0.7361733333333333, + "nauc_map_at_1000_diff1": 0.7340461524722205, + "nauc_map_at_1000_max": 0.2746846594310973, + "nauc_map_at_1000_std": -0.13044406423475827, + "nauc_map_at_100_diff1": 0.7339980317575249, + "nauc_map_at_100_max": 0.27472942919912974, + "nauc_map_at_100_std": -0.13037642751865008, + "nauc_map_at_10_diff1": 0.7332598764253798, + "nauc_map_at_10_max": 0.27416948317996004, + "nauc_map_at_10_std": -0.13137450890592223, + "nauc_map_at_1_diff1": 0.7655951380500258, + "nauc_map_at_1_max": 0.2551353397678101, + "nauc_map_at_1_std": -0.14688300341878768, + "nauc_map_at_20_diff1": 0.7336489171269032, + "nauc_map_at_20_max": 0.2744720004459579, + "nauc_map_at_20_std": -0.13086678468419324, + "nauc_map_at_3_diff1": 0.7329219382222375, + "nauc_map_at_3_max": 0.2746252378024666, + "nauc_map_at_3_std": -0.1375373724873289, + "nauc_map_at_5_diff1": 0.7330713576191396, + "nauc_map_at_5_max": 0.2729418963842226, + "nauc_map_at_5_std": -0.13366386335928515, + "nauc_mrr_at_1000_diff1": 0.7340461524722205, + "nauc_mrr_at_1000_max": 0.2746846594310973, + "nauc_mrr_at_1000_std": -0.13044406423475827, + "nauc_mrr_at_100_diff1": 0.7339980317575249, + "nauc_mrr_at_100_max": 0.27472942919912974, + "nauc_mrr_at_100_std": -0.13037642751865008, + "nauc_mrr_at_10_diff1": 0.7332598764253798, + "nauc_mrr_at_10_max": 0.27416948317996004, + "nauc_mrr_at_10_std": -0.13137450890592223, + "nauc_mrr_at_1_diff1": 0.7655951380500258, + "nauc_mrr_at_1_max": 0.2551353397678101, + "nauc_mrr_at_1_std": -0.14688300341878768, + "nauc_mrr_at_20_diff1": 0.7336489171269032, + "nauc_mrr_at_20_max": 0.2744720004459579, + "nauc_mrr_at_20_std": -0.13086678468419324, + "nauc_mrr_at_3_diff1": 0.7329219382222375, + "nauc_mrr_at_3_max": 0.2746252378024666, + "nauc_mrr_at_3_std": -0.1375373724873289, + "nauc_mrr_at_5_diff1": 0.7330713576191396, + "nauc_mrr_at_5_max": 0.2729418963842226, + "nauc_mrr_at_5_std": -0.13366386335928515, + "nauc_ndcg_at_1000_diff1": 0.7249970249037964, + "nauc_ndcg_at_1000_max": 0.2861136005430832, + "nauc_ndcg_at_1000_std": -0.115117791977539, + "nauc_ndcg_at_100_diff1": 0.7237824871038682, + "nauc_ndcg_at_100_max": 0.287628576066114, + "nauc_ndcg_at_100_std": -0.11196060868316768, + "nauc_ndcg_at_10_diff1": 0.7202900291242293, + "nauc_ndcg_at_10_max": 0.283062715974235, + "nauc_ndcg_at_10_std": -0.11834884575533453, + "nauc_ndcg_at_1_diff1": 0.7655951380500258, + "nauc_ndcg_at_1_max": 0.2551353397678101, + "nauc_ndcg_at_1_std": -0.14688300341878768, + "nauc_ndcg_at_20_diff1": 0.7214454689796568, + "nauc_ndcg_at_20_max": 0.2850288130249375, + "nauc_ndcg_at_20_std": -0.11570565002064016, + "nauc_ndcg_at_3_diff1": 0.7211684470046514, + "nauc_ndcg_at_3_max": 0.28245055344388675, + "nauc_ndcg_at_3_std": -0.13265775798091967, + "nauc_ndcg_at_5_diff1": 0.7205641923842616, + "nauc_ndcg_at_5_max": 0.279472265175996, + "nauc_ndcg_at_5_std": -0.12498809189064163, + "nauc_precision_at_1000_diff1": 0.5365019571046774, + "nauc_precision_at_1000_max": 0.6542824026162215, + "nauc_precision_at_1000_std": 0.3361804491911592, + "nauc_precision_at_100_diff1": 0.600846659205445, + "nauc_precision_at_100_max": 0.5061713172853765, + "nauc_precision_at_100_std": 0.20403917774784816, + "nauc_precision_at_10_diff1": 0.6455445735443166, + "nauc_precision_at_10_max": 0.335841238732868, + "nauc_precision_at_10_std": -0.03580083506501928, + "nauc_precision_at_1_diff1": 0.7655951380500258, + "nauc_precision_at_1_max": 0.2551353397678101, + "nauc_precision_at_1_std": -0.14688300341878768, + "nauc_precision_at_20_diff1": 0.6318902314377433, + "nauc_precision_at_20_max": 0.3716981861778245, + "nauc_precision_at_20_std": 0.013990685031406555, + "nauc_precision_at_3_diff1": 0.6777065530714855, + "nauc_precision_at_3_max": 0.3117338191938825, + "nauc_precision_at_3_std": -0.11396732943046223, + "nauc_precision_at_5_diff1": 0.664958315859036, + "nauc_precision_at_5_max": 0.3078980644833314, + "nauc_precision_at_5_std": -0.08424993963425, + "nauc_recall_at_1000_diff1": 0.5365019571046783, + "nauc_recall_at_1000_max": 0.6542824026162174, + "nauc_recall_at_1000_std": 0.33618044919116485, + "nauc_recall_at_100_diff1": 0.6008466592054419, + "nauc_recall_at_100_max": 0.506171317285377, + "nauc_recall_at_100_std": 0.20403917774784716, + "nauc_recall_at_10_diff1": 0.6455445735443142, + "nauc_recall_at_10_max": 0.3358412387328666, + "nauc_recall_at_10_std": -0.03580083506501958, + "nauc_recall_at_1_diff1": 0.7655951380500258, + "nauc_recall_at_1_max": 0.2551353397678101, + "nauc_recall_at_1_std": -0.14688300341878768, + "nauc_recall_at_20_diff1": 0.6318902314377425, + "nauc_recall_at_20_max": 0.3716981861778246, + "nauc_recall_at_20_std": 0.013990685031408804, + "nauc_recall_at_3_diff1": 0.6777065530714869, + "nauc_recall_at_3_max": 0.31173381919388315, + "nauc_recall_at_3_std": -0.11396732943046385, + "nauc_recall_at_5_diff1": 0.664958315859037, + "nauc_recall_at_5_max": 0.307898064483334, + "nauc_recall_at_5_std": -0.08424993963424714, + "ndcg_at_1": 0.6616, + "ndcg_at_10": 0.77718, + "ndcg_at_100": 0.79201, + "ndcg_at_1000": 0.79503, + "ndcg_at_20": 0.78446, + "ndcg_at_3": 0.74709, + "ndcg_at_5": 0.76387, + "precision_at_1": 0.6616, + "precision_at_10": 0.08869, "precision_at_100": 0.00956, "precision_at_1000": 0.00098, - "precision_at_20": 0.04584, - "precision_at_3": 0.26857, - "precision_at_5": 0.16922, - "recall_at_1": 0.6628, - "recall_at_10": 0.8876, - "recall_at_100": 0.956, - "recall_at_1000": 0.9798, - "recall_at_20": 0.9168, - "recall_at_3": 0.8057, - "recall_at_5": 0.8461 + "precision_at_20": 0.04578, + "precision_at_3": 0.26847, + "precision_at_5": 0.1692, + "recall_at_1": 0.6616, + "recall_at_10": 0.8869, + "recall_at_100": 0.9557, + "recall_at_1000": 0.9797, + "recall_at_20": 0.9155, + "recall_at_3": 0.8054, + "recall_at_5": 0.846 } ] }, diff --git a/results/deepvk__USER-base/436a489a2087d61aa670b3496a9915f84e46c861/RuReviewsClassification.json b/results/deepvk__USER-base/436a489a2087d61aa670b3496a9915f84e46c861/RuReviewsClassification.json index 0b4aef5be..01d1cd4d6 100644 --- a/results/deepvk__USER-base/436a489a2087d61aa670b3496a9915f84e46c861/RuReviewsClassification.json +++ b/results/deepvk__USER-base/436a489a2087d61aa670b3496a9915f84e46c861/RuReviewsClassification.json @@ -6,64 +6,64 @@ "scores": { "test": [ { - "accuracy": 0.66435546875, - "f1": 0.657877473772173, - "f1_weighted": 0.6578877970867311, + "accuracy": 0.65478515625, + "f1": 0.6492869603071028, + "f1_weighted": 0.6492990225772213, "hf_subset": "default", "languages": [ "rus-Cyrl" ], - "main_score": 0.66435546875, + "main_score": 0.65478515625, "scores_per_experiment": [ { - "accuracy": 0.69921875, - "f1": 0.697448814737963, - "f1_weighted": 0.6974754144832702 + "accuracy": 0.6904296875, + "f1": 0.689213236208475, + "f1_weighted": 0.6892363397255682 }, { - "accuracy": 0.62451171875, - "f1": 0.6157070184301395, - "f1_weighted": 0.6157315359158633 + "accuracy": 0.61962890625, + "f1": 0.6105553404323656, + "f1_weighted": 0.6105700685493429 }, { - "accuracy": 0.671875, - "f1": 0.6739936104359646, - "f1_weighted": 0.6740011578114872 + "accuracy": 0.6611328125, + "f1": 0.6622486059952563, + "f1_weighted": 0.6622589892957631 }, { - "accuracy": 0.6748046875, - "f1": 0.6732986604868606, - "f1_weighted": 0.6733199832558339 + "accuracy": 0.6826171875, + "f1": 0.6823776954378467, + "f1_weighted": 0.6824047552268061 }, { - "accuracy": 0.6904296875, - "f1": 0.6914455513707439, - "f1_weighted": 0.6914606164568013 + "accuracy": 0.68505859375, + "f1": 0.6845222736377687, + "f1_weighted": 0.6845314684516515 }, { - "accuracy": 0.6591796875, - "f1": 0.6524028196627085, - "f1_weighted": 0.6524145734829577 + "accuracy": 0.64453125, + "f1": 0.6385887045770767, + "f1_weighted": 0.6386088241979979 }, { - "accuracy": 0.64892578125, - "f1": 0.6386933379170996, - "f1_weighted": 0.6387073112684147 + "accuracy": 0.62890625, + "f1": 0.619814870917239, + "f1_weighted": 0.6198399728044428 }, { - "accuracy": 0.65234375, - "f1": 0.6444962922896867, - "f1_weighted": 0.6444884602065921 + "accuracy": 0.6318359375, + "f1": 0.6263736816247104, + "f1_weighted": 0.6263656602014703 }, { - "accuracy": 0.64453125, - "f1": 0.6202387265054201, - "f1_weighted": 0.6201997451113108 + "accuracy": 0.62451171875, + "f1": 0.6052958463300698, + "f1_weighted": 0.6052617099101317 }, { - "accuracy": 0.677734375, - "f1": 0.6710499058851435, - "f1_weighted": 0.6710791728747794 + "accuracy": 0.67919921875, + "f1": 0.6738793479102191, + "f1_weighted": 0.6739124374090378 } ] } diff --git a/results/deepvk__USER-base/436a489a2087d61aa670b3496a9915f84e46c861/RuSTSBenchmarkSTS.json b/results/deepvk__USER-base/436a489a2087d61aa670b3496a9915f84e46c861/RuSTSBenchmarkSTS.json index 477701642..6249f224c 100644 --- a/results/deepvk__USER-base/436a489a2087d61aa670b3496a9915f84e46c861/RuSTSBenchmarkSTS.json +++ b/results/deepvk__USER-base/436a489a2087d61aa670b3496a9915f84e46c861/RuSTSBenchmarkSTS.json @@ -6,19 +6,19 @@ "scores": { "test": [ { - "cosine_pearson": 0.8271871635858065, - "cosine_spearman": 0.8225600754552332, - "euclidean_pearson": 0.8050630795071115, - "euclidean_spearman": 0.8225620021900173, + "cosine_pearson": 0.8243011714527109, + "cosine_spearman": 0.8190961090247312, + "euclidean_pearson": 0.8011375272866301, + "euclidean_spearman": 0.8191048894772341, "hf_subset": "default", "languages": [ "rus-Cyrl" ], - "main_score": 0.8225600754552332, - "manhattan_pearson": 0.8045913726803143, - "manhattan_spearman": 0.8215451425334964, - "pearson": 0.8271871635858065, - "spearman": 0.8225600754552332 + "main_score": 0.8190961090247312, + "manhattan_pearson": 0.8007316200028387, + "manhattan_spearman": 0.8179452258948954, + "pearson": 0.8243011714527109, + "spearman": 0.8190961090247312 } ] }, diff --git a/results/deepvk__USER-base/436a489a2087d61aa670b3496a9915f84e46c861/RuSciBenchGRNTIClusteringP2P.json b/results/deepvk__USER-base/436a489a2087d61aa670b3496a9915f84e46c861/RuSciBenchGRNTIClusteringP2P.json index 5f6c18dc4..a873421e0 100644 --- a/results/deepvk__USER-base/436a489a2087d61aa670b3496a9915f84e46c861/RuSciBenchGRNTIClusteringP2P.json +++ b/results/deepvk__USER-base/436a489a2087d61aa670b3496a9915f84e46c861/RuSciBenchGRNTIClusteringP2P.json @@ -10,21 +10,21 @@ "languages": [ "rus-Cyrl" ], - "main_score": 0.5137832792769835, - "v_measure": 0.5137832792769835, - "v_measure_std": 0.009216164856261736, + "main_score": 0.5149507067467738, + "v_measure": 0.5149507067467738, + "v_measure_std": 0.009516599627655886, "v_measures": { "Level 0": [ - 0.518253997602414, - 0.5148031921165572, - 0.5045484929222809, - 0.5328384885508178, - 0.5016025416719033, - 0.5079383261695173, - 0.5174635077883575, - 0.5201658183109488, - 0.502488358331398, - 0.5177300693056409 + 0.5179851048982849, + 0.5128944891602332, + 0.5036238856702459, + 0.5084383302994167, + 0.5206725207189712, + 0.5291534123879419, + 0.521018330963934, + 0.5173079561458096, + 0.5229668433429769, + 0.49544619387992356 ] } } diff --git a/results/deepvk__USER-base/436a489a2087d61aa670b3496a9915f84e46c861/RuSciBenchOECDClusteringP2P.json b/results/deepvk__USER-base/436a489a2087d61aa670b3496a9915f84e46c861/RuSciBenchOECDClusteringP2P.json index 34e1c17d7..61d265e61 100644 --- a/results/deepvk__USER-base/436a489a2087d61aa670b3496a9915f84e46c861/RuSciBenchOECDClusteringP2P.json +++ b/results/deepvk__USER-base/436a489a2087d61aa670b3496a9915f84e46c861/RuSciBenchOECDClusteringP2P.json @@ -10,21 +10,21 @@ "languages": [ "rus-Cyrl" ], - "main_score": 0.4473151957029938, - "v_measure": 0.4473151957029938, - "v_measure_std": 0.008651382972765657, + "main_score": 0.44874036521309246, + "v_measure": 0.44874036521309246, + "v_measure_std": 0.006126863523830374, "v_measures": { "Level 0": [ - 0.45132379543041806, - 0.46581753154267247, - 0.4418338156124625, - 0.4409123458522766, - 0.4478474890591896, - 0.4419341352851398, - 0.43955278295946976, - 0.4593944378318499, - 0.43822661779669686, - 0.44630900565976195 + 0.4458119638704562, + 0.4570674047737045, + 0.4506933908952024, + 0.4507064405319279, + 0.4478557734580001, + 0.4426578469717941, + 0.4599680562851473, + 0.4385319132534394, + 0.44422182844414665, + 0.4498890336471064 ] } } diff --git a/results/deepvk__USER-base/436a489a2087d61aa670b3496a9915f84e46c861/STS22.json b/results/deepvk__USER-base/436a489a2087d61aa670b3496a9915f84e46c861/STS22.json index cd6db2ed5..6e6eb9a67 100644 --- a/results/deepvk__USER-base/436a489a2087d61aa670b3496a9915f84e46c861/STS22.json +++ b/results/deepvk__USER-base/436a489a2087d61aa670b3496a9915f84e46c861/STS22.json @@ -6,19 +6,19 @@ "scores": { "test": [ { - "cosine_pearson": 0.570358494628828, - "cosine_spearman": 0.6339243874572952, - "euclidean_pearson": 0.5640103943291542, - "euclidean_spearman": 0.6338684637439985, + "cosine_pearson": 0.6465830742059284, + "cosine_spearman": 0.6805523075749293, + "euclidean_pearson": 0.6491664446397086, + "euclidean_spearman": 0.6805523075749293, "hf_subset": "ru", "languages": [ "rus-Cyrl" ], - "main_score": 0.6339243874572952, - "manhattan_pearson": 0.5647376834532587, - "manhattan_spearman": 0.63567556756953, - "pearson": 0.570358494628828, - "spearman": 0.6339243874572952 + "main_score": 0.6805523075749293, + "manhattan_pearson": 0.648437588245472, + "manhattan_spearman": 0.6817601210607527, + "pearson": 0.6465830742059284, + "spearman": 0.6805523075749293 } ] }, diff --git a/results/deepvk__USER-base/436a489a2087d61aa670b3496a9915f84e46c861/SensitiveTopicsClassification.json b/results/deepvk__USER-base/436a489a2087d61aa670b3496a9915f84e46c861/SensitiveTopicsClassification.json index 09ad63203..8b67b3586 100644 --- a/results/deepvk__USER-base/436a489a2087d61aa670b3496a9915f84e46c861/SensitiveTopicsClassification.json +++ b/results/deepvk__USER-base/436a489a2087d61aa670b3496a9915f84e46c861/SensitiveTopicsClassification.json @@ -6,64 +6,64 @@ "scores": { "test": [ { - "accuracy": 0.275, - "f1": 0.294228647603761, + "accuracy": 0.286474609375, + "f1": 0.31344367604012535, "hf_subset": "default", "languages": [ "rus-Cyrl" ], - "lrap": 0.42499457465277024, - "main_score": 0.275, + "lrap": 0.43825276692707577, + "main_score": 0.286474609375, "scores_per_experiment": [ { - "accuracy": 0.27490234375, - "f1": 0.27596784407747565, - "lrap": 0.41220431857638107 + "accuracy": 0.24658203125, + "f1": 0.2649084049399284, + "lrap": 0.3908013237847149 }, { - "accuracy": 0.26611328125, - "f1": 0.28554914501383477, - "lrap": 0.4115804036458254 + "accuracy": 0.28125, + "f1": 0.3300402547687023, + "lrap": 0.4378390842013814 }, { - "accuracy": 0.2470703125, - "f1": 0.28269837321252544, - "lrap": 0.4038764105902703 + "accuracy": 0.27001953125, + "f1": 0.2624618752005571, + "lrap": 0.41437445746527 }, { - "accuracy": 0.26708984375, - "f1": 0.3022005074621278, - "lrap": 0.41071234809027035 + "accuracy": 0.28662109375, + "f1": 0.31980006539529515, + "lrap": 0.4348415798611036 }, { - "accuracy": 0.25732421875, - "f1": 0.26148957870819634, - "lrap": 0.4088609483506867 + "accuracy": 0.29345703125, + "f1": 0.3398979036142473, + "lrap": 0.444898817274298 }, { - "accuracy": 0.3017578125, - "f1": 0.344533097100572, - "lrap": 0.4612019856770756 + "accuracy": 0.2998046875, + "f1": 0.3382285743229926, + "lrap": 0.46346028645832593 }, { - "accuracy": 0.27197265625, - "f1": 0.29769587523960134, - "lrap": 0.4316948784722146 + "accuracy": 0.31640625, + "f1": 0.3298451231083622, + "lrap": 0.47119818793402035 }, { - "accuracy": 0.28857421875, - "f1": 0.30240471017547066, - "lrap": 0.4267713758680478 + "accuracy": 0.28515625, + "f1": 0.28646566356438896, + "lrap": 0.4251166449652698 }, { - "accuracy": 0.28076171875, - "f1": 0.29421543720545684, - "lrap": 0.44064670138888173 + "accuracy": 0.2939453125, + "f1": 0.3351168895695391, + "lrap": 0.45534939236110333 }, { - "accuracy": 0.29443359375, - "f1": 0.2955319078423487, - "lrap": 0.44239637586804825 + "accuracy": 0.29150390625, + "f1": 0.3276720059172403, + "lrap": 0.44464789496527035 } ] } diff --git a/results/deepvk__USER-base/436a489a2087d61aa670b3496a9915f84e46c861/TERRa.json b/results/deepvk__USER-base/436a489a2087d61aa670b3496a9915f84e46c861/TERRa.json index b62711f03..38c80c378 100644 --- a/results/deepvk__USER-base/436a489a2087d61aa670b3496a9915f84e46c861/TERRa.json +++ b/results/deepvk__USER-base/436a489a2087d61aa670b3496a9915f84e46c861/TERRa.json @@ -6,48 +6,49 @@ "scores": { "dev": [ { - "cosine_accuracy": 0.6123778501628665, - "cosine_accuracy_threshold": 0.6209567785263062, - "cosine_ap": 0.6001782200359489, + "cosine_accuracy": 0.6188925081433225, + "cosine_accuracy_threshold": 0.6009500026702881, + "cosine_ap": 0.59756668260072, "cosine_f1": 0.6681222707423581, - "cosine_f1_threshold": 0.26439931988716125, + "cosine_f1_threshold": 0.23748540878295898, "cosine_precision": 0.5016393442622951, "cosine_recall": 1.0, - "dot_accuracy": 0.6123778501628665, - "dot_accuracy_threshold": 0.6209567785263062, - "dot_ap": 0.6001782200359489, + "dot_accuracy": 0.6188925081433225, + "dot_accuracy_threshold": 0.6009500026702881, + "dot_ap": 0.59756668260072, "dot_f1": 0.6681222707423581, - "dot_f1_threshold": 0.26439931988716125, + "dot_f1_threshold": 0.23748543858528137, "dot_precision": 0.5016393442622951, "dot_recall": 1.0, - "euclidean_accuracy": 0.6123778501628665, - "euclidean_accuracy_threshold": 0.8706815242767334, - "euclidean_ap": 0.6001782200359489, + "euclidean_accuracy": 0.6188925081433225, + "euclidean_accuracy_threshold": 0.8933643102645874, + "euclidean_ap": 0.59756668260072, "euclidean_f1": 0.6681222707423581, - "euclidean_f1_threshold": 1.2129271030426025, + "euclidean_f1_threshold": 1.2348787784576416, "euclidean_precision": 0.5016393442622951, "euclidean_recall": 1.0, "hf_subset": "default", "languages": [ "rus-Cyrl" ], - "main_score": 0.6010737523228904, - "manhattan_accuracy": 0.6123778501628665, - "manhattan_accuracy_threshold": 20.636640548706055, - "manhattan_ap": 0.6010737523228904, - "manhattan_f1": 0.6695842450765863, - "manhattan_f1_threshold": 26.682357788085938, - "manhattan_precision": 0.5032894736842105, + "main_score": 0.59756668260072, + "manhattan_accuracy": 0.6091205211726385, + "manhattan_accuracy_threshold": 19.89917755126953, + "manhattan_ap": 0.5975310094029704, + "manhattan_f1": 0.6681222707423581, + "manhattan_f1_threshold": 27.12159538269043, + "manhattan_precision": 0.5016393442622951, "manhattan_recall": 1.0, - "max_ap": 0.6010737523228904, - "max_f1": 0.6695842450765863, - "max_precision": 0.5032894736842105, + "max_accuracy": 0.6188925081433225, + "max_ap": 0.59756668260072, + "max_f1": 0.6681222707423581, + "max_precision": 0.5016393442622951, "max_recall": 1.0, - "similarity_accuracy": 0.6123778501628665, - "similarity_accuracy_threshold": 0.6209567785263062, - "similarity_ap": 0.6001782200359489, + "similarity_accuracy": 0.6188925081433225, + "similarity_accuracy_threshold": 0.6009500026702881, + "similarity_ap": 0.59756668260072, "similarity_f1": 0.6681222707423581, - "similarity_f1_threshold": 0.26439931988716125, + "similarity_f1_threshold": 0.23748540878295898, "similarity_precision": 0.5016393442622951, "similarity_recall": 1.0 } diff --git a/results/deepvk__USER-base/436a489a2087d61aa670b3496a9915f84e46c861/model_meta.json b/results/deepvk__USER-base/436a489a2087d61aa670b3496a9915f84e46c861/model_meta.json index f5bcd5d2a..e3c878033 100644 --- a/results/deepvk__USER-base/436a489a2087d61aa670b3496a9915f84e46c861/model_meta.json +++ b/results/deepvk__USER-base/436a489a2087d61aa670b3496a9915f84e46c861/model_meta.json @@ -1 +1,17 @@ -{"name": "deepvk/USER-base", "revision": "436a489a2087d61aa670b3496a9915f84e46c861", "release_date": "2024-06-10", "languages": ["rus_Cyrl"], "n_parameters": null, "memory_usage": null, "max_tokens": null, "embed_dim": null, "license": null, "open_source": true, "similarity_fn_name": null, "framework": [], "loader": "E5Wrapper"} \ No newline at end of file +{ + "name": "deepvk/USER-base", + "revision": "436a489a2087d61aa670b3496a9915f84e46c861", + "release_date": "2024-06-10", + "languages": [ + "rus_Cyrl" + ], + "n_parameters": null, + "memory_usage": null, + "max_tokens": null, + "embed_dim": null, + "license": null, + "open_source": true, + "similarity_fn_name": null, + "framework": [], + "loader": "E5Wrapper" +} \ No newline at end of file diff --git a/results/deepvk__USER-bge-m3/0cc6cfe48e260fb0474c753087a69369e88709ae/GeoreviewClusteringP2P.json b/results/deepvk__USER-bge-m3/0cc6cfe48e260fb0474c753087a69369e88709ae/GeoreviewClusteringP2P.json index 2a77a4043..ffbd50e87 100644 --- a/results/deepvk__USER-bge-m3/0cc6cfe48e260fb0474c753087a69369e88709ae/GeoreviewClusteringP2P.json +++ b/results/deepvk__USER-bge-m3/0cc6cfe48e260fb0474c753087a69369e88709ae/GeoreviewClusteringP2P.json @@ -10,21 +10,21 @@ "languages": [ "rus-Cyrl" ], - "main_score": 0.6278847022889213, - "v_measure": 0.6278847022889213, - "v_measure_std": 0.014019269145121784, + "main_score": 0.6293866005919149, + "v_measure": 0.6293866005919149, + "v_measure_std": 0.01071287843027738, "v_measures": { "Level 0": [ - 0.6228266121500853, - 0.6259264279216733, - 0.6585205888028158, - 0.6291140245857286, - 0.636175821814589, - 0.614494182356848, - 0.6011407773776926, - 0.6335139975168036, - 0.6285217654782769, - 0.6286128248846997 + 0.6305793656344673, + 0.613612817219088, + 0.6400684979271899, + 0.6233558037986263, + 0.6181890139721261, + 0.6446521504002639, + 0.6386839828620182, + 0.6169510484940528, + 0.6265743129211448, + 0.641199012690171 ] } } diff --git a/results/deepvk__USER-bge-m3/0cc6cfe48e260fb0474c753087a69369e88709ae/STS22.json b/results/deepvk__USER-bge-m3/0cc6cfe48e260fb0474c753087a69369e88709ae/STS22.json index 1f146a56d..492606df8 100644 --- a/results/deepvk__USER-bge-m3/0cc6cfe48e260fb0474c753087a69369e88709ae/STS22.json +++ b/results/deepvk__USER-bge-m3/0cc6cfe48e260fb0474c753087a69369e88709ae/STS22.json @@ -6,25 +6,19 @@ "scores": { "test": [ { - "cosine_pearson": 0.640653431531091, - "cosine_spearman": 0.6641859247967075, - "euclidean_pearson": 0.6238114119414466, - "euclidean_spearman": 0.6641299873988215, + "cosine_pearson": 0.6875225197530586, + "cosine_spearman": 0.7055469538106395, + "euclidean_pearson": 0.6898391961978994, + "euclidean_spearman": 0.7055469538106395, "hf_subset": "ru", "languages": [ "rus-Cyrl" ], - "main_score": 0.6641859247967075, - "manhattan_pearson": 0.6263564525091572, - "manhattan_spearman": 0.6649170787154212, - "pearson": [ - 0.6406534231296672, - 1.0329950505912451e-32 - ], - "spearman": [ - 0.6641859247967075, - 7.48708778631243e-36 - ] + "main_score": 0.7055469538106395, + "manhattan_pearson": 0.6903204132599111, + "manhattan_spearman": 0.705027926807547, + "pearson": 0.6875225197530586, + "spearman": 0.7055469538106395 } ] }, diff --git a/results/deepvk__USER-bge-m3/0cc6cfe48e260fb0474c753087a69369e88709ae/SensitiveTopicsClassification.json b/results/deepvk__USER-bge-m3/0cc6cfe48e260fb0474c753087a69369e88709ae/SensitiveTopicsClassification.json index 40cf097b9..28d47252b 100644 --- a/results/deepvk__USER-bge-m3/0cc6cfe48e260fb0474c753087a69369e88709ae/SensitiveTopicsClassification.json +++ b/results/deepvk__USER-bge-m3/0cc6cfe48e260fb0474c753087a69369e88709ae/SensitiveTopicsClassification.json @@ -6,64 +6,64 @@ "scores": { "test": [ { - "accuracy": 0.262890625, - "f1": 0.27148406010537085, + "accuracy": 0.26884765625, + "f1": 0.2848861764380188, "hf_subset": "default", "languages": [ "rus-Cyrl" ], - "lrap": 0.394974772135409, - "main_score": 0.262890625, + "lrap": 0.40690171983506174, + "main_score": 0.26884765625, "scores_per_experiment": [ { - "accuracy": 0.28955078125, - "f1": 0.27823582233773564, - "lrap": 0.41912163628471444 + "accuracy": 0.255859375, + "f1": 0.26870824851267416, + "lrap": 0.38203938802082565 }, { - "accuracy": 0.244140625, - "f1": 0.269156439011958, - "lrap": 0.3757595486111034 + "accuracy": 0.25927734375, + "f1": 0.283705175345863, + "lrap": 0.4002346462673535 }, { - "accuracy": 0.25390625, - "f1": 0.23320463312365158, - "lrap": 0.366794162326381 + "accuracy": 0.26220703125, + "f1": 0.24820884007599478, + "lrap": 0.38256835937499195 }, { - "accuracy": 0.2646484375, - "f1": 0.27489843808620285, - "lrap": 0.39321560329860367 + "accuracy": 0.23779296875, + "f1": 0.24744606169853295, + "lrap": 0.3751085069444369 }, { - "accuracy": 0.25537109375, - "f1": 0.2531200219620444, - "lrap": 0.3924153645833257 + "accuracy": 0.2490234375, + "f1": 0.3004871367097983, + "lrap": 0.39134385850693665 }, { - "accuracy": 0.26904296875, - "f1": 0.29471755576279496, - "lrap": 0.41078016493054825 + "accuracy": 0.27099609375, + "f1": 0.2756999605812786, + "lrap": 0.41296386718749223 }, { - "accuracy": 0.24658203125, - "f1": 0.26009324073579915, - "lrap": 0.3899332682291589 + "accuracy": 0.302734375, + "f1": 0.3057480955244404, + "lrap": 0.4429796006944369 }, { - "accuracy": 0.26953125, - "f1": 0.27832515261294577, - "lrap": 0.40261501736110317 + "accuracy": 0.27880859375, + "f1": 0.3007367854094889, + "lrap": 0.4206678602430478 }, { - "accuracy": 0.267578125, - "f1": 0.2894346304004547, - "lrap": 0.3985188802083254 + "accuracy": 0.2998046875, + "f1": 0.31458868856247124, + "lrap": 0.43253580729165886 }, { - "accuracy": 0.2685546875, - "f1": 0.2836546670201223, - "lrap": 0.40059407552082577 + "accuracy": 0.27197265625, + "f1": 0.3035327719596455, + "lrap": 0.42857530381943704 } ] } diff --git a/results/deepvk__USER-bge-m3/0cc6cfe48e260fb0474c753087a69369e88709ae/model_meta.json b/results/deepvk__USER-bge-m3/0cc6cfe48e260fb0474c753087a69369e88709ae/model_meta.json index baa769362..e74110ccb 100644 --- a/results/deepvk__USER-bge-m3/0cc6cfe48e260fb0474c753087a69369e88709ae/model_meta.json +++ b/results/deepvk__USER-bge-m3/0cc6cfe48e260fb0474c753087a69369e88709ae/model_meta.json @@ -1 +1,17 @@ -{"name": "deepvk/USER-bge-m3", "revision": "0cc6cfe48e260fb0474c753087a69369e88709ae", "release_date": "2024-07-05", "languages": ["rus_Cyrl"], "n_parameters": null, "memory_usage": null, "max_tokens": null, "embed_dim": null, "license": null, "open_source": true, "similarity_fn_name": null, "framework": [], "loader": null} \ No newline at end of file +{ + "name": "deepvk/USER-bge-m3", + "revision": "0cc6cfe48e260fb0474c753087a69369e88709ae", + "release_date": "2024-07-05", + "languages": [ + "rus_Cyrl" + ], + "n_parameters": null, + "memory_usage": null, + "max_tokens": null, + "embed_dim": null, + "license": null, + "open_source": true, + "similarity_fn_name": null, + "framework": [], + "loader": null +} \ No newline at end of file diff --git a/results/dunzhang__stella_en_1.5B_v5/d03be74b361d4eb24f42a2fe5bd2e29917df4604/CEDRClassification.json b/results/dunzhang__stella_en_1.5B_v5/d03be74b361d4eb24f42a2fe5bd2e29917df4604/CEDRClassification.json index cb4a53d8c..a5bf2632c 100644 --- a/results/dunzhang__stella_en_1.5B_v5/d03be74b361d4eb24f42a2fe5bd2e29917df4604/CEDRClassification.json +++ b/results/dunzhang__stella_en_1.5B_v5/d03be74b361d4eb24f42a2fe5bd2e29917df4604/CEDRClassification.json @@ -6,64 +6,64 @@ "scores": { "test": [ { - "accuracy": 0.4636556854410202, - "f1": 0.4155312839113403, + "accuracy": 0.439798087141339, + "f1": 0.41608330185744513, "hf_subset": "default", "languages": [ "rus-Cyrl" ], - "lrap": 0.7181057385759934, - "main_score": 0.4636556854410202, + "lrap": 0.7105127523910837, + "main_score": 0.439798087141339, "scores_per_experiment": [ { - "accuracy": 0.46014877789585545, - "f1": 0.3986340031472954, - "lrap": 0.6980871413390123 + "accuracy": 0.4038257173219979, + "f1": 0.3977950372469544, + "lrap": 0.7331030818278527 }, { - "accuracy": 0.4442082890541977, - "f1": 0.3993609475775795, - "lrap": 0.74067481402764 + "accuracy": 0.3942614240170032, + "f1": 0.4169774151429394, + "lrap": 0.7318544102019221 }, { - "accuracy": 0.359192348565356, - "f1": 0.2884625690436526, - "lrap": 0.6198724760892789 + "accuracy": 0.40488841657810837, + "f1": 0.42215582907708227, + "lrap": 0.7176408076514447 }, { - "accuracy": 0.4527098831030818, - "f1": 0.4065288983724738, - "lrap": 0.7133900106270034 + "accuracy": 0.4436769394261424, + "f1": 0.45907739270761877, + "lrap": 0.7317747077577141 }, { - "accuracy": 0.5005313496280552, - "f1": 0.4317989183065774, - "lrap": 0.7423485653560141 + "accuracy": 0.4957492029755579, + "f1": 0.4390706271597142, + "lrap": 0.7128586609989479 }, { - "accuracy": 0.5074388947927736, - "f1": 0.45649852090127874, - "lrap": 0.7444208289054292 + "accuracy": 0.32146652497343253, + "f1": 0.3330327008810168, + "lrap": 0.6885759829968227 }, { - "accuracy": 0.49946865037194477, - "f1": 0.4129024742906573, - "lrap": 0.7150371944739745 + "accuracy": 0.4909670563230606, + "f1": 0.44353780766732775, + "lrap": 0.7058448459086184 }, { - "accuracy": 0.4867162592986185, - "f1": 0.4554814601702543, - "lrap": 0.7540913921360352 + "accuracy": 0.4691817215727949, + "f1": 0.4078955373848836, + "lrap": 0.6868756641870464 }, { - "accuracy": 0.4404888416578108, - "f1": 0.4066394931589155, - "lrap": 0.6884165781084065 + "accuracy": 0.4771519659936238, + "f1": 0.3944142884922425, + "lrap": 0.6841126461211589 }, { - "accuracy": 0.485653560042508, - "f1": 0.4990055541447185, - "lrap": 0.7647183846971397 + "accuracy": 0.49681190223166843, + "f1": 0.44687638281467185, + "lrap": 0.7124867162593093 } ] } diff --git a/results/dunzhang__stella_en_1.5B_v5/d03be74b361d4eb24f42a2fe5bd2e29917df4604/GeoreviewClassification.json b/results/dunzhang__stella_en_1.5B_v5/d03be74b361d4eb24f42a2fe5bd2e29917df4604/GeoreviewClassification.json new file mode 100644 index 000000000..bb6819425 --- /dev/null +++ b/results/dunzhang__stella_en_1.5B_v5/d03be74b361d4eb24f42a2fe5bd2e29917df4604/GeoreviewClassification.json @@ -0,0 +1,73 @@ +{ + "dataset_revision": "3765c0d1de6b7d264bc459433c45e5a75513839c", + "evaluation_time": 110.65320181846619, + "kg_co2_emissions": null, + "mteb_version": "1.19.0", + "scores": { + "test": [ + { + "accuracy": 0.556787109375, + "f1": 0.5249213526406058, + "f1_weighted": 0.5248778223637784, + "hf_subset": "default", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.556787109375, + "scores_per_experiment": [ + { + "accuracy": 0.51953125, + "f1": 0.46552153773533256, + "f1_weighted": 0.46544799878915954 + }, + { + "accuracy": 0.537109375, + "f1": 0.49339854206427897, + "f1_weighted": 0.4933045668799658 + }, + { + "accuracy": 0.5869140625, + "f1": 0.5649070461578163, + "f1_weighted": 0.564871040529292 + }, + { + "accuracy": 0.58544921875, + "f1": 0.5711262538585702, + "f1_weighted": 0.5711006659268909 + }, + { + "accuracy": 0.54150390625, + "f1": 0.4854357216411678, + "f1_weighted": 0.48534360721744807 + }, + { + "accuracy": 0.5615234375, + "f1": 0.5385812840220969, + "f1_weighted": 0.5385598487879071 + }, + { + "accuracy": 0.55517578125, + "f1": 0.527948881583678, + "f1_weighted": 0.5279255105411433 + }, + { + "accuracy": 0.54296875, + "f1": 0.5051996859972497, + "f1_weighted": 0.5051335468624031 + }, + { + "accuracy": 0.5791015625, + "f1": 0.5665672056102067, + "f1_weighted": 0.5665712969135116 + }, + { + "accuracy": 0.55859375, + "f1": 0.530527367735661, + "f1_weighted": 0.5305201411900635 + } + ] + } + ] + }, + "task_name": "GeoreviewClassification" +} \ No newline at end of file diff --git a/results/dunzhang__stella_en_1.5B_v5/d03be74b361d4eb24f42a2fe5bd2e29917df4604/GeoreviewClusteringP2P.json b/results/dunzhang__stella_en_1.5B_v5/d03be74b361d4eb24f42a2fe5bd2e29917df4604/GeoreviewClusteringP2P.json new file mode 100644 index 000000000..81eebb0d9 --- /dev/null +++ b/results/dunzhang__stella_en_1.5B_v5/d03be74b361d4eb24f42a2fe5bd2e29917df4604/GeoreviewClusteringP2P.json @@ -0,0 +1,34 @@ +{ + "dataset_revision": "97a313c8fc85b47f13f33e7e9a95c1ad888c7fec", + "evaluation_time": 207.2279450893402, + "kg_co2_emissions": null, + "mteb_version": "1.19.0", + "scores": { + "test": [ + { + "hf_subset": "default", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.6674618837018939, + "v_measure": 0.6674618837018939, + "v_measure_std": 0.005795056232048515, + "v_measures": { + "Level 0": [ + 0.6719711826021157, + 0.6650226093274467, + 0.6666274540312642, + 0.6762540228601276, + 0.6692737446862568, + 0.6577169242484231, + 0.6617479895798549, + 0.6656619852364654, + 0.6766028703274033, + 0.6637400541195807 + ] + } + } + ] + }, + "task_name": "GeoreviewClusteringP2P" +} \ No newline at end of file diff --git a/results/dunzhang__stella_en_1.5B_v5/d03be74b361d4eb24f42a2fe5bd2e29917df4604/HeadlineClassification.json b/results/dunzhang__stella_en_1.5B_v5/d03be74b361d4eb24f42a2fe5bd2e29917df4604/HeadlineClassification.json new file mode 100644 index 000000000..d40f415ec --- /dev/null +++ b/results/dunzhang__stella_en_1.5B_v5/d03be74b361d4eb24f42a2fe5bd2e29917df4604/HeadlineClassification.json @@ -0,0 +1,73 @@ +{ + "dataset_revision": "2fe05ee6b5832cda29f2ef7aaad7b7fe6a3609eb", + "evaluation_time": 38.21867632865906, + "kg_co2_emissions": null, + "mteb_version": "1.19.0", + "scores": { + "test": [ + { + "accuracy": 0.780712890625, + "f1": 0.7817797313414938, + "f1_weighted": 0.7817509893497769, + "hf_subset": "default", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.780712890625, + "scores_per_experiment": [ + { + "accuracy": 0.7841796875, + "f1": 0.784825473786143, + "f1_weighted": 0.7848105565920948 + }, + { + "accuracy": 0.73583984375, + "f1": 0.735088332659144, + "f1_weighted": 0.7351218935372469 + }, + { + "accuracy": 0.7705078125, + "f1": 0.7724188713603399, + "f1_weighted": 0.7723823808242635 + }, + { + "accuracy": 0.80712890625, + "f1": 0.8081353341721281, + "f1_weighted": 0.8081066806847195 + }, + { + "accuracy": 0.7568359375, + "f1": 0.7623577169313503, + "f1_weighted": 0.7623263234388495 + }, + { + "accuracy": 0.7939453125, + "f1": 0.7942766248044051, + "f1_weighted": 0.7942255224162281 + }, + { + "accuracy": 0.7802734375, + "f1": 0.7817254034974565, + "f1_weighted": 0.7816561142409766 + }, + { + "accuracy": 0.81787109375, + "f1": 0.8176249913316506, + "f1_weighted": 0.817602439409845 + }, + { + "accuracy": 0.7705078125, + "f1": 0.7713578800616062, + "f1_weighted": 0.7713331400636014 + }, + { + "accuracy": 0.7900390625, + "f1": 0.7899866848107148, + "f1_weighted": 0.7899448422899443 + } + ] + } + ] + }, + "task_name": "HeadlineClassification" +} \ No newline at end of file diff --git a/results/dunzhang__stella_en_1.5B_v5/d03be74b361d4eb24f42a2fe5bd2e29917df4604/InappropriatenessClassification.json b/results/dunzhang__stella_en_1.5B_v5/d03be74b361d4eb24f42a2fe5bd2e29917df4604/InappropriatenessClassification.json new file mode 100644 index 000000000..539de5e37 --- /dev/null +++ b/results/dunzhang__stella_en_1.5B_v5/d03be74b361d4eb24f42a2fe5bd2e29917df4604/InappropriatenessClassification.json @@ -0,0 +1,95 @@ +{ + "dataset_revision": "601651fdc45ef243751676e62dd7a19f491c0285", + "evaluation_time": 40.399051904678345, + "kg_co2_emissions": null, + "mteb_version": "1.19.0", + "scores": { + "test": [ + { + "accuracy": 0.612646484375, + "ap": 0.5709322265336014, + "ap_weighted": 0.5709322265336014, + "f1": 0.6049092458230604, + "f1_weighted": 0.6049092458230604, + "hf_subset": "default", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.612646484375, + "scores_per_experiment": [ + { + "accuracy": 0.62548828125, + "ap": 0.5833383221583652, + "ap_weighted": 0.5833383221583652, + "f1": 0.6202293735129851, + "f1_weighted": 0.6202293735129851 + }, + { + "accuracy": 0.5908203125, + "ap": 0.552124217507949, + "ap_weighted": 0.552124217507949, + "f1": 0.5854078754492036, + "f1_weighted": 0.5854078754492036 + }, + { + "accuracy": 0.625, + "ap": 0.584478021978022, + "ap_weighted": 0.584478021978022, + "f1": 0.6169993922109496, + "f1_weighted": 0.6169993922109496 + }, + { + "accuracy": 0.58056640625, + "ap": 0.546342208152917, + "ap_weighted": 0.546342208152917, + "f1": 0.5800328242589177, + "f1_weighted": 0.5800328242589177 + }, + { + "accuracy": 0.57177734375, + "ap": 0.5439185573630136, + "ap_weighted": 0.5439185573630136, + "f1": 0.5575698680786232, + "f1_weighted": 0.5575698680786232 + }, + { + "accuracy": 0.58447265625, + "ap": 0.547685160910701, + "ap_weighted": 0.547685160910701, + "f1": 0.574272897041024, + "f1_weighted": 0.574272897041024 + }, + { + "accuracy": 0.7041015625, + "ap": 0.6438715916053922, + "ap_weighted": 0.6438715916053922, + "f1": 0.7041004337327337, + "f1_weighted": 0.7041004337327337 + }, + { + "accuracy": 0.62451171875, + "ap": 0.5724127654150672, + "ap_weighted": 0.5724127654150672, + "f1": 0.5965677395490476, + "f1_weighted": 0.5965677395490476 + }, + { + "accuracy": 0.6123046875, + "ap": 0.5715273902529762, + "ap_weighted": 0.5715273902529762, + "f1": 0.6091497838694294, + "f1_weighted": 0.6091497838694294 + }, + { + "accuracy": 0.607421875, + "ap": 0.5636240299916107, + "ap_weighted": 0.5636240299916107, + "f1": 0.6047622705276903, + "f1_weighted": 0.6047622705276903 + } + ] + } + ] + }, + "task_name": "InappropriatenessClassification" +} \ No newline at end of file diff --git a/results/dunzhang__stella_en_1.5B_v5/d03be74b361d4eb24f42a2fe5bd2e29917df4604/KinopoiskClassification.json b/results/dunzhang__stella_en_1.5B_v5/d03be74b361d4eb24f42a2fe5bd2e29917df4604/KinopoiskClassification.json new file mode 100644 index 000000000..ffd888854 --- /dev/null +++ b/results/dunzhang__stella_en_1.5B_v5/d03be74b361d4eb24f42a2fe5bd2e29917df4604/KinopoiskClassification.json @@ -0,0 +1,73 @@ +{ + "dataset_revision": "5911f26666ac11af46cb9c6849d0dc80a378af24", + "evaluation_time": 153.17531180381775, + "kg_co2_emissions": null, + "mteb_version": "1.19.0", + "scores": { + "test": [ + { + "accuracy": 0.6738, + "f1": 0.6447041199505383, + "f1_weighted": 0.6447041199505383, + "hf_subset": "default", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.6738, + "scores_per_experiment": [ + { + "accuracy": 0.674, + "f1": 0.6390605439801077, + "f1_weighted": 0.6390605439801077 + }, + { + "accuracy": 0.6613333333333333, + "f1": 0.6337059104237756, + "f1_weighted": 0.6337059104237757 + }, + { + "accuracy": 0.6873333333333334, + "f1": 0.6619648464375417, + "f1_weighted": 0.6619648464375417 + }, + { + "accuracy": 0.6753333333333333, + "f1": 0.6394989984389966, + "f1_weighted": 0.6394989984389965 + }, + { + "accuracy": 0.6613333333333333, + "f1": 0.630519814464736, + "f1_weighted": 0.630519814464736 + }, + { + "accuracy": 0.678, + "f1": 0.6550823556401769, + "f1_weighted": 0.6550823556401769 + }, + { + "accuracy": 0.672, + "f1": 0.6456237729671205, + "f1_weighted": 0.6456237729671205 + }, + { + "accuracy": 0.6773333333333333, + "f1": 0.6484588923288613, + "f1_weighted": 0.6484588923288613 + }, + { + "accuracy": 0.688, + "f1": 0.659516366658306, + "f1_weighted": 0.659516366658306 + }, + { + "accuracy": 0.6633333333333333, + "f1": 0.6336096981657602, + "f1_weighted": 0.6336096981657602 + } + ] + } + ] + }, + "task_name": "KinopoiskClassification" +} \ No newline at end of file diff --git a/results/dunzhang__stella_en_1.5B_v5/d03be74b361d4eb24f42a2fe5bd2e29917df4604/MIRACLReranking.json b/results/dunzhang__stella_en_1.5B_v5/d03be74b361d4eb24f42a2fe5bd2e29917df4604/MIRACLReranking.json new file mode 100644 index 000000000..3a20d69d4 --- /dev/null +++ b/results/dunzhang__stella_en_1.5B_v5/d03be74b361d4eb24f42a2fe5bd2e29917df4604/MIRACLReranking.json @@ -0,0 +1,130 @@ +{ + "dataset_revision": "6d1962c527217f8927fca80f890f14f36b2802af", + "evaluation_time": 5382.259135484695, + "kg_co2_emissions": null, + "mteb_version": "1.19.0", + "scores": { + "dev": [ + { + "MAP@1(MIRACL)": 0.33667, + "MAP@10(MIRACL)": 0.52761, + "MAP@100(MIRACL)": 0.54698, + "MAP@1000(MIRACL)": 0.54698, + "MAP@20(MIRACL)": 0.54096, + "MAP@3(MIRACL)": 0.46614, + "MAP@5(MIRACL)": 0.49964, + "NDCG@1(MIRACL)": 0.55413, + "NDCG@10(MIRACL)": 0.59929, + "NDCG@100(MIRACL)": 0.6389, + "NDCG@1000(MIRACL)": 0.6389, + "NDCG@20(MIRACL)": 0.6224, + "NDCG@3(MIRACL)": 0.55431, + "NDCG@5(MIRACL)": 0.5689, + "P@1(MIRACL)": 0.55413, + "P@10(MIRACL)": 0.1506, + "P@100(MIRACL)": 0.01953, + "P@1000(MIRACL)": 0.00195, + "P@20(MIRACL)": 0.08693, + "P@3(MIRACL)": 0.33948, + "P@5(MIRACL)": 0.24667, + "Recall@1(MIRACL)": 0.33667, + "Recall@10(MIRACL)": 0.68192, + "Recall@100(MIRACL)": 0.79952, + "Recall@1000(MIRACL)": 0.79952, + "Recall@20(MIRACL)": 0.74312, + "Recall@3(MIRACL)": 0.53339, + "Recall@5(MIRACL)": 0.60178, + "hf_subset": "ru", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.59929, + "nAUC_MAP@1000_diff1(MIRACL)": 0.2391540601429222, + "nAUC_MAP@1000_max(MIRACL)": 0.3843259353484255, + "nAUC_MAP@1000_std(MIRACL)": 0.13253814503975128, + "nAUC_MAP@100_diff1(MIRACL)": 0.2391540601429222, + "nAUC_MAP@100_max(MIRACL)": 0.3843259353484255, + "nAUC_MAP@100_std(MIRACL)": 0.13253814503975128, + "nAUC_MAP@10_diff1(MIRACL)": 0.2483320817832952, + "nAUC_MAP@10_max(MIRACL)": 0.36015010323032753, + "nAUC_MAP@10_std(MIRACL)": 0.11830706583787767, + "nAUC_MAP@1_diff1(MIRACL)": 0.4047644140429935, + "nAUC_MAP@1_max(MIRACL)": 0.21061188958672308, + "nAUC_MAP@1_std(MIRACL)": 0.005235766381523269, + "nAUC_MAP@20_diff1(MIRACL)": 0.24195376202464114, + "nAUC_MAP@20_max(MIRACL)": 0.3770172215447117, + "nAUC_MAP@20_std(MIRACL)": 0.13207122811148578, + "nAUC_MAP@3_diff1(MIRACL)": 0.29195550258850556, + "nAUC_MAP@3_max(MIRACL)": 0.2945350293360231, + "nAUC_MAP@3_std(MIRACL)": 0.0718910686806699, + "nAUC_MAP@5_diff1(MIRACL)": 0.266119547314597, + "nAUC_MAP@5_max(MIRACL)": 0.32720655364181966, + "nAUC_MAP@5_std(MIRACL)": 0.09772526856764836, + "nAUC_NDCG@1000_diff1(MIRACL)": 0.17428812576641672, + "nAUC_NDCG@1000_max(MIRACL)": 0.47085571997467524, + "nAUC_NDCG@1000_std(MIRACL)": 0.17777498889304028, + "nAUC_NDCG@100_diff1(MIRACL)": 0.17428812576641672, + "nAUC_NDCG@100_max(MIRACL)": 0.47085571997467524, + "nAUC_NDCG@100_std(MIRACL)": 0.17777498889304028, + "nAUC_NDCG@10_diff1(MIRACL)": 0.1989411889299795, + "nAUC_NDCG@10_max(MIRACL)": 0.42202823838085163, + "nAUC_NDCG@10_std(MIRACL)": 0.15507060896353367, + "nAUC_NDCG@1_diff1(MIRACL)": 0.2926696563994893, + "nAUC_NDCG@1_max(MIRACL)": 0.4477668006149943, + "nAUC_NDCG@1_std(MIRACL)": 0.14904720926015, + "nAUC_NDCG@20_diff1(MIRACL)": 0.18356981710922823, + "nAUC_NDCG@20_max(MIRACL)": 0.4496733253937478, + "nAUC_NDCG@20_std(MIRACL)": 0.1775567065668807, + "nAUC_NDCG@3_diff1(MIRACL)": 0.23174484482404872, + "nAUC_NDCG@3_max(MIRACL)": 0.39015401191243726, + "nAUC_NDCG@3_std(MIRACL)": 0.13729867432999338, + "nAUC_NDCG@5_diff1(MIRACL)": 0.22436451599817944, + "nAUC_NDCG@5_max(MIRACL)": 0.3930764100428907, + "nAUC_NDCG@5_std(MIRACL)": 0.14305726689307952, + "nAUC_P@1000_diff1(MIRACL)": -0.19307741626204192, + "nAUC_P@1000_max(MIRACL)": 0.31600284236823045, + "nAUC_P@1000_std(MIRACL)": 0.18286753102035241, + "nAUC_P@100_diff1(MIRACL)": -0.1930774162620416, + "nAUC_P@100_max(MIRACL)": 0.31600284236823106, + "nAUC_P@100_std(MIRACL)": 0.18286753102035289, + "nAUC_P@10_diff1(MIRACL)": -0.16559524572240802, + "nAUC_P@10_max(MIRACL)": 0.3595458195640155, + "nAUC_P@10_std(MIRACL)": 0.22305393306745766, + "nAUC_P@1_diff1(MIRACL)": 0.2926696563994893, + "nAUC_P@1_max(MIRACL)": 0.4477668006149943, + "nAUC_P@1_std(MIRACL)": 0.14904720926015, + "nAUC_P@20_diff1(MIRACL)": -0.18394946995380826, + "nAUC_P@20_max(MIRACL)": 0.34099217604058796, + "nAUC_P@20_std(MIRACL)": 0.22797752861867038, + "nAUC_P@3_diff1(MIRACL)": -0.07091827476214699, + "nAUC_P@3_max(MIRACL)": 0.40023668080484087, + "nAUC_P@3_std(MIRACL)": 0.225842195100354, + "nAUC_P@5_diff1(MIRACL)": -0.1306831284548672, + "nAUC_P@5_max(MIRACL)": 0.3801283841677278, + "nAUC_P@5_std(MIRACL)": 0.2374198067837887, + "nAUC_Recall@1000_diff1(MIRACL)": -0.045386297065176315, + "nAUC_Recall@1000_max(MIRACL)": 0.6423448955621966, + "nAUC_Recall@1000_std(MIRACL)": 0.257964517512322, + "nAUC_Recall@100_diff1(MIRACL)": -0.045386297065176315, + "nAUC_Recall@100_max(MIRACL)": 0.6423448955621966, + "nAUC_Recall@100_std(MIRACL)": 0.257964517512322, + "nAUC_Recall@10_diff1(MIRACL)": 0.08255677639913413, + "nAUC_Recall@10_max(MIRACL)": 0.421744479394554, + "nAUC_Recall@10_std(MIRACL)": 0.16561300411915256, + "nAUC_Recall@1_diff1(MIRACL)": 0.4047644140429935, + "nAUC_Recall@1_max(MIRACL)": 0.21061188958672308, + "nAUC_Recall@1_std(MIRACL)": 0.005235766381523269, + "nAUC_Recall@20_diff1(MIRACL)": 0.019917848013351603, + "nAUC_Recall@20_max(MIRACL)": 0.5116319723677314, + "nAUC_Recall@20_std(MIRACL)": 0.24016569429785548, + "nAUC_Recall@3_diff1(MIRACL)": 0.21457739423036967, + "nAUC_Recall@3_max(MIRACL)": 0.28191262582342663, + "nAUC_Recall@3_std(MIRACL)": 0.09039521380155778, + "nAUC_Recall@5_diff1(MIRACL)": 0.15999800167699244, + "nAUC_Recall@5_max(MIRACL)": 0.33428406152416956, + "nAUC_Recall@5_std(MIRACL)": 0.1297965759629048 + } + ] + }, + "task_name": "MIRACLReranking" +} \ No newline at end of file diff --git a/results/dunzhang__stella_en_1.5B_v5/d03be74b361d4eb24f42a2fe5bd2e29917df4604/MIRACLRetrieval.json b/results/dunzhang__stella_en_1.5B_v5/d03be74b361d4eb24f42a2fe5bd2e29917df4604/MIRACLRetrieval.json new file mode 100644 index 000000000..3e1282d60 --- /dev/null +++ b/results/dunzhang__stella_en_1.5B_v5/d03be74b361d4eb24f42a2fe5bd2e29917df4604/MIRACLRetrieval.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "main", + "evaluation_time": 691313.8391666412, + "kg_co2_emissions": null, + "mteb_version": "1.19.4", + "scores": { + "dev": [ + { + "hf_subset": "ru", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.58129, + "map_at_1": 0.27646, + "map_at_10": 0.47687, + "map_at_100": 0.50278, + "map_at_1000": 0.50395, + "map_at_20": 0.49236, + "map_at_3": 0.4087, + "map_at_5": 0.44476, + "mrr_at_1": 0.5567092651757188, + "mrr_at_10": 0.673411430599929, + "mrr_at_100": 0.6772310934021565, + "mrr_at_1000": 0.6773653423270617, + "mrr_at_20": 0.6763908562027856, + "mrr_at_3": 0.6522896698615548, + "mrr_at_5": 0.6656682641107561, + "nauc_map_at_1000_diff1": 0.29695588395752365, + "nauc_map_at_1000_max": 0.30930539292447173, + "nauc_map_at_1000_std": 0.09921807808694715, + "nauc_map_at_100_diff1": 0.2968123342805225, + "nauc_map_at_100_max": 0.30875845574274996, + "nauc_map_at_100_std": 0.09892552616442056, + "nauc_map_at_10_diff1": 0.2936807855213625, + "nauc_map_at_10_max": 0.28884969638745034, + "nauc_map_at_10_std": 0.07298415179342725, + "nauc_map_at_1_diff1": 0.33968283088944456, + "nauc_map_at_1_max": 0.1971136263056469, + "nauc_map_at_1_std": 0.01413897364401697, + "nauc_map_at_20_diff1": 0.29691635312902676, + "nauc_map_at_20_max": 0.30152821145351066, + "nauc_map_at_20_std": 0.08682843546421597, + "nauc_map_at_3_diff1": 0.2913703736909007, + "nauc_map_at_3_max": 0.24069302801811437, + "nauc_map_at_3_std": 0.03327022062729413, + "nauc_map_at_5_diff1": 0.29268536712841403, + "nauc_map_at_5_max": 0.26759075583955466, + "nauc_map_at_5_std": 0.05521681026423832, + "nauc_mrr_at_1000_diff1": 0.36059946662718856, + "nauc_mrr_at_1000_max": 0.41238712201982725, + "nauc_mrr_at_1000_std": 0.21344552429320648, + "nauc_mrr_at_100_diff1": 0.36058631872463565, + "nauc_mrr_at_100_max": 0.41254731587146865, + "nauc_mrr_at_100_std": 0.21367524027541673, + "nauc_mrr_at_10_diff1": 0.3596066267608852, + "nauc_mrr_at_10_max": 0.4135585962302046, + "nauc_mrr_at_10_std": 0.21500242139263012, + "nauc_mrr_at_1_diff1": 0.3660440022858535, + "nauc_mrr_at_1_max": 0.3754645678768974, + "nauc_mrr_at_1_std": 0.1803397542565535, + "nauc_mrr_at_20_diff1": 0.3606582515164157, + "nauc_mrr_at_20_max": 0.41312421158852347, + "nauc_mrr_at_20_std": 0.21430810329909253, + "nauc_mrr_at_3_diff1": 0.36447960668209894, + "nauc_mrr_at_3_max": 0.41225234467986777, + "nauc_mrr_at_3_std": 0.20136983014955365, + "nauc_mrr_at_5_diff1": 0.36150408388728494, + "nauc_mrr_at_5_max": 0.41484063944848076, + "nauc_mrr_at_5_std": 0.21485372872499608, + "nauc_ndcg_at_1000_diff1": 0.30998977519302906, + "nauc_ndcg_at_1000_max": 0.37510229781623583, + "nauc_ndcg_at_1000_std": 0.18099084218140027, + "nauc_ndcg_at_100_diff1": 0.30700700149956134, + "nauc_ndcg_at_100_max": 0.3746666789185031, + "nauc_ndcg_at_100_std": 0.18814501398830583, + "nauc_ndcg_at_10_diff1": 0.2990035407305346, + "nauc_ndcg_at_10_max": 0.3322476842774391, + "nauc_ndcg_at_10_std": 0.12411101615991835, + "nauc_ndcg_at_1_diff1": 0.3660440022858535, + "nauc_ndcg_at_1_max": 0.3754645678768974, + "nauc_ndcg_at_1_std": 0.1803397542565535, + "nauc_ndcg_at_20_diff1": 0.3086879489540479, + "nauc_ndcg_at_20_max": 0.35525813898159603, + "nauc_ndcg_at_20_std": 0.15106056440202498, + "nauc_ndcg_at_3_diff1": 0.29793061737192045, + "nauc_ndcg_at_3_max": 0.327316912419475, + "nauc_ndcg_at_3_std": 0.12074846133627425, + "nauc_ndcg_at_5_diff1": 0.3011796739806996, + "nauc_ndcg_at_5_max": 0.32238520466047316, + "nauc_ndcg_at_5_std": 0.11695822040597702, + "nauc_precision_at_1000_diff1": -0.06336765921897028, + "nauc_precision_at_1000_max": 0.1674528429111938, + "nauc_precision_at_1000_std": 0.23636199503064767, + "nauc_precision_at_100_diff1": -0.04469378044080892, + "nauc_precision_at_100_max": 0.21151314467568066, + "nauc_precision_at_100_std": 0.28285203828930455, + "nauc_precision_at_10_diff1": 0.03014872848939142, + "nauc_precision_at_10_max": 0.2877222763485144, + "nauc_precision_at_10_std": 0.23873811810472648, + "nauc_precision_at_1_diff1": 0.3660440022858535, + "nauc_precision_at_1_max": 0.3754645678768974, + "nauc_precision_at_1_std": 0.1803397542565535, + "nauc_precision_at_20_diff1": 0.007870929929902412, + "nauc_precision_at_20_max": 0.27129761200879976, + "nauc_precision_at_20_std": 0.2648480581053351, + "nauc_precision_at_3_diff1": 0.12076326624800245, + "nauc_precision_at_3_max": 0.3325119720529147, + "nauc_precision_at_3_std": 0.19191752455654915, + "nauc_precision_at_5_diff1": 0.07761462325070048, + "nauc_precision_at_5_max": 0.32200194180703107, + "nauc_precision_at_5_std": 0.21987406837977125, + "nauc_recall_at_1000_diff1": 0.20360092901819063, + "nauc_recall_at_1000_max": 0.6032627635865067, + "nauc_recall_at_1000_std": 0.6655007655375861, + "nauc_recall_at_100_diff1": 0.19228465526510666, + "nauc_recall_at_100_max": 0.4186834364812096, + "nauc_recall_at_100_std": 0.43886513170921027, + "nauc_recall_at_10_diff1": 0.20722939550011651, + "nauc_recall_at_10_max": 0.25582936806551554, + "nauc_recall_at_10_std": 0.10440554605686661, + "nauc_recall_at_1_diff1": 0.33968283088944456, + "nauc_recall_at_1_max": 0.1971136263056469, + "nauc_recall_at_1_std": 0.01413897364401697, + "nauc_recall_at_20_diff1": 0.22498231250739312, + "nauc_recall_at_20_max": 0.3011509447141894, + "nauc_recall_at_20_std": 0.17278474996370927, + "nauc_recall_at_3_diff1": 0.24813934944462843, + "nauc_recall_at_3_max": 0.20917179838492386, + "nauc_recall_at_3_std": 0.027163812143893774, + "nauc_recall_at_5_diff1": 0.23293592196601137, + "nauc_recall_at_5_max": 0.23418318049117073, + "nauc_recall_at_5_std": 0.06805324177235571, + "ndcg_at_1": 0.55671, + "ndcg_at_10": 0.58129, + "ndcg_at_100": 0.64895, + "ndcg_at_1000": 0.66286, + "ndcg_at_20": 0.61403, + "ndcg_at_3": 0.53095, + "ndcg_at_5": 0.54394, + "precision_at_1": 0.55671, + "precision_at_10": 0.16925, + "precision_at_100": 0.02487, + "precision_at_1000": 0.00275, + "precision_at_20": 0.0998, + "precision_at_3": 0.36129, + "precision_at_5": 0.26821, + "recall_at_1": 0.27646, + "recall_at_10": 0.67821, + "recall_at_100": 0.89435, + "recall_at_1000": 0.97117, + "recall_at_20": 0.77044, + "recall_at_3": 0.48742, + "recall_at_5": 0.5703 + } + ] + }, + "task_name": "MIRACLRetrieval" +} \ No newline at end of file diff --git a/results/dunzhang__stella_en_1.5B_v5/d03be74b361d4eb24f42a2fe5bd2e29917df4604/MassiveScenarioClassification.json b/results/dunzhang__stella_en_1.5B_v5/d03be74b361d4eb24f42a2fe5bd2e29917df4604/MassiveScenarioClassification.json new file mode 100644 index 000000000..75a71b08a --- /dev/null +++ b/results/dunzhang__stella_en_1.5B_v5/d03be74b361d4eb24f42a2fe5bd2e29917df4604/MassiveScenarioClassification.json @@ -0,0 +1,137 @@ +{ + "dataset_revision": "fad2c6e8459f9e1c45d9315f4953d921437d70f8", + "evaluation_time": 112.84681367874146, + "kg_co2_emissions": null, + "mteb_version": "1.19.0", + "scores": { + "test": [ + { + "accuracy": 0.8198049764626765, + "f1": 0.808026990412551, + "f1_weighted": 0.8185025264273482, + "hf_subset": "ru", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.8198049764626765, + "scores_per_experiment": [ + { + "accuracy": 0.8244788164088769, + "f1": 0.810746153648465, + "f1_weighted": 0.8235653928182464 + }, + { + "accuracy": 0.8285137861466039, + "f1": 0.8140915347671699, + "f1_weighted": 0.8273935088202502 + }, + { + "accuracy": 0.8032952252858103, + "f1": 0.7840919432205451, + "f1_weighted": 0.8034379854262542 + }, + { + "accuracy": 0.8308675184936113, + "f1": 0.8190981595921863, + "f1_weighted": 0.8290181288548112 + }, + { + "accuracy": 0.8342299932750504, + "f1": 0.8195552462215963, + "f1_weighted": 0.8290211176431735 + }, + { + "accuracy": 0.8120376597175522, + "f1": 0.8023514695744962, + "f1_weighted": 0.8108611450560325 + }, + { + "accuracy": 0.7844653665097512, + "f1": 0.7823005598592876, + "f1_weighted": 0.7807781869452115 + }, + { + "accuracy": 0.8328850033624747, + "f1": 0.8280273581287563, + "f1_weighted": 0.8346274674379812 + }, + { + "accuracy": 0.8130464021519839, + "f1": 0.8024353768573464, + "f1_weighted": 0.81204419961264 + }, + { + "accuracy": 0.8342299932750504, + "f1": 0.8175721022556615, + "f1_weighted": 0.8342781316588819 + } + ] + } + ], + "validation": [ + { + "accuracy": 0.8122970978848991, + "f1": 0.7996607226981363, + "f1_weighted": 0.8111794196352209, + "hf_subset": "ru", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.8122970978848991, + "scores_per_experiment": [ + { + "accuracy": 0.82439744220364, + "f1": 0.8114872609692291, + "f1_weighted": 0.8234432280090804 + }, + { + "accuracy": 0.8184948352188883, + "f1": 0.8036198979165203, + "f1_weighted": 0.8182730073857466 + }, + { + "accuracy": 0.8037383177570093, + "f1": 0.7805975041516406, + "f1_weighted": 0.8043538521493093 + }, + { + "accuracy": 0.8258730939498279, + "f1": 0.8158988334700096, + "f1_weighted": 0.8237133448999737 + }, + { + "accuracy": 0.8214461387112642, + "f1": 0.8046316839685284, + "f1_weighted": 0.8157575403770211 + }, + { + "accuracy": 0.8022626660108214, + "f1": 0.7935005265402225, + "f1_weighted": 0.8015926987449585 + }, + { + "accuracy": 0.778652238071815, + "f1": 0.773401819325035, + "f1_weighted": 0.7763615851574013 + }, + { + "accuracy": 0.8184948352188883, + "f1": 0.8103726871056602, + "f1_weighted": 0.8200518770134431 + }, + { + "accuracy": 0.8047220855878012, + "f1": 0.7960968908284956, + "f1_weighted": 0.8034527795850329 + }, + { + "accuracy": 0.8248893261190359, + "f1": 0.8070001227060204, + "f1_weighted": 0.8247942830302423 + } + ] + } + ] + }, + "task_name": "MassiveScenarioClassification" +} \ No newline at end of file diff --git a/results/dunzhang__stella_en_1.5B_v5/d03be74b361d4eb24f42a2fe5bd2e29917df4604/RUParaPhraserSTS.json b/results/dunzhang__stella_en_1.5B_v5/d03be74b361d4eb24f42a2fe5bd2e29917df4604/RUParaPhraserSTS.json new file mode 100644 index 000000000..0bf7b0a77 --- /dev/null +++ b/results/dunzhang__stella_en_1.5B_v5/d03be74b361d4eb24f42a2fe5bd2e29917df4604/RUParaPhraserSTS.json @@ -0,0 +1,26 @@ +{ + "dataset_revision": "43265056790b8f7c59e0139acb4be0a8dad2c8f4", + "evaluation_time": 46.697439193725586, + "kg_co2_emissions": null, + "mteb_version": "1.19.0", + "scores": { + "test": [ + { + "cosine_pearson": 0.6555992017671803, + "cosine_spearman": 0.712568934864605, + "euclidean_pearson": 0.6909165406828682, + "euclidean_spearman": 0.7125696377993136, + "hf_subset": "default", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.712568934864605, + "manhattan_pearson": 0.6909908138687297, + "manhattan_spearman": 0.7128097817668758, + "pearson": 0.6555992017671803, + "spearman": 0.712568934864605 + } + ] + }, + "task_name": "RUParaPhraserSTS" +} \ No newline at end of file diff --git a/results/dunzhang__stella_en_1.5B_v5/d03be74b361d4eb24f42a2fe5bd2e29917df4604/RiaNewsRetrieval.json b/results/dunzhang__stella_en_1.5B_v5/d03be74b361d4eb24f42a2fe5bd2e29917df4604/RiaNewsRetrieval.json new file mode 100644 index 000000000..ea8bd4c97 --- /dev/null +++ b/results/dunzhang__stella_en_1.5B_v5/d03be74b361d4eb24f42a2fe5bd2e29917df4604/RiaNewsRetrieval.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "82374b0bbacda6114f39ff9c5b925fa1512ca5d7", + "evaluation_time": 54462.471239089966, + "kg_co2_emissions": null, + "mteb_version": "1.19.2", + "scores": { + "test": [ + { + "hf_subset": "default", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.69308, + "map_at_1": 0.5605, + "map_at_10": 0.65075, + "map_at_100": 0.65528, + "map_at_1000": 0.65544, + "map_at_20": 0.65354, + "map_at_3": 0.6311, + "map_at_5": 0.64329, + "mrr_at_1": 0.5605, + "mrr_at_10": 0.6507471428571429, + "mrr_at_100": 0.6552799300068545, + "mrr_at_1000": 0.6554428868350467, + "mrr_at_20": 0.6535428113600366, + "mrr_at_3": 0.6311, + "mrr_at_5": 0.6432899999999999, + "nauc_map_at_1000_diff1": 0.6597614156855645, + "nauc_map_at_1000_max": 0.2916825266925678, + "nauc_map_at_1000_std": 0.0393316081871174, + "nauc_map_at_100_diff1": 0.6596968286303069, + "nauc_map_at_100_max": 0.2916957900517394, + "nauc_map_at_100_std": 0.03943202417178389, + "nauc_map_at_10_diff1": 0.6590270551835647, + "nauc_map_at_10_max": 0.2911932893616769, + "nauc_map_at_10_std": 0.03807012581118365, + "nauc_map_at_1_diff1": 0.6972280078085887, + "nauc_map_at_1_max": 0.2684658992055774, + "nauc_map_at_1_std": 0.013945673639905584, + "nauc_map_at_20_diff1": 0.6593803341401233, + "nauc_map_at_20_max": 0.2917473461658606, + "nauc_map_at_20_std": 0.039237814476398514, + "nauc_map_at_3_diff1": 0.6604334124537607, + "nauc_map_at_3_max": 0.28977237993191834, + "nauc_map_at_3_std": 0.03035389139650655, + "nauc_map_at_5_diff1": 0.6589234069743621, + "nauc_map_at_5_max": 0.29147830126145413, + "nauc_map_at_5_std": 0.03540660994515181, + "nauc_mrr_at_1000_diff1": 0.6597614156855645, + "nauc_mrr_at_1000_max": 0.2916825266925678, + "nauc_mrr_at_1000_std": 0.0393316081871174, + "nauc_mrr_at_100_diff1": 0.6596968286303069, + "nauc_mrr_at_100_max": 0.2916957900517394, + "nauc_mrr_at_100_std": 0.03943202417178389, + "nauc_mrr_at_10_diff1": 0.6590270551835647, + "nauc_mrr_at_10_max": 0.2911932893616769, + "nauc_mrr_at_10_std": 0.03807012581118365, + "nauc_mrr_at_1_diff1": 0.6972280078085887, + "nauc_mrr_at_1_max": 0.2684658992055774, + "nauc_mrr_at_1_std": 0.013945673639905584, + "nauc_mrr_at_20_diff1": 0.6593803341401233, + "nauc_mrr_at_20_max": 0.2917473461658606, + "nauc_mrr_at_20_std": 0.039237814476398514, + "nauc_mrr_at_3_diff1": 0.6604334124537607, + "nauc_mrr_at_3_max": 0.28977237993191834, + "nauc_mrr_at_3_std": 0.03035389139650655, + "nauc_mrr_at_5_diff1": 0.6589234069743621, + "nauc_mrr_at_5_max": 0.29147830126145413, + "nauc_mrr_at_5_std": 0.03540660994515181, + "nauc_ndcg_at_1000_diff1": 0.6491332621705926, + "nauc_ndcg_at_1000_max": 0.3014853155266885, + "nauc_ndcg_at_1000_std": 0.05733945074922103, + "nauc_ndcg_at_100_diff1": 0.6472978534517118, + "nauc_ndcg_at_100_max": 0.302440257028989, + "nauc_ndcg_at_100_std": 0.0614765937360659, + "nauc_ndcg_at_10_diff1": 0.644103217922291, + "nauc_ndcg_at_10_max": 0.3003347670655498, + "nauc_ndcg_at_10_std": 0.05466003375794747, + "nauc_ndcg_at_1_diff1": 0.6972280078085887, + "nauc_ndcg_at_1_max": 0.2684658992055774, + "nauc_ndcg_at_1_std": 0.013945673639905584, + "nauc_ndcg_at_20_diff1": 0.6450247184469647, + "nauc_ndcg_at_20_max": 0.3026195808516587, + "nauc_ndcg_at_20_std": 0.059780738897096, + "nauc_ndcg_at_3_diff1": 0.6479271561061314, + "nauc_ndcg_at_3_max": 0.297085886779037, + "nauc_ndcg_at_3_std": 0.036696905118796384, + "nauc_ndcg_at_5_diff1": 0.6443543830863401, + "nauc_ndcg_at_5_max": 0.30072587861217115, + "nauc_ndcg_at_5_std": 0.04692486829176859, + "nauc_precision_at_1000_diff1": 0.48020511136050187, + "nauc_precision_at_1000_max": 0.5192808936821168, + "nauc_precision_at_1000_std": 0.4570859831446126, + "nauc_precision_at_100_diff1": 0.5242975978270092, + "nauc_precision_at_100_max": 0.43044158034670243, + "nauc_precision_at_100_std": 0.35731330655429544, + "nauc_precision_at_10_diff1": 0.5717202447906187, + "nauc_precision_at_10_max": 0.34472736037587554, + "nauc_precision_at_10_std": 0.14298558098165137, + "nauc_precision_at_1_diff1": 0.6972280078085887, + "nauc_precision_at_1_max": 0.2684658992055774, + "nauc_precision_at_1_std": 0.013945673639905584, + "nauc_precision_at_20_diff1": 0.5594905272273013, + "nauc_precision_at_20_max": 0.3717940932080547, + "nauc_precision_at_20_std": 0.20242340368222458, + "nauc_precision_at_3_diff1": 0.6055492243680125, + "nauc_precision_at_3_max": 0.32189593420410395, + "nauc_precision_at_3_std": 0.058489573603510134, + "nauc_precision_at_5_diff1": 0.5869817692504921, + "nauc_precision_at_5_max": 0.3374369552783646, + "nauc_precision_at_5_std": 0.0943604445807085, + "nauc_recall_at_1000_diff1": 0.48020511136050265, + "nauc_recall_at_1000_max": 0.5192808936821178, + "nauc_recall_at_1000_std": 0.4570859831446148, + "nauc_recall_at_100_diff1": 0.5242975978270084, + "nauc_recall_at_100_max": 0.43044158034670227, + "nauc_recall_at_100_std": 0.3573133065542948, + "nauc_recall_at_10_diff1": 0.571720244790618, + "nauc_recall_at_10_max": 0.3447273603758772, + "nauc_recall_at_10_std": 0.14298558098165173, + "nauc_recall_at_1_diff1": 0.6972280078085887, + "nauc_recall_at_1_max": 0.2684658992055774, + "nauc_recall_at_1_std": 0.013945673639905584, + "nauc_recall_at_20_diff1": 0.5594905272273003, + "nauc_recall_at_20_max": 0.37179409320805495, + "nauc_recall_at_20_std": 0.20242340368222347, + "nauc_recall_at_3_diff1": 0.6055492243680124, + "nauc_recall_at_3_max": 0.32189593420410345, + "nauc_recall_at_3_std": 0.058489573603508704, + "nauc_recall_at_5_diff1": 0.5869817692504916, + "nauc_recall_at_5_max": 0.3374369552783643, + "nauc_recall_at_5_std": 0.0943604445807093, + "ndcg_at_1": 0.5605, + "ndcg_at_10": 0.69308, + "ndcg_at_100": 0.71563, + "ndcg_at_1000": 0.7203, + "ndcg_at_20": 0.70314, + "ndcg_at_3": 0.6533, + "ndcg_at_5": 0.67522, + "precision_at_1": 0.5605, + "precision_at_10": 0.08252, + "precision_at_100": 0.00932, + "precision_at_1000": 0.00097, + "precision_at_20": 0.04324, + "precision_at_3": 0.2391, + "precision_at_5": 0.15408, + "recall_at_1": 0.5605, + "recall_at_10": 0.8252, + "recall_at_100": 0.9318, + "recall_at_1000": 0.9693, + "recall_at_20": 0.8647, + "recall_at_3": 0.7173, + "recall_at_5": 0.7704 + } + ] + }, + "task_name": "RiaNewsRetrieval" +} \ No newline at end of file diff --git a/results/dunzhang__stella_en_1.5B_v5/d03be74b361d4eb24f42a2fe5bd2e29917df4604/RuBQRetrieval.json b/results/dunzhang__stella_en_1.5B_v5/d03be74b361d4eb24f42a2fe5bd2e29917df4604/RuBQRetrieval.json new file mode 100644 index 000000000..1cc4d3309 --- /dev/null +++ b/results/dunzhang__stella_en_1.5B_v5/d03be74b361d4eb24f42a2fe5bd2e29917df4604/RuBQRetrieval.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "e19b6ffa60b3bc248e0b41f4cc37c26a55c2a67b", + "evaluation_time": 2340.2597711086273, + "kg_co2_emissions": null, + "mteb_version": "1.19.0", + "scores": { + "test": [ + { + "hf_subset": "default", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.69711, + "map_at_1": 0.42655, + "map_at_10": 0.62082, + "map_at_100": 0.63055, + "map_at_1000": 0.63084, + "map_at_20": 0.62754, + "map_at_3": 0.56602, + "map_at_5": 0.60104, + "mrr_at_1": 0.6046099290780141, + "mrr_at_10": 0.708497270066419, + "mrr_at_100": 0.711774425300996, + "mrr_at_1000": 0.7118886812990914, + "mrr_at_20": 0.7106574628057074, + "mrr_at_3": 0.6881402679275019, + "mrr_at_5": 0.7010539795114263, + "nauc_map_at_1000_diff1": 0.4361545445290786, + "nauc_map_at_1000_max": 0.3857877555639102, + "nauc_map_at_1000_std": -0.01609014048925164, + "nauc_map_at_100_diff1": 0.43604978903337094, + "nauc_map_at_100_max": 0.3858818023955993, + "nauc_map_at_100_std": -0.01572895210928839, + "nauc_map_at_10_diff1": 0.43411489441095114, + "nauc_map_at_10_max": 0.3847168322449776, + "nauc_map_at_10_std": -0.01884896029836748, + "nauc_map_at_1_diff1": 0.4672280000006232, + "nauc_map_at_1_max": 0.2728573344017962, + "nauc_map_at_1_std": -0.05815351013282631, + "nauc_map_at_20_diff1": 0.43461387639600846, + "nauc_map_at_20_max": 0.38569664336165627, + "nauc_map_at_20_std": -0.01498805481492192, + "nauc_map_at_3_diff1": 0.4298985645849098, + "nauc_map_at_3_max": 0.35018489163666844, + "nauc_map_at_3_std": -0.047280920606469844, + "nauc_map_at_5_diff1": 0.4334641589557385, + "nauc_map_at_5_max": 0.37204885801770277, + "nauc_map_at_5_std": -0.03365812684986057, + "nauc_mrr_at_1000_diff1": 0.5153512849300359, + "nauc_mrr_at_1000_max": 0.4615961637524889, + "nauc_mrr_at_1000_std": -0.02131454583243616, + "nauc_mrr_at_100_diff1": 0.5153865142802733, + "nauc_mrr_at_100_max": 0.4617521181261791, + "nauc_mrr_at_100_std": -0.02105868028043579, + "nauc_mrr_at_10_diff1": 0.5148493536382918, + "nauc_mrr_at_10_max": 0.4619719166388346, + "nauc_mrr_at_10_std": -0.020579171218226885, + "nauc_mrr_at_1_diff1": 0.5405741942577544, + "nauc_mrr_at_1_max": 0.4292371560847846, + "nauc_mrr_at_1_std": -0.045756795367605914, + "nauc_mrr_at_20_diff1": 0.5155153789674889, + "nauc_mrr_at_20_max": 0.46235825285400417, + "nauc_mrr_at_20_std": -0.020575410654767784, + "nauc_mrr_at_3_diff1": 0.5063492100977116, + "nauc_mrr_at_3_max": 0.4587929336258869, + "nauc_mrr_at_3_std": -0.02985834431752308, + "nauc_mrr_at_5_diff1": 0.512409464529452, + "nauc_mrr_at_5_max": 0.4618020558389936, + "nauc_mrr_at_5_std": -0.025468712011599018, + "nauc_ndcg_at_1000_diff1": 0.45038556519331313, + "nauc_ndcg_at_1000_max": 0.4221002910805701, + "nauc_ndcg_at_1000_std": 0.004428752777482846, + "nauc_ndcg_at_100_diff1": 0.44840976597223564, + "nauc_ndcg_at_100_max": 0.42576257339108725, + "nauc_ndcg_at_100_std": 0.015633915276243415, + "nauc_ndcg_at_10_diff1": 0.4393588142236606, + "nauc_ndcg_at_10_max": 0.42274197929705604, + "nauc_ndcg_at_10_std": 0.006059683072983812, + "nauc_ndcg_at_1_diff1": 0.5392460132384942, + "nauc_ndcg_at_1_max": 0.42945544501424265, + "nauc_ndcg_at_1_std": -0.0425035687075792, + "nauc_ndcg_at_20_diff1": 0.4418277376117853, + "nauc_ndcg_at_20_max": 0.426692385697961, + "nauc_ndcg_at_20_std": 0.01782021239535179, + "nauc_ndcg_at_3_diff1": 0.4303015701937597, + "nauc_ndcg_at_3_max": 0.38243576793904777, + "nauc_ndcg_at_3_std": -0.03806549362851626, + "nauc_ndcg_at_5_diff1": 0.4378338648524465, + "nauc_ndcg_at_5_max": 0.4024363987575144, + "nauc_ndcg_at_5_std": -0.022188825914044943, + "nauc_precision_at_1000_diff1": -0.11101482337784857, + "nauc_precision_at_1000_max": 0.06526095583737274, + "nauc_precision_at_1000_std": 0.059320881574561286, + "nauc_precision_at_100_diff1": -0.0839147595395992, + "nauc_precision_at_100_max": 0.11330107749970646, + "nauc_precision_at_100_std": 0.10263561590227024, + "nauc_precision_at_10_diff1": 0.003630670139364784, + "nauc_precision_at_10_max": 0.22672717754457447, + "nauc_precision_at_10_std": 0.09995644441314884, + "nauc_precision_at_1_diff1": 0.5392460132384942, + "nauc_precision_at_1_max": 0.42945544501424265, + "nauc_precision_at_1_std": -0.0425035687075792, + "nauc_precision_at_20_diff1": -0.04137464058694073, + "nauc_precision_at_20_max": 0.17596498954860587, + "nauc_precision_at_20_std": 0.12199576511204042, + "nauc_precision_at_3_diff1": 0.15357095068210552, + "nauc_precision_at_3_max": 0.31247374689269153, + "nauc_precision_at_3_std": 0.012848560941469713, + "nauc_precision_at_5_diff1": 0.08109910746060119, + "nauc_precision_at_5_max": 0.2720558129353094, + "nauc_precision_at_5_std": 0.0492873434023708, + "nauc_recall_at_1000_diff1": 0.17127675507818169, + "nauc_recall_at_1000_max": 0.6438864586784443, + "nauc_recall_at_1000_std": 0.6762918792645474, + "nauc_recall_at_100_diff1": 0.2959588962962824, + "nauc_recall_at_100_max": 0.5551830882794535, + "nauc_recall_at_100_std": 0.489005128198763, + "nauc_recall_at_10_diff1": 0.3167742850497026, + "nauc_recall_at_10_max": 0.4343588721347347, + "nauc_recall_at_10_std": 0.11386525275829033, + "nauc_recall_at_1_diff1": 0.4672280000006232, + "nauc_recall_at_1_max": 0.2728573344017962, + "nauc_recall_at_1_std": -0.05815351013282631, + "nauc_recall_at_20_diff1": 0.29927115031576634, + "nauc_recall_at_20_max": 0.47558995314995334, + "nauc_recall_at_20_std": 0.22756546497773786, + "nauc_recall_at_3_diff1": 0.34285280177520755, + "nauc_recall_at_3_max": 0.33665099213094823, + "nauc_recall_at_3_std": -0.03094504186218613, + "nauc_recall_at_5_diff1": 0.34449954880775274, + "nauc_recall_at_5_max": 0.3766868889853373, + "nauc_recall_at_5_std": 0.00484617539423338, + "ndcg_at_1": 0.6052, + "ndcg_at_10": 0.69711, + "ndcg_at_100": 0.72818, + "ndcg_at_1000": 0.73381, + "ndcg_at_20": 0.71339, + "ndcg_at_3": 0.6247, + "ndcg_at_5": 0.66384, + "precision_at_1": 0.6052, + "precision_at_10": 0.13558, + "precision_at_100": 0.01585, + "precision_at_1000": 0.00166, + "precision_at_20": 0.07308, + "precision_at_3": 0.33629, + "precision_at_5": 0.23735, + "recall_at_1": 0.42655, + "recall_at_10": 0.82986, + "recall_at_100": 0.95023, + "recall_at_1000": 0.98809, + "recall_at_20": 0.88312, + "recall_at_3": 0.65421, + "recall_at_5": 0.74413 + } + ] + }, + "task_name": "RuBQRetrieval" +} \ No newline at end of file diff --git a/results/dunzhang__stella_en_1.5B_v5/d03be74b361d4eb24f42a2fe5bd2e29917df4604/RuReviewsClassification.json b/results/dunzhang__stella_en_1.5B_v5/d03be74b361d4eb24f42a2fe5bd2e29917df4604/RuReviewsClassification.json new file mode 100644 index 000000000..0498e5fc3 --- /dev/null +++ b/results/dunzhang__stella_en_1.5B_v5/d03be74b361d4eb24f42a2fe5bd2e29917df4604/RuReviewsClassification.json @@ -0,0 +1,73 @@ +{ + "dataset_revision": "f6d2c31f4dc6b88f468552750bfec05b4b41b05a", + "evaluation_time": 55.22489595413208, + "kg_co2_emissions": null, + "mteb_version": "1.19.0", + "scores": { + "test": [ + { + "accuracy": 0.6646484375, + "f1": 0.6437810537871659, + "f1_weighted": 0.6437682049172256, + "hf_subset": "default", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.6646484375, + "scores_per_experiment": [ + { + "accuracy": 0.68603515625, + "f1": 0.6697910969513613, + "f1_weighted": 0.6697762169928466 + }, + { + "accuracy": 0.59326171875, + "f1": 0.5785320285960983, + "f1_weighted": 0.5785762269994019 + }, + { + "accuracy": 0.6767578125, + "f1": 0.6519627193186621, + "f1_weighted": 0.6519389769530224 + }, + { + "accuracy": 0.70166015625, + "f1": 0.6931068481250136, + "f1_weighted": 0.6931019778201464 + }, + { + "accuracy": 0.69091796875, + "f1": 0.6791757290270289, + "f1_weighted": 0.6791580289279626 + }, + { + "accuracy": 0.65771484375, + "f1": 0.6356179870847556, + "f1_weighted": 0.6355991101063047 + }, + { + "accuracy": 0.630859375, + "f1": 0.6124806096742343, + "f1_weighted": 0.6124794995723366 + }, + { + "accuracy": 0.68505859375, + "f1": 0.6630551594226458, + "f1_weighted": 0.6630355307765263 + }, + { + "accuracy": 0.65087890625, + "f1": 0.6019724240454174, + "f1_weighted": 0.6019206085225457 + }, + { + "accuracy": 0.67333984375, + "f1": 0.6521159356264419, + "f1_weighted": 0.652095872501163 + } + ] + } + ] + }, + "task_name": "RuReviewsClassification" +} \ No newline at end of file diff --git a/results/dunzhang__stella_en_1.5B_v5/d03be74b361d4eb24f42a2fe5bd2e29917df4604/RuSTSBenchmarkSTS.json b/results/dunzhang__stella_en_1.5B_v5/d03be74b361d4eb24f42a2fe5bd2e29917df4604/RuSTSBenchmarkSTS.json new file mode 100644 index 000000000..c3aa86cd5 --- /dev/null +++ b/results/dunzhang__stella_en_1.5B_v5/d03be74b361d4eb24f42a2fe5bd2e29917df4604/RuSTSBenchmarkSTS.json @@ -0,0 +1,26 @@ +{ + "dataset_revision": "7cf24f325c6da6195df55bef3d86b5e0616f3018", + "evaluation_time": 32.034199714660645, + "kg_co2_emissions": null, + "mteb_version": "1.19.0", + "scores": { + "test": [ + { + "cosine_pearson": 0.7928549061537362, + "cosine_spearman": 0.7885969948761932, + "euclidean_pearson": 0.7849423563746483, + "euclidean_spearman": 0.7885886368275632, + "hf_subset": "default", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.7885969948761932, + "manhattan_pearson": 0.7845712561152632, + "manhattan_spearman": 0.7882597529933683, + "pearson": 0.7928549061537362, + "spearman": 0.7885969948761932 + } + ] + }, + "task_name": "RuSTSBenchmarkSTS" +} \ No newline at end of file diff --git a/results/dunzhang__stella_en_1.5B_v5/d03be74b361d4eb24f42a2fe5bd2e29917df4604/RuSciBenchGRNTIClassification.json b/results/dunzhang__stella_en_1.5B_v5/d03be74b361d4eb24f42a2fe5bd2e29917df4604/RuSciBenchGRNTIClassification.json new file mode 100644 index 000000000..0c46dc16c --- /dev/null +++ b/results/dunzhang__stella_en_1.5B_v5/d03be74b361d4eb24f42a2fe5bd2e29917df4604/RuSciBenchGRNTIClassification.json @@ -0,0 +1,73 @@ +{ + "dataset_revision": "673a610d6d3dd91a547a0d57ae1b56f37ebbf6a1", + "evaluation_time": 303.87458539009094, + "kg_co2_emissions": null, + "mteb_version": "1.19.0", + "scores": { + "test": [ + { + "accuracy": 0.64931640625, + "f1": 0.6332526497243622, + "f1_weighted": 0.6334126848801434, + "hf_subset": "default", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.64931640625, + "scores_per_experiment": [ + { + "accuracy": 0.65966796875, + "f1": 0.6485028761894557, + "f1_weighted": 0.6486759739451547 + }, + { + "accuracy": 0.6513671875, + "f1": 0.6359415326869692, + "f1_weighted": 0.6360874309043886 + }, + { + "accuracy": 0.6455078125, + "f1": 0.6285585061236539, + "f1_weighted": 0.6287557762090282 + }, + { + "accuracy": 0.64990234375, + "f1": 0.6316959498092652, + "f1_weighted": 0.6318366529155865 + }, + { + "accuracy": 0.65869140625, + "f1": 0.6441286111153469, + "f1_weighted": 0.644274145436476 + }, + { + "accuracy": 0.6259765625, + "f1": 0.6113615930122668, + "f1_weighted": 0.6115264468795295 + }, + { + "accuracy": 0.658203125, + "f1": 0.6401689567351637, + "f1_weighted": 0.6403526623967492 + }, + { + "accuracy": 0.64404296875, + "f1": 0.6268064991439148, + "f1_weighted": 0.6269523880432311 + }, + { + "accuracy": 0.66455078125, + "f1": 0.6476486304819608, + "f1_weighted": 0.6477980944244626 + }, + { + "accuracy": 0.63525390625, + "f1": 0.6177133419456255, + "f1_weighted": 0.6178672776468269 + } + ] + } + ] + }, + "task_name": "RuSciBenchGRNTIClassification" +} \ No newline at end of file diff --git a/results/dunzhang__stella_en_1.5B_v5/d03be74b361d4eb24f42a2fe5bd2e29917df4604/RuSciBenchGRNTIClusteringP2P.json b/results/dunzhang__stella_en_1.5B_v5/d03be74b361d4eb24f42a2fe5bd2e29917df4604/RuSciBenchGRNTIClusteringP2P.json new file mode 100644 index 000000000..f212ff964 --- /dev/null +++ b/results/dunzhang__stella_en_1.5B_v5/d03be74b361d4eb24f42a2fe5bd2e29917df4604/RuSciBenchGRNTIClusteringP2P.json @@ -0,0 +1,34 @@ +{ + "dataset_revision": "673a610d6d3dd91a547a0d57ae1b56f37ebbf6a1", + "evaluation_time": 200.33459424972534, + "kg_co2_emissions": null, + "mteb_version": "1.19.0", + "scores": { + "test": [ + { + "hf_subset": "default", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.5944973954891928, + "v_measure": 0.5944973954891928, + "v_measure_std": 0.00787997590126789, + "v_measures": { + "Level 0": [ + 0.5914259966749548, + 0.5788758801175734, + 0.6071950836098392, + 0.5888645641057759, + 0.593280327873349, + 0.6012556844048756, + 0.595042660228372, + 0.5979780854638821, + 0.5880170759012284, + 0.6030385965120786 + ] + } + } + ] + }, + "task_name": "RuSciBenchGRNTIClusteringP2P" +} \ No newline at end of file diff --git a/results/dunzhang__stella_en_1.5B_v5/d03be74b361d4eb24f42a2fe5bd2e29917df4604/RuSciBenchOECDClassification.json b/results/dunzhang__stella_en_1.5B_v5/d03be74b361d4eb24f42a2fe5bd2e29917df4604/RuSciBenchOECDClassification.json new file mode 100644 index 000000000..d336ce864 --- /dev/null +++ b/results/dunzhang__stella_en_1.5B_v5/d03be74b361d4eb24f42a2fe5bd2e29917df4604/RuSciBenchOECDClassification.json @@ -0,0 +1,73 @@ +{ + "dataset_revision": "26c88e99dcaba32bb45d0e1bfc21902337f6d471", + "evaluation_time": 301.23367834091187, + "kg_co2_emissions": null, + "mteb_version": "1.19.0", + "scores": { + "test": [ + { + "accuracy": 0.524072265625, + "f1": 0.5020159478401244, + "f1_weighted": 0.5020389467786961, + "hf_subset": "default", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.524072265625, + "scores_per_experiment": [ + { + "accuracy": 0.533203125, + "f1": 0.5098413691150165, + "f1_weighted": 0.5099466466326619 + }, + { + "accuracy": 0.54638671875, + "f1": 0.5249506912077743, + "f1_weighted": 0.524991828244169 + }, + { + "accuracy": 0.51953125, + "f1": 0.5034403578764758, + "f1_weighted": 0.5034777055278902 + }, + { + "accuracy": 0.5078125, + "f1": 0.47397549256659616, + "f1_weighted": 0.47391868853322927 + }, + { + "accuracy": 0.5234375, + "f1": 0.4970942073345913, + "f1_weighted": 0.4971192879551676 + }, + { + "accuracy": 0.53515625, + "f1": 0.5167888144715264, + "f1_weighted": 0.5168635418144083 + }, + { + "accuracy": 0.5185546875, + "f1": 0.49012447417152016, + "f1_weighted": 0.4902150200665909 + }, + { + "accuracy": 0.5361328125, + "f1": 0.5249565523152754, + "f1_weighted": 0.5249615836578992 + }, + { + "accuracy": 0.49560546875, + "f1": 0.4734159968336826, + "f1_weighted": 0.47336214148275685 + }, + { + "accuracy": 0.52490234375, + "f1": 0.5055715225087856, + "f1_weighted": 0.5055330238721879 + } + ] + } + ] + }, + "task_name": "RuSciBenchOECDClassification" +} \ No newline at end of file diff --git a/results/dunzhang__stella_en_1.5B_v5/d03be74b361d4eb24f42a2fe5bd2e29917df4604/RuSciBenchOECDClusteringP2P.json b/results/dunzhang__stella_en_1.5B_v5/d03be74b361d4eb24f42a2fe5bd2e29917df4604/RuSciBenchOECDClusteringP2P.json new file mode 100644 index 000000000..228390536 --- /dev/null +++ b/results/dunzhang__stella_en_1.5B_v5/d03be74b361d4eb24f42a2fe5bd2e29917df4604/RuSciBenchOECDClusteringP2P.json @@ -0,0 +1,34 @@ +{ + "dataset_revision": "26c88e99dcaba32bb45d0e1bfc21902337f6d471", + "evaluation_time": 191.39618635177612, + "kg_co2_emissions": null, + "mteb_version": "1.19.0", + "scores": { + "test": [ + { + "hf_subset": "default", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.5263573213221638, + "v_measure": 0.5263573213221638, + "v_measure_std": 0.011544136253692728, + "v_measures": { + "Level 0": [ + 0.5071847510805272, + 0.5404804839019417, + 0.5147516748574964, + 0.5383722064400731, + 0.5310425625292059, + 0.5239279717223291, + 0.5418874060642938, + 0.5117707754734192, + 0.5284431965240929, + 0.5257121846282585 + ] + } + } + ] + }, + "task_name": "RuSciBenchOECDClusteringP2P" +} \ No newline at end of file diff --git a/results/dunzhang__stella_en_1.5B_v5/d03be74b361d4eb24f42a2fe5bd2e29917df4604/STS22.json b/results/dunzhang__stella_en_1.5B_v5/d03be74b361d4eb24f42a2fe5bd2e29917df4604/STS22.json new file mode 100644 index 000000000..de96808e4 --- /dev/null +++ b/results/dunzhang__stella_en_1.5B_v5/d03be74b361d4eb24f42a2fe5bd2e29917df4604/STS22.json @@ -0,0 +1,26 @@ +{ + "dataset_revision": "de9d86b3b84231dc21f76c7b7af1f28e2f57f6e3", + "evaluation_time": 43.042364835739136, + "kg_co2_emissions": null, + "mteb_version": "1.19.0", + "scores": { + "test": [ + { + "cosine_pearson": 0.48395700595956387, + "cosine_spearman": 0.6252534913763196, + "euclidean_pearson": 0.5203418727597087, + "euclidean_spearman": 0.6252253901070653, + "hf_subset": "ru", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.6252534913763196, + "manhattan_pearson": 0.5204510099405986, + "manhattan_spearman": 0.6243842334124884, + "pearson": 0.48395700595956387, + "spearman": 0.6252534913763196 + } + ] + }, + "task_name": "STS22" +} \ No newline at end of file diff --git a/results/dunzhang__stella_en_1.5B_v5/d03be74b361d4eb24f42a2fe5bd2e29917df4604/SensitiveTopicsClassification.json b/results/dunzhang__stella_en_1.5B_v5/d03be74b361d4eb24f42a2fe5bd2e29917df4604/SensitiveTopicsClassification.json new file mode 100644 index 000000000..51a7ceaee --- /dev/null +++ b/results/dunzhang__stella_en_1.5B_v5/d03be74b361d4eb24f42a2fe5bd2e29917df4604/SensitiveTopicsClassification.json @@ -0,0 +1,73 @@ +{ + "dataset_revision": "416b34a802308eac30e4192afc0ff99bb8dcc7f2", + "evaluation_time": 63.605530977249146, + "kg_co2_emissions": null, + "mteb_version": "1.19.0", + "scores": { + "test": [ + { + "accuracy": 0.297216796875, + "f1": 0.34230144302756715, + "hf_subset": "default", + "languages": [ + "rus-Cyrl" + ], + "lrap": 0.4446533203124924, + "main_score": 0.297216796875, + "scores_per_experiment": [ + { + "accuracy": 0.30126953125, + "f1": 0.3317694323064797, + "lrap": 0.42758517795138107 + }, + { + "accuracy": 0.30322265625, + "f1": 0.3736626597995294, + "lrap": 0.47759331597221466 + }, + { + "accuracy": 0.28857421875, + "f1": 0.29934833331449934, + "lrap": 0.4128282335069364 + }, + { + "accuracy": 0.27099609375, + "f1": 0.30412195323872526, + "lrap": 0.4095323350694364 + }, + { + "accuracy": 0.27783203125, + "f1": 0.32263026731008887, + "lrap": 0.43358696831596455 + }, + { + "accuracy": 0.30615234375, + "f1": 0.36136353974934116, + "lrap": 0.4714830186631869 + }, + { + "accuracy": 0.31640625, + "f1": 0.3649735431321617, + "lrap": 0.46371799045138146 + }, + { + "accuracy": 0.2919921875, + "f1": 0.3644435439523085, + "lrap": 0.4401177300347148 + }, + { + "accuracy": 0.31640625, + "f1": 0.3483101044873775, + "lrap": 0.46487087673610333 + }, + { + "accuracy": 0.29931640625, + "f1": 0.35239105298516, + "lrap": 0.44521755642360367 + } + ] + } + ] + }, + "task_name": "SensitiveTopicsClassification" +} \ No newline at end of file diff --git a/results/dunzhang__stella_en_1.5B_v5/d03be74b361d4eb24f42a2fe5bd2e29917df4604/model_meta.json b/results/dunzhang__stella_en_1.5B_v5/d03be74b361d4eb24f42a2fe5bd2e29917df4604/model_meta.json index 05cd339a6..e7acfcdc0 100644 --- a/results/dunzhang__stella_en_1.5B_v5/d03be74b361d4eb24f42a2fe5bd2e29917df4604/model_meta.json +++ b/results/dunzhang__stella_en_1.5B_v5/d03be74b361d4eb24f42a2fe5bd2e29917df4604/model_meta.json @@ -1 +1,26 @@ -{"name": "dunzhang/stella_en_1.5B_v5", "revision": "d03be74b361d4eb24f42a2fe5bd2e29917df4604", "release_date": "2024-07-12", "languages": ["eng_Latn"], "n_parameters": 1540000, "memory_usage": null, "max_tokens": 131072, "embed_dim": 8960, "license": "mit", "open_weights": true, "public_training_data": null, "public_training_code": null, "framework": ["Sentence Transformers", "PyTorch", "GritLM"], "reference": "https://huggingface.co/dunzhang/stella_en_1.5B_v5", "similarity_fn_name": "cosine", "use_instructions": true, "zero_shot_benchmarks": null, "loader": "instruct_wrapper"} \ No newline at end of file +{ + "name": "dunzhang/stella_en_1.5B_v5", + "revision": "d03be74b361d4eb24f42a2fe5bd2e29917df4604", + "release_date": "2024-07-12", + "languages": [ + "eng_Latn" + ], + "n_parameters": 1540000, + "memory_usage": null, + "max_tokens": 131072, + "embed_dim": 8960, + "license": "mit", + "open_weights": true, + "public_training_data": null, + "public_training_code": null, + "framework": [ + "Sentence Transformers", + "PyTorch", + "GritLM" + ], + "reference": "https://huggingface.co/dunzhang/stella_en_1.5B_v5", + "similarity_fn_name": "cosine", + "use_instructions": true, + "zero_shot_benchmarks": null, + "loader": "instruct_wrapper" +} \ No newline at end of file diff --git a/results/dunzhang__stella_en_400M_v5/1bb50bc7bb726810eac2140e62155b88b0df198f/CEDRClassification.json b/results/dunzhang__stella_en_400M_v5/1bb50bc7bb726810eac2140e62155b88b0df198f/CEDRClassification.json index 4aaf64a75..51bd3384f 100644 --- a/results/dunzhang__stella_en_400M_v5/1bb50bc7bb726810eac2140e62155b88b0df198f/CEDRClassification.json +++ b/results/dunzhang__stella_en_400M_v5/1bb50bc7bb726810eac2140e62155b88b0df198f/CEDRClassification.json @@ -6,64 +6,64 @@ "scores": { "test": [ { - "accuracy": 0.3092454835281615, - "f1": 0.09007556223042724, + "accuracy": 0.33405951115834215, + "f1": 0.09164807314338298, "hf_subset": "default", "languages": [ "rus-Cyrl" ], - "lrap": 0.5605366631243485, - "main_score": 0.3092454835281615, + "lrap": 0.5666896918172282, + "main_score": 0.33405951115834215, "scores_per_experiment": [ { - "accuracy": 0.3645058448459086, - "f1": 0.08894768523138336, - "lrap": 0.5812433581296614 + "accuracy": 0.32624867162592985, + "f1": 0.08456934440187251, + "lrap": 0.5539851222104272 }, { - "accuracy": 0.3873538788522848, - "f1": 0.08500193246899265, - "lrap": 0.5528692879915105 + "accuracy": 0.2502656748140276, + "f1": 0.09333029732259474, + "lrap": 0.579171094580246 }, { - "accuracy": 0.3655685441020191, - "f1": 0.059610329542515605, - "lrap": 0.5338469713071332 + "accuracy": 0.2970244420828905, + "f1": 0.06564961390003135, + "lrap": 0.5442082890542105 }, { - "accuracy": 0.3374070138150903, - "f1": 0.09773813346794628, - "lrap": 0.5537194473963994 + "accuracy": 0.34643995749202977, + "f1": 0.0774469571015492, + "lrap": 0.5612646121147844 }, { - "accuracy": 0.2667375132837407, - "f1": 0.1173637603538337, - "lrap": 0.5806057385759955 + "accuracy": 0.30499468650371947, + "f1": 0.09932813434967311, + "lrap": 0.5623273113708948 }, { - "accuracy": 0.20828905419766205, - "f1": 0.05969813136416128, - "lrap": 0.5420828905419898 + "accuracy": 0.37300743889479276, + "f1": 0.07877962707955213, + "lrap": 0.5638150903294495 }, { - "accuracy": 0.3140276301806589, - "f1": 0.10000709032456541, - "lrap": 0.5837407013815215 + "accuracy": 0.3257173219978746, + "f1": 0.07249101836262041, + "lrap": 0.5638682252922548 }, { - "accuracy": 0.26567481402763016, - "f1": 0.0824476911976912, - "lrap": 0.5543039319872607 + "accuracy": 0.3703506907545165, + "f1": 0.13866531483279781, + "lrap": 0.604303931987259 }, { - "accuracy": 0.34962805526036134, - "f1": 0.11211560025322871, - "lrap": 0.5664718384697255 + "accuracy": 0.36291179596174283, + "f1": 0.10280512468083809, + "lrap": 0.571572794899056 }, { - "accuracy": 0.2332624867162593, - "f1": 0.09782526809995422, - "lrap": 0.5564824654622869 + "accuracy": 0.383634431455898, + "f1": 0.10341529940230043, + "lrap": 0.5623804463336998 } ] } diff --git a/results/dunzhang__stella_en_400M_v5/1bb50bc7bb726810eac2140e62155b88b0df198f/GeoreviewClassification.json b/results/dunzhang__stella_en_400M_v5/1bb50bc7bb726810eac2140e62155b88b0df198f/GeoreviewClassification.json new file mode 100644 index 000000000..6aab8b701 --- /dev/null +++ b/results/dunzhang__stella_en_400M_v5/1bb50bc7bb726810eac2140e62155b88b0df198f/GeoreviewClassification.json @@ -0,0 +1,73 @@ +{ + "dataset_revision": "3765c0d1de6b7d264bc459433c45e5a75513839c", + "evaluation_time": 61.33358645439148, + "kg_co2_emissions": null, + "mteb_version": "1.19.0", + "scores": { + "test": [ + { + "accuracy": 0.3505859375, + "f1": 0.3084283176964876, + "f1_weighted": 0.3085108596874553, + "hf_subset": "default", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.3505859375, + "scores_per_experiment": [ + { + "accuracy": 0.35791015625, + "f1": 0.3207156811645603, + "f1_weighted": 0.32071128345721284 + }, + { + "accuracy": 0.37744140625, + "f1": 0.3400761555082674, + "f1_weighted": 0.3401926965559934 + }, + { + "accuracy": 0.36181640625, + "f1": 0.32329532076750545, + "f1_weighted": 0.3234004050172064 + }, + { + "accuracy": 0.3310546875, + "f1": 0.29097087029046725, + "f1_weighted": 0.2911423455004266 + }, + { + "accuracy": 0.31494140625, + "f1": 0.2738191957480771, + "f1_weighted": 0.27394848460691795 + }, + { + "accuracy": 0.32958984375, + "f1": 0.29790850173313976, + "f1_weighted": 0.29798349456619844 + }, + { + "accuracy": 0.357421875, + "f1": 0.31922694241268623, + "f1_weighted": 0.31931914920317245 + }, + { + "accuracy": 0.3544921875, + "f1": 0.30834154202292563, + "f1_weighted": 0.3083968788463111 + }, + { + "accuracy": 0.3623046875, + "f1": 0.3188023055690569, + "f1_weighted": 0.3188036530228295 + }, + { + "accuracy": 0.35888671875, + "f1": 0.2911266617481899, + "f1_weighted": 0.2912102060982841 + } + ] + } + ] + }, + "task_name": "GeoreviewClassification" +} \ No newline at end of file diff --git a/results/dunzhang__stella_en_400M_v5/1bb50bc7bb726810eac2140e62155b88b0df198f/GeoreviewClusteringP2P.json b/results/dunzhang__stella_en_400M_v5/1bb50bc7bb726810eac2140e62155b88b0df198f/GeoreviewClusteringP2P.json new file mode 100644 index 000000000..e71156b60 --- /dev/null +++ b/results/dunzhang__stella_en_400M_v5/1bb50bc7bb726810eac2140e62155b88b0df198f/GeoreviewClusteringP2P.json @@ -0,0 +1,34 @@ +{ + "dataset_revision": "97a313c8fc85b47f13f33e7e9a95c1ad888c7fec", + "evaluation_time": 129.96525406837463, + "kg_co2_emissions": null, + "mteb_version": "1.19.0", + "scores": { + "test": [ + { + "hf_subset": "default", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.21862321999672094, + "v_measure": 0.21862321999672094, + "v_measure_std": 0.005958490084441652, + "v_measures": { + "Level 0": [ + 0.2171759138432109, + 0.21921645912059098, + 0.22224786150642226, + 0.22369044009678743, + 0.20557732030770515, + 0.21391129846489637, + 0.21598171319104223, + 0.22403719821236448, + 0.22765339170789237, + 0.2167406035162969 + ] + } + } + ] + }, + "task_name": "GeoreviewClusteringP2P" +} \ No newline at end of file diff --git a/results/dunzhang__stella_en_400M_v5/1bb50bc7bb726810eac2140e62155b88b0df198f/HeadlineClassification.json b/results/dunzhang__stella_en_400M_v5/1bb50bc7bb726810eac2140e62155b88b0df198f/HeadlineClassification.json new file mode 100644 index 000000000..1d280f79a --- /dev/null +++ b/results/dunzhang__stella_en_400M_v5/1bb50bc7bb726810eac2140e62155b88b0df198f/HeadlineClassification.json @@ -0,0 +1,73 @@ +{ + "dataset_revision": "2fe05ee6b5832cda29f2ef7aaad7b7fe6a3609eb", + "evaluation_time": 27.35127305984497, + "kg_co2_emissions": null, + "mteb_version": "1.19.0", + "scores": { + "test": [ + { + "accuracy": 0.3921875, + "f1": 0.38935669996959643, + "f1_weighted": 0.3893189397803015, + "hf_subset": "default", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.3921875, + "scores_per_experiment": [ + { + "accuracy": 0.4228515625, + "f1": 0.42410062487131367, + "f1_weighted": 0.4240771917388367 + }, + { + "accuracy": 0.41064453125, + "f1": 0.41365224991220084, + "f1_weighted": 0.41365436841166814 + }, + { + "accuracy": 0.37744140625, + "f1": 0.376784341844705, + "f1_weighted": 0.3767665056415442 + }, + { + "accuracy": 0.44189453125, + "f1": 0.44312578084633425, + "f1_weighted": 0.44311247333412795 + }, + { + "accuracy": 0.36767578125, + "f1": 0.35470589738870345, + "f1_weighted": 0.3546603491913746 + }, + { + "accuracy": 0.37451171875, + "f1": 0.3655660406305097, + "f1_weighted": 0.3654847636198891 + }, + { + "accuracy": 0.38330078125, + "f1": 0.3874576637450018, + "f1_weighted": 0.3874293934102657 + }, + { + "accuracy": 0.3486328125, + "f1": 0.3443256453562111, + "f1_weighted": 0.3443399307898149 + }, + { + "accuracy": 0.3818359375, + "f1": 0.38619254737453784, + "f1_weighted": 0.3861431283982333 + }, + { + "accuracy": 0.4130859375, + "f1": 0.39765620772644644, + "f1_weighted": 0.39752129326726104 + } + ] + } + ] + }, + "task_name": "HeadlineClassification" +} \ No newline at end of file diff --git a/results/dunzhang__stella_en_400M_v5/1bb50bc7bb726810eac2140e62155b88b0df198f/InappropriatenessClassification.json b/results/dunzhang__stella_en_400M_v5/1bb50bc7bb726810eac2140e62155b88b0df198f/InappropriatenessClassification.json new file mode 100644 index 000000000..66a348823 --- /dev/null +++ b/results/dunzhang__stella_en_400M_v5/1bb50bc7bb726810eac2140e62155b88b0df198f/InappropriatenessClassification.json @@ -0,0 +1,95 @@ +{ + "dataset_revision": "601651fdc45ef243751676e62dd7a19f491c0285", + "evaluation_time": 26.26379108428955, + "kg_co2_emissions": null, + "mteb_version": "1.19.0", + "scores": { + "test": [ + { + "accuracy": 0.52470703125, + "ap": 0.5139557556695517, + "ap_weighted": 0.5139557556695517, + "f1": 0.5210637908725082, + "f1_weighted": 0.5210637908725082, + "hf_subset": "default", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.52470703125, + "scores_per_experiment": [ + { + "accuracy": 0.55322265625, + "ap": 0.5298378406562848, + "ap_weighted": 0.5298378406562848, + "f1": 0.5515520574803664, + "f1_weighted": 0.5515520574803664 + }, + { + "accuracy": 0.5498046875, + "ap": 0.5269507623487903, + "ap_weighted": 0.5269507623487903, + "f1": 0.5447405372876387, + "f1_weighted": 0.5447405372876387 + }, + { + "accuracy": 0.5087890625, + "ap": 0.5044759114583334, + "ap_weighted": 0.5044759114583334, + "f1": 0.5084721824601584, + "f1_weighted": 0.5084721824601584 + }, + { + "accuracy": 0.537109375, + "ap": 0.5203173828125001, + "ap_weighted": 0.5203173828125001, + "f1": 0.5315048183839881, + "f1_weighted": 0.5315048183839881 + }, + { + "accuracy": 0.4794921875, + "ap": 0.49020620492788464, + "ap_weighted": 0.49020620492788464, + "f1": 0.4785293892838034, + "f1_weighted": 0.4785293892838034 + }, + { + "accuracy": 0.55419921875, + "ap": 0.5295235067485898, + "ap_weighted": 0.5295235067485898, + "f1": 0.5491374331931187, + "f1_weighted": 0.5491374331931187 + }, + { + "accuracy": 0.55029296875, + "ap": 0.5272747406532456, + "ap_weighted": 0.5272747406532456, + "f1": 0.5462634004120706, + "f1_weighted": 0.5462634004120706 + }, + { + "accuracy": 0.5380859375, + "ap": 0.5202088647959184, + "ap_weighted": 0.5202088647959184, + "f1": 0.5310987646074209, + "f1_weighted": 0.5310987646074209 + }, + { + "accuracy": 0.45703125, + "ap": 0.4808848292606516, + "ap_weighted": 0.4808848292606516, + "f1": 0.4503377559719137, + "f1_weighted": 0.4503377559719137 + }, + { + "accuracy": 0.51904296875, + "ap": 0.5098775130333173, + "ap_weighted": 0.5098775130333173, + "f1": 0.5190015696446041, + "f1_weighted": 0.5190015696446041 + } + ] + } + ] + }, + "task_name": "InappropriatenessClassification" +} \ No newline at end of file diff --git a/results/dunzhang__stella_en_400M_v5/1bb50bc7bb726810eac2140e62155b88b0df198f/KinopoiskClassification.json b/results/dunzhang__stella_en_400M_v5/1bb50bc7bb726810eac2140e62155b88b0df198f/KinopoiskClassification.json new file mode 100644 index 000000000..ea89abd12 --- /dev/null +++ b/results/dunzhang__stella_en_400M_v5/1bb50bc7bb726810eac2140e62155b88b0df198f/KinopoiskClassification.json @@ -0,0 +1,73 @@ +{ + "dataset_revision": "5911f26666ac11af46cb9c6849d0dc80a378af24", + "evaluation_time": 60.10074496269226, + "kg_co2_emissions": null, + "mteb_version": "1.19.0", + "scores": { + "test": [ + { + "accuracy": 0.3735333333333334, + "f1": 0.3387028194253951, + "f1_weighted": 0.3387028194253951, + "hf_subset": "default", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.3735333333333334, + "scores_per_experiment": [ + { + "accuracy": 0.3566666666666667, + "f1": 0.34634084019056105, + "f1_weighted": 0.34634084019056105 + }, + { + "accuracy": 0.4206666666666667, + "f1": 0.3710282860684859, + "f1_weighted": 0.3710282860684859 + }, + { + "accuracy": 0.2813333333333333, + "f1": 0.2809446996208807, + "f1_weighted": 0.28094469962088076 + }, + { + "accuracy": 0.38133333333333336, + "f1": 0.344976913151717, + "f1_weighted": 0.344976913151717 + }, + { + "accuracy": 0.36533333333333334, + "f1": 0.3071914583299131, + "f1_weighted": 0.3071914583299131 + }, + { + "accuracy": 0.374, + "f1": 0.31937378151846857, + "f1_weighted": 0.3193737815184686 + }, + { + "accuracy": 0.386, + "f1": 0.35004155773563167, + "f1_weighted": 0.3500415577356316 + }, + { + "accuracy": 0.37266666666666665, + "f1": 0.3327100251844643, + "f1_weighted": 0.33271002518446435 + }, + { + "accuracy": 0.382, + "f1": 0.33051255820762665, + "f1_weighted": 0.33051255820762665 + }, + { + "accuracy": 0.41533333333333333, + "f1": 0.403908074246202, + "f1_weighted": 0.40390807424620195 + } + ] + } + ] + }, + "task_name": "KinopoiskClassification" +} \ No newline at end of file diff --git a/results/dunzhang__stella_en_400M_v5/1bb50bc7bb726810eac2140e62155b88b0df198f/MIRACLReranking.json b/results/dunzhang__stella_en_400M_v5/1bb50bc7bb726810eac2140e62155b88b0df198f/MIRACLReranking.json new file mode 100644 index 000000000..68f01eea7 --- /dev/null +++ b/results/dunzhang__stella_en_400M_v5/1bb50bc7bb726810eac2140e62155b88b0df198f/MIRACLReranking.json @@ -0,0 +1,130 @@ +{ + "dataset_revision": "6d1962c527217f8927fca80f890f14f36b2802af", + "evaluation_time": 2793.295126914978, + "kg_co2_emissions": null, + "mteb_version": "1.19.0", + "scores": { + "dev": [ + { + "MAP@1(MIRACL)": 0.11552, + "MAP@10(MIRACL)": 0.20918, + "MAP@100(MIRACL)": 0.24114, + "MAP@1000(MIRACL)": 0.24114, + "MAP@20(MIRACL)": 0.22358, + "MAP@3(MIRACL)": 0.16879, + "MAP@5(MIRACL)": 0.1872, + "NDCG@1(MIRACL)": 0.18605, + "NDCG@10(MIRACL)": 0.27729, + "NDCG@100(MIRACL)": 0.3892, + "NDCG@1000(MIRACL)": 0.3892, + "NDCG@20(MIRACL)": 0.31631, + "NDCG@3(MIRACL)": 0.2152, + "NDCG@5(MIRACL)": 0.23664, + "P@1(MIRACL)": 0.18605, + "P@10(MIRACL)": 0.08284, + "P@100(MIRACL)": 0.01953, + "P@1000(MIRACL)": 0.00195, + "P@20(MIRACL)": 0.0567, + "P@3(MIRACL)": 0.13472, + "P@5(MIRACL)": 0.1097, + "Recall@1(MIRACL)": 0.11552, + "Recall@10(MIRACL)": 0.38732, + "Recall@100(MIRACL)": 0.79952, + "Recall@1000(MIRACL)": 0.79952, + "Recall@20(MIRACL)": 0.5069, + "Recall@3(MIRACL)": 0.22068, + "Recall@5(MIRACL)": 0.28261, + "hf_subset": "ru", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.27729, + "nAUC_MAP@1000_diff1(MIRACL)": 0.18294643103872485, + "nAUC_MAP@1000_max(MIRACL)": 0.19079121419868578, + "nAUC_MAP@1000_std(MIRACL)": 0.10548341975711983, + "nAUC_MAP@100_diff1(MIRACL)": 0.18294643103872485, + "nAUC_MAP@100_max(MIRACL)": 0.19079121419868578, + "nAUC_MAP@100_std(MIRACL)": 0.10548341975711983, + "nAUC_MAP@10_diff1(MIRACL)": 0.20074514367396934, + "nAUC_MAP@10_max(MIRACL)": 0.18104426758035524, + "nAUC_MAP@10_std(MIRACL)": 0.09898923016197665, + "nAUC_MAP@1_diff1(MIRACL)": 0.3242954869406802, + "nAUC_MAP@1_max(MIRACL)": 0.16047111952272786, + "nAUC_MAP@1_std(MIRACL)": 0.058804793037164065, + "nAUC_MAP@20_diff1(MIRACL)": 0.1924033087077666, + "nAUC_MAP@20_max(MIRACL)": 0.18697631018023336, + "nAUC_MAP@20_std(MIRACL)": 0.1073349712232071, + "nAUC_MAP@3_diff1(MIRACL)": 0.23107686517241327, + "nAUC_MAP@3_max(MIRACL)": 0.15825989207412486, + "nAUC_MAP@3_std(MIRACL)": 0.08262923772417825, + "nAUC_MAP@5_diff1(MIRACL)": 0.21498867569166247, + "nAUC_MAP@5_max(MIRACL)": 0.16496059393468349, + "nAUC_MAP@5_std(MIRACL)": 0.08718577141137802, + "nAUC_NDCG@1000_diff1(MIRACL)": 0.08989583968543814, + "nAUC_NDCG@1000_max(MIRACL)": 0.22781006807404225, + "nAUC_NDCG@1000_std(MIRACL)": 0.1077465896139074, + "nAUC_NDCG@100_diff1(MIRACL)": 0.08989583968543814, + "nAUC_NDCG@100_max(MIRACL)": 0.22781006807404225, + "nAUC_NDCG@100_std(MIRACL)": 0.1077465896139074, + "nAUC_NDCG@10_diff1(MIRACL)": 0.14787238152395066, + "nAUC_NDCG@10_max(MIRACL)": 0.19920423505813126, + "nAUC_NDCG@10_std(MIRACL)": 0.1098941124097528, + "nAUC_NDCG@1_diff1(MIRACL)": 0.23482368690556896, + "nAUC_NDCG@1_max(MIRACL)": 0.18836465279954823, + "nAUC_NDCG@1_std(MIRACL)": 0.06533806189818456, + "nAUC_NDCG@20_diff1(MIRACL)": 0.1287406348314595, + "nAUC_NDCG@20_max(MIRACL)": 0.2103388014865725, + "nAUC_NDCG@20_std(MIRACL)": 0.1255221470875142, + "nAUC_NDCG@3_diff1(MIRACL)": 0.18140658540139493, + "nAUC_NDCG@3_max(MIRACL)": 0.17332778146331856, + "nAUC_NDCG@3_std(MIRACL)": 0.09641889063179436, + "nAUC_NDCG@5_diff1(MIRACL)": 0.1717075147547138, + "nAUC_NDCG@5_max(MIRACL)": 0.17419407778793772, + "nAUC_NDCG@5_std(MIRACL)": 0.09706179735672864, + "nAUC_P@1000_diff1(MIRACL)": -0.12298427492787627, + "nAUC_P@1000_max(MIRACL)": 0.12428031309794739, + "nAUC_P@1000_std(MIRACL)": 0.012687856502524459, + "nAUC_P@100_diff1(MIRACL)": -0.12298427492787596, + "nAUC_P@100_max(MIRACL)": 0.12428031309794761, + "nAUC_P@100_std(MIRACL)": 0.012687856502524693, + "nAUC_P@10_diff1(MIRACL)": -0.019570335760913035, + "nAUC_P@10_max(MIRACL)": 0.2036877335818553, + "nAUC_P@10_std(MIRACL)": 0.11589271852537715, + "nAUC_P@1_diff1(MIRACL)": 0.23482368690556896, + "nAUC_P@1_max(MIRACL)": 0.18836465279954823, + "nAUC_P@1_std(MIRACL)": 0.06533806189818456, + "nAUC_P@20_diff1(MIRACL)": -0.06007994503761798, + "nAUC_P@20_max(MIRACL)": 0.18882794698230124, + "nAUC_P@20_std(MIRACL)": 0.11486347479129365, + "nAUC_P@3_diff1(MIRACL)": 0.0685585533275335, + "nAUC_P@3_max(MIRACL)": 0.1980598259653799, + "nAUC_P@3_std(MIRACL)": 0.11009267721021743, + "nAUC_P@5_diff1(MIRACL)": 0.026725728589669925, + "nAUC_P@5_max(MIRACL)": 0.18344132640553593, + "nAUC_P@5_std(MIRACL)": 0.09730682781879942, + "nAUC_Recall@1000_diff1(MIRACL)": -0.251963848430677, + "nAUC_Recall@1000_max(MIRACL)": 0.3812956749224815, + "nAUC_Recall@1000_std(MIRACL)": 0.132630727181112, + "nAUC_Recall@100_diff1(MIRACL)": -0.251963848430677, + "nAUC_Recall@100_max(MIRACL)": 0.3812956749224815, + "nAUC_Recall@100_std(MIRACL)": 0.132630727181112, + "nAUC_Recall@10_diff1(MIRACL)": 0.0693407787457681, + "nAUC_Recall@10_max(MIRACL)": 0.1936289879909027, + "nAUC_Recall@10_std(MIRACL)": 0.13197827289869787, + "nAUC_Recall@1_diff1(MIRACL)": 0.3242954869406802, + "nAUC_Recall@1_max(MIRACL)": 0.16047111952272786, + "nAUC_Recall@1_std(MIRACL)": 0.058804793037164065, + "nAUC_Recall@20_diff1(MIRACL)": 0.015493134117015273, + "nAUC_Recall@20_max(MIRACL)": 0.2184274958380227, + "nAUC_Recall@20_std(MIRACL)": 0.17133869608475605, + "nAUC_Recall@3_diff1(MIRACL)": 0.15798907088354655, + "nAUC_Recall@3_max(MIRACL)": 0.14430391069663573, + "nAUC_Recall@3_std(MIRACL)": 0.1044448107685899, + "nAUC_Recall@5_diff1(MIRACL)": 0.12202809759779475, + "nAUC_Recall@5_max(MIRACL)": 0.15135041256145423, + "nAUC_Recall@5_std(MIRACL)": 0.1079970191990127 + } + ] + }, + "task_name": "MIRACLReranking" +} \ No newline at end of file diff --git a/results/dunzhang__stella_en_400M_v5/1bb50bc7bb726810eac2140e62155b88b0df198f/MIRACLRetrieval.json b/results/dunzhang__stella_en_400M_v5/1bb50bc7bb726810eac2140e62155b88b0df198f/MIRACLRetrieval.json new file mode 100644 index 000000000..2498449fe --- /dev/null +++ b/results/dunzhang__stella_en_400M_v5/1bb50bc7bb726810eac2140e62155b88b0df198f/MIRACLRetrieval.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "main", + "evaluation_time": 113264.00490522385, + "kg_co2_emissions": null, + "mteb_version": "1.19.4", + "scores": { + "dev": [ + { + "hf_subset": "ru", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.09889, + "map_at_1": 0.03346, + "map_at_10": 0.06744, + "map_at_100": 0.0762, + "map_at_1000": 0.07726, + "map_at_20": 0.072, + "map_at_3": 0.05384, + "map_at_5": 0.06094, + "mrr_at_1": 0.07108626198083066, + "mrr_at_10": 0.12313853390131346, + "mrr_at_100": 0.1299679579531057, + "mrr_at_1000": 0.1306076999618469, + "mrr_at_20": 0.12732413686574337, + "mrr_at_3": 0.10742811501597443, + "mrr_at_5": 0.11681309904153354, + "nauc_map_at_1000_diff1": 0.2433034819250918, + "nauc_map_at_1000_max": 0.32901262643628215, + "nauc_map_at_1000_std": 0.49548057856715055, + "nauc_map_at_100_diff1": 0.24358924368152268, + "nauc_map_at_100_max": 0.329179834896921, + "nauc_map_at_100_std": 0.4936324943391382, + "nauc_map_at_10_diff1": 0.24929551689333523, + "nauc_map_at_10_max": 0.3124297909375344, + "nauc_map_at_10_std": 0.4675104019393471, + "nauc_map_at_1_diff1": 0.3383970612005789, + "nauc_map_at_1_max": 0.278589526569681, + "nauc_map_at_1_std": 0.4145522265294242, + "nauc_map_at_20_diff1": 0.24990462372511751, + "nauc_map_at_20_max": 0.3209716468798788, + "nauc_map_at_20_std": 0.479839204324279, + "nauc_map_at_3_diff1": 0.27737460625064775, + "nauc_map_at_3_max": 0.2912707512449207, + "nauc_map_at_3_std": 0.4533397674927968, + "nauc_map_at_5_diff1": 0.2486852308638612, + "nauc_map_at_5_max": 0.2904745324952268, + "nauc_map_at_5_std": 0.4476791060472492, + "nauc_mrr_at_1000_diff1": 0.2085603493426865, + "nauc_mrr_at_1000_max": 0.28720019654888723, + "nauc_mrr_at_1000_std": 0.44920880771034366, + "nauc_mrr_at_100_diff1": 0.20819219035872083, + "nauc_mrr_at_100_max": 0.2873940856549141, + "nauc_mrr_at_100_std": 0.448906178417097, + "nauc_mrr_at_10_diff1": 0.20718711339475385, + "nauc_mrr_at_10_max": 0.2840494734811338, + "nauc_mrr_at_10_std": 0.441598682976741, + "nauc_mrr_at_1_diff1": 0.2786336037933114, + "nauc_mrr_at_1_max": 0.2603234067641529, + "nauc_mrr_at_1_std": 0.39502518997813935, + "nauc_mrr_at_20_diff1": 0.2083342786637419, + "nauc_mrr_at_20_max": 0.2862130323120075, + "nauc_mrr_at_20_std": 0.44628901501680396, + "nauc_mrr_at_3_diff1": 0.21871723696320447, + "nauc_mrr_at_3_max": 0.27839394785981514, + "nauc_mrr_at_3_std": 0.43371313246753057, + "nauc_mrr_at_5_diff1": 0.20973870141068723, + "nauc_mrr_at_5_max": 0.278861437650677, + "nauc_mrr_at_5_std": 0.4312756916948983, + "nauc_ndcg_at_1000_diff1": 0.20645599420979655, + "nauc_ndcg_at_1000_max": 0.3508252613603072, + "nauc_ndcg_at_1000_std": 0.5607385953986749, + "nauc_ndcg_at_100_diff1": 0.20629712467170339, + "nauc_ndcg_at_100_max": 0.35528080114906446, + "nauc_ndcg_at_100_std": 0.5444312461823263, + "nauc_ndcg_at_10_diff1": 0.21549114747506296, + "nauc_ndcg_at_10_max": 0.31891299985592575, + "nauc_ndcg_at_10_std": 0.47705669733353134, + "nauc_ndcg_at_1_diff1": 0.2786336037933114, + "nauc_ndcg_at_1_max": 0.2603234067641529, + "nauc_ndcg_at_1_std": 0.39502518997813935, + "nauc_ndcg_at_20_diff1": 0.22042606374738688, + "nauc_ndcg_at_20_max": 0.33215335082348085, + "nauc_ndcg_at_20_std": 0.500873760454895, + "nauc_ndcg_at_3_diff1": 0.2514098220292235, + "nauc_ndcg_at_3_max": 0.2876179909048953, + "nauc_ndcg_at_3_std": 0.44488289083623594, + "nauc_ndcg_at_5_diff1": 0.22146896174185843, + "nauc_ndcg_at_5_max": 0.28985208722045464, + "nauc_ndcg_at_5_std": 0.4458075774239965, + "nauc_precision_at_1000_diff1": 0.12696263271741687, + "nauc_precision_at_1000_max": 0.26500224220951507, + "nauc_precision_at_1000_std": 0.4796772965554239, + "nauc_precision_at_100_diff1": 0.12961480743769707, + "nauc_precision_at_100_max": 0.34501027971495823, + "nauc_precision_at_100_std": 0.5349354247064961, + "nauc_precision_at_10_diff1": 0.18097824735567955, + "nauc_precision_at_10_max": 0.36263966133901887, + "nauc_precision_at_10_std": 0.4797766094890311, + "nauc_precision_at_1_diff1": 0.2786336037933114, + "nauc_precision_at_1_max": 0.2603234067641529, + "nauc_precision_at_1_std": 0.39502518997813935, + "nauc_precision_at_20_diff1": 0.1776920077183219, + "nauc_precision_at_20_max": 0.3481384608628716, + "nauc_precision_at_20_std": 0.4939161028722111, + "nauc_precision_at_3_diff1": 0.2052105402913774, + "nauc_precision_at_3_max": 0.3166169741405959, + "nauc_precision_at_3_std": 0.4636205636550047, + "nauc_precision_at_5_diff1": 0.17464935607219853, + "nauc_precision_at_5_max": 0.3257353743897456, + "nauc_precision_at_5_std": 0.44660360805741, + "nauc_recall_at_1000_diff1": 0.15760329684479082, + "nauc_recall_at_1000_max": 0.37255181610798954, + "nauc_recall_at_1000_std": 0.6503298895175857, + "nauc_recall_at_100_diff1": 0.16195649592561887, + "nauc_recall_at_100_max": 0.38271012794611625, + "nauc_recall_at_100_std": 0.5975324881802806, + "nauc_recall_at_10_diff1": 0.1740097667083196, + "nauc_recall_at_10_max": 0.315815866395565, + "nauc_recall_at_10_std": 0.4828819772657213, + "nauc_recall_at_1_diff1": 0.3383970612005789, + "nauc_recall_at_1_max": 0.278589526569681, + "nauc_recall_at_1_std": 0.4145522265294242, + "nauc_recall_at_20_diff1": 0.18947110579551632, + "nauc_recall_at_20_max": 0.3336061798944577, + "nauc_recall_at_20_std": 0.5139221384574942, + "nauc_recall_at_3_diff1": 0.23872568980787512, + "nauc_recall_at_3_max": 0.2886217481987023, + "nauc_recall_at_3_std": 0.46796585408095576, + "nauc_recall_at_5_diff1": 0.1767364293132168, + "nauc_recall_at_5_max": 0.2719283813780604, + "nauc_recall_at_5_std": 0.4388391081272781, + "ndcg_at_1": 0.07109, + "ndcg_at_10": 0.09889, + "ndcg_at_100": 0.13878, + "ndcg_at_1000": 0.16304, + "ndcg_at_20": 0.11383, + "ndcg_at_3": 0.07827, + "ndcg_at_5": 0.08619, + "precision_at_1": 0.07109, + "precision_at_10": 0.03227, + "precision_at_100": 0.0074, + "precision_at_1000": 0.00118, + "precision_at_20": 0.02236, + "precision_at_3": 0.05618, + "precision_at_5": 0.04537, + "recall_at_1": 0.03346, + "recall_at_10": 0.13636, + "recall_at_100": 0.28589, + "recall_at_1000": 0.43202, + "recall_at_20": 0.18194, + "recall_at_3": 0.07523, + "recall_at_5": 0.10105 + } + ] + }, + "task_name": "MIRACLRetrieval" +} \ No newline at end of file diff --git a/results/dunzhang__stella_en_400M_v5/1bb50bc7bb726810eac2140e62155b88b0df198f/MassiveScenarioClassification.json b/results/dunzhang__stella_en_400M_v5/1bb50bc7bb726810eac2140e62155b88b0df198f/MassiveScenarioClassification.json new file mode 100644 index 000000000..74e4c3415 --- /dev/null +++ b/results/dunzhang__stella_en_400M_v5/1bb50bc7bb726810eac2140e62155b88b0df198f/MassiveScenarioClassification.json @@ -0,0 +1,137 @@ +{ + "dataset_revision": "fad2c6e8459f9e1c45d9315f4953d921437d70f8", + "evaluation_time": 89.19464540481567, + "kg_co2_emissions": null, + "mteb_version": "1.19.0", + "scores": { + "test": [ + { + "accuracy": 0.25581708137188974, + "f1": 0.24791678115855142, + "f1_weighted": 0.24233995545325743, + "hf_subset": "ru", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.25581708137188974, + "scores_per_experiment": [ + { + "accuracy": 0.26698049764626763, + "f1": 0.2616889293337436, + "f1_weighted": 0.270465588702382 + }, + { + "accuracy": 0.2824478816408877, + "f1": 0.2662015088772411, + "f1_weighted": 0.2776146553692399 + }, + { + "accuracy": 0.2535305985205111, + "f1": 0.2467491293734047, + "f1_weighted": 0.24393712308472768 + }, + { + "accuracy": 0.2767316745124412, + "f1": 0.25593058524616097, + "f1_weighted": 0.26199165890343346 + }, + { + "accuracy": 0.22326832548755884, + "f1": 0.2109829459631756, + "f1_weighted": 0.18200101473617783 + }, + { + "accuracy": 0.22562205783456624, + "f1": 0.21151704042492317, + "f1_weighted": 0.1950320177338689 + }, + { + "accuracy": 0.234364492266308, + "f1": 0.22875368996590056, + "f1_weighted": 0.23211038461948835 + }, + { + "accuracy": 0.285137861466039, + "f1": 0.2806482241283775, + "f1_weighted": 0.2656776433788408 + }, + { + "accuracy": 0.24243443174176194, + "f1": 0.24118605795212578, + "f1_weighted": 0.22253806698101705 + }, + { + "accuracy": 0.2676529926025555, + "f1": 0.27550970032046124, + "f1_weighted": 0.27203140102339846 + } + ] + } + ], + "validation": [ + { + "accuracy": 0.26463354648303, + "f1": 0.2594593881134233, + "f1_weighted": 0.2521639527219338, + "hf_subset": "ru", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.26463354648303, + "scores_per_experiment": [ + { + "accuracy": 0.2867683226758485, + "f1": 0.2854671994759544, + "f1_weighted": 0.28747991832214115 + }, + { + "accuracy": 0.2852926709296606, + "f1": 0.27530194886632436, + "f1_weighted": 0.28192149937486627 + }, + { + "accuracy": 0.25922282341367436, + "f1": 0.25072858551626376, + "f1_weighted": 0.2448911657748885 + }, + { + "accuracy": 0.28726020659124446, + "f1": 0.2733132500420075, + "f1_weighted": 0.2740581096429117 + }, + { + "accuracy": 0.22429906542056074, + "f1": 0.22104709449741577, + "f1_weighted": 0.19461550822648138 + }, + { + "accuracy": 0.24495818986719134, + "f1": 0.23043421311884676, + "f1_weighted": 0.21995713480411022 + }, + { + "accuracy": 0.24741760944417118, + "f1": 0.24068562479427044, + "f1_weighted": 0.24856981196990646 + }, + { + "accuracy": 0.27594687653713723, + "f1": 0.26860188174011024, + "f1_weighted": 0.25667133887061194 + }, + { + "accuracy": 0.26512543039842595, + "f1": 0.26620926696063824, + "f1_weighted": 0.24396240653447399 + }, + { + "accuracy": 0.27004426955238564, + "f1": 0.28280481612240116, + "f1_weighted": 0.2695126336989469 + } + ] + } + ] + }, + "task_name": "MassiveScenarioClassification" +} \ No newline at end of file diff --git a/results/dunzhang__stella_en_400M_v5/1bb50bc7bb726810eac2140e62155b88b0df198f/RUParaPhraserSTS.json b/results/dunzhang__stella_en_400M_v5/1bb50bc7bb726810eac2140e62155b88b0df198f/RUParaPhraserSTS.json new file mode 100644 index 000000000..fc7de7d58 --- /dev/null +++ b/results/dunzhang__stella_en_400M_v5/1bb50bc7bb726810eac2140e62155b88b0df198f/RUParaPhraserSTS.json @@ -0,0 +1,26 @@ +{ + "dataset_revision": "43265056790b8f7c59e0139acb4be0a8dad2c8f4", + "evaluation_time": 32.50419354438782, + "kg_co2_emissions": null, + "mteb_version": "1.19.0", + "scores": { + "test": [ + { + "cosine_pearson": 0.48876615361856335, + "cosine_spearman": 0.4979407108181118, + "euclidean_pearson": 0.505746555393063, + "euclidean_spearman": 0.4979414138432267, + "hf_subset": "default", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.4979407108181118, + "manhattan_pearson": 0.5065076680340204, + "manhattan_spearman": 0.4991235840927585, + "pearson": 0.48876615361856335, + "spearman": 0.4979407108181118 + } + ] + }, + "task_name": "RUParaPhraserSTS" +} \ No newline at end of file diff --git a/results/dunzhang__stella_en_400M_v5/1bb50bc7bb726810eac2140e62155b88b0df198f/RiaNewsRetrieval.json b/results/dunzhang__stella_en_400M_v5/1bb50bc7bb726810eac2140e62155b88b0df198f/RiaNewsRetrieval.json new file mode 100644 index 000000000..9695594b4 --- /dev/null +++ b/results/dunzhang__stella_en_400M_v5/1bb50bc7bb726810eac2140e62155b88b0df198f/RiaNewsRetrieval.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "82374b0bbacda6114f39ff9c5b925fa1512ca5d7", + "evaluation_time": 21814.083217144012, + "kg_co2_emissions": null, + "mteb_version": "1.19.0", + "scores": { + "test": [ + { + "hf_subset": "default", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.15389, + "map_at_1": 0.0998, + "map_at_10": 0.13415, + "map_at_100": 0.13934, + "map_at_1000": 0.14001, + "map_at_20": 0.13674, + "map_at_3": 0.1229, + "map_at_5": 0.129, + "mrr_at_1": 0.0996, + "mrr_at_10": 0.1340849603174603, + "mrr_at_100": 0.13927883318086795, + "mrr_at_1000": 0.1399478164137755, + "mrr_at_20": 0.1366811297268263, + "mrr_at_3": 0.12283333333333332, + "mrr_at_5": 0.12893833333333335, + "nauc_map_at_1000_diff1": 0.3963088312423281, + "nauc_map_at_1000_max": 0.10059418448184779, + "nauc_map_at_1000_std": 0.25500737238336535, + "nauc_map_at_100_diff1": 0.39647836566788547, + "nauc_map_at_100_max": 0.10056047509915335, + "nauc_map_at_100_std": 0.25470052222391243, + "nauc_map_at_10_diff1": 0.40221913452785657, + "nauc_map_at_10_max": 0.10254534710895805, + "nauc_map_at_10_std": 0.25049686827136325, + "nauc_map_at_1_diff1": 0.4876019815321548, + "nauc_map_at_1_max": 0.12740775352690906, + "nauc_map_at_1_std": 0.22545385915377855, + "nauc_map_at_20_diff1": 0.3994938134942822, + "nauc_map_at_20_max": 0.10143483688518909, + "nauc_map_at_20_std": 0.25272301408984355, + "nauc_map_at_3_diff1": 0.4233998798481889, + "nauc_map_at_3_max": 0.10997852789404385, + "nauc_map_at_3_std": 0.2455868525671397, + "nauc_map_at_5_diff1": 0.4092179238053635, + "nauc_map_at_5_max": 0.10745346383948368, + "nauc_map_at_5_std": 0.24840930055218166, + "nauc_mrr_at_1000_diff1": 0.3971296053906526, + "nauc_mrr_at_1000_max": 0.09982076376964599, + "nauc_mrr_at_1000_std": 0.25500800026511233, + "nauc_mrr_at_100_diff1": 0.39729913757122876, + "nauc_mrr_at_100_max": 0.0997871496001195, + "nauc_mrr_at_100_std": 0.2547009310291848, + "nauc_mrr_at_10_diff1": 0.4030564444050217, + "nauc_mrr_at_10_max": 0.10176087593431704, + "nauc_mrr_at_10_std": 0.25049610146349277, + "nauc_mrr_at_1_diff1": 0.4897351682016501, + "nauc_mrr_at_1_max": 0.12540745218599106, + "nauc_mrr_at_1_std": 0.22567490245392624, + "nauc_mrr_at_20_diff1": 0.40032052455938527, + "nauc_mrr_at_20_max": 0.10065811109198394, + "nauc_mrr_at_20_std": 0.25272217758049786, + "nauc_mrr_at_3_diff1": 0.4243209090022143, + "nauc_mrr_at_3_max": 0.10917514781353345, + "nauc_mrr_at_3_std": 0.24558258025620866, + "nauc_mrr_at_5_diff1": 0.41006606898454745, + "nauc_mrr_at_5_max": 0.1066552790004448, + "nauc_mrr_at_5_std": 0.24840425103072616, + "nauc_ndcg_at_1000_diff1": 0.34177137753383763, + "nauc_ndcg_at_1000_max": 0.08553120148458279, + "nauc_ndcg_at_1000_std": 0.28954761566711, + "nauc_ndcg_at_100_diff1": 0.34521681198040016, + "nauc_ndcg_at_100_max": 0.08324473830188885, + "nauc_ndcg_at_100_std": 0.28037291804434966, + "nauc_ndcg_at_10_diff1": 0.36932478543944897, + "nauc_ndcg_at_10_max": 0.0914030446032406, + "nauc_ndcg_at_10_std": 0.2610015338570018, + "nauc_ndcg_at_1_diff1": 0.4876019815321548, + "nauc_ndcg_at_1_max": 0.12740775352690906, + "nauc_ndcg_at_1_std": 0.22545385915377855, + "nauc_ndcg_at_20_diff1": 0.36109012773964205, + "nauc_ndcg_at_20_max": 0.08799278116287539, + "nauc_ndcg_at_20_std": 0.26756196172660973, + "nauc_ndcg_at_3_diff1": 0.4056159399383812, + "nauc_ndcg_at_3_max": 0.10520400193340342, + "nauc_ndcg_at_3_std": 0.2514763540674069, + "nauc_ndcg_at_5_diff1": 0.38268647062761324, + "nauc_ndcg_at_5_max": 0.10124035961131686, + "nauc_ndcg_at_5_std": 0.2558999131899075, + "nauc_precision_at_1000_diff1": 0.20365251868391673, + "nauc_precision_at_1000_max": 0.056384835991144105, + "nauc_precision_at_1000_std": 0.406207503100537, + "nauc_precision_at_100_diff1": 0.22895960156913234, + "nauc_precision_at_100_max": 0.043442524770407255, + "nauc_precision_at_100_std": 0.347310608699394, + "nauc_precision_at_10_diff1": 0.2961981056268111, + "nauc_precision_at_10_max": 0.06584631710903982, + "nauc_precision_at_10_std": 0.2851887674269497, + "nauc_precision_at_1_diff1": 0.4876019815321548, + "nauc_precision_at_1_max": 0.12740775352690906, + "nauc_precision_at_1_std": 0.22545385915377855, + "nauc_precision_at_20_diff1": 0.27661781911688, + "nauc_precision_at_20_max": 0.05740460194057458, + "nauc_precision_at_20_std": 0.30217501125780694, + "nauc_precision_at_3_diff1": 0.3615212279279991, + "nauc_precision_at_3_max": 0.09337993365351137, + "nauc_precision_at_3_std": 0.26616249391398866, + "nauc_precision_at_5_diff1": 0.3199363422304861, + "nauc_precision_at_5_max": 0.0867272775022943, + "nauc_precision_at_5_std": 0.2734627863088206, + "nauc_recall_at_1000_diff1": 0.2036525186839176, + "nauc_recall_at_1000_max": 0.05638483599114502, + "nauc_recall_at_1000_std": 0.40620750310053805, + "nauc_recall_at_100_diff1": 0.22895960156913236, + "nauc_recall_at_100_max": 0.043442524770407234, + "nauc_recall_at_100_std": 0.3473106086993939, + "nauc_recall_at_10_diff1": 0.29619810562681115, + "nauc_recall_at_10_max": 0.0658463171090399, + "nauc_recall_at_10_std": 0.2851887674269497, + "nauc_recall_at_1_diff1": 0.4876019815321548, + "nauc_recall_at_1_max": 0.12740775352690906, + "nauc_recall_at_1_std": 0.22545385915377855, + "nauc_recall_at_20_diff1": 0.27661781911688005, + "nauc_recall_at_20_max": 0.057404601940574645, + "nauc_recall_at_20_std": 0.30217501125780716, + "nauc_recall_at_3_diff1": 0.3615212279279991, + "nauc_recall_at_3_max": 0.09337993365351145, + "nauc_recall_at_3_std": 0.2661624939139886, + "nauc_recall_at_5_diff1": 0.319936342230486, + "nauc_recall_at_5_max": 0.08672727750229443, + "nauc_recall_at_5_std": 0.27346278630882054, + "ndcg_at_1": 0.0998, + "ndcg_at_10": 0.15389, + "ndcg_at_100": 0.18268, + "ndcg_at_1000": 0.20415, + "ndcg_at_20": 0.16331, + "ndcg_at_3": 0.13047, + "ndcg_at_5": 0.14143, + "precision_at_1": 0.0998, + "precision_at_10": 0.02174, + "precision_at_100": 0.0036, + "precision_at_1000": 0.00054, + "precision_at_20": 0.01273, + "precision_at_3": 0.0508, + "precision_at_5": 0.03578, + "recall_at_1": 0.0998, + "recall_at_10": 0.2174, + "recall_at_100": 0.3603, + "recall_at_1000": 0.5373, + "recall_at_20": 0.2546, + "recall_at_3": 0.1524, + "recall_at_5": 0.1789 + } + ] + }, + "task_name": "RiaNewsRetrieval" +} \ No newline at end of file diff --git a/results/dunzhang__stella_en_400M_v5/1bb50bc7bb726810eac2140e62155b88b0df198f/RuBQRetrieval.json b/results/dunzhang__stella_en_400M_v5/1bb50bc7bb726810eac2140e62155b88b0df198f/RuBQRetrieval.json new file mode 100644 index 000000000..6ab1ef2ff --- /dev/null +++ b/results/dunzhang__stella_en_400M_v5/1bb50bc7bb726810eac2140e62155b88b0df198f/RuBQRetrieval.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "e19b6ffa60b3bc248e0b41f4cc37c26a55c2a67b", + "evaluation_time": 1176.20521068573, + "kg_co2_emissions": null, + "mteb_version": "1.19.0", + "scores": { + "test": [ + { + "hf_subset": "default", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.14796, + "map_at_1": 0.07613, + "map_at_10": 0.11853, + "map_at_100": 0.12413, + "map_at_1000": 0.1251, + "map_at_20": 0.1215, + "map_at_3": 0.1046, + "map_at_5": 0.11297, + "mrr_at_1": 0.11583924349881797, + "mrr_at_10": 0.1601788528650231, + "mrr_at_100": 0.16544462943102664, + "mrr_at_1000": 0.16620997054844033, + "mrr_at_20": 0.16305852341055335, + "mrr_at_3": 0.14716312056737588, + "mrr_at_5": 0.154580378250591, + "nauc_map_at_1000_diff1": 0.35273543462832496, + "nauc_map_at_1000_max": 0.2171960658420851, + "nauc_map_at_1000_std": 0.39073080394828863, + "nauc_map_at_100_diff1": 0.35359146052888096, + "nauc_map_at_100_max": 0.2168885706146736, + "nauc_map_at_100_std": 0.38985375278033313, + "nauc_map_at_10_diff1": 0.35680461400519753, + "nauc_map_at_10_max": 0.21927846115600527, + "nauc_map_at_10_std": 0.3883513374217442, + "nauc_map_at_1_diff1": 0.4163388155836285, + "nauc_map_at_1_max": 0.21821687514835017, + "nauc_map_at_1_std": 0.34753859378760665, + "nauc_map_at_20_diff1": 0.35603448080735917, + "nauc_map_at_20_max": 0.21934303043023118, + "nauc_map_at_20_std": 0.3881958400979315, + "nauc_map_at_3_diff1": 0.36415903721613363, + "nauc_map_at_3_max": 0.21839881134719963, + "nauc_map_at_3_std": 0.378076669055852, + "nauc_map_at_5_diff1": 0.35852217178082685, + "nauc_map_at_5_max": 0.21929596533420426, + "nauc_map_at_5_std": 0.37939872379714507, + "nauc_mrr_at_1000_diff1": 0.32316744521611723, + "nauc_mrr_at_1000_max": 0.22801284009223566, + "nauc_mrr_at_1000_std": 0.3542753900095467, + "nauc_mrr_at_100_diff1": 0.32336199774836405, + "nauc_mrr_at_100_max": 0.22782735790540423, + "nauc_mrr_at_100_std": 0.35386821807786806, + "nauc_mrr_at_10_diff1": 0.3274117765672043, + "nauc_mrr_at_10_max": 0.23155248213275978, + "nauc_mrr_at_10_std": 0.353861834753534, + "nauc_mrr_at_1_diff1": 0.3768947326621428, + "nauc_mrr_at_1_max": 0.2481640239868733, + "nauc_mrr_at_1_std": 0.3491395196569146, + "nauc_mrr_at_20_diff1": 0.3247830013463993, + "nauc_mrr_at_20_max": 0.2300616264277655, + "nauc_mrr_at_20_std": 0.35282766103633423, + "nauc_mrr_at_3_diff1": 0.33292402832341206, + "nauc_mrr_at_3_max": 0.23710953384440053, + "nauc_mrr_at_3_std": 0.3480921517089299, + "nauc_mrr_at_5_diff1": 0.32875715246005827, + "nauc_mrr_at_5_max": 0.23441692527919564, + "nauc_mrr_at_5_std": 0.35006332479308794, + "nauc_ndcg_at_1000_diff1": 0.30446151450855363, + "nauc_ndcg_at_1000_max": 0.20692722090450497, + "nauc_ndcg_at_1000_std": 0.4084910645927704, + "nauc_ndcg_at_100_diff1": 0.31533822892632873, + "nauc_ndcg_at_100_max": 0.2020769033254006, + "nauc_ndcg_at_100_std": 0.39362094764226685, + "nauc_ndcg_at_10_diff1": 0.33503327750575074, + "nauc_ndcg_at_10_max": 0.21838821708988923, + "nauc_ndcg_at_10_std": 0.3889825498828006, + "nauc_ndcg_at_1_diff1": 0.3813188154293476, + "nauc_ndcg_at_1_max": 0.2454921818306341, + "nauc_ndcg_at_1_std": 0.34500984861364925, + "nauc_ndcg_at_20_diff1": 0.3287927729462185, + "nauc_ndcg_at_20_max": 0.21611674669676287, + "nauc_ndcg_at_20_std": 0.38624030488553357, + "nauc_ndcg_at_3_diff1": 0.3436461476847011, + "nauc_ndcg_at_3_max": 0.2229846038234378, + "nauc_ndcg_at_3_std": 0.3719989343346257, + "nauc_ndcg_at_5_diff1": 0.33773298372611993, + "nauc_ndcg_at_5_max": 0.220017381558542, + "nauc_ndcg_at_5_std": 0.37479908705795323, + "nauc_precision_at_1000_diff1": 0.11922921916672098, + "nauc_precision_at_1000_max": 0.1782965270538377, + "nauc_precision_at_1000_std": 0.34471316811314384, + "nauc_precision_at_100_diff1": 0.19256294740673752, + "nauc_precision_at_100_max": 0.17028116338776336, + "nauc_precision_at_100_std": 0.3642258678475493, + "nauc_precision_at_10_diff1": 0.27393020950529534, + "nauc_precision_at_10_max": 0.22717202358024738, + "nauc_precision_at_10_std": 0.3897494181786337, + "nauc_precision_at_1_diff1": 0.3813188154293476, + "nauc_precision_at_1_max": 0.2454921818306341, + "nauc_precision_at_1_std": 0.34500984861364925, + "nauc_precision_at_20_diff1": 0.2548070601563504, + "nauc_precision_at_20_max": 0.22198938804689555, + "nauc_precision_at_20_std": 0.3703706272103639, + "nauc_precision_at_3_diff1": 0.29262473117033605, + "nauc_precision_at_3_max": 0.241595482332749, + "nauc_precision_at_3_std": 0.3743972367661125, + "nauc_precision_at_5_diff1": 0.2876147666453023, + "nauc_precision_at_5_max": 0.23887457408120572, + "nauc_precision_at_5_std": 0.37198678968715326, + "nauc_recall_at_1000_diff1": 0.1813733310428525, + "nauc_recall_at_1000_max": 0.1468088939868214, + "nauc_recall_at_1000_std": 0.4626524584858094, + "nauc_recall_at_100_diff1": 0.23447978142035858, + "nauc_recall_at_100_max": 0.13713594714812866, + "nauc_recall_at_100_std": 0.3979592617439332, + "nauc_recall_at_10_diff1": 0.2986165331750267, + "nauc_recall_at_10_max": 0.1976239315736399, + "nauc_recall_at_10_std": 0.4017526865835479, + "nauc_recall_at_1_diff1": 0.4163388155836285, + "nauc_recall_at_1_max": 0.21821687514835017, + "nauc_recall_at_1_std": 0.34753859378760665, + "nauc_recall_at_20_diff1": 0.27882273218508574, + "nauc_recall_at_20_max": 0.18640398346296955, + "nauc_recall_at_20_std": 0.38731021363055446, + "nauc_recall_at_3_diff1": 0.3150735193903634, + "nauc_recall_at_3_max": 0.20512119761860523, + "nauc_recall_at_3_std": 0.3744443515029672, + "nauc_recall_at_5_diff1": 0.30238926691962964, + "nauc_recall_at_5_max": 0.1982685241077943, + "nauc_recall_at_5_std": 0.3738434582589644, + "ndcg_at_1": 0.11525, + "ndcg_at_10": 0.14796, + "ndcg_at_100": 0.17694, + "ndcg_at_1000": 0.20349, + "ndcg_at_20": 0.15809, + "ndcg_at_3": 0.12289, + "ndcg_at_5": 0.13516, + "precision_at_1": 0.11525, + "precision_at_10": 0.03197, + "precision_at_100": 0.00543, + "precision_at_1000": 0.00087, + "precision_at_20": 0.01921, + "precision_at_3": 0.06994, + "precision_at_5": 0.05154, + "recall_at_1": 0.07613, + "recall_at_10": 0.19473, + "recall_at_100": 0.31911, + "recall_at_1000": 0.50872, + "recall_at_20": 0.22899, + "recall_at_3": 0.13123, + "recall_at_5": 0.16001 + } + ] + }, + "task_name": "RuBQRetrieval" +} \ No newline at end of file diff --git a/results/dunzhang__stella_en_400M_v5/1bb50bc7bb726810eac2140e62155b88b0df198f/RuReviewsClassification.json b/results/dunzhang__stella_en_400M_v5/1bb50bc7bb726810eac2140e62155b88b0df198f/RuReviewsClassification.json new file mode 100644 index 000000000..b7ff8b571 --- /dev/null +++ b/results/dunzhang__stella_en_400M_v5/1bb50bc7bb726810eac2140e62155b88b0df198f/RuReviewsClassification.json @@ -0,0 +1,73 @@ +{ + "dataset_revision": "f6d2c31f4dc6b88f468552750bfec05b4b41b05a", + "evaluation_time": 26.117958545684814, + "kg_co2_emissions": null, + "mteb_version": "1.19.0", + "scores": { + "test": [ + { + "accuracy": 0.4830078125, + "f1": 0.4465546962577661, + "f1_weighted": 0.44651628404669463, + "hf_subset": "default", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.4830078125, + "scores_per_experiment": [ + { + "accuracy": 0.50439453125, + "f1": 0.4806577515385508, + "f1_weighted": 0.48061567095643765 + }, + { + "accuracy": 0.50830078125, + "f1": 0.48233343866453987, + "f1_weighted": 0.4823021976298275 + }, + { + "accuracy": 0.50830078125, + "f1": 0.4681120868799528, + "f1_weighted": 0.4680636005256095 + }, + { + "accuracy": 0.50537109375, + "f1": 0.49728773915680663, + "f1_weighted": 0.49725021365040123 + }, + { + "accuracy": 0.49853515625, + "f1": 0.4512745030615936, + "f1_weighted": 0.45121617004660397 + }, + { + "accuracy": 0.5048828125, + "f1": 0.44820043710243845, + "f1_weighted": 0.44813749081788856 + }, + { + "accuracy": 0.43505859375, + "f1": 0.3857260199616885, + "f1_weighted": 0.3858506215964739 + }, + { + "accuracy": 0.494140625, + "f1": 0.473210382086201, + "f1_weighted": 0.47316394867368006 + }, + { + "accuracy": 0.39111328125, + "f1": 0.3731429905568897, + "f1_weighted": 0.37304471284517066 + }, + { + "accuracy": 0.47998046875, + "f1": 0.40560161356899993, + "f1_weighted": 0.4055182137248529 + } + ] + } + ] + }, + "task_name": "RuReviewsClassification" +} \ No newline at end of file diff --git a/results/dunzhang__stella_en_400M_v5/1bb50bc7bb726810eac2140e62155b88b0df198f/RuSTSBenchmarkSTS.json b/results/dunzhang__stella_en_400M_v5/1bb50bc7bb726810eac2140e62155b88b0df198f/RuSTSBenchmarkSTS.json new file mode 100644 index 000000000..7d32738c3 --- /dev/null +++ b/results/dunzhang__stella_en_400M_v5/1bb50bc7bb726810eac2140e62155b88b0df198f/RuSTSBenchmarkSTS.json @@ -0,0 +1,26 @@ +{ + "dataset_revision": "7cf24f325c6da6195df55bef3d86b5e0616f3018", + "evaluation_time": 21.668428897857666, + "kg_co2_emissions": null, + "mteb_version": "1.19.0", + "scores": { + "test": [ + { + "cosine_pearson": 0.5557938313811188, + "cosine_spearman": 0.5543277494201082, + "euclidean_pearson": 0.5642461636445986, + "euclidean_spearman": 0.5543256559281592, + "hf_subset": "default", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.5543277494201082, + "manhattan_pearson": 0.5647835644931885, + "manhattan_spearman": 0.5555875960535721, + "pearson": 0.5557938313811188, + "spearman": 0.5543277494201082 + } + ] + }, + "task_name": "RuSTSBenchmarkSTS" +} \ No newline at end of file diff --git a/results/dunzhang__stella_en_400M_v5/1bb50bc7bb726810eac2140e62155b88b0df198f/RuSciBenchGRNTIClassification.json b/results/dunzhang__stella_en_400M_v5/1bb50bc7bb726810eac2140e62155b88b0df198f/RuSciBenchGRNTIClassification.json new file mode 100644 index 000000000..ce9536a02 --- /dev/null +++ b/results/dunzhang__stella_en_400M_v5/1bb50bc7bb726810eac2140e62155b88b0df198f/RuSciBenchGRNTIClassification.json @@ -0,0 +1,73 @@ +{ + "dataset_revision": "673a610d6d3dd91a547a0d57ae1b56f37ebbf6a1", + "evaluation_time": 134.35021543502808, + "kg_co2_emissions": null, + "mteb_version": "1.19.0", + "scores": { + "test": [ + { + "accuracy": 0.256005859375, + "f1": 0.20961875888567366, + "f1_weighted": 0.2098790156816459, + "hf_subset": "default", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.256005859375, + "scores_per_experiment": [ + { + "accuracy": 0.2509765625, + "f1": 0.20671388849208366, + "f1_weighted": 0.20697636505383124 + }, + { + "accuracy": 0.2646484375, + "f1": 0.22099398938388998, + "f1_weighted": 0.22118316983034003 + }, + { + "accuracy": 0.25, + "f1": 0.2114197098553578, + "f1_weighted": 0.2116740636720412 + }, + { + "accuracy": 0.26025390625, + "f1": 0.20453041664486643, + "f1_weighted": 0.2047842965843544 + }, + { + "accuracy": 0.251953125, + "f1": 0.20915084533621453, + "f1_weighted": 0.20944741884209792 + }, + { + "accuracy": 0.24853515625, + "f1": 0.1925901142309042, + "f1_weighted": 0.19279619700064587 + }, + { + "accuracy": 0.267578125, + "f1": 0.23069206245966928, + "f1_weighted": 0.2310191682201786 + }, + { + "accuracy": 0.240234375, + "f1": 0.18700023022492748, + "f1_weighted": 0.18735093063560632 + }, + { + "accuracy": 0.2626953125, + "f1": 0.21825561861222312, + "f1_weighted": 0.21853471482947232 + }, + { + "accuracy": 0.26318359375, + "f1": 0.21484071361660034, + "f1_weighted": 0.2150238321478914 + } + ] + } + ] + }, + "task_name": "RuSciBenchGRNTIClassification" +} \ No newline at end of file diff --git a/results/dunzhang__stella_en_400M_v5/1bb50bc7bb726810eac2140e62155b88b0df198f/RuSciBenchGRNTIClusteringP2P.json b/results/dunzhang__stella_en_400M_v5/1bb50bc7bb726810eac2140e62155b88b0df198f/RuSciBenchGRNTIClusteringP2P.json new file mode 100644 index 000000000..6cebaeace --- /dev/null +++ b/results/dunzhang__stella_en_400M_v5/1bb50bc7bb726810eac2140e62155b88b0df198f/RuSciBenchGRNTIClusteringP2P.json @@ -0,0 +1,34 @@ +{ + "dataset_revision": "673a610d6d3dd91a547a0d57ae1b56f37ebbf6a1", + "evaluation_time": 132.52715277671814, + "kg_co2_emissions": null, + "mteb_version": "1.19.0", + "scores": { + "test": [ + { + "hf_subset": "default", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.31915601135326366, + "v_measure": 0.31915601135326366, + "v_measure_std": 0.004871843366474794, + "v_measures": { + "Level 0": [ + 0.31855295408373313, + 0.3277702302677735, + 0.31850103943035896, + 0.3223989091060155, + 0.31991862616255534, + 0.3213933326477595, + 0.32174890910313264, + 0.3119327177424906, + 0.30981730054822104, + 0.31952609444059576 + ] + } + } + ] + }, + "task_name": "RuSciBenchGRNTIClusteringP2P" +} \ No newline at end of file diff --git a/results/dunzhang__stella_en_400M_v5/1bb50bc7bb726810eac2140e62155b88b0df198f/RuSciBenchOECDClassification.json b/results/dunzhang__stella_en_400M_v5/1bb50bc7bb726810eac2140e62155b88b0df198f/RuSciBenchOECDClassification.json new file mode 100644 index 000000000..93c967383 --- /dev/null +++ b/results/dunzhang__stella_en_400M_v5/1bb50bc7bb726810eac2140e62155b88b0df198f/RuSciBenchOECDClassification.json @@ -0,0 +1,73 @@ +{ + "dataset_revision": "26c88e99dcaba32bb45d0e1bfc21902337f6d471", + "evaluation_time": 140.34889245033264, + "kg_co2_emissions": null, + "mteb_version": "1.19.0", + "scores": { + "test": [ + { + "accuracy": 0.2041015625, + "f1": 0.16711612935174366, + "f1_weighted": 0.16708221329094303, + "hf_subset": "default", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.2041015625, + "scores_per_experiment": [ + { + "accuracy": 0.2001953125, + "f1": 0.15659717371657775, + "f1_weighted": 0.15643614985721752 + }, + { + "accuracy": 0.21923828125, + "f1": 0.18080693391787322, + "f1_weighted": 0.18054953248413383 + }, + { + "accuracy": 0.20068359375, + "f1": 0.1677093344557966, + "f1_weighted": 0.1676030699288583 + }, + { + "accuracy": 0.2060546875, + "f1": 0.17183775896854836, + "f1_weighted": 0.17174598746717454 + }, + { + "accuracy": 0.20361328125, + "f1": 0.16298837465465668, + "f1_weighted": 0.16301600734542726 + }, + { + "accuracy": 0.19580078125, + "f1": 0.15712663742051514, + "f1_weighted": 0.15710180869510376 + }, + { + "accuracy": 0.1943359375, + "f1": 0.15980974805335146, + "f1_weighted": 0.15981266815379278 + }, + { + "accuracy": 0.19775390625, + "f1": 0.1620875901629356, + "f1_weighted": 0.16211126620295838 + }, + { + "accuracy": 0.201171875, + "f1": 0.16195424407154724, + "f1_weighted": 0.16213355710871794 + }, + { + "accuracy": 0.22216796875, + "f1": 0.19024349809563468, + "f1_weighted": 0.19031208566604596 + } + ] + } + ] + }, + "task_name": "RuSciBenchOECDClassification" +} \ No newline at end of file diff --git a/results/dunzhang__stella_en_400M_v5/1bb50bc7bb726810eac2140e62155b88b0df198f/RuSciBenchOECDClusteringP2P.json b/results/dunzhang__stella_en_400M_v5/1bb50bc7bb726810eac2140e62155b88b0df198f/RuSciBenchOECDClusteringP2P.json new file mode 100644 index 000000000..b61d6c4c4 --- /dev/null +++ b/results/dunzhang__stella_en_400M_v5/1bb50bc7bb726810eac2140e62155b88b0df198f/RuSciBenchOECDClusteringP2P.json @@ -0,0 +1,34 @@ +{ + "dataset_revision": "26c88e99dcaba32bb45d0e1bfc21902337f6d471", + "evaluation_time": 110.61871933937073, + "kg_co2_emissions": null, + "mteb_version": "1.19.0", + "scores": { + "test": [ + { + "hf_subset": "default", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.2696387099780032, + "v_measure": 0.2696387099780032, + "v_measure_std": 0.004976624529019471, + "v_measures": { + "Level 0": [ + 0.2658192704707403, + 0.2664443384655662, + 0.2688309761957671, + 0.26475062737658345, + 0.2726739313269988, + 0.27815953213884165, + 0.2658468762060901, + 0.26437262103133496, + 0.27808107049883257, + 0.27140785606927725 + ] + } + } + ] + }, + "task_name": "RuSciBenchOECDClusteringP2P" +} \ No newline at end of file diff --git a/results/dunzhang__stella_en_400M_v5/1bb50bc7bb726810eac2140e62155b88b0df198f/STS22.json b/results/dunzhang__stella_en_400M_v5/1bb50bc7bb726810eac2140e62155b88b0df198f/STS22.json new file mode 100644 index 000000000..5bfce1706 --- /dev/null +++ b/results/dunzhang__stella_en_400M_v5/1bb50bc7bb726810eac2140e62155b88b0df198f/STS22.json @@ -0,0 +1,26 @@ +{ + "dataset_revision": "de9d86b3b84231dc21f76c7b7af1f28e2f57f6e3", + "evaluation_time": 17.20219326019287, + "kg_co2_emissions": null, + "mteb_version": "1.19.0", + "scores": { + "test": [ + { + "cosine_pearson": 0.2641395974970053, + "cosine_spearman": 0.37373400961354913, + "euclidean_pearson": 0.27910062074228287, + "euclidean_spearman": 0.37373400961354913, + "hf_subset": "ru", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.37373400961354913, + "manhattan_pearson": 0.2813173765101985, + "manhattan_spearman": 0.3739376978738526, + "pearson": 0.2641395974970053, + "spearman": 0.37373400961354913 + } + ] + }, + "task_name": "STS22" +} \ No newline at end of file diff --git a/results/dunzhang__stella_en_400M_v5/1bb50bc7bb726810eac2140e62155b88b0df198f/SensitiveTopicsClassification.json b/results/dunzhang__stella_en_400M_v5/1bb50bc7bb726810eac2140e62155b88b0df198f/SensitiveTopicsClassification.json new file mode 100644 index 000000000..03dda1234 --- /dev/null +++ b/results/dunzhang__stella_en_400M_v5/1bb50bc7bb726810eac2140e62155b88b0df198f/SensitiveTopicsClassification.json @@ -0,0 +1,73 @@ +{ + "dataset_revision": "416b34a802308eac30e4192afc0ff99bb8dcc7f2", + "evaluation_time": 47.77882170677185, + "kg_co2_emissions": null, + "mteb_version": "1.19.0", + "scores": { + "test": [ + { + "accuracy": 0.179833984375, + "f1": 0.019114291074852217, + "hf_subset": "default", + "languages": [ + "rus-Cyrl" + ], + "lrap": 0.261232503255202, + "main_score": 0.179833984375, + "scores_per_experiment": [ + { + "accuracy": 0.18115234375, + "f1": 0.007582174358512373, + "lrap": 0.25646972656249367 + }, + { + "accuracy": 0.1875, + "f1": 0.03497039561697098, + "lrap": 0.2713216145833268 + }, + { + "accuracy": 0.1806640625, + "f1": 0.02785031533309173, + "lrap": 0.2587348090277714 + }, + { + "accuracy": 0.18505859375, + "f1": 0.01748443255481736, + "lrap": 0.26171874999999356 + }, + { + "accuracy": 0.1708984375, + "f1": 0.011360719001969496, + "lrap": 0.2570393880208269 + }, + { + "accuracy": 0.1748046875, + "f1": 0.021436353935820697, + "lrap": 0.26424153645832693 + }, + { + "accuracy": 0.18212890625, + "f1": 0.021102917152422105, + "lrap": 0.2633599175347158 + }, + { + "accuracy": 0.18017578125, + "f1": 0.014892885463763593, + "lrap": 0.2591824001736047 + }, + { + "accuracy": 0.171875, + "f1": 0.0202603068519639, + "lrap": 0.26077609592013246 + }, + { + "accuracy": 0.18408203125, + "f1": 0.014202410479189936, + "lrap": 0.25948079427082693 + } + ] + } + ] + }, + "task_name": "SensitiveTopicsClassification" +} \ No newline at end of file diff --git a/results/dunzhang__stella_en_400M_v5/1bb50bc7bb726810eac2140e62155b88b0df198f/model_meta.json b/results/dunzhang__stella_en_400M_v5/1bb50bc7bb726810eac2140e62155b88b0df198f/model_meta.json index fb31f0b4b..95d12b033 100644 --- a/results/dunzhang__stella_en_400M_v5/1bb50bc7bb726810eac2140e62155b88b0df198f/model_meta.json +++ b/results/dunzhang__stella_en_400M_v5/1bb50bc7bb726810eac2140e62155b88b0df198f/model_meta.json @@ -1 +1,26 @@ -{"name": "dunzhang/stella_en_400M_v5", "revision": "1bb50bc7bb726810eac2140e62155b88b0df198f", "release_date": "2024-07-12", "languages": ["eng_Latn"], "n_parameters": 435000, "memory_usage": null, "max_tokens": 8192, "embed_dim": 4096, "license": "mit", "open_weights": true, "public_training_data": null, "public_training_code": null, "framework": ["Sentence Transformers", "PyTorch", "GritLM"], "reference": "https://huggingface.co/dunzhang/stella_en_400M_v5", "similarity_fn_name": "cosine", "use_instructions": true, "zero_shot_benchmarks": null, "loader": "instruct_wrapper"} \ No newline at end of file +{ + "name": "dunzhang/stella_en_400M_v5", + "revision": "1bb50bc7bb726810eac2140e62155b88b0df198f", + "release_date": "2024-07-12", + "languages": [ + "eng_Latn" + ], + "n_parameters": 435000, + "memory_usage": null, + "max_tokens": 8192, + "embed_dim": 4096, + "license": "mit", + "open_weights": true, + "public_training_data": null, + "public_training_code": null, + "framework": [ + "Sentence Transformers", + "PyTorch", + "GritLM" + ], + "reference": "https://huggingface.co/dunzhang/stella_en_400M_v5", + "similarity_fn_name": "cosine", + "use_instructions": true, + "zero_shot_benchmarks": null, + "loader": "instruct_wrapper" +} \ No newline at end of file diff --git a/results/intfloat__multilingual-e5-large-instruct/baa7be480a7de1539afce709c8f13f833a510e0a/MIRACLReranking.json b/results/intfloat__multilingual-e5-large-instruct/baa7be480a7de1539afce709c8f13f833a510e0a/MIRACLReranking.json index 3f129943c..86bd61705 100644 --- a/results/intfloat__multilingual-e5-large-instruct/baa7be480a7de1539afce709c8f13f833a510e0a/MIRACLReranking.json +++ b/results/intfloat__multilingual-e5-large-instruct/baa7be480a7de1539afce709c8f13f833a510e0a/MIRACLReranking.json @@ -6,123 +6,123 @@ "scores": { "dev": [ { - "MAP@1(MIRACL)": 0.35248, - "MAP@10(MIRACL)": 0.55613, - "MAP@100(MIRACL)": 0.57337, - "MAP@1000(MIRACL)": 0.57337, - "MAP@20(MIRACL)": 0.56876, - "MAP@3(MIRACL)": 0.48691, - "MAP@5(MIRACL)": 0.5259, - "NDCG@1(MIRACL)": 0.57819, - "NDCG@10(MIRACL)": 0.62495, - "NDCG@100(MIRACL)": 0.65745, - "NDCG@1000(MIRACL)": 0.65745, - "NDCG@20(MIRACL)": 0.64591, - "NDCG@3(MIRACL)": 0.57803, - "NDCG@5(MIRACL)": 0.59467, - "P@1(MIRACL)": 0.57819, - "P@10(MIRACL)": 0.1591, + "MAP@1(MIRACL)": 0.30323, + "MAP@10(MIRACL)": 0.49095, + "MAP@100(MIRACL)": 0.51077, + "MAP@1000(MIRACL)": 0.51077, + "MAP@20(MIRACL)": 0.5045, + "MAP@3(MIRACL)": 0.42348, + "MAP@5(MIRACL)": 0.46013, + "NDCG@1(MIRACL)": 0.49318, + "NDCG@10(MIRACL)": 0.57031, + "NDCG@100(MIRACL)": 0.61121, + "NDCG@1000(MIRACL)": 0.61121, + "NDCG@20(MIRACL)": 0.59418, + "NDCG@3(MIRACL)": 0.50932, + "NDCG@5(MIRACL)": 0.5323, + "P@1(MIRACL)": 0.49318, + "P@10(MIRACL)": 0.14892, "P@100(MIRACL)": 0.01953, "P@1000(MIRACL)": 0.00195, - "P@20(MIRACL)": 0.0899, - "P@3(MIRACL)": 0.35365, - "P@5(MIRACL)": 0.26127, - "Recall@1(MIRACL)": 0.35248, - "Recall@10(MIRACL)": 0.70566, + "P@20(MIRACL)": 0.08645, + "P@3(MIRACL)": 0.31168, + "P@5(MIRACL)": 0.23545, + "Recall@1(MIRACL)": 0.30323, + "Recall@10(MIRACL)": 0.67844, "Recall@100(MIRACL)": 0.79952, "Recall@1000(MIRACL)": 0.79952, - "Recall@20(MIRACL)": 0.76094, - "Recall@3(MIRACL)": 0.55047, - "Recall@5(MIRACL)": 0.6238, + "Recall@20(MIRACL)": 0.74187, + "Recall@3(MIRACL)": 0.49799, + "Recall@5(MIRACL)": 0.57847, "hf_subset": "ru", "languages": [ "rus-Cyrl" ], - "main_score": 0.62495, - "nAUC_MAP@1000_diff1(MIRACL)": 0.0913556026226369, - "nAUC_MAP@1000_max(MIRACL)": 0.3738092003641524, - "nAUC_MAP@1000_std(MIRACL)": 0.13124771374425342, - "nAUC_MAP@100_diff1(MIRACL)": 0.0913556026226369, - "nAUC_MAP@100_max(MIRACL)": 0.3738092003641524, - "nAUC_MAP@100_std(MIRACL)": 0.13124771374425342, - "nAUC_MAP@10_diff1(MIRACL)": 0.10408240470247715, - "nAUC_MAP@10_max(MIRACL)": 0.3515707223490397, - "nAUC_MAP@10_std(MIRACL)": 0.12571005089649392, - "nAUC_MAP@1_diff1(MIRACL)": 0.2899925786584375, - "nAUC_MAP@1_max(MIRACL)": 0.1977528961679013, - "nAUC_MAP@1_std(MIRACL)": 0.03565151836181529, - "nAUC_MAP@20_diff1(MIRACL)": 0.09530449716090438, - "nAUC_MAP@20_max(MIRACL)": 0.36686607653852377, - "nAUC_MAP@20_std(MIRACL)": 0.13149518237778732, - "nAUC_MAP@3_diff1(MIRACL)": 0.1622783507922158, - "nAUC_MAP@3_max(MIRACL)": 0.2711817831633491, - "nAUC_MAP@3_std(MIRACL)": 0.07635163395523538, - "nAUC_MAP@5_diff1(MIRACL)": 0.13162515305168498, - "nAUC_MAP@5_max(MIRACL)": 0.3179855579788043, - "nAUC_MAP@5_std(MIRACL)": 0.11025428546468255, - "nAUC_NDCG@1000_diff1(MIRACL)": 0.0061445829074449655, - "nAUC_NDCG@1000_max(MIRACL)": 0.4568903371543913, - "nAUC_NDCG@1000_std(MIRACL)": 0.16194856078641434, - "nAUC_NDCG@100_diff1(MIRACL)": 0.0061445829074449655, - "nAUC_NDCG@100_max(MIRACL)": 0.4568903371543913, - "nAUC_NDCG@100_std(MIRACL)": 0.16194856078641434, - "nAUC_NDCG@10_diff1(MIRACL)": 0.03839047468406367, - "nAUC_NDCG@10_max(MIRACL)": 0.413531103293431, - "nAUC_NDCG@10_std(MIRACL)": 0.15628049871064897, - "nAUC_NDCG@1_diff1(MIRACL)": 0.15948730551976256, - "nAUC_NDCG@1_max(MIRACL)": 0.43654621802539767, - "nAUC_NDCG@1_std(MIRACL)": 0.12841837679030227, - "nAUC_NDCG@20_diff1(MIRACL)": 0.018602543093889028, - "nAUC_NDCG@20_max(MIRACL)": 0.440483088174477, - "nAUC_NDCG@20_std(MIRACL)": 0.1639973640202779, - "nAUC_NDCG@3_diff1(MIRACL)": 0.08445958553342472, - "nAUC_NDCG@3_max(MIRACL)": 0.37222040294070724, - "nAUC_NDCG@3_std(MIRACL)": 0.11937330620388391, - "nAUC_NDCG@5_diff1(MIRACL)": 0.07217201024018231, - "nAUC_NDCG@5_max(MIRACL)": 0.3886561955590856, - "nAUC_NDCG@5_std(MIRACL)": 0.14277281261414868, - "nAUC_P@1000_diff1(MIRACL)": -0.2421159509358296, - "nAUC_P@1000_max(MIRACL)": 0.2997974476719383, - "nAUC_P@1000_std(MIRACL)": 0.11351581770974536, - "nAUC_P@100_diff1(MIRACL)": -0.24211595093582922, - "nAUC_P@100_max(MIRACL)": 0.29979744767193833, - "nAUC_P@100_std(MIRACL)": 0.11351581770974592, - "nAUC_P@10_diff1(MIRACL)": -0.23325259424137176, - "nAUC_P@10_max(MIRACL)": 0.32628237887836453, - "nAUC_P@10_std(MIRACL)": 0.1541315490552596, - "nAUC_P@1_diff1(MIRACL)": 0.15948730551976256, - "nAUC_P@1_max(MIRACL)": 0.43654621802539767, - "nAUC_P@1_std(MIRACL)": 0.12841837679030227, - "nAUC_P@20_diff1(MIRACL)": -0.23954387246720693, - "nAUC_P@20_max(MIRACL)": 0.31339175174362177, - "nAUC_P@20_std(MIRACL)": 0.14245785444204978, - "nAUC_P@3_diff1(MIRACL)": -0.17471417553656193, - "nAUC_P@3_max(MIRACL)": 0.35152682997668444, - "nAUC_P@3_std(MIRACL)": 0.14442991204633387, - "nAUC_P@5_diff1(MIRACL)": -0.2054776818690083, - "nAUC_P@5_max(MIRACL)": 0.35022359677322046, - "nAUC_P@5_std(MIRACL)": 0.17089192885055757, - "nAUC_Recall@1000_diff1(MIRACL)": -0.2689130421794338, - "nAUC_Recall@1000_max(MIRACL)": 0.6192091460765856, - "nAUC_Recall@1000_std(MIRACL)": 0.21323332780709123, - "nAUC_Recall@100_diff1(MIRACL)": -0.2689130421794338, - "nAUC_Recall@100_max(MIRACL)": 0.6192091460765856, - "nAUC_Recall@100_std(MIRACL)": 0.21323332780709123, - "nAUC_Recall@10_diff1(MIRACL)": -0.09513535480701664, - "nAUC_Recall@10_max(MIRACL)": 0.42499125842626606, - "nAUC_Recall@10_std(MIRACL)": 0.18469004981761036, - "nAUC_Recall@1_diff1(MIRACL)": 0.2899925786584375, - "nAUC_Recall@1_max(MIRACL)": 0.1977528961679013, - "nAUC_Recall@1_std(MIRACL)": 0.03565151836181529, - "nAUC_Recall@20_diff1(MIRACL)": -0.18251768217699588, - "nAUC_Recall@20_max(MIRACL)": 0.5223762876683883, - "nAUC_Recall@20_std(MIRACL)": 0.21308962654362823, - "nAUC_Recall@3_diff1(MIRACL)": 0.07954645645353128, - "nAUC_Recall@3_max(MIRACL)": 0.25483613750341055, - "nAUC_Recall@3_std(MIRACL)": 0.08771928703096613, - "nAUC_Recall@5_diff1(MIRACL)": 0.01601791121671337, - "nAUC_Recall@5_max(MIRACL)": 0.3353281556010821, - "nAUC_Recall@5_std(MIRACL)": 0.1487808286281421 + "main_score": 0.57031, + "nAUC_MAP@1000_diff1(MIRACL)": 0.1312614887738495, + "nAUC_MAP@1000_max(MIRACL)": 0.261577250034097, + "nAUC_MAP@1000_std(MIRACL)": 0.11308228599258063, + "nAUC_MAP@100_diff1(MIRACL)": 0.1312614887738495, + "nAUC_MAP@100_max(MIRACL)": 0.261577250034097, + "nAUC_MAP@100_std(MIRACL)": 0.11308228599258063, + "nAUC_MAP@10_diff1(MIRACL)": 0.14268237838221554, + "nAUC_MAP@10_max(MIRACL)": 0.240374671591113, + "nAUC_MAP@10_std(MIRACL)": 0.11073835711374107, + "nAUC_MAP@1_diff1(MIRACL)": 0.26659281096159554, + "nAUC_MAP@1_max(MIRACL)": 0.10895912570931304, + "nAUC_MAP@1_std(MIRACL)": 0.05511451927849174, + "nAUC_MAP@20_diff1(MIRACL)": 0.1345622773342837, + "nAUC_MAP@20_max(MIRACL)": 0.2536374478155574, + "nAUC_MAP@20_std(MIRACL)": 0.11517986613475917, + "nAUC_MAP@3_diff1(MIRACL)": 0.1881353358282121, + "nAUC_MAP@3_max(MIRACL)": 0.1817458941230111, + "nAUC_MAP@3_std(MIRACL)": 0.0654471435432486, + "nAUC_MAP@5_diff1(MIRACL)": 0.16216272911706817, + "nAUC_MAP@5_max(MIRACL)": 0.2084349796821478, + "nAUC_MAP@5_std(MIRACL)": 0.09104604039420143, + "nAUC_NDCG@1000_diff1(MIRACL)": 0.06150254286649549, + "nAUC_NDCG@1000_max(MIRACL)": 0.3473450194579663, + "nAUC_NDCG@1000_std(MIRACL)": 0.12791807045419396, + "nAUC_NDCG@100_diff1(MIRACL)": 0.06150254286649549, + "nAUC_NDCG@100_max(MIRACL)": 0.3473450194579663, + "nAUC_NDCG@100_std(MIRACL)": 0.12791807045419396, + "nAUC_NDCG@10_diff1(MIRACL)": 0.09016377309747622, + "nAUC_NDCG@10_max(MIRACL)": 0.30171512206103157, + "nAUC_NDCG@10_std(MIRACL)": 0.13171483935493025, + "nAUC_NDCG@1_diff1(MIRACL)": 0.18492564328096106, + "nAUC_NDCG@1_max(MIRACL)": 0.2597014352360243, + "nAUC_NDCG@1_std(MIRACL)": 0.10670933404171296, + "nAUC_NDCG@20_diff1(MIRACL)": 0.07364635731057032, + "nAUC_NDCG@20_max(MIRACL)": 0.32494672076311193, + "nAUC_NDCG@20_std(MIRACL)": 0.13606437628891482, + "nAUC_NDCG@3_diff1(MIRACL)": 0.13701235310099316, + "nAUC_NDCG@3_max(MIRACL)": 0.25077247440887673, + "nAUC_NDCG@3_std(MIRACL)": 0.0827969673230326, + "nAUC_NDCG@5_diff1(MIRACL)": 0.11471842259483842, + "nAUC_NDCG@5_max(MIRACL)": 0.2610715093579639, + "nAUC_NDCG@5_std(MIRACL)": 0.107855407321909, + "nAUC_P@1000_diff1(MIRACL)": -0.18246042603760462, + "nAUC_P@1000_max(MIRACL)": 0.29448204543193796, + "nAUC_P@1000_std(MIRACL)": 0.07588219260610417, + "nAUC_P@100_diff1(MIRACL)": -0.18246042603760443, + "nAUC_P@100_max(MIRACL)": 0.2944820454319382, + "nAUC_P@100_std(MIRACL)": 0.07588219260610429, + "nAUC_P@10_diff1(MIRACL)": -0.16635424763503295, + "nAUC_P@10_max(MIRACL)": 0.3137930290196717, + "nAUC_P@10_std(MIRACL)": 0.1408949463871209, + "nAUC_P@1_diff1(MIRACL)": 0.18492564328096106, + "nAUC_P@1_max(MIRACL)": 0.2597014352360243, + "nAUC_P@1_std(MIRACL)": 0.10670933404171296, + "nAUC_P@20_diff1(MIRACL)": -0.18180377500503817, + "nAUC_P@20_max(MIRACL)": 0.2978232025476432, + "nAUC_P@20_std(MIRACL)": 0.12151912613986732, + "nAUC_P@3_diff1(MIRACL)": -0.062407009466557105, + "nAUC_P@3_max(MIRACL)": 0.30999736013563284, + "nAUC_P@3_std(MIRACL)": 0.08839023443460986, + "nAUC_P@5_diff1(MIRACL)": -0.1277248575511655, + "nAUC_P@5_max(MIRACL)": 0.3034170582568701, + "nAUC_P@5_std(MIRACL)": 0.1319263025134623, + "nAUC_Recall@1000_diff1(MIRACL)": -0.19156455263975775, + "nAUC_Recall@1000_max(MIRACL)": 0.5766759773578772, + "nAUC_Recall@1000_std(MIRACL)": 0.13788374718862703, + "nAUC_Recall@100_diff1(MIRACL)": -0.19156455263975775, + "nAUC_Recall@100_max(MIRACL)": 0.5766759773578772, + "nAUC_Recall@100_std(MIRACL)": 0.13788374718862703, + "nAUC_Recall@10_diff1(MIRACL)": -0.028772874028217404, + "nAUC_Recall@10_max(MIRACL)": 0.3452814553802481, + "nAUC_Recall@10_std(MIRACL)": 0.15025650863282378, + "nAUC_Recall@1_diff1(MIRACL)": 0.26659281096159554, + "nAUC_Recall@1_max(MIRACL)": 0.10895912570931304, + "nAUC_Recall@1_std(MIRACL)": 0.05511451927849174, + "nAUC_Recall@20_diff1(MIRACL)": -0.09695158767619454, + "nAUC_Recall@20_max(MIRACL)": 0.43188154130236145, + "nAUC_Recall@20_std(MIRACL)": 0.16594739636583677, + "nAUC_Recall@3_diff1(MIRACL)": 0.12266485140952804, + "nAUC_Recall@3_max(MIRACL)": 0.18652654576797129, + "nAUC_Recall@3_std(MIRACL)": 0.05403440495241685, + "nAUC_Recall@5_diff1(MIRACL)": 0.055648357575505326, + "nAUC_Recall@5_max(MIRACL)": 0.23251829097487708, + "nAUC_Recall@5_std(MIRACL)": 0.1019021157414967 } ] }, diff --git a/results/intfloat__multilingual-e5-large-instruct/baa7be480a7de1539afce709c8f13f833a510e0a/MIRACLRetrieval.json b/results/intfloat__multilingual-e5-large-instruct/baa7be480a7de1539afce709c8f13f833a510e0a/MIRACLRetrieval.json index b13e59c52..9c367b116 100644 --- a/results/intfloat__multilingual-e5-large-instruct/baa7be480a7de1539afce709c8f13f833a510e0a/MIRACLRetrieval.json +++ b/results/intfloat__multilingual-e5-large-instruct/baa7be480a7de1539afce709c8f13f833a510e0a/MIRACLRetrieval.json @@ -10,147 +10,147 @@ "languages": [ "rus-Cyrl" ], - "main_score": 0.66077, - "map_at_1": 0.3203, - "map_at_10": 0.55553, - "map_at_100": 0.58279, - "map_at_1000": 0.58353, - "map_at_20": 0.573, - "map_at_3": 0.47215, - "map_at_5": 0.51853, - "mrr_at_1": 0.6269968051118211, - "mrr_at_10": 0.744924945484051, - "mrr_at_100": 0.7473438451102236, - "mrr_at_1000": 0.7473845596123778, - "mrr_at_20": 0.746663284702886, - "mrr_at_3": 0.7257720979765712, - "mrr_at_5": 0.7387912673056448, - "nauc_map_at_1000_diff1": 0.25651130831009217, - "nauc_map_at_1000_max": 0.26608147562635653, - "nauc_map_at_1000_std": 0.010943176426008451, - "nauc_map_at_100_diff1": 0.25607191778875493, - "nauc_map_at_100_max": 0.2660046074914393, - "nauc_map_at_100_std": 0.010729304719374522, - "nauc_map_at_10_diff1": 0.25133649322378243, - "nauc_map_at_10_max": 0.2489553497559216, - "nauc_map_at_10_std": -0.011480829996309519, - "nauc_map_at_1_diff1": 0.3001558541239315, - "nauc_map_at_1_max": 0.15195557681770389, - "nauc_map_at_1_std": -0.06640237737451227, - "nauc_map_at_20_diff1": 0.2532184530655113, - "nauc_map_at_20_max": 0.2592505692891478, - "nauc_map_at_20_std": 0.00203828890886048, - "nauc_map_at_3_diff1": 0.2510562056898608, - "nauc_map_at_3_max": 0.1920136429397892, - "nauc_map_at_3_std": -0.05240420199731552, - "nauc_map_at_5_diff1": 0.24269594569942857, - "nauc_map_at_5_max": 0.22260353930241236, - "nauc_map_at_5_std": -0.030968639256134863, - "nauc_mrr_at_1000_diff1": 0.34348385906924994, - "nauc_mrr_at_1000_max": 0.3843860494574848, - "nauc_mrr_at_1000_std": 0.13134056215313822, - "nauc_mrr_at_100_diff1": 0.3434286650685133, - "nauc_mrr_at_100_max": 0.3844722754587728, - "nauc_mrr_at_100_std": 0.13146315946644851, - "nauc_mrr_at_10_diff1": 0.3417621264452389, - "nauc_mrr_at_10_max": 0.3843526797152263, - "nauc_mrr_at_10_std": 0.13000275143241427, - "nauc_mrr_at_1_diff1": 0.3656926455789233, - "nauc_mrr_at_1_max": 0.3569753501667631, - "nauc_mrr_at_1_std": 0.11039546532518289, - "nauc_mrr_at_20_diff1": 0.3438604503594018, - "nauc_mrr_at_20_max": 0.3852853119533693, - "nauc_mrr_at_20_std": 0.13139288815922817, - "nauc_mrr_at_3_diff1": 0.3364958692397905, - "nauc_mrr_at_3_max": 0.3836967482444759, - "nauc_mrr_at_3_std": 0.12963700290787736, - "nauc_mrr_at_5_diff1": 0.3398919113715298, - "nauc_mrr_at_5_max": 0.38643044488252754, - "nauc_mrr_at_5_std": 0.13133030983414312, - "nauc_ndcg_at_1000_diff1": 0.2774514340272379, - "nauc_ndcg_at_1000_max": 0.3235467781934033, - "nauc_ndcg_at_1000_std": 0.07323898634640161, - "nauc_ndcg_at_100_diff1": 0.27276980657424166, - "nauc_ndcg_at_100_max": 0.32731852849134924, - "nauc_ndcg_at_100_std": 0.07745094307099266, - "nauc_ndcg_at_10_diff1": 0.2592141115418782, - "nauc_ndcg_at_10_max": 0.289473579282218, - "nauc_ndcg_at_10_std": 0.021929748628662166, - "nauc_ndcg_at_1_diff1": 0.3656926455789233, - "nauc_ndcg_at_1_max": 0.3569753501667631, - "nauc_ndcg_at_1_std": 0.11039546532518289, - "nauc_ndcg_at_20_diff1": 0.2637551647871148, - "nauc_ndcg_at_20_max": 0.3079099630106372, - "nauc_ndcg_at_20_std": 0.04839107257210489, - "nauc_ndcg_at_3_diff1": 0.25559897239189466, - "nauc_ndcg_at_3_max": 0.2758478222391575, - "nauc_ndcg_at_3_std": 0.03301367583405843, - "nauc_ndcg_at_5_diff1": 0.24965446508765288, - "nauc_ndcg_at_5_max": 0.27637665167869513, - "nauc_ndcg_at_5_std": 0.020536588664830508, - "nauc_precision_at_1000_diff1": -0.05154029056922919, - "nauc_precision_at_1000_max": 0.11674074334091578, - "nauc_precision_at_1000_std": 0.21698981492936475, - "nauc_precision_at_100_diff1": -0.049842631680673044, - "nauc_precision_at_100_max": 0.1503255477225557, - "nauc_precision_at_100_std": 0.23599280958555266, - "nauc_precision_at_10_diff1": -0.00957978926541625, - "nauc_precision_at_10_max": 0.2261800471566172, - "nauc_precision_at_10_std": 0.20697193103933523, - "nauc_precision_at_1_diff1": 0.3656926455789233, - "nauc_precision_at_1_max": 0.3569753501667631, - "nauc_precision_at_1_std": 0.11039546532518289, - "nauc_precision_at_20_diff1": -0.03332326374108059, - "nauc_precision_at_20_max": 0.1842857339377062, - "nauc_precision_at_20_std": 0.22231923886483995, - "nauc_precision_at_3_diff1": 0.055809128972329744, - "nauc_precision_at_3_max": 0.2593506821199334, - "nauc_precision_at_3_std": 0.1476808775371136, - "nauc_precision_at_5_diff1": 0.005294020026938451, - "nauc_precision_at_5_max": 0.25522120139960686, - "nauc_precision_at_5_std": 0.18902166233162848, - "nauc_recall_at_1000_diff1": 0.35800605130315466, - "nauc_recall_at_1000_max": 0.7135419525926031, - "nauc_recall_at_1000_std": 0.652217397520194, - "nauc_recall_at_100_diff1": 0.17929212157418473, - "nauc_recall_at_100_max": 0.46699618364791856, - "nauc_recall_at_100_std": 0.3388832882209966, - "nauc_recall_at_10_diff1": 0.1695062989798401, - "nauc_recall_at_10_max": 0.21487309684891126, - "nauc_recall_at_10_std": -0.02644976751926065, - "nauc_recall_at_1_diff1": 0.3001558541239315, - "nauc_recall_at_1_max": 0.15195557681770389, - "nauc_recall_at_1_std": -0.06640237737451227, - "nauc_recall_at_20_diff1": 0.16397781275214718, - "nauc_recall_at_20_max": 0.26119403634013033, - "nauc_recall_at_20_std": 0.03852359094045782, - "nauc_recall_at_3_diff1": 0.1847693047120627, - "nauc_recall_at_3_max": 0.14802595366360966, - "nauc_recall_at_3_std": -0.07565272686010868, - "nauc_recall_at_5_diff1": 0.16301505991390536, - "nauc_recall_at_5_max": 0.17687415892167838, - "nauc_recall_at_5_std": -0.05642658192040872, - "ndcg_at_1": 0.627, - "ndcg_at_10": 0.66077, - "ndcg_at_100": 0.7227, - "ndcg_at_1000": 0.73045, - "ndcg_at_20": 0.69246, - "ndcg_at_3": 0.60259, - "ndcg_at_5": 0.62205, - "precision_at_1": 0.627, - "precision_at_10": 0.19297, - "precision_at_100": 0.02661, - "precision_at_1000": 0.00282, - "precision_at_20": 0.11254, - "precision_at_3": 0.40708, - "precision_at_5": 0.30719, - "recall_at_1": 0.3203, - "recall_at_10": 0.75877, - "recall_at_100": 0.94838, - "recall_at_1000": 0.98977, - "recall_at_20": 0.8419, - "recall_at_3": 0.54969, - "recall_at_5": 0.6479 + "main_score": 0.53082, + "map_at_1": 0.23, + "map_at_10": 0.42051, + "map_at_100": 0.44965, + "map_at_1000": 0.45083, + "map_at_20": 0.43805, + "map_at_3": 0.35127, + "map_at_5": 0.38729, + "mrr_at_1": 0.44568690095846647, + "mrr_at_10": 0.5995432704498199, + "mrr_at_100": 0.6043225093600862, + "mrr_at_1000": 0.6044024980788916, + "mrr_at_20": 0.6030750425149727, + "mrr_at_3": 0.5704206602768903, + "mrr_at_5": 0.5899494142705005, + "nauc_map_at_1000_diff1": 0.20246937035587098, + "nauc_map_at_1000_max": 0.16933040577535954, + "nauc_map_at_1000_std": -0.011841570282573214, + "nauc_map_at_100_diff1": 0.20206232831777704, + "nauc_map_at_100_max": 0.16874367535481147, + "nauc_map_at_100_std": -0.01231811369883067, + "nauc_map_at_10_diff1": 0.2005789654175029, + "nauc_map_at_10_max": 0.1553362336900638, + "nauc_map_at_10_std": -0.03227298923033208, + "nauc_map_at_1_diff1": 0.2474498525210096, + "nauc_map_at_1_max": 0.0952420196836924, + "nauc_map_at_1_std": -0.07951349033396907, + "nauc_map_at_20_diff1": 0.20043836209335952, + "nauc_map_at_20_max": 0.16403363510702496, + "nauc_map_at_20_std": -0.02070008550122574, + "nauc_map_at_3_diff1": 0.20835412563570585, + "nauc_map_at_3_max": 0.11706422781688205, + "nauc_map_at_3_std": -0.07383035423282713, + "nauc_map_at_5_diff1": 0.19740497223777884, + "nauc_map_at_5_max": 0.13251558972152291, + "nauc_map_at_5_std": -0.05373953901042484, + "nauc_mrr_at_1000_diff1": 0.235408469683452, + "nauc_mrr_at_1000_max": 0.21512301295632288, + "nauc_mrr_at_1000_std": 0.05839593417341141, + "nauc_mrr_at_100_diff1": 0.23537826770889214, + "nauc_mrr_at_100_max": 0.21523777749686543, + "nauc_mrr_at_100_std": 0.058532504512171846, + "nauc_mrr_at_10_diff1": 0.23471692825332247, + "nauc_mrr_at_10_max": 0.21585397225773825, + "nauc_mrr_at_10_std": 0.05854290490903898, + "nauc_mrr_at_1_diff1": 0.2639298035378703, + "nauc_mrr_at_1_max": 0.18417074567355884, + "nauc_mrr_at_1_std": 0.027381422792082718, + "nauc_mrr_at_20_diff1": 0.2352359143747651, + "nauc_mrr_at_20_max": 0.2161078565459678, + "nauc_mrr_at_20_std": 0.05964157069820256, + "nauc_mrr_at_3_diff1": 0.23574607247867752, + "nauc_mrr_at_3_max": 0.21149825121005303, + "nauc_mrr_at_3_std": 0.04724040821931203, + "nauc_mrr_at_5_diff1": 0.2326667382721074, + "nauc_mrr_at_5_max": 0.21221478999244406, + "nauc_mrr_at_5_std": 0.054264868257810135, + "nauc_ndcg_at_1000_diff1": 0.2110977888687846, + "nauc_ndcg_at_1000_max": 0.21647219768176562, + "nauc_ndcg_at_1000_std": 0.050548017521547715, + "nauc_ndcg_at_100_diff1": 0.20650254954228728, + "nauc_ndcg_at_100_max": 0.21445315567027817, + "nauc_ndcg_at_100_std": 0.052080032926675145, + "nauc_ndcg_at_10_diff1": 0.20086646748050044, + "nauc_ndcg_at_10_max": 0.18801294144964983, + "nauc_ndcg_at_10_std": 0.006085350318764736, + "nauc_ndcg_at_1_diff1": 0.2639298035378703, + "nauc_ndcg_at_1_max": 0.18417074567355884, + "nauc_ndcg_at_1_std": 0.027381422792082718, + "nauc_ndcg_at_20_diff1": 0.1999918290066272, + "nauc_ndcg_at_20_max": 0.20219804354765325, + "nauc_ndcg_at_20_std": 0.02958913722710025, + "nauc_ndcg_at_3_diff1": 0.20254008876647384, + "nauc_ndcg_at_3_max": 0.16310630233279735, + "nauc_ndcg_at_3_std": -0.007907108242035457, + "nauc_ndcg_at_5_diff1": 0.19019308051005201, + "nauc_ndcg_at_5_max": 0.16107664903382626, + "nauc_ndcg_at_5_std": -0.00966286251833291, + "nauc_precision_at_1000_diff1": -0.0330176931691381, + "nauc_precision_at_1000_max": 0.15134623499866776, + "nauc_precision_at_1000_std": 0.23911733884569272, + "nauc_precision_at_100_diff1": -0.030648680466573014, + "nauc_precision_at_100_max": 0.17511693183035457, + "nauc_precision_at_100_std": 0.26685748508003726, + "nauc_precision_at_10_diff1": 0.010483421851997368, + "nauc_precision_at_10_max": 0.22832689354137875, + "nauc_precision_at_10_std": 0.21191630233413672, + "nauc_precision_at_1_diff1": 0.2639298035378703, + "nauc_precision_at_1_max": 0.18417074567355884, + "nauc_precision_at_1_std": 0.027381422792082718, + "nauc_precision_at_20_diff1": -0.012923501968053738, + "nauc_precision_at_20_max": 0.2107257922850281, + "nauc_precision_at_20_std": 0.2426474853222564, + "nauc_precision_at_3_diff1": 0.0852169428938435, + "nauc_precision_at_3_max": 0.20494713299740983, + "nauc_precision_at_3_std": 0.10067046647245116, + "nauc_precision_at_5_diff1": 0.011897310239773943, + "nauc_precision_at_5_max": 0.20261027939911647, + "nauc_precision_at_5_std": 0.150641599719867, + "nauc_recall_at_1000_diff1": 0.27868085097091727, + "nauc_recall_at_1000_max": 0.5130171981099101, + "nauc_recall_at_1000_std": 0.4925250073907747, + "nauc_recall_at_100_diff1": 0.13537056543731096, + "nauc_recall_at_100_max": 0.26013553551901786, + "nauc_recall_at_100_std": 0.21417296229489302, + "nauc_recall_at_10_diff1": 0.13696809880902164, + "nauc_recall_at_10_max": 0.15593916263583762, + "nauc_recall_at_10_std": 0.007104426147937375, + "nauc_recall_at_1_diff1": 0.2474498525210096, + "nauc_recall_at_1_max": 0.0952420196836924, + "nauc_recall_at_1_std": -0.07951349033396907, + "nauc_recall_at_20_diff1": 0.12010102944845759, + "nauc_recall_at_20_max": 0.1812196451140207, + "nauc_recall_at_20_std": 0.06572303978029614, + "nauc_recall_at_3_diff1": 0.15872435343209812, + "nauc_recall_at_3_max": 0.09102448065600338, + "nauc_recall_at_3_std": -0.07883316144058132, + "nauc_recall_at_5_diff1": 0.1326794557788197, + "nauc_recall_at_5_max": 0.10149476723277895, + "nauc_recall_at_5_std": -0.0470684656868848, + "ndcg_at_1": 0.44569, + "ndcg_at_10": 0.53082, + "ndcg_at_100": 0.60924, + "ndcg_at_1000": 0.62289, + "ndcg_at_20": 0.5683, + "ndcg_at_3": 0.46394, + "ndcg_at_5": 0.48769, + "precision_at_1": 0.44569, + "precision_at_10": 0.16054, + "precision_at_100": 0.02514, + "precision_at_1000": 0.00278, + "precision_at_20": 0.09816, + "precision_at_3": 0.31683, + "precision_at_5": 0.24457, + "recall_at_1": 0.23, + "recall_at_10": 0.65389, + "recall_at_100": 0.90686, + "recall_at_1000": 0.97947, + "recall_at_20": 0.75777, + "recall_at_3": 0.44363, + "recall_at_5": 0.53538 } ] }, diff --git a/results/intfloat__multilingual-e5-large-instruct/baa7be480a7de1539afce709c8f13f833a510e0a/RiaNewsRetrieval.json b/results/intfloat__multilingual-e5-large-instruct/baa7be480a7de1539afce709c8f13f833a510e0a/RiaNewsRetrieval.json index 4e7bf7dca..b9bc50811 100644 --- a/results/intfloat__multilingual-e5-large-instruct/baa7be480a7de1539afce709c8f13f833a510e0a/RiaNewsRetrieval.json +++ b/results/intfloat__multilingual-e5-large-instruct/baa7be480a7de1539afce709c8f13f833a510e0a/RiaNewsRetrieval.json @@ -10,147 +10,147 @@ "languages": [ "rus-Cyrl" ], - "main_score": 0.8326, - "map_at_1": 0.7334, - "map_at_10": 0.80372, - "map_at_100": 0.80582, - "map_at_1000": 0.80591, - "map_at_20": 0.80511, - "map_at_3": 0.79283, - "map_at_5": 0.80006, - "mrr_at_1": 0.7335, - "mrr_at_10": 0.8037739285714252, - "mrr_at_100": 0.8058744742297735, - "mrr_at_1000": 0.8059571366191401, - "mrr_at_20": 0.8051606268559398, - "mrr_at_3": 0.7928833333333292, - "mrr_at_5": 0.8001133333333277, - "nauc_map_at_1000_diff1": 0.7814778752132965, - "nauc_map_at_1000_max": 0.33150599253379254, - "nauc_map_at_1000_std": -0.17621451046355077, - "nauc_map_at_100_diff1": 0.7814443330690705, - "nauc_map_at_100_max": 0.3316048292999798, - "nauc_map_at_100_std": -0.17609895364326303, - "nauc_map_at_10_diff1": 0.7807811107737767, - "nauc_map_at_10_max": 0.3316308634607904, - "nauc_map_at_10_std": -0.17823300275053414, - "nauc_map_at_1_diff1": 0.8080883904064546, - "nauc_map_at_1_max": 0.2950550046461256, - "nauc_map_at_1_std": -0.1852310409665532, - "nauc_map_at_20_diff1": 0.7812675348771089, - "nauc_map_at_20_max": 0.3319903833309077, - "nauc_map_at_20_std": -0.17657728848405616, - "nauc_map_at_3_diff1": 0.7818084428828918, - "nauc_map_at_3_max": 0.32928652642725087, - "nauc_map_at_3_std": -0.18804176350025112, - "nauc_map_at_5_diff1": 0.7796046387817263, - "nauc_map_at_5_max": 0.3307454968442525, - "nauc_map_at_5_std": -0.18164881745822922, - "nauc_mrr_at_1000_diff1": 0.781354283141064, - "nauc_mrr_at_1000_max": 0.33132728309433584, - "nauc_mrr_at_1000_std": -0.17640784923809844, - "nauc_mrr_at_100_diff1": 0.7813207920734118, - "nauc_mrr_at_100_max": 0.33142622389283966, - "nauc_mrr_at_100_std": -0.17629217810188216, - "nauc_mrr_at_10_diff1": 0.7806588900890844, - "nauc_mrr_at_10_max": 0.3314543661951199, - "nauc_mrr_at_10_std": -0.17842438416267886, - "nauc_mrr_at_1_diff1": 0.8079111738997816, - "nauc_mrr_at_1_max": 0.2947757242305656, - "nauc_mrr_at_1_std": -0.18552383336760844, - "nauc_mrr_at_20_diff1": 0.7811444560016378, - "nauc_mrr_at_20_max": 0.33181257681024867, - "nauc_mrr_at_20_std": -0.17676983133154964, - "nauc_mrr_at_3_diff1": 0.7816927937733283, - "nauc_mrr_at_3_max": 0.32911878470065037, - "nauc_mrr_at_3_std": -0.18822504514037589, - "nauc_mrr_at_5_diff1": 0.779484557300717, - "nauc_mrr_at_5_max": 0.33057225024833725, - "nauc_mrr_at_5_std": -0.18183716148903808, - "nauc_ndcg_at_1000_diff1": 0.7735973587782945, - "nauc_ndcg_at_1000_max": 0.3461123524009483, - "nauc_ndcg_at_1000_std": -0.1590375977025701, - "nauc_ndcg_at_100_diff1": 0.7728148376231733, - "nauc_ndcg_at_100_max": 0.3491160669126663, - "nauc_ndcg_at_100_std": -0.15451023750785087, - "nauc_ndcg_at_10_diff1": 0.7696360689997127, - "nauc_ndcg_at_10_max": 0.34906679496736426, - "nauc_ndcg_at_10_std": -0.16685619322621542, - "nauc_ndcg_at_1_diff1": 0.8080883904064546, - "nauc_ndcg_at_1_max": 0.2950550046461256, - "nauc_ndcg_at_1_std": -0.1852310409665532, - "nauc_ndcg_at_20_diff1": 0.7713273129844826, - "nauc_ndcg_at_20_max": 0.35115639739259547, - "nauc_ndcg_at_20_std": -0.1593505600313524, - "nauc_ndcg_at_3_diff1": 0.7721180753873903, - "nauc_ndcg_at_3_max": 0.34187008470729396, - "nauc_ndcg_at_3_std": -0.18968525420034155, - "nauc_ndcg_at_5_diff1": 0.7669692401936944, - "nauc_ndcg_at_5_max": 0.34550747885126887, - "nauc_ndcg_at_5_std": -0.17712057379012125, - "nauc_precision_at_1000_diff1": 0.611856971585477, - "nauc_precision_at_1000_max": 0.676360090387237, - "nauc_precision_at_1000_std": 0.37724241570395467, - "nauc_precision_at_100_diff1": 0.6695576189659513, - "nauc_precision_at_100_max": 0.6098153547133169, - "nauc_precision_at_100_std": 0.2656817829172501, - "nauc_precision_at_10_diff1": 0.6989567712606916, - "nauc_precision_at_10_max": 0.46529840892585816, - "nauc_precision_at_10_std": -0.08017853606089333, - "nauc_precision_at_1_diff1": 0.8080883904064546, - "nauc_precision_at_1_max": 0.2950550046461256, - "nauc_precision_at_1_std": -0.1852310409665532, - "nauc_precision_at_20_diff1": 0.6929448429399495, - "nauc_precision_at_20_max": 0.5224439193314594, - "nauc_precision_at_20_std": 0.020914411244251113, - "nauc_precision_at_3_diff1": 0.732979925882572, - "nauc_precision_at_3_max": 0.39267481454757575, - "nauc_precision_at_3_std": -0.19661451806373872, - "nauc_precision_at_5_diff1": 0.7034524379423565, - "nauc_precision_at_5_max": 0.41878928422662515, - "nauc_precision_at_5_std": -0.15200114749686783, - "nauc_recall_at_1000_diff1": 0.6118569715854769, - "nauc_recall_at_1000_max": 0.676360090387241, - "nauc_recall_at_1000_std": 0.377242415703956, - "nauc_recall_at_100_diff1": 0.6695576189659566, - "nauc_recall_at_100_max": 0.6098153547133157, - "nauc_recall_at_100_std": 0.26568178291724726, - "nauc_recall_at_10_diff1": 0.6989567712606909, - "nauc_recall_at_10_max": 0.46529840892586016, - "nauc_recall_at_10_std": -0.08017853606088883, - "nauc_recall_at_1_diff1": 0.8080883904064546, - "nauc_recall_at_1_max": 0.2950550046461256, - "nauc_recall_at_1_std": -0.1852310409665532, - "nauc_recall_at_20_diff1": 0.6929448429399518, - "nauc_recall_at_20_max": 0.522443919331458, - "nauc_recall_at_20_std": 0.020914411244256536, - "nauc_recall_at_3_diff1": 0.732979925882572, - "nauc_recall_at_3_max": 0.39267481454757475, - "nauc_recall_at_3_std": -0.19661451806373972, - "nauc_recall_at_5_diff1": 0.7034524379423579, - "nauc_recall_at_5_max": 0.41878928422662565, - "nauc_recall_at_5_std": -0.15200114749686747, - "ndcg_at_1": 0.7334, - "ndcg_at_10": 0.8326, - "ndcg_at_100": 0.84239, - "ndcg_at_1000": 0.84459, - "ndcg_at_20": 0.8375, - "ndcg_at_3": 0.81093, - "ndcg_at_5": 0.82387, - "precision_at_1": 0.7334, - "precision_at_10": 0.09208, - "precision_at_100": 0.00966, - "precision_at_1000": 0.00098, - "precision_at_20": 0.047, - "precision_at_3": 0.28763, - "precision_at_5": 0.17882, - "recall_at_1": 0.7334, - "recall_at_10": 0.9208, - "recall_at_100": 0.9657, - "recall_at_1000": 0.9831, - "recall_at_20": 0.9399, - "recall_at_3": 0.8629, - "recall_at_5": 0.8941 + "main_score": 0.82444, + "map_at_1": 0.7152, + "map_at_10": 0.79258, + "map_at_100": 0.7947, + "map_at_1000": 0.79477, + "map_at_20": 0.79402, + "map_at_3": 0.78095, + "map_at_5": 0.78856, + "mrr_at_1": 0.7158, + "mrr_at_10": 0.7929000396825396, + "mrr_at_100": 0.7950206402626601, + "mrr_at_1000": 0.7950935723465302, + "mrr_at_20": 0.7943407488473858, + "mrr_at_3": 0.7813, + "mrr_at_5": 0.7888850000000001, + "nauc_map_at_1000_diff1": 0.7448076044635364, + "nauc_map_at_1000_max": 0.25620475423329425, + "nauc_map_at_1000_std": -0.20848800758098918, + "nauc_map_at_100_diff1": 0.7447770251313918, + "nauc_map_at_100_max": 0.2562530676648632, + "nauc_map_at_100_std": -0.208415139197629, + "nauc_map_at_10_diff1": 0.7440033041001333, + "nauc_map_at_10_max": 0.25603425089556114, + "nauc_map_at_10_std": -0.20961801127556542, + "nauc_map_at_1_diff1": 0.7734267068182435, + "nauc_map_at_1_max": 0.22912396423078507, + "nauc_map_at_1_std": -0.21998094004051244, + "nauc_map_at_20_diff1": 0.744516879143539, + "nauc_map_at_20_max": 0.25652866403340685, + "nauc_map_at_20_std": -0.20885286918031837, + "nauc_map_at_3_diff1": 0.743098925142951, + "nauc_map_at_3_max": 0.25510322984117384, + "nauc_map_at_3_std": -0.21803907126632932, + "nauc_map_at_5_diff1": 0.743199850031989, + "nauc_map_at_5_max": 0.25579593604420126, + "nauc_map_at_5_std": -0.21187233395088761, + "nauc_mrr_at_1000_diff1": 0.7440752415823455, + "nauc_mrr_at_1000_max": 0.2571526977865483, + "nauc_mrr_at_1000_std": -0.20729990496032, + "nauc_mrr_at_100_diff1": 0.7440449122712941, + "nauc_mrr_at_100_max": 0.25720069890506614, + "nauc_mrr_at_100_std": -0.20722741207127449, + "nauc_mrr_at_10_diff1": 0.7432785795313153, + "nauc_mrr_at_10_max": 0.25697064799619285, + "nauc_mrr_at_10_std": -0.2084448707459271, + "nauc_mrr_at_1_diff1": 0.7723576454412964, + "nauc_mrr_at_1_max": 0.23034409771728423, + "nauc_mrr_at_1_std": -0.21850783575240879, + "nauc_mrr_at_20_diff1": 0.7437871766514825, + "nauc_mrr_at_20_max": 0.2574729869126586, + "nauc_mrr_at_20_std": -0.2076696110392776, + "nauc_mrr_at_3_diff1": 0.742357788142687, + "nauc_mrr_at_3_max": 0.2562642307838455, + "nauc_mrr_at_3_std": -0.21681665398372654, + "nauc_mrr_at_5_diff1": 0.7424855224612325, + "nauc_mrr_at_5_max": 0.2567300909848355, + "nauc_mrr_at_5_std": -0.2107119618557157, + "nauc_ndcg_at_1000_diff1": 0.7369659501097652, + "nauc_ndcg_at_1000_max": 0.2669254648302882, + "nauc_ndcg_at_1000_std": -0.193017039769233, + "nauc_ndcg_at_100_diff1": 0.7358690819029983, + "nauc_ndcg_at_100_max": 0.26849109068870053, + "nauc_ndcg_at_100_std": -0.18918611059910645, + "nauc_ndcg_at_10_diff1": 0.7321093853421414, + "nauc_ndcg_at_10_max": 0.2676750036397339, + "nauc_ndcg_at_10_std": -0.1970693226477929, + "nauc_ndcg_at_1_diff1": 0.7734267068182435, + "nauc_ndcg_at_1_max": 0.22912396423078507, + "nauc_ndcg_at_1_std": -0.21998094004051244, + "nauc_ndcg_at_20_diff1": 0.7339597980313309, + "nauc_ndcg_at_20_max": 0.2700207107786095, + "nauc_ndcg_at_20_std": -0.19332008483549196, + "nauc_ndcg_at_3_diff1": 0.7313359646293657, + "nauc_ndcg_at_3_max": 0.2649147354188418, + "nauc_ndcg_at_3_std": -0.2161784699693373, + "nauc_ndcg_at_5_diff1": 0.7305769165185565, + "nauc_ndcg_at_5_max": 0.266897283198301, + "nauc_ndcg_at_5_std": -0.20361686025017536, + "nauc_precision_at_1000_diff1": 0.5711695637159063, + "nauc_precision_at_1000_max": 0.575425604031578, + "nauc_precision_at_1000_std": 0.34083496412265013, + "nauc_precision_at_100_diff1": 0.6208022454649502, + "nauc_precision_at_100_max": 0.4634398101748332, + "nauc_precision_at_100_std": 0.2001716078888182, + "nauc_precision_at_10_diff1": 0.6520336182213702, + "nauc_precision_at_10_max": 0.3469007381298652, + "nauc_precision_at_10_std": -0.10060631243119607, + "nauc_precision_at_1_diff1": 0.7734267068182435, + "nauc_precision_at_1_max": 0.22912396423078507, + "nauc_precision_at_1_std": -0.21998094004051244, + "nauc_precision_at_20_diff1": 0.6438238513796348, + "nauc_precision_at_20_max": 0.398088315786086, + "nauc_precision_at_20_std": -0.027498470652630443, + "nauc_precision_at_3_diff1": 0.682482859363674, + "nauc_precision_at_3_max": 0.3055385239293006, + "nauc_precision_at_3_std": -0.2079298332249354, + "nauc_precision_at_5_diff1": 0.665926452300585, + "nauc_precision_at_5_max": 0.3236936273887752, + "nauc_precision_at_5_std": -0.15754270417008645, + "nauc_recall_at_1000_diff1": 0.5711695637158906, + "nauc_recall_at_1000_max": 0.5754256040315625, + "nauc_recall_at_1000_std": 0.34083496412263514, + "nauc_recall_at_100_diff1": 0.6208022454649564, + "nauc_recall_at_100_max": 0.4634398101748313, + "nauc_recall_at_100_std": 0.20017160788882343, + "nauc_recall_at_10_diff1": 0.6520336182213716, + "nauc_recall_at_10_max": 0.3469007381298643, + "nauc_recall_at_10_std": -0.10060631243119561, + "nauc_recall_at_1_diff1": 0.7734267068182435, + "nauc_recall_at_1_max": 0.22912396423078507, + "nauc_recall_at_1_std": -0.21998094004051244, + "nauc_recall_at_20_diff1": 0.6438238513796317, + "nauc_recall_at_20_max": 0.398088315786084, + "nauc_recall_at_20_std": -0.027498470652627564, + "nauc_recall_at_3_diff1": 0.6824828593636737, + "nauc_recall_at_3_max": 0.30553852392930136, + "nauc_recall_at_3_std": -0.20792983322493563, + "nauc_recall_at_5_diff1": 0.6659264523005867, + "nauc_recall_at_5_max": 0.32369362738877777, + "nauc_recall_at_5_std": -0.15754270417008193, + "ndcg_at_1": 0.7152, + "ndcg_at_10": 0.82444, + "ndcg_at_100": 0.83435, + "ndcg_at_1000": 0.83657, + "ndcg_at_20": 0.82959, + "ndcg_at_3": 0.80117, + "ndcg_at_5": 0.81482, + "precision_at_1": 0.7152, + "precision_at_10": 0.09218, + "precision_at_100": 0.00967, + "precision_at_1000": 0.00099, + "precision_at_20": 0.0471, + "precision_at_3": 0.28643, + "precision_at_5": 0.17846, + "recall_at_1": 0.7152, + "recall_at_10": 0.9218, + "recall_at_100": 0.9673, + "recall_at_1000": 0.9854, + "recall_at_20": 0.942, + "recall_at_3": 0.8593, + "recall_at_5": 0.8923 } ] }, diff --git a/results/intfloat__multilingual-e5-large-instruct/baa7be480a7de1539afce709c8f13f833a510e0a/RuBQReranking.json b/results/intfloat__multilingual-e5-large-instruct/baa7be480a7de1539afce709c8f13f833a510e0a/RuBQReranking.json index 9910b7f09..bdd119267 100644 --- a/results/intfloat__multilingual-e5-large-instruct/baa7be480a7de1539afce709c8f13f833a510e0a/RuBQReranking.json +++ b/results/intfloat__multilingual-e5-large-instruct/baa7be480a7de1539afce709c8f13f833a510e0a/RuBQReranking.json @@ -10,15 +10,15 @@ "languages": [ "rus-Cyrl" ], - "main_score": 0.7583624195682908, - "map": 0.7583624195682908, - "mrr": 0.8157545567120035, - "nAUC_map_diff1": 0.410527584442837, - "nAUC_map_max": 0.2512342651095511, - "nAUC_map_std": 0.1306873131678, - "nAUC_mrr_diff1": 0.47913183777433416, - "nAUC_mrr_max": 0.3557683398641048, - "nAUC_mrr_std": 0.17973777222092463 + "main_score": 0.716591133500063, + "map": 0.716591133500063, + "mrr": 0.76903073286052, + "nAUC_map_diff1": 0.370258661399668, + "nAUC_map_max": 0.11303543827245233, + "nAUC_map_std": 0.06026492006550841, + "nAUC_mrr_diff1": 0.3946576923351406, + "nAUC_mrr_max": 0.14359390125072474, + "nAUC_mrr_std": 0.07264588612204545 } ] }, diff --git a/results/intfloat__multilingual-e5-large-instruct/baa7be480a7de1539afce709c8f13f833a510e0a/RuBQRetrieval.json b/results/intfloat__multilingual-e5-large-instruct/baa7be480a7de1539afce709c8f13f833a510e0a/RuBQRetrieval.json index 0055e645a..4c6a8a6b6 100644 --- a/results/intfloat__multilingual-e5-large-instruct/baa7be480a7de1539afce709c8f13f833a510e0a/RuBQRetrieval.json +++ b/results/intfloat__multilingual-e5-large-instruct/baa7be480a7de1539afce709c8f13f833a510e0a/RuBQRetrieval.json @@ -10,147 +10,147 @@ "languages": [ "rus-Cyrl" ], - "main_score": 0.73902, - "map_at_1": 0.46932, - "map_at_10": 0.6664, - "map_at_100": 0.67564, - "map_at_1000": 0.67591, - "map_at_20": 0.67303, - "map_at_3": 0.61397, - "map_at_5": 0.64636, - "mrr_at_1": 0.6601654846335697, - "mrr_at_10": 0.7559544917257681, - "mrr_at_100": 0.7586607116883776, - "mrr_at_1000": 0.758720452896963, - "mrr_at_20": 0.7579402369766007, - "mrr_at_3": 0.7381796690307328, - "mrr_at_5": 0.7490248226950357, - "nauc_map_at_1000_diff1": 0.39310804502992686, - "nauc_map_at_1000_max": 0.3350774882352565, - "nauc_map_at_1000_std": -0.0783821403940471, - "nauc_map_at_100_diff1": 0.392749946359186, - "nauc_map_at_100_max": 0.3350304323843582, - "nauc_map_at_100_std": -0.07830653553256185, - "nauc_map_at_10_diff1": 0.3912894483482967, - "nauc_map_at_10_max": 0.33228744193450266, - "nauc_map_at_10_std": -0.08203830585471199, - "nauc_map_at_1_diff1": 0.42860174404789153, - "nauc_map_at_1_max": 0.26835017006823886, - "nauc_map_at_1_std": -0.07584800410518629, - "nauc_map_at_20_diff1": 0.3912700391141, - "nauc_map_at_20_max": 0.33521497609690565, - "nauc_map_at_20_std": -0.07977780998702264, - "nauc_map_at_3_diff1": 0.39959151863507847, - "nauc_map_at_3_max": 0.3123544708649225, - "nauc_map_at_3_std": -0.08531518136181697, - "nauc_map_at_5_diff1": 0.3898125381765358, - "nauc_map_at_5_max": 0.3209399515836407, - "nauc_map_at_5_std": -0.08769857464922982, - "nauc_mrr_at_1000_diff1": 0.4837807503521872, - "nauc_mrr_at_1000_max": 0.42613142793890985, - "nauc_mrr_at_1000_std": -0.09023667016239204, - "nauc_mrr_at_100_diff1": 0.48372218460756466, - "nauc_mrr_at_100_max": 0.42618156138414903, - "nauc_mrr_at_100_std": -0.09021997756601169, - "nauc_mrr_at_10_diff1": 0.4819812498559355, - "nauc_mrr_at_10_max": 0.4262883095250716, - "nauc_mrr_at_10_std": -0.08919521987037732, - "nauc_mrr_at_1_diff1": 0.5173062495979291, - "nauc_mrr_at_1_max": 0.41835030689767727, - "nauc_mrr_at_1_std": -0.10726601873758122, - "nauc_mrr_at_20_diff1": 0.48312983833560735, - "nauc_mrr_at_20_max": 0.42682221718937013, - "nauc_mrr_at_20_std": -0.08981054232230391, - "nauc_mrr_at_3_diff1": 0.48178479811150016, - "nauc_mrr_at_3_max": 0.4189334665107361, - "nauc_mrr_at_3_std": -0.09221931553446863, - "nauc_mrr_at_5_diff1": 0.479237042684324, - "nauc_mrr_at_5_max": 0.4247000730973994, - "nauc_mrr_at_5_std": -0.09230414205898824, - "nauc_ndcg_at_1000_diff1": 0.4079753907547425, - "nauc_ndcg_at_1000_max": 0.36672464048458414, - "nauc_ndcg_at_1000_std": -0.06709279385347566, - "nauc_ndcg_at_100_diff1": 0.4005067476999608, - "nauc_ndcg_at_100_max": 0.36689256928395936, - "nauc_ndcg_at_100_std": -0.06264368526298046, - "nauc_ndcg_at_10_diff1": 0.39231711990772655, - "nauc_ndcg_at_10_max": 0.3624371693357732, - "nauc_ndcg_at_10_std": -0.07343888212077378, - "nauc_ndcg_at_1_diff1": 0.5173062495979291, - "nauc_ndcg_at_1_max": 0.41835030689767727, - "nauc_ndcg_at_1_std": -0.10726601873758122, - "nauc_ndcg_at_20_diff1": 0.3931888603651246, - "nauc_ndcg_at_20_max": 0.3697673043896615, - "nauc_ndcg_at_20_std": -0.06728409505101884, - "nauc_ndcg_at_3_diff1": 0.40234410222854167, - "nauc_ndcg_at_3_max": 0.3315615057635194, - "nauc_ndcg_at_3_std": -0.08725881807857037, - "nauc_ndcg_at_5_diff1": 0.38943938877579515, - "nauc_ndcg_at_5_max": 0.3435533800191916, - "nauc_ndcg_at_5_std": -0.08662960191400126, - "nauc_precision_at_1000_diff1": -0.12360100150225584, - "nauc_precision_at_1000_max": 0.006597934420036655, - "nauc_precision_at_1000_std": 0.03980738249200735, - "nauc_precision_at_100_diff1": -0.130138769139958, - "nauc_precision_at_100_max": 0.02977127272574392, - "nauc_precision_at_100_std": 0.05540051952293426, - "nauc_precision_at_10_diff1": -0.058519520033513406, - "nauc_precision_at_10_max": 0.1108906220575976, - "nauc_precision_at_10_std": 0.014277929222216577, - "nauc_precision_at_1_diff1": 0.5173062495979291, - "nauc_precision_at_1_max": 0.41835030689767727, - "nauc_precision_at_1_std": -0.10726601873758122, - "nauc_precision_at_20_diff1": -0.10330454617302505, - "nauc_precision_at_20_max": 0.0773986628178308, - "nauc_precision_at_20_std": 0.038030670641286894, - "nauc_precision_at_3_diff1": 0.10764940676176303, - "nauc_precision_at_3_max": 0.20359779908392878, - "nauc_precision_at_3_std": -0.04512537044942992, - "nauc_precision_at_5_diff1": 0.007556603374458451, - "nauc_precision_at_5_max": 0.15044212285526792, - "nauc_precision_at_5_std": -0.018737533081410306, - "nauc_recall_at_1000_diff1": 0.2742381634796684, - "nauc_recall_at_1000_max": 0.4827269172391828, - "nauc_recall_at_1000_std": 0.7252136032139339, - "nauc_recall_at_100_diff1": 0.08945770537317706, - "nauc_recall_at_100_max": 0.3695179929855661, - "nauc_recall_at_100_std": 0.2712072501263794, - "nauc_recall_at_10_diff1": 0.24267113493870207, - "nauc_recall_at_10_max": 0.3437844943580729, - "nauc_recall_at_10_std": -0.018295739083107557, - "nauc_recall_at_1_diff1": 0.42860174404789153, - "nauc_recall_at_1_max": 0.26835017006823886, - "nauc_recall_at_1_std": -0.07584800410518629, - "nauc_recall_at_20_diff1": 0.18250867012780755, - "nauc_recall_at_20_max": 0.38649236559282535, - "nauc_recall_at_20_std": 0.0452553575861491, - "nauc_recall_at_3_diff1": 0.3326140320558434, - "nauc_recall_at_3_max": 0.28419697594029, - "nauc_recall_at_3_std": -0.06433418555491488, - "nauc_recall_at_5_diff1": 0.2769660445436549, - "nauc_recall_at_5_max": 0.29261890277912445, - "nauc_recall_at_5_std": -0.07528618321381433, - "ndcg_at_1": 0.66017, - "ndcg_at_10": 0.73902, - "ndcg_at_100": 0.76682, - "ndcg_at_1000": 0.77118, - "ndcg_at_20": 0.75491, - "ndcg_at_3": 0.67157, - "ndcg_at_5": 0.70609, - "precision_at_1": 0.66017, - "precision_at_10": 0.14054, - "precision_at_100": 0.01608, + "main_score": 0.69176, + "map_at_1": 0.40086, + "map_at_10": 0.60898, + "map_at_100": 0.61927, + "map_at_1000": 0.61954, + "map_at_20": 0.61614, + "map_at_3": 0.55283, + "map_at_5": 0.58576, + "mrr_at_1": 0.5673758865248227, + "mrr_at_10": 0.6940670850688581, + "mrr_at_100": 0.6970898163430719, + "mrr_at_1000": 0.6971653611009756, + "mrr_at_20": 0.696224837476649, + "mrr_at_3": 0.6700157604412924, + "mrr_at_5": 0.6848798266351458, + "nauc_map_at_1000_diff1": 0.3451889520960452, + "nauc_map_at_1000_max": 0.18855632342520304, + "nauc_map_at_1000_std": -0.10243504064051992, + "nauc_map_at_100_diff1": 0.3448660376620878, + "nauc_map_at_100_max": 0.18869998884373215, + "nauc_map_at_100_std": -0.10227544854712893, + "nauc_map_at_10_diff1": 0.34434680977556936, + "nauc_map_at_10_max": 0.1896264332137727, + "nauc_map_at_10_std": -0.10708153094029998, + "nauc_map_at_1_diff1": 0.38580722584804555, + "nauc_map_at_1_max": 0.13333022181910123, + "nauc_map_at_1_std": -0.09351152581427404, + "nauc_map_at_20_diff1": 0.34424401251136927, + "nauc_map_at_20_max": 0.18935276841750082, + "nauc_map_at_20_std": -0.10329119218084819, + "nauc_map_at_3_diff1": 0.3433134416286612, + "nauc_map_at_3_max": 0.16488803363479773, + "nauc_map_at_3_std": -0.11601438130350093, + "nauc_map_at_5_diff1": 0.34497145388215805, + "nauc_map_at_5_max": 0.17808065872637557, + "nauc_map_at_5_std": -0.11082335392922679, + "nauc_mrr_at_1000_diff1": 0.38870004420352394, + "nauc_mrr_at_1000_max": 0.20759273546678567, + "nauc_mrr_at_1000_std": -0.10933014085667585, + "nauc_mrr_at_100_diff1": 0.38862777675954363, + "nauc_mrr_at_100_max": 0.2076920013717805, + "nauc_mrr_at_100_std": -0.10927537258482942, + "nauc_mrr_at_10_diff1": 0.38794091872631475, + "nauc_mrr_at_10_max": 0.20977851859702293, + "nauc_mrr_at_10_std": -0.10899887031734305, + "nauc_mrr_at_1_diff1": 0.4314015201290902, + "nauc_mrr_at_1_max": 0.1757585327492127, + "nauc_mrr_at_1_std": -0.11026711472678012, + "nauc_mrr_at_20_diff1": 0.38865722369672673, + "nauc_mrr_at_20_max": 0.20910529231378316, + "nauc_mrr_at_20_std": -0.10886763711363066, + "nauc_mrr_at_3_diff1": 0.38240111529509613, + "nauc_mrr_at_3_max": 0.2022533080638829, + "nauc_mrr_at_3_std": -0.12003100356912472, + "nauc_mrr_at_5_diff1": 0.38462603518839444, + "nauc_mrr_at_5_max": 0.20553671220879366, + "nauc_mrr_at_5_std": -0.11454623065679513, + "nauc_ndcg_at_1000_diff1": 0.3483467030579176, + "nauc_ndcg_at_1000_max": 0.20637722577795864, + "nauc_ndcg_at_1000_std": -0.09259061272961588, + "nauc_ndcg_at_100_diff1": 0.3416343967881553, + "nauc_ndcg_at_100_max": 0.21069141950187223, + "nauc_ndcg_at_100_std": -0.08682277066715591, + "nauc_ndcg_at_10_diff1": 0.33839049320254927, + "nauc_ndcg_at_10_max": 0.2179061187580867, + "nauc_ndcg_at_10_std": -0.10006235017089225, + "nauc_ndcg_at_1_diff1": 0.43293217938044315, + "nauc_ndcg_at_1_max": 0.17249293849472974, + "nauc_ndcg_at_1_std": -0.10997343787737572, + "nauc_ndcg_at_20_diff1": 0.3393108700633641, + "nauc_ndcg_at_20_max": 0.21688067634778072, + "nauc_ndcg_at_20_std": -0.08962556946171202, + "nauc_ndcg_at_3_diff1": 0.33390318619966297, + "nauc_ndcg_at_3_max": 0.17435209108756433, + "nauc_ndcg_at_3_std": -0.12330957642287138, + "nauc_ndcg_at_5_diff1": 0.3382847675085889, + "nauc_ndcg_at_5_max": 0.19598230885782586, + "nauc_ndcg_at_5_std": -0.11164169942210181, + "nauc_precision_at_1000_diff1": -0.10564460424753937, + "nauc_precision_at_1000_max": 0.005299238799398134, + "nauc_precision_at_1000_std": 0.04221027538282351, + "nauc_precision_at_100_diff1": -0.1104921712322409, + "nauc_precision_at_100_max": 0.03330756728161014, + "nauc_precision_at_100_std": 0.059807341198580825, + "nauc_precision_at_10_diff1": -0.03478410591720867, + "nauc_precision_at_10_max": 0.10699546469820291, + "nauc_precision_at_10_std": 0.009932316908315785, + "nauc_precision_at_1_diff1": 0.43293217938044315, + "nauc_precision_at_1_max": 0.17249293849472974, + "nauc_precision_at_1_std": -0.10997343787737572, + "nauc_precision_at_20_diff1": -0.07406106668254143, + "nauc_precision_at_20_max": 0.0764621983142494, + "nauc_precision_at_20_std": 0.04810370055890343, + "nauc_precision_at_3_diff1": 0.0953835981294087, + "nauc_precision_at_3_max": 0.12665258574465446, + "nauc_precision_at_3_std": -0.07282440522292559, + "nauc_precision_at_5_diff1": 0.03730067112097692, + "nauc_precision_at_5_max": 0.118819052499945, + "nauc_precision_at_5_std": -0.03322221490175339, + "nauc_recall_at_1000_diff1": 0.3137752279084665, + "nauc_recall_at_1000_max": 0.630834955322045, + "nauc_recall_at_1000_std": 0.7339098145683902, + "nauc_recall_at_100_diff1": 0.06215275360574706, + "nauc_recall_at_100_max": 0.4111903795975184, + "nauc_recall_at_100_std": 0.27416255632226305, + "nauc_recall_at_10_diff1": 0.21463505175037018, + "nauc_recall_at_10_max": 0.3041427662024013, + "nauc_recall_at_10_std": -0.04666364155484872, + "nauc_recall_at_1_diff1": 0.38580722584804555, + "nauc_recall_at_1_max": 0.13333022181910123, + "nauc_recall_at_1_std": -0.09351152581427404, + "nauc_recall_at_20_diff1": 0.18128768346932664, + "nauc_recall_at_20_max": 0.3447412440952482, + "nauc_recall_at_20_std": 0.04162680664699626, + "nauc_recall_at_3_diff1": 0.26148635682246985, + "nauc_recall_at_3_max": 0.16636161117305281, + "nauc_recall_at_3_std": -0.12062189812453104, + "nauc_recall_at_5_diff1": 0.24757177654561052, + "nauc_recall_at_5_max": 0.20776106951470089, + "nauc_recall_at_5_std": -0.09561904550288293, + "ndcg_at_1": 0.56678, + "ndcg_at_10": 0.69176, + "ndcg_at_100": 0.72299, + "ndcg_at_1000": 0.72779, + "ndcg_at_20": 0.70891, + "ndcg_at_3": 0.61363, + "ndcg_at_5": 0.65251, + "precision_at_1": 0.56678, + "precision_at_10": 0.13771, + "precision_at_100": 0.01606, "precision_at_1000": 0.00167, - "precision_at_20": 0.0753, - "precision_at_3": 0.35835, - "precision_at_5": 0.24799, - "recall_at_1": 0.46932, - "recall_at_10": 0.85943, - "recall_at_100": 0.96497, - "recall_at_1000": 0.99325, - "recall_at_20": 0.91166, - "recall_at_3": 0.69349, - "recall_at_5": 0.77521 + "precision_at_20": 0.07423, + "precision_at_3": 0.3353, + "precision_at_5": 0.23629, + "recall_at_1": 0.40086, + "recall_at_10": 0.84434, + "recall_at_100": 0.96261, + "recall_at_1000": 0.99424, + "recall_at_20": 0.90063, + "recall_at_3": 0.65286, + "recall_at_5": 0.74382 } ] }, diff --git a/results/intfloat__multilingual-e5-large-instruct/baa7be480a7de1539afce709c8f13f833a510e0a/RuSciBenchOECDClusteringP2P.json b/results/intfloat__multilingual-e5-large-instruct/baa7be480a7de1539afce709c8f13f833a510e0a/RuSciBenchOECDClusteringP2P.json index 3f1b800f5..bdbde27f2 100644 --- a/results/intfloat__multilingual-e5-large-instruct/baa7be480a7de1539afce709c8f13f833a510e0a/RuSciBenchOECDClusteringP2P.json +++ b/results/intfloat__multilingual-e5-large-instruct/baa7be480a7de1539afce709c8f13f833a510e0a/RuSciBenchOECDClusteringP2P.json @@ -10,21 +10,21 @@ "languages": [ "rus-Cyrl" ], - "main_score": 0.5308960921068835, - "v_measure": 0.5308960921068835, - "v_measure_std": 0.005066224169215475, + "main_score": 0.5283097250210953, + "v_measure": 0.5283097250210953, + "v_measure_std": 0.005346480006299713, "v_measures": { "Level 0": [ - 0.5259535461100251, - 0.5313805452007462, - 0.529557561682473, - 0.5211629276977965, - 0.5347544628862441, - 0.5305633739801393, - 0.5355140167365859, - 0.5393322077039413, - 0.5340748210420564, - 0.5266674580288273 + 0.526964508507581, + 0.5302779253598838, + 0.5345105579125883, + 0.5274608171330081, + 0.5174544384692139, + 0.5336539388590341, + 0.5364721230807232, + 0.524276491147096, + 0.525252738095793, + 0.5267737116460323 ] } } diff --git a/results/intfloat__multilingual-e5-large-instruct/baa7be480a7de1539afce709c8f13f833a510e0a/STS22.json b/results/intfloat__multilingual-e5-large-instruct/baa7be480a7de1539afce709c8f13f833a510e0a/STS22.json index c72150ed1..6874e9173 100644 --- a/results/intfloat__multilingual-e5-large-instruct/baa7be480a7de1539afce709c8f13f833a510e0a/STS22.json +++ b/results/intfloat__multilingual-e5-large-instruct/baa7be480a7de1539afce709c8f13f833a510e0a/STS22.json @@ -98,19 +98,19 @@ "spearman": 0.409671211166876 }, { - "cosine_pearson": 0.5702442606358379, - "cosine_spearman": 0.6517442812302365, - "euclidean_pearson": 0.5456016137695938, - "euclidean_spearman": 0.6475246651711097, + "cosine_pearson": 0.6582336494368964, + "cosine_spearman": 0.7006054402449277, + "euclidean_pearson": 0.674411358600615, + "euclidean_spearman": 0.7006054402449277, "hf_subset": "ru", "languages": [ "rus-Cyrl" ], - "main_score": 0.6517442812302365, - "manhattan_pearson": 0.5488527923421918, - "manhattan_spearman": 0.6464307837731835, - "pearson": 0.5702442606358379, - "spearman": 0.6517442812302365 + "main_score": 0.7006054402449277, + "manhattan_pearson": 0.6731196863624583, + "manhattan_spearman": 0.698370329789253, + "pearson": 0.6582336494368964, + "spearman": 0.7006054402449277 }, { "cosine_pearson": 0.6869269801164152, diff --git a/results/intfloat__multilingual-e5-large-instruct/baa7be480a7de1539afce709c8f13f833a510e0a/model_meta.json b/results/intfloat__multilingual-e5-large-instruct/baa7be480a7de1539afce709c8f13f833a510e0a/model_meta.json index e6984db3d..504be0dc6 100644 --- a/results/intfloat__multilingual-e5-large-instruct/baa7be480a7de1539afce709c8f13f833a510e0a/model_meta.json +++ b/results/intfloat__multilingual-e5-large-instruct/baa7be480a7de1539afce709c8f13f833a510e0a/model_meta.json @@ -1 +1,115 @@ -{"name": "intfloat/multilingual-e5-large-instruct", "revision": "baa7be480a7de1539afce709c8f13f833a510e0a", "release_date": "2024-02-08", "languages": ["afr_Latn", "amh_Latn", "ara_Latn", "asm_Latn", "aze_Latn", "bel_Latn", "bul_Latn", "ben_Latn", "ben_Beng", "bre_Latn", "bos_Latn", "cat_Latn", "ces_Latn", "cym_Latn", "dan_Latn", "deu_Latn", "ell_Latn", "eng_Latn", "epo_Latn", "spa_Latn", "est_Latn", "eus_Latn", "fas_Latn", "fin_Latn", "fra_Latn", "fry_Latn", "gle_Latn", "gla_Latn", "glg_Latn", "guj_Latn", "hau_Latn", "heb_Latn", "hin_Latn", "hin_Deva", "hrv_Latn", "hun_Latn", "hye_Latn", "ind_Latn", "isl_Latn", "ita_Latn", "jpn_Latn", "jav_Latn", "kat_Latn", "kaz_Latn", "khm_Latn", "kan_Latn", "kor_Latn", "kur_Latn", "kir_Latn", "lat_Latn", "lao_Latn", "lit_Latn", "lav_Latn", "mlg_Latn", "mkd_Latn", "mal_Latn", "mon_Latn", "mar_Latn", "msa_Latn", "mya_Latn", "nep_Latn", "nld_Latn", "nob_Latn", "orm_Latn", "ori_Latn", "pan_Latn", "pol_Latn", "pus_Latn", "por_Latn", "ron_Latn", "rus_Latn", "san_Latn", "snd_Latn", "sin_Latn", "slk_Latn", "slv_Latn", "som_Latn", "sqi_Latn", "srp_Latn", "sun_Latn", "swe_Latn", "swa_Latn", "tam_Latn", "tam_Taml", "tel_Latn", "tel_Telu", "tha_Latn", "tgl_Latn", "tur_Latn", "uig_Latn", "ukr_Latn", "urd_Latn", "urd_Arab", "uzb_Latn", "vie_Latn", "xho_Latn", "yid_Latn", "zho_Hant", "zho_Hans"], "n_parameters": null, "memory_usage": null, "max_tokens": null, "embed_dim": null, "license": null, "open_source": true, "similarity_fn_name": null, "framework": [], "loader": "e5_loader"} \ No newline at end of file +{ + "name": "intfloat/multilingual-e5-large-instruct", + "revision": "baa7be480a7de1539afce709c8f13f833a510e0a", + "release_date": "2024-02-08", + "languages": [ + "afr_Latn", + "amh_Latn", + "ara_Latn", + "asm_Latn", + "aze_Latn", + "bel_Latn", + "bul_Latn", + "ben_Latn", + "ben_Beng", + "bre_Latn", + "bos_Latn", + "cat_Latn", + "ces_Latn", + "cym_Latn", + "dan_Latn", + "deu_Latn", + "ell_Latn", + "eng_Latn", + "epo_Latn", + "spa_Latn", + "est_Latn", + "eus_Latn", + "fas_Latn", + "fin_Latn", + "fra_Latn", + "fry_Latn", + "gle_Latn", + "gla_Latn", + "glg_Latn", + "guj_Latn", + "hau_Latn", + "heb_Latn", + "hin_Latn", + "hin_Deva", + "hrv_Latn", + "hun_Latn", + "hye_Latn", + "ind_Latn", + "isl_Latn", + "ita_Latn", + "jpn_Latn", + "jav_Latn", + "kat_Latn", + "kaz_Latn", + "khm_Latn", + "kan_Latn", + "kor_Latn", + "kur_Latn", + "kir_Latn", + "lat_Latn", + "lao_Latn", + "lit_Latn", + "lav_Latn", + "mlg_Latn", + "mkd_Latn", + "mal_Latn", + "mon_Latn", + "mar_Latn", + "msa_Latn", + "mya_Latn", + "nep_Latn", + "nld_Latn", + "nob_Latn", + "orm_Latn", + "ori_Latn", + "pan_Latn", + "pol_Latn", + "pus_Latn", + "por_Latn", + "ron_Latn", + "rus_Latn", + "san_Latn", + "snd_Latn", + "sin_Latn", + "slk_Latn", + "slv_Latn", + "som_Latn", + "sqi_Latn", + "srp_Latn", + "sun_Latn", + "swe_Latn", + "swa_Latn", + "tam_Latn", + "tam_Taml", + "tel_Latn", + "tel_Telu", + "tha_Latn", + "tgl_Latn", + "tur_Latn", + "uig_Latn", + "ukr_Latn", + "urd_Latn", + "urd_Arab", + "uzb_Latn", + "vie_Latn", + "xho_Latn", + "yid_Latn", + "zho_Hant", + "zho_Hans" + ], + "n_parameters": null, + "memory_usage": null, + "max_tokens": null, + "embed_dim": null, + "license": null, + "open_source": true, + "similarity_fn_name": null, + "framework": [], + "loader": "e5_loader" +} \ No newline at end of file diff --git a/results/intfloat__multilingual-e5-small/fd1525a9fd15316a2d503bf26ab031a61d056e98/CEDRClassification.json b/results/intfloat__multilingual-e5-small/fd1525a9fd15316a2d503bf26ab031a61d056e98/CEDRClassification.json new file mode 100644 index 000000000..19262878a --- /dev/null +++ b/results/intfloat__multilingual-e5-small/fd1525a9fd15316a2d503bf26ab031a61d056e98/CEDRClassification.json @@ -0,0 +1,73 @@ +{ + "dataset_revision": "c0ba03d058e3e1b2f3fd20518875a4563dd12db4", + "evaluation_time": 6.30263090133667, + "kg_co2_emissions": null, + "mteb_version": "1.19.4", + "scores": { + "test": [ + { + "accuracy": 0.4039319872476089, + "f1": 0.2606861030501914, + "hf_subset": "default", + "languages": [ + "rus-Cyrl" + ], + "lrap": 0.6377550478214786, + "main_score": 0.4039319872476089, + "scores_per_experiment": [ + { + "accuracy": 0.34537725823591925, + "f1": 0.20125244368654954, + "lrap": 0.5980871413390134 + }, + { + "accuracy": 0.43783209351753455, + "f1": 0.3126131310626406, + "lrap": 0.6590329436769511 + }, + { + "accuracy": 0.38257173219978746, + "f1": 0.2233345762631005, + "lrap": 0.6222104144527216 + }, + { + "accuracy": 0.44739638682252925, + "f1": 0.28615115613761116, + "lrap": 0.6350159404888538 + }, + { + "accuracy": 0.40170031880977686, + "f1": 0.2556544016794049, + "lrap": 0.6630180658873657 + }, + { + "accuracy": 0.3655685441020191, + "f1": 0.24531651350669104, + "lrap": 0.6149309245483653 + }, + { + "accuracy": 0.38629117959617426, + "f1": 0.24148434643427713, + "lrap": 0.6419234856535726 + }, + { + "accuracy": 0.44739638682252925, + "f1": 0.3123380935880936, + "lrap": 0.6834218916046877 + }, + { + "accuracy": 0.41445270988310307, + "f1": 0.22538627207748893, + "lrap": 0.6206695005313608 + }, + { + "accuracy": 0.41073326248671627, + "f1": 0.303330096066057, + "lrap": 0.6392401700318939 + } + ] + } + ] + }, + "task_name": "CEDRClassification" +} \ No newline at end of file diff --git a/results/intfloat__multilingual-e5-small/fd1525a9fd15316a2d503bf26ab031a61d056e98/GeoreviewClassification.json b/results/intfloat__multilingual-e5-small/fd1525a9fd15316a2d503bf26ab031a61d056e98/GeoreviewClassification.json new file mode 100644 index 000000000..20a1b6f5e --- /dev/null +++ b/results/intfloat__multilingual-e5-small/fd1525a9fd15316a2d503bf26ab031a61d056e98/GeoreviewClassification.json @@ -0,0 +1,73 @@ +{ + "dataset_revision": "3765c0d1de6b7d264bc459433c45e5a75513839c", + "evaluation_time": 14.698610305786133, + "kg_co2_emissions": null, + "mteb_version": "1.19.4", + "scores": { + "test": [ + { + "accuracy": 0.423046875, + "f1": 0.40627294178104983, + "f1_weighted": 0.4062463245387987, + "hf_subset": "default", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.423046875, + "scores_per_experiment": [ + { + "accuracy": 0.42529296875, + "f1": 0.4062613411708762, + "f1_weighted": 0.40629576931046285 + }, + { + "accuracy": 0.4267578125, + "f1": 0.4211003062181492, + "f1_weighted": 0.4210603827772984 + }, + { + "accuracy": 0.43505859375, + "f1": 0.40564165138267916, + "f1_weighted": 0.40560529671595674 + }, + { + "accuracy": 0.4560546875, + "f1": 0.43623452643616645, + "f1_weighted": 0.4362295209197421 + }, + { + "accuracy": 0.400390625, + "f1": 0.38445812042139876, + "f1_weighted": 0.38448944250014877 + }, + { + "accuracy": 0.40478515625, + "f1": 0.3920306534479027, + "f1_weighted": 0.39195746716882585 + }, + { + "accuracy": 0.419921875, + "f1": 0.40308357429464775, + "f1_weighted": 0.40305583003546464 + }, + { + "accuracy": 0.44580078125, + "f1": 0.4419482016465282, + "f1_weighted": 0.44190668021905966 + }, + { + "accuracy": 0.36279296875, + "f1": 0.35565502536226984, + "f1_weighted": 0.3556243356621771 + }, + { + "accuracy": 0.45361328125, + "f1": 0.41631601742988017, + "f1_weighted": 0.41623852007885087 + } + ] + } + ] + }, + "task_name": "GeoreviewClassification" +} \ No newline at end of file diff --git a/results/intfloat__multilingual-e5-small/fd1525a9fd15316a2d503bf26ab031a61d056e98/GeoreviewClusteringP2P.json b/results/intfloat__multilingual-e5-small/fd1525a9fd15316a2d503bf26ab031a61d056e98/GeoreviewClusteringP2P.json new file mode 100644 index 000000000..7f560e24f --- /dev/null +++ b/results/intfloat__multilingual-e5-small/fd1525a9fd15316a2d503bf26ab031a61d056e98/GeoreviewClusteringP2P.json @@ -0,0 +1,34 @@ +{ + "dataset_revision": "97a313c8fc85b47f13f33e7e9a95c1ad888c7fec", + "evaluation_time": 68.02217268943787, + "kg_co2_emissions": null, + "mteb_version": "1.19.4", + "scores": { + "test": [ + { + "hf_subset": "default", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.6156149904418479, + "v_measure": 0.6156149904418479, + "v_measure_std": 0.012696880352120467, + "v_measures": { + "Level 0": [ + 0.6270385062951819, + 0.6032317073878031, + 0.6243229012338016, + 0.6057687859892861, + 0.6264471448013151, + 0.6154021384654684, + 0.6330586640633025, + 0.6194591679959552, + 0.6124313338290535, + 0.5889895543573123 + ] + } + } + ] + }, + "task_name": "GeoreviewClusteringP2P" +} \ No newline at end of file diff --git a/results/intfloat__multilingual-e5-small/fd1525a9fd15316a2d503bf26ab031a61d056e98/HeadlineClassification.json b/results/intfloat__multilingual-e5-small/fd1525a9fd15316a2d503bf26ab031a61d056e98/HeadlineClassification.json new file mode 100644 index 000000000..5f1b5438c --- /dev/null +++ b/results/intfloat__multilingual-e5-small/fd1525a9fd15316a2d503bf26ab031a61d056e98/HeadlineClassification.json @@ -0,0 +1,73 @@ +{ + "dataset_revision": "2fe05ee6b5832cda29f2ef7aaad7b7fe6a3609eb", + "evaluation_time": 12.799273252487183, + "kg_co2_emissions": null, + "mteb_version": "1.19.4", + "scores": { + "test": [ + { + "accuracy": 0.73740234375, + "f1": 0.7368256804848748, + "f1_weighted": 0.7368454844803385, + "hf_subset": "default", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.73740234375, + "scores_per_experiment": [ + { + "accuracy": 0.72412109375, + "f1": 0.7238105970908896, + "f1_weighted": 0.7238507233275504 + }, + { + "accuracy": 0.75732421875, + "f1": 0.756323973256969, + "f1_weighted": 0.7563419037964596 + }, + { + "accuracy": 0.724609375, + "f1": 0.7244978411773731, + "f1_weighted": 0.7245059968344618 + }, + { + "accuracy": 0.73779296875, + "f1": 0.7387013633644486, + "f1_weighted": 0.7387007919552269 + }, + { + "accuracy": 0.7421875, + "f1": 0.7448734427343854, + "f1_weighted": 0.7448946958424718 + }, + { + "accuracy": 0.75048828125, + "f1": 0.7511093727244752, + "f1_weighted": 0.7511128967563264 + }, + { + "accuracy": 0.73095703125, + "f1": 0.7287251210809632, + "f1_weighted": 0.7287420646620745 + }, + { + "accuracy": 0.72705078125, + "f1": 0.7253848114297723, + "f1_weighted": 0.7254210262558589 + }, + { + "accuracy": 0.7099609375, + "f1": 0.7049781360594113, + "f1_weighted": 0.7050216716832429 + }, + { + "accuracy": 0.76953125, + "f1": 0.7698521459300599, + "f1_weighted": 0.769863073689712 + } + ] + } + ] + }, + "task_name": "HeadlineClassification" +} \ No newline at end of file diff --git a/results/intfloat__multilingual-e5-small/fd1525a9fd15316a2d503bf26ab031a61d056e98/InappropriatenessClassification.json b/results/intfloat__multilingual-e5-small/fd1525a9fd15316a2d503bf26ab031a61d056e98/InappropriatenessClassification.json new file mode 100644 index 000000000..a25bce31e --- /dev/null +++ b/results/intfloat__multilingual-e5-small/fd1525a9fd15316a2d503bf26ab031a61d056e98/InappropriatenessClassification.json @@ -0,0 +1,95 @@ +{ + "dataset_revision": "601651fdc45ef243751676e62dd7a19f491c0285", + "evaluation_time": 12.428212642669678, + "kg_co2_emissions": null, + "mteb_version": "1.19.4", + "scores": { + "test": [ + { + "accuracy": 0.584423828125, + "ap": 0.5502987014659555, + "ap_weighted": 0.5502987014659555, + "f1": 0.5800244669851191, + "f1_weighted": 0.5800244669851191, + "hf_subset": "default", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.584423828125, + "scores_per_experiment": [ + { + "accuracy": 0.60546875, + "ap": 0.5672262643129771, + "ap_weighted": 0.5672262643129771, + "f1": 0.6000676710991237, + "f1_weighted": 0.6000676710991237 + }, + { + "accuracy": 0.6103515625, + "ap": 0.5657792902317177, + "ap_weighted": 0.5657792902317177, + "f1": 0.60819332566168, + "f1_weighted": 0.60819332566168 + }, + { + "accuracy": 0.62158203125, + "ap": 0.5790063740974729, + "ap_weighted": 0.5790063740974729, + "f1": 0.6181912435606457, + "f1_weighted": 0.6181912435606457 + }, + { + "accuracy": 0.53466796875, + "ap": 0.5185253811713456, + "ap_weighted": 0.5185253811713456, + "f1": 0.5346589821285135, + "f1_weighted": 0.5346589821285135 + }, + { + "accuracy": 0.56298828125, + "ap": 0.5352946496433584, + "ap_weighted": 0.5352946496433584, + "f1": 0.5627771911172896, + "f1_weighted": 0.5627771911172896 + }, + { + "accuracy": 0.56396484375, + "ap": 0.5351492376464475, + "ap_weighted": 0.5351492376464475, + "f1": 0.5544684003495333, + "f1_weighted": 0.5544684003495333 + }, + { + "accuracy": 0.6171875, + "ap": 0.57232666015625, + "ap_weighted": 0.57232666015625, + "f1": 0.6171875, + "f1_weighted": 0.6171875 + }, + { + "accuracy": 0.56201171875, + "ap": 0.5337010983487338, + "ap_weighted": 0.5337010983487338, + "f1": 0.5411185923736085, + "f1_weighted": 0.5411185923736085 + }, + { + "accuracy": 0.57568359375, + "ap": 0.543724924774323, + "ap_weighted": 0.543724924774323, + "f1": 0.5756098317068372, + "f1_weighted": 0.5756098317068372 + }, + { + "accuracy": 0.59033203125, + "ap": 0.5522531342769296, + "ap_weighted": 0.5522531342769296, + "f1": 0.5879719318539599, + "f1_weighted": 0.5879719318539599 + } + ] + } + ] + }, + "task_name": "InappropriatenessClassification" +} \ No newline at end of file diff --git a/results/intfloat__multilingual-e5-small/fd1525a9fd15316a2d503bf26ab031a61d056e98/KinopoiskClassification.json b/results/intfloat__multilingual-e5-small/fd1525a9fd15316a2d503bf26ab031a61d056e98/KinopoiskClassification.json new file mode 100644 index 000000000..84adfe1b0 --- /dev/null +++ b/results/intfloat__multilingual-e5-small/fd1525a9fd15316a2d503bf26ab031a61d056e98/KinopoiskClassification.json @@ -0,0 +1,73 @@ +{ + "dataset_revision": "5911f26666ac11af46cb9c6849d0dc80a378af24", + "evaluation_time": 15.080833196640015, + "kg_co2_emissions": null, + "mteb_version": "1.19.4", + "scores": { + "test": [ + { + "accuracy": 0.47573333333333323, + "f1": 0.46508168540748, + "f1_weighted": 0.46508168540748, + "hf_subset": "default", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.47573333333333323, + "scores_per_experiment": [ + { + "accuracy": 0.404, + "f1": 0.40434158024390915, + "f1_weighted": 0.40434158024390915 + }, + { + "accuracy": 0.4866666666666667, + "f1": 0.47198679046105146, + "f1_weighted": 0.47198679046105146 + }, + { + "accuracy": 0.49466666666666664, + "f1": 0.4811370783030105, + "f1_weighted": 0.4811370783030105 + }, + { + "accuracy": 0.44733333333333336, + "f1": 0.44844883362018234, + "f1_weighted": 0.4484488336201823 + }, + { + "accuracy": 0.4806666666666667, + "f1": 0.47808004908479934, + "f1_weighted": 0.4780800490847993 + }, + { + "accuracy": 0.48933333333333334, + "f1": 0.4711961973486585, + "f1_weighted": 0.4711961973486585 + }, + { + "accuracy": 0.4866666666666667, + "f1": 0.4706516940071077, + "f1_weighted": 0.47065169400710766 + }, + { + "accuracy": 0.47733333333333333, + "f1": 0.478896312108459, + "f1_weighted": 0.47889631210845907 + }, + { + "accuracy": 0.5033333333333333, + "f1": 0.47733021888146904, + "f1_weighted": 0.47733021888146904 + }, + { + "accuracy": 0.48733333333333334, + "f1": 0.468748100016153, + "f1_weighted": 0.468748100016153 + } + ] + } + ] + }, + "task_name": "KinopoiskClassification" +} \ No newline at end of file diff --git a/results/intfloat__multilingual-e5-small/fd1525a9fd15316a2d503bf26ab031a61d056e98/MIRACLReranking.json b/results/intfloat__multilingual-e5-small/fd1525a9fd15316a2d503bf26ab031a61d056e98/MIRACLReranking.json new file mode 100644 index 000000000..63eebcc1b --- /dev/null +++ b/results/intfloat__multilingual-e5-small/fd1525a9fd15316a2d503bf26ab031a61d056e98/MIRACLReranking.json @@ -0,0 +1,130 @@ +{ + "dataset_revision": "6d1962c527217f8927fca80f890f14f36b2802af", + "evaluation_time": 1749.6024370193481, + "kg_co2_emissions": null, + "mteb_version": "1.19.4", + "scores": { + "dev": [ + { + "MAP@1(MIRACL)": 0.33212, + "MAP@10(MIRACL)": 0.51881, + "MAP@100(MIRACL)": 0.53879, + "MAP@1000(MIRACL)": 0.53879, + "MAP@20(MIRACL)": 0.53315, + "MAP@3(MIRACL)": 0.45409, + "MAP@5(MIRACL)": 0.4871, + "NDCG@1(MIRACL)": 0.54451, + "NDCG@10(MIRACL)": 0.59115, + "NDCG@100(MIRACL)": 0.63157, + "NDCG@1000(MIRACL)": 0.63157, + "NDCG@20(MIRACL)": 0.61587, + "NDCG@3(MIRACL)": 0.54029, + "NDCG@5(MIRACL)": 0.55452, + "P@1(MIRACL)": 0.54451, + "P@10(MIRACL)": 0.15108, + "P@100(MIRACL)": 0.01953, + "P@1000(MIRACL)": 0.00195, + "P@20(MIRACL)": 0.08781, + "P@3(MIRACL)": 0.33039, + "P@5(MIRACL)": 0.23962, + "Recall@1(MIRACL)": 0.33212, + "Recall@10(MIRACL)": 0.67895, + "Recall@100(MIRACL)": 0.79952, + "Recall@1000(MIRACL)": 0.79952, + "Recall@20(MIRACL)": 0.74472, + "Recall@3(MIRACL)": 0.51496, + "Recall@5(MIRACL)": 0.58437, + "hf_subset": "ru", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.59115, + "nAUC_MAP@1000_diff1(MIRACL)": 0.19774147136543177, + "nAUC_MAP@1000_max(MIRACL)": 0.3817323586052673, + "nAUC_MAP@1000_std(MIRACL)": 0.16797549526351718, + "nAUC_MAP@100_diff1(MIRACL)": 0.19774147136543177, + "nAUC_MAP@100_max(MIRACL)": 0.3817323586052673, + "nAUC_MAP@100_std(MIRACL)": 0.16797549526351718, + "nAUC_MAP@10_diff1(MIRACL)": 0.21162488648844058, + "nAUC_MAP@10_max(MIRACL)": 0.35981862277052157, + "nAUC_MAP@10_std(MIRACL)": 0.15717577141459818, + "nAUC_MAP@1_diff1(MIRACL)": 0.35001402631999684, + "nAUC_MAP@1_max(MIRACL)": 0.22099771003727614, + "nAUC_MAP@1_std(MIRACL)": 0.05037931081239784, + "nAUC_MAP@20_diff1(MIRACL)": 0.20237478329457756, + "nAUC_MAP@20_max(MIRACL)": 0.3753269010331992, + "nAUC_MAP@20_std(MIRACL)": 0.16666265631852667, + "nAUC_MAP@3_diff1(MIRACL)": 0.2543886605853371, + "nAUC_MAP@3_max(MIRACL)": 0.3062889193889672, + "nAUC_MAP@3_std(MIRACL)": 0.10823082511008097, + "nAUC_MAP@5_diff1(MIRACL)": 0.23062823635124635, + "nAUC_MAP@5_max(MIRACL)": 0.3265414309434945, + "nAUC_MAP@5_std(MIRACL)": 0.12936886907872588, + "nAUC_NDCG@1000_diff1(MIRACL)": 0.12324890517056367, + "nAUC_NDCG@1000_max(MIRACL)": 0.4583142131488544, + "nAUC_NDCG@1000_std(MIRACL)": 0.21471817248927164, + "nAUC_NDCG@100_diff1(MIRACL)": 0.12324890517056367, + "nAUC_NDCG@100_max(MIRACL)": 0.4583142131488544, + "nAUC_NDCG@100_std(MIRACL)": 0.21471817248927164, + "nAUC_NDCG@10_diff1(MIRACL)": 0.15751430164579522, + "nAUC_NDCG@10_max(MIRACL)": 0.4124869356715717, + "nAUC_NDCG@10_std(MIRACL)": 0.20160891462644273, + "nAUC_NDCG@1_diff1(MIRACL)": 0.27560611621680764, + "nAUC_NDCG@1_max(MIRACL)": 0.4506486772279199, + "nAUC_NDCG@1_std(MIRACL)": 0.15476045922295853, + "nAUC_NDCG@20_diff1(MIRACL)": 0.1402170739238101, + "nAUC_NDCG@20_max(MIRACL)": 0.44062115916351247, + "nAUC_NDCG@20_std(MIRACL)": 0.21374939295659431, + "nAUC_NDCG@3_diff1(MIRACL)": 0.19629204268224856, + "nAUC_NDCG@3_max(MIRACL)": 0.3902641954426259, + "nAUC_NDCG@3_std(MIRACL)": 0.15506584996465952, + "nAUC_NDCG@5_diff1(MIRACL)": 0.18359952397621251, + "nAUC_NDCG@5_max(MIRACL)": 0.38033526861211825, + "nAUC_NDCG@5_std(MIRACL)": 0.17174050014711903, + "nAUC_P@1000_diff1(MIRACL)": -0.20935781091387437, + "nAUC_P@1000_max(MIRACL)": 0.2882366727487267, + "nAUC_P@1000_std(MIRACL)": 0.16104207831084227, + "nAUC_P@100_diff1(MIRACL)": -0.20935781091387398, + "nAUC_P@100_max(MIRACL)": 0.2882366727487272, + "nAUC_P@100_std(MIRACL)": 0.16104207831084233, + "nAUC_P@10_diff1(MIRACL)": -0.16483982896764182, + "nAUC_P@10_max(MIRACL)": 0.3247720555223653, + "nAUC_P@10_std(MIRACL)": 0.21440114555134304, + "nAUC_P@1_diff1(MIRACL)": 0.27560611621680764, + "nAUC_P@1_max(MIRACL)": 0.4506486772279199, + "nAUC_P@1_std(MIRACL)": 0.15476045922295853, + "nAUC_P@20_diff1(MIRACL)": -0.19007001343055138, + "nAUC_P@20_max(MIRACL)": 0.3064447297463435, + "nAUC_P@20_std(MIRACL)": 0.19848848635514418, + "nAUC_P@3_diff1(MIRACL)": -0.050207137942680544, + "nAUC_P@3_max(MIRACL)": 0.3914830765671082, + "nAUC_P@3_std(MIRACL)": 0.18372314385940566, + "nAUC_P@5_diff1(MIRACL)": -0.10903438589897832, + "nAUC_P@5_max(MIRACL)": 0.35232166222547523, + "nAUC_P@5_std(MIRACL)": 0.20127676826093152, + "nAUC_Recall@1000_diff1(MIRACL)": -0.1538038509040113, + "nAUC_Recall@1000_max(MIRACL)": 0.6128823385482446, + "nAUC_Recall@1000_std(MIRACL)": 0.31931084830494916, + "nAUC_Recall@100_diff1(MIRACL)": -0.1538038509040113, + "nAUC_Recall@100_max(MIRACL)": 0.6128823385482446, + "nAUC_Recall@100_std(MIRACL)": 0.31931084830494916, + "nAUC_Recall@10_diff1(MIRACL)": 0.030445181023422058, + "nAUC_Recall@10_max(MIRACL)": 0.39897541420658444, + "nAUC_Recall@10_std(MIRACL)": 0.249506602369794, + "nAUC_Recall@1_diff1(MIRACL)": 0.35001402631999684, + "nAUC_Recall@1_max(MIRACL)": 0.22099771003727614, + "nAUC_Recall@1_std(MIRACL)": 0.05037931081239784, + "nAUC_Recall@20_diff1(MIRACL)": -0.03531252982615509, + "nAUC_Recall@20_max(MIRACL)": 0.5018020716948332, + "nAUC_Recall@20_std(MIRACL)": 0.2955064260547226, + "nAUC_Recall@3_diff1(MIRACL)": 0.18220225237185056, + "nAUC_Recall@3_max(MIRACL)": 0.2923966104399364, + "nAUC_Recall@3_std(MIRACL)": 0.1246293811732446, + "nAUC_Recall@5_diff1(MIRACL)": 0.11808973148407408, + "nAUC_Recall@5_max(MIRACL)": 0.3124270870043942, + "nAUC_Recall@5_std(MIRACL)": 0.17449145779385813 + } + ] + }, + "task_name": "MIRACLReranking" +} \ No newline at end of file diff --git a/results/intfloat__multilingual-e5-small/fd1525a9fd15316a2d503bf26ab031a61d056e98/MIRACLRetrieval.json b/results/intfloat__multilingual-e5-small/fd1525a9fd15316a2d503bf26ab031a61d056e98/MIRACLRetrieval.json new file mode 100644 index 000000000..05f06e205 --- /dev/null +++ b/results/intfloat__multilingual-e5-small/fd1525a9fd15316a2d503bf26ab031a61d056e98/MIRACLRetrieval.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "main", + "evaluation_time": 63888.87465238571, + "kg_co2_emissions": null, + "mteb_version": "1.19.4", + "scores": { + "dev": [ + { + "hf_subset": "ru", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.59011, + "map_at_1": 0.28373, + "map_at_10": 0.48535, + "map_at_100": 0.51384, + "map_at_1000": 0.51475, + "map_at_20": 0.50212, + "map_at_3": 0.41545, + "map_at_5": 0.45248, + "mrr_at_1": 0.5575079872204473, + "mrr_at_10": 0.677545134134591, + "mrr_at_100": 0.6811707690274026, + "mrr_at_1000": 0.681229074110741, + "mrr_at_20": 0.680040115492432, + "mrr_at_3": 0.6542864749733759, + "mrr_at_5": 0.6690228966986156, + "nauc_map_at_1000_diff1": 0.2553546152319778, + "nauc_map_at_1000_max": 0.285323703950145, + "nauc_map_at_1000_std": 0.026984361006023527, + "nauc_map_at_100_diff1": 0.2552585506929465, + "nauc_map_at_100_max": 0.2851599705941512, + "nauc_map_at_100_std": 0.026565693128775235, + "nauc_map_at_10_diff1": 0.25146616083216383, + "nauc_map_at_10_max": 0.2679468482647983, + "nauc_map_at_10_std": 0.004522309471081264, + "nauc_map_at_1_diff1": 0.32475115413531297, + "nauc_map_at_1_max": 0.1894625439703188, + "nauc_map_at_1_std": -0.05416591785746409, + "nauc_map_at_20_diff1": 0.2523943244660881, + "nauc_map_at_20_max": 0.27637192714109127, + "nauc_map_at_20_std": 0.015386830722495906, + "nauc_map_at_3_diff1": 0.25257148529542844, + "nauc_map_at_3_max": 0.23053500864480583, + "nauc_map_at_3_std": -0.02164213839442764, + "nauc_map_at_5_diff1": 0.24363579142474037, + "nauc_map_at_5_max": 0.24427531809464095, + "nauc_map_at_5_std": -0.008167246153509269, + "nauc_mrr_at_1000_diff1": 0.36018062805039824, + "nauc_mrr_at_1000_max": 0.40546838901697835, + "nauc_mrr_at_1000_std": 0.12864595119949643, + "nauc_mrr_at_100_diff1": 0.36016310546629726, + "nauc_mrr_at_100_max": 0.40554154078547605, + "nauc_mrr_at_100_std": 0.12872639706386563, + "nauc_mrr_at_10_diff1": 0.35641760929579247, + "nauc_mrr_at_10_max": 0.40634672663994376, + "nauc_mrr_at_10_std": 0.1286405910320357, + "nauc_mrr_at_1_diff1": 0.409203823751519, + "nauc_mrr_at_1_max": 0.377174458901218, + "nauc_mrr_at_1_std": 0.09467529555705953, + "nauc_mrr_at_20_diff1": 0.3598040248926976, + "nauc_mrr_at_20_max": 0.40621939871001206, + "nauc_mrr_at_20_std": 0.12883746787928932, + "nauc_mrr_at_3_diff1": 0.35791703951524406, + "nauc_mrr_at_3_max": 0.40949675495921833, + "nauc_mrr_at_3_std": 0.1357189131702993, + "nauc_mrr_at_5_diff1": 0.35367656250002394, + "nauc_mrr_at_5_max": 0.402545961448335, + "nauc_mrr_at_5_std": 0.12904093994135563, + "nauc_ndcg_at_1000_diff1": 0.2752239233567687, + "nauc_ndcg_at_1000_max": 0.3467783776510215, + "nauc_ndcg_at_1000_std": 0.09274884968522296, + "nauc_ndcg_at_100_diff1": 0.2742124261496497, + "nauc_ndcg_at_100_max": 0.34788216068763084, + "nauc_ndcg_at_100_std": 0.0944070164790139, + "nauc_ndcg_at_10_diff1": 0.2551459381807844, + "nauc_ndcg_at_10_max": 0.3049898130347873, + "nauc_ndcg_at_10_std": 0.032187964196876494, + "nauc_ndcg_at_1_diff1": 0.409203823751519, + "nauc_ndcg_at_1_max": 0.377174458901218, + "nauc_ndcg_at_1_std": 0.09467529555705953, + "nauc_ndcg_at_20_diff1": 0.2639829999732871, + "nauc_ndcg_at_20_max": 0.32259131654617357, + "nauc_ndcg_at_20_std": 0.05370086107839136, + "nauc_ndcg_at_3_diff1": 0.2630824139596539, + "nauc_ndcg_at_3_max": 0.31240210785381084, + "nauc_ndcg_at_3_std": 0.05476404088829716, + "nauc_ndcg_at_5_diff1": 0.24672028262902054, + "nauc_ndcg_at_5_max": 0.2904432727434337, + "nauc_ndcg_at_5_std": 0.03464930558887216, + "nauc_precision_at_1000_diff1": -0.045643281439324425, + "nauc_precision_at_1000_max": 0.14965137093272662, + "nauc_precision_at_1000_std": 0.24108503657752728, + "nauc_precision_at_100_diff1": -0.03444876643253895, + "nauc_precision_at_100_max": 0.18497942923278216, + "nauc_precision_at_100_std": 0.2613619778269954, + "nauc_precision_at_10_diff1": 0.023224780492930044, + "nauc_precision_at_10_max": 0.2625301596609032, + "nauc_precision_at_10_std": 0.1944611596691056, + "nauc_precision_at_1_diff1": 0.409203823751519, + "nauc_precision_at_1_max": 0.377174458901218, + "nauc_precision_at_1_std": 0.09467529555705953, + "nauc_precision_at_20_diff1": -0.0015154695439441907, + "nauc_precision_at_20_max": 0.22554078161333785, + "nauc_precision_at_20_std": 0.22173973690621945, + "nauc_precision_at_3_diff1": 0.0935677165192493, + "nauc_precision_at_3_max": 0.31627375837779825, + "nauc_precision_at_3_std": 0.1706792981984227, + "nauc_precision_at_5_diff1": 0.045720261875154315, + "nauc_precision_at_5_max": 0.278229058877701, + "nauc_precision_at_5_std": 0.1807534420379424, + "nauc_recall_at_1000_diff1": 0.11960317777014781, + "nauc_recall_at_1000_max": 0.6416641514198436, + "nauc_recall_at_1000_std": 0.6612434473855183, + "nauc_recall_at_100_diff1": 0.18037814677895117, + "nauc_recall_at_100_max": 0.41362783480152265, + "nauc_recall_at_100_std": 0.314401658630538, + "nauc_recall_at_10_diff1": 0.13836782712173767, + "nauc_recall_at_10_max": 0.2127325684031103, + "nauc_recall_at_10_std": -0.02605941779115589, + "nauc_recall_at_1_diff1": 0.32475115413531297, + "nauc_recall_at_1_max": 0.1894625439703188, + "nauc_recall_at_1_std": -0.05416591785746409, + "nauc_recall_at_20_diff1": 0.1551508451776284, + "nauc_recall_at_20_max": 0.24487348179528964, + "nauc_recall_at_20_std": 0.008745830804905291, + "nauc_recall_at_3_diff1": 0.17996839357545893, + "nauc_recall_at_3_max": 0.1897967311488864, + "nauc_recall_at_3_std": -0.032145628747650465, + "nauc_recall_at_5_diff1": 0.13465591805356367, + "nauc_recall_at_5_max": 0.17467324990202773, + "nauc_recall_at_5_std": -0.03389593555329314, + "ndcg_at_1": 0.55751, + "ndcg_at_10": 0.59011, + "ndcg_at_100": 0.66318, + "ndcg_at_1000": 0.67385, + "ndcg_at_20": 0.62335, + "ndcg_at_3": 0.53588, + "ndcg_at_5": 0.55138, + "precision_at_1": 0.55751, + "precision_at_10": 0.17204, + "precision_at_100": 0.02585, + "precision_at_1000": 0.00279, + "precision_at_20": 0.10272, + "precision_at_3": 0.36209, + "precision_at_5": 0.27093, + "recall_at_1": 0.28373, + "recall_at_10": 0.68864, + "recall_at_100": 0.92238, + "recall_at_1000": 0.9814, + "recall_at_20": 0.77767, + "recall_at_3": 0.4893, + "recall_at_5": 0.57646 + } + ] + }, + "task_name": "MIRACLRetrieval" +} \ No newline at end of file diff --git a/results/intfloat__multilingual-e5-small/fd1525a9fd15316a2d503bf26ab031a61d056e98/MassiveIntentClassification.json b/results/intfloat__multilingual-e5-small/fd1525a9fd15316a2d503bf26ab031a61d056e98/MassiveIntentClassification.json new file mode 100644 index 000000000..e5d147d36 --- /dev/null +++ b/results/intfloat__multilingual-e5-small/fd1525a9fd15316a2d503bf26ab031a61d056e98/MassiveIntentClassification.json @@ -0,0 +1,137 @@ +{ + "dataset_revision": "4672e20407010da34463acc759c162ca9734bca6", + "evaluation_time": 1373.1658415794373, + "kg_co2_emissions": null, + "mteb_version": "1.19.4", + "scores": { + "test": [ + { + "accuracy": 0.5694687289845326, + "f1": 0.5307147141095014, + "f1_weighted": 0.5584758128832608, + "hf_subset": "ru", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.5694687289845326, + "scores_per_experiment": [ + { + "accuracy": 0.5907868190988568, + "f1": 0.5611148056227053, + "f1_weighted": 0.5822466034831201 + }, + { + "accuracy": 0.5931405514458642, + "f1": 0.5556655711302849, + "f1_weighted": 0.590700393707666 + }, + { + "accuracy": 0.5652320107599192, + "f1": 0.5246905043777108, + "f1_weighted": 0.5594738025764997 + }, + { + "accuracy": 0.5880968392737055, + "f1": 0.5419410324050167, + "f1_weighted": 0.5850596839006869 + }, + { + "accuracy": 0.5628782784129119, + "f1": 0.5194489911828216, + "f1_weighted": 0.5495969276682834 + }, + { + "accuracy": 0.5332885003362475, + "f1": 0.5190966851512107, + "f1_weighted": 0.5195621594413293 + }, + { + "accuracy": 0.5790181573638198, + "f1": 0.5357590955504744, + "f1_weighted": 0.5671428428541702 + }, + { + "accuracy": 0.554808338937458, + "f1": 0.509002510348647, + "f1_weighted": 0.5393986975038177 + }, + { + "accuracy": 0.5497646267652992, + "f1": 0.5109282403227248, + "f1_weighted": 0.5269291942718756 + }, + { + "accuracy": 0.5776731674512441, + "f1": 0.5294997050034175, + "f1_weighted": 0.5646478234251583 + } + ] + } + ], + "validation": [ + { + "accuracy": 0.5757009345794393, + "f1": 0.5188823893319428, + "f1_weighted": 0.5669502392200523, + "hf_subset": "ru", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.5757009345794393, + "scores_per_experiment": [ + { + "accuracy": 0.5804230201672406, + "f1": 0.5296776585193796, + "f1_weighted": 0.573056109873572 + }, + { + "accuracy": 0.602557796360059, + "f1": 0.5395899047953502, + "f1_weighted": 0.5991227258611797 + }, + { + "accuracy": 0.5779636005902608, + "f1": 0.5125653220780408, + "f1_weighted": 0.5744180592969034 + }, + { + "accuracy": 0.6000983767830792, + "f1": 0.5443706159810089, + "f1_weighted": 0.604142719334455 + }, + { + "accuracy": 0.5681259222823414, + "f1": 0.4991333663204531, + "f1_weighted": 0.5578109649457884 + }, + { + "accuracy": 0.5518937530742745, + "f1": 0.5123670381359348, + "f1_weighted": 0.5407090999681644 + }, + { + "accuracy": 0.5715691096901131, + "f1": 0.5153659027514236, + "f1_weighted": 0.5589157773990072 + }, + { + "accuracy": 0.5558288243974422, + "f1": 0.49925779962495037, + "f1_weighted": 0.5448447638537954 + }, + { + "accuracy": 0.5607476635514018, + "f1": 0.5110426859222787, + "f1_weighted": 0.5420738292518411 + }, + { + "accuracy": 0.58780127889818, + "f1": 0.5254535991906079, + "f1_weighted": 0.5744083424158165 + } + ] + } + ] + }, + "task_name": "MassiveIntentClassification" +} \ No newline at end of file diff --git a/results/intfloat__multilingual-e5-small/fd1525a9fd15316a2d503bf26ab031a61d056e98/MassiveScenarioClassification.json b/results/intfloat__multilingual-e5-small/fd1525a9fd15316a2d503bf26ab031a61d056e98/MassiveScenarioClassification.json new file mode 100644 index 000000000..6f257ba16 --- /dev/null +++ b/results/intfloat__multilingual-e5-small/fd1525a9fd15316a2d503bf26ab031a61d056e98/MassiveScenarioClassification.json @@ -0,0 +1,137 @@ +{ + "dataset_revision": "fad2c6e8459f9e1c45d9315f4953d921437d70f8", + "evaluation_time": 273.4347870349884, + "kg_co2_emissions": null, + "mteb_version": "1.19.4", + "scores": { + "test": [ + { + "accuracy": 0.6274714189643579, + "f1": 0.6232500105216028, + "f1_weighted": 0.6177231229561672, + "hf_subset": "ru", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.6274714189643579, + "scores_per_experiment": [ + { + "accuracy": 0.6577000672494956, + "f1": 0.6576322752104935, + "f1_weighted": 0.6493740224497979 + }, + { + "accuracy": 0.6422326832548756, + "f1": 0.6408440644199842, + "f1_weighted": 0.6326577758515171 + }, + { + "accuracy": 0.6472763954270343, + "f1": 0.6305996886519358, + "f1_weighted": 0.6425527348672134 + }, + { + "accuracy": 0.6264290517821116, + "f1": 0.6204764988284478, + "f1_weighted": 0.6199108830257698 + }, + { + "accuracy": 0.617014122394082, + "f1": 0.603089923839662, + "f1_weighted": 0.5966542452337902 + }, + { + "accuracy": 0.574310692669805, + "f1": 0.5695770003122406, + "f1_weighted": 0.5495348145157877 + }, + { + "accuracy": 0.6284465366509752, + "f1": 0.6193681673903165, + "f1_weighted": 0.6212949557775685 + }, + { + "accuracy": 0.644250168123739, + "f1": 0.6430675420392971, + "f1_weighted": 0.6392111062281107 + }, + { + "accuracy": 0.6264290517821116, + "f1": 0.6323223501500985, + "f1_weighted": 0.6261800264557543 + }, + { + "accuracy": 0.6106254203093476, + "f1": 0.6155225943735526, + "f1_weighted": 0.5998606651563613 + } + ] + } + ], + "validation": [ + { + "accuracy": 0.6265617314313822, + "f1": 0.6235587885501526, + "f1_weighted": 0.6165170759513978, + "hf_subset": "ru", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.6265617314313822, + "scores_per_experiment": [ + { + "accuracy": 0.6689621249385145, + "f1": 0.6666305846554667, + "f1_weighted": 0.6636893748230799 + }, + { + "accuracy": 0.6301032956222331, + "f1": 0.6323778007926625, + "f1_weighted": 0.6216310538552833 + }, + { + "accuracy": 0.6409247417609444, + "f1": 0.6217412500421139, + "f1_weighted": 0.6353967322731422 + }, + { + "accuracy": 0.617314313821938, + "f1": 0.6151965956303116, + "f1_weighted": 0.607135576315962 + }, + { + "accuracy": 0.6089522872602066, + "f1": 0.5975153486980335, + "f1_weighted": 0.5879762966147705 + }, + { + "accuracy": 0.5636989670437776, + "f1": 0.571808066723555, + "f1_weighted": 0.5401520697291556 + }, + { + "accuracy": 0.6404328578455485, + "f1": 0.6307970090546159, + "f1_weighted": 0.6343348109097958 + }, + { + "accuracy": 0.6414166256763404, + "f1": 0.6392897437434121, + "f1_weighted": 0.634562398636677 + }, + { + "accuracy": 0.6266601082144614, + "f1": 0.635225572558537, + "f1_weighted": 0.6226767108776795 + }, + { + "accuracy": 0.6271519921298574, + "f1": 0.625005913602817, + "f1_weighted": 0.6176157354784336 + } + ] + } + ] + }, + "task_name": "MassiveScenarioClassification" +} \ No newline at end of file diff --git a/results/intfloat__multilingual-e5-small/fd1525a9fd15316a2d503bf26ab031a61d056e98/RUParaPhraserSTS.json b/results/intfloat__multilingual-e5-small/fd1525a9fd15316a2d503bf26ab031a61d056e98/RUParaPhraserSTS.json new file mode 100644 index 000000000..314920259 --- /dev/null +++ b/results/intfloat__multilingual-e5-small/fd1525a9fd15316a2d503bf26ab031a61d056e98/RUParaPhraserSTS.json @@ -0,0 +1,26 @@ +{ + "dataset_revision": "43265056790b8f7c59e0139acb4be0a8dad2c8f4", + "evaluation_time": 1.9035377502441406, + "kg_co2_emissions": null, + "mteb_version": "1.19.4", + "scores": { + "test": [ + { + "cosine_pearson": 0.655870724458462, + "cosine_spearman": 0.7144624426376325, + "euclidean_pearson": 0.6881974875250008, + "euclidean_spearman": 0.7144624426376325, + "hf_subset": "default", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.7144624426376325, + "manhattan_pearson": 0.6868163089463737, + "manhattan_spearman": 0.7125525565771754, + "pearson": 0.655870724458462, + "spearman": 0.7144624426376325 + } + ] + }, + "task_name": "RUParaPhraserSTS" +} \ No newline at end of file diff --git a/results/intfloat__multilingual-e5-small/fd1525a9fd15316a2d503bf26ab031a61d056e98/RiaNewsRetrieval.json b/results/intfloat__multilingual-e5-small/fd1525a9fd15316a2d503bf26ab031a61d056e98/RiaNewsRetrieval.json new file mode 100644 index 000000000..371c64343 --- /dev/null +++ b/results/intfloat__multilingual-e5-small/fd1525a9fd15316a2d503bf26ab031a61d056e98/RiaNewsRetrieval.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "82374b0bbacda6114f39ff9c5b925fa1512ca5d7", + "evaluation_time": 1126.6029734611511, + "kg_co2_emissions": null, + "mteb_version": "1.19.4", + "scores": { + "test": [ + { + "hf_subset": "default", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.70001, + "map_at_1": 0.5594, + "map_at_10": 0.65585, + "map_at_100": 0.66045, + "map_at_1000": 0.66061, + "map_at_20": 0.65879, + "map_at_3": 0.63737, + "map_at_5": 0.64831, + "mrr_at_1": 0.5599, + "mrr_at_10": 0.6561349206349206, + "mrr_at_100": 0.6607380118967974, + "mrr_at_1000": 0.6609026980667922, + "mrr_at_20": 0.6590839414361008, + "mrr_at_3": 0.63765, + "mrr_at_5": 0.6486149999999999, + "nauc_map_at_1000_diff1": 0.6599933806389039, + "nauc_map_at_1000_max": 0.28173426373323296, + "nauc_map_at_1000_std": -0.1322057973223156, + "nauc_map_at_100_diff1": 0.6599361239369486, + "nauc_map_at_100_max": 0.28174241167912817, + "nauc_map_at_100_std": -0.13212711154715356, + "nauc_map_at_10_diff1": 0.6592975416059462, + "nauc_map_at_10_max": 0.2809754689772448, + "nauc_map_at_10_std": -0.13407013951023197, + "nauc_map_at_1_diff1": 0.69381289363268, + "nauc_map_at_1_max": 0.2632013325385613, + "nauc_map_at_1_std": -0.1456136279361001, + "nauc_map_at_20_diff1": 0.6595799630775688, + "nauc_map_at_20_max": 0.28167001840177874, + "nauc_map_at_20_std": -0.132719801108789, + "nauc_map_at_3_diff1": 0.6604634961809234, + "nauc_map_at_3_max": 0.282148792118497, + "nauc_map_at_3_std": -0.13969459383063138, + "nauc_map_at_5_diff1": 0.6594205963271544, + "nauc_map_at_5_max": 0.2822729578616017, + "nauc_map_at_5_std": -0.13584946742081913, + "nauc_mrr_at_1000_diff1": 0.6592635818683523, + "nauc_mrr_at_1000_max": 0.28242755519206025, + "nauc_mrr_at_1000_std": -0.13193475502892094, + "nauc_mrr_at_100_diff1": 0.6592067464817776, + "nauc_mrr_at_100_max": 0.2824352833385982, + "nauc_mrr_at_100_std": -0.13185620144040464, + "nauc_mrr_at_10_diff1": 0.658579169525927, + "nauc_mrr_at_10_max": 0.2816605238866574, + "nauc_mrr_at_10_std": -0.1338006471207937, + "nauc_mrr_at_1_diff1": 0.692670440860189, + "nauc_mrr_at_1_max": 0.26415943528746516, + "nauc_mrr_at_1_std": -0.14512629004876368, + "nauc_mrr_at_20_diff1": 0.6588548284853936, + "nauc_mrr_at_20_max": 0.2823596520676295, + "nauc_mrr_at_20_std": -0.13244969271145313, + "nauc_mrr_at_3_diff1": 0.6597824468476589, + "nauc_mrr_at_3_max": 0.2827967454518486, + "nauc_mrr_at_3_std": -0.1394324933759171, + "nauc_mrr_at_5_diff1": 0.6587099211969666, + "nauc_mrr_at_5_max": 0.2829159614137189, + "nauc_mrr_at_5_std": -0.13560975190012725, + "nauc_ndcg_at_1000_diff1": 0.6507085111729919, + "nauc_ndcg_at_1000_max": 0.2907499645874372, + "nauc_ndcg_at_1000_std": -0.11638868873416415, + "nauc_ndcg_at_100_diff1": 0.6489964012075488, + "nauc_ndcg_at_100_max": 0.291662150431441, + "nauc_ndcg_at_100_std": -0.11246791008974727, + "nauc_ndcg_at_10_diff1": 0.645762255286488, + "nauc_ndcg_at_10_max": 0.28717805875707775, + "nauc_ndcg_at_10_std": -0.12397669675282848, + "nauc_ndcg_at_1_diff1": 0.69381289363268, + "nauc_ndcg_at_1_max": 0.2632013325385613, + "nauc_ndcg_at_1_std": -0.1456136279361001, + "nauc_ndcg_at_20_diff1": 0.6462959681259094, + "nauc_ndcg_at_20_max": 0.290151577552033, + "nauc_ndcg_at_20_std": -0.11815523692703374, + "nauc_ndcg_at_3_diff1": 0.6491112344304072, + "nauc_ndcg_at_3_max": 0.2892236646642829, + "nauc_ndcg_at_3_std": -0.13699185949682374, + "nauc_ndcg_at_5_diff1": 0.6465030270759105, + "nauc_ndcg_at_5_max": 0.2898979945179426, + "nauc_ndcg_at_5_std": -0.12915998022414235, + "nauc_precision_at_1000_diff1": 0.4885481417002382, + "nauc_precision_at_1000_max": 0.5712983732669638, + "nauc_precision_at_1000_std": 0.3788952499990643, + "nauc_precision_at_100_diff1": 0.5337467232678826, + "nauc_precision_at_100_max": 0.43819333062207977, + "nauc_precision_at_100_std": 0.2138706488576936, + "nauc_precision_at_10_diff1": 0.5766571169774445, + "nauc_precision_at_10_max": 0.31779694837242034, + "nauc_precision_at_10_std": -0.06624839914718025, + "nauc_precision_at_1_diff1": 0.69381289363268, + "nauc_precision_at_1_max": 0.2632013325385613, + "nauc_precision_at_1_std": -0.1456136279361001, + "nauc_precision_at_20_diff1": 0.5593570036001682, + "nauc_precision_at_20_max": 0.34986401733887434, + "nauc_precision_at_20_std": -0.003651846515932617, + "nauc_precision_at_3_diff1": 0.6094100093991508, + "nauc_precision_at_3_max": 0.31422239034357674, + "nauc_precision_at_3_std": -0.1272576556537896, + "nauc_precision_at_5_diff1": 0.5945050519543406, + "nauc_precision_at_5_max": 0.3207638712418377, + "nauc_precision_at_5_std": -0.10024459103498598, + "nauc_recall_at_1000_diff1": 0.48854814170024186, + "nauc_recall_at_1000_max": 0.5712983732669716, + "nauc_recall_at_1000_std": 0.3788952499990672, + "nauc_recall_at_100_diff1": 0.5337467232678822, + "nauc_recall_at_100_max": 0.43819333062207905, + "nauc_recall_at_100_std": 0.21387064885769397, + "nauc_recall_at_10_diff1": 0.5766571169774447, + "nauc_recall_at_10_max": 0.31779694837242134, + "nauc_recall_at_10_std": -0.0662483991471789, + "nauc_recall_at_1_diff1": 0.69381289363268, + "nauc_recall_at_1_max": 0.2632013325385613, + "nauc_recall_at_1_std": -0.1456136279361001, + "nauc_recall_at_20_diff1": 0.5593570036001682, + "nauc_recall_at_20_max": 0.34986401733887557, + "nauc_recall_at_20_std": -0.003651846515931506, + "nauc_recall_at_3_diff1": 0.6094100093991499, + "nauc_recall_at_3_max": 0.3142223903435761, + "nauc_recall_at_3_std": -0.12725765565378966, + "nauc_recall_at_5_diff1": 0.5945050519543412, + "nauc_recall_at_5_max": 0.32076387124183864, + "nauc_recall_at_5_std": -0.10024459103498472, + "ndcg_at_1": 0.5594, + "ndcg_at_10": 0.70001, + "ndcg_at_100": 0.72192, + "ndcg_at_1000": 0.72646, + "ndcg_at_20": 0.71059, + "ndcg_at_3": 0.66218, + "ndcg_at_5": 0.68182, + "precision_at_1": 0.5594, + "precision_at_10": 0.08374, + "precision_at_100": 0.00939, + "precision_at_1000": 0.00097, + "precision_at_20": 0.04395, + "precision_at_3": 0.2446, + "precision_at_5": 0.15626, + "recall_at_1": 0.5594, + "recall_at_10": 0.8374, + "recall_at_100": 0.9387, + "recall_at_1000": 0.9749, + "recall_at_20": 0.8789, + "recall_at_3": 0.7338, + "recall_at_5": 0.7813 + } + ] + }, + "task_name": "RiaNewsRetrieval" +} \ No newline at end of file diff --git a/results/intfloat__multilingual-e5-small/fd1525a9fd15316a2d503bf26ab031a61d056e98/RuBQReranking.json b/results/intfloat__multilingual-e5-small/fd1525a9fd15316a2d503bf26ab031a61d056e98/RuBQReranking.json new file mode 100644 index 000000000..61feeeb61 --- /dev/null +++ b/results/intfloat__multilingual-e5-small/fd1525a9fd15316a2d503bf26ab031a61d056e98/RuBQReranking.json @@ -0,0 +1,26 @@ +{ + "dataset_revision": "2e96b8f098fa4b0950fc58eacadeb31c0d0c7fa2", + "evaluation_time": 40.56157875061035, + "kg_co2_emissions": null, + "mteb_version": "1.19.4", + "scores": { + "test": [ + { + "hf_subset": "default", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.7145735496700822, + "map": 0.7145735496700822, + "mrr": 0.7781615547572995, + "nAUC_map_diff1": 0.3811431904511653, + "nAUC_map_max": 0.27537544321915625, + "nAUC_map_std": 0.16793659347828613, + "nAUC_mrr_diff1": 0.4438746320013376, + "nAUC_mrr_max": 0.3646508109947836, + "nAUC_mrr_std": 0.22371386645066563 + } + ] + }, + "task_name": "RuBQReranking" +} \ No newline at end of file diff --git a/results/intfloat__multilingual-e5-small/fd1525a9fd15316a2d503bf26ab031a61d056e98/RuBQRetrieval.json b/results/intfloat__multilingual-e5-small/fd1525a9fd15316a2d503bf26ab031a61d056e98/RuBQRetrieval.json new file mode 100644 index 000000000..e2f94d76e --- /dev/null +++ b/results/intfloat__multilingual-e5-small/fd1525a9fd15316a2d503bf26ab031a61d056e98/RuBQRetrieval.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "e19b6ffa60b3bc248e0b41f4cc37c26a55c2a67b", + "evaluation_time": 47.21118927001953, + "kg_co2_emissions": null, + "mteb_version": "1.19.4", + "scores": { + "test": [ + { + "hf_subset": "default", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.68529, + "map_at_1": 0.42529, + "map_at_10": 0.60864, + "map_at_100": 0.61868, + "map_at_1000": 0.61907, + "map_at_20": 0.61596, + "map_at_3": 0.55701, + "map_at_5": 0.5878, + "mrr_at_1": 0.6057919621749409, + "mrr_at_10": 0.7055614188149649, + "mrr_at_100": 0.70883838166645, + "mrr_at_1000": 0.7089719252668835, + "mrr_at_20": 0.7079839750105351, + "mrr_at_3": 0.6845941686367217, + "mrr_at_5": 0.6967100078802206, + "nauc_map_at_1000_diff1": 0.4067438787885424, + "nauc_map_at_1000_max": 0.32799822709339854, + "nauc_map_at_1000_std": -0.06873393559338634, + "nauc_map_at_100_diff1": 0.40656436644432836, + "nauc_map_at_100_max": 0.3281594799919249, + "nauc_map_at_100_std": -0.06847324679449819, + "nauc_map_at_10_diff1": 0.4039048268484908, + "nauc_map_at_10_max": 0.3240324216147953, + "nauc_map_at_10_std": -0.07344413799841244, + "nauc_map_at_1_diff1": 0.4436306892906905, + "nauc_map_at_1_max": 0.2561348630699028, + "nauc_map_at_1_std": -0.08713074613333902, + "nauc_map_at_20_diff1": 0.40530326570124614, + "nauc_map_at_20_max": 0.3274028319323205, + "nauc_map_at_20_std": -0.07008180779820569, + "nauc_map_at_3_diff1": 0.40764924859364043, + "nauc_map_at_3_max": 0.29809671682025335, + "nauc_map_at_3_std": -0.09205620202725563, + "nauc_map_at_5_diff1": 0.4088599496021476, + "nauc_map_at_5_max": 0.321701894666848, + "nauc_map_at_5_std": -0.07801251849010624, + "nauc_mrr_at_1000_diff1": 0.48641813735407285, + "nauc_mrr_at_1000_max": 0.40136947990653543, + "nauc_mrr_at_1000_std": -0.07250260497468805, + "nauc_mrr_at_100_diff1": 0.4863349902496212, + "nauc_mrr_at_100_max": 0.4014510559704008, + "nauc_mrr_at_100_std": -0.07228702374801103, + "nauc_mrr_at_10_diff1": 0.48585805601948134, + "nauc_mrr_at_10_max": 0.4015075599433366, + "nauc_mrr_at_10_std": -0.07267928771548689, + "nauc_mrr_at_1_diff1": 0.5147535097164919, + "nauc_mrr_at_1_max": 0.3823579750430856, + "nauc_mrr_at_1_std": -0.09187785187137633, + "nauc_mrr_at_20_diff1": 0.48586883783362217, + "nauc_mrr_at_20_max": 0.40134087440882993, + "nauc_mrr_at_20_std": -0.07283132775160146, + "nauc_mrr_at_3_diff1": 0.4866833005454742, + "nauc_mrr_at_3_max": 0.40079873336380384, + "nauc_mrr_at_3_std": -0.07738819947521418, + "nauc_mrr_at_5_diff1": 0.48765363059415373, + "nauc_mrr_at_5_max": 0.4038192973952218, + "nauc_mrr_at_5_std": -0.07592858318378928, + "nauc_ndcg_at_1000_diff1": 0.4167304096882066, + "nauc_ndcg_at_1000_max": 0.3584127926915261, + "nauc_ndcg_at_1000_std": -0.04789690555800973, + "nauc_ndcg_at_100_diff1": 0.41169188501857834, + "nauc_ndcg_at_100_max": 0.3608246196232657, + "nauc_ndcg_at_100_std": -0.040924422516972696, + "nauc_ndcg_at_10_diff1": 0.40300065598615203, + "nauc_ndcg_at_10_max": 0.3487866296788365, + "nauc_ndcg_at_10_std": -0.05866529277842453, + "nauc_ndcg_at_1_diff1": 0.5174612915209496, + "nauc_ndcg_at_1_max": 0.3771907067970078, + "nauc_ndcg_at_1_std": -0.09064124266098696, + "nauc_ndcg_at_20_diff1": 0.40493949850214583, + "nauc_ndcg_at_20_max": 0.3569331503650286, + "nauc_ndcg_at_20_std": -0.04995310342975384, + "nauc_ndcg_at_3_diff1": 0.4126944321211236, + "nauc_ndcg_at_3_max": 0.3257284446095334, + "nauc_ndcg_at_3_std": -0.09063015396458791, + "nauc_ndcg_at_5_diff1": 0.41370396525228875, + "nauc_ndcg_at_5_max": 0.34674160113935715, + "nauc_ndcg_at_5_std": -0.07106845569862319, + "nauc_precision_at_1000_diff1": -0.09571769961090155, + "nauc_precision_at_1000_max": 0.055747825834171885, + "nauc_precision_at_1000_std": 0.0728333847923847, + "nauc_precision_at_100_diff1": -0.07740501200338373, + "nauc_precision_at_100_max": 0.09677453550703531, + "nauc_precision_at_100_std": 0.09327890294080993, + "nauc_precision_at_10_diff1": -0.010068796475329309, + "nauc_precision_at_10_max": 0.15899825481231064, + "nauc_precision_at_10_std": 0.042284084852153105, + "nauc_precision_at_1_diff1": 0.5174612915209496, + "nauc_precision_at_1_max": 0.3771907067970078, + "nauc_precision_at_1_std": -0.09064124266098696, + "nauc_precision_at_20_diff1": -0.04982301544401409, + "nauc_precision_at_20_max": 0.13241674471380568, + "nauc_precision_at_20_std": 0.0705228013382154, + "nauc_precision_at_3_diff1": 0.15442614376387373, + "nauc_precision_at_3_max": 0.2512695418083, + "nauc_precision_at_3_std": -0.03115006669792064, + "nauc_precision_at_5_diff1": 0.08381026072692445, + "nauc_precision_at_5_max": 0.22839056540604824, + "nauc_precision_at_5_std": 0.01512690548652433, + "nauc_recall_at_1000_diff1": -0.008869709920433502, + "nauc_recall_at_1000_max": 0.4509232443337726, + "nauc_recall_at_1000_std": 0.6221264093315108, + "nauc_recall_at_100_diff1": 0.16036715011075714, + "nauc_recall_at_100_max": 0.3979963411771158, + "nauc_recall_at_100_std": 0.2841850069503361, + "nauc_recall_at_10_diff1": 0.2518962279448, + "nauc_recall_at_10_max": 0.3082355277039427, + "nauc_recall_at_10_std": 0.000964544736531047, + "nauc_recall_at_1_diff1": 0.4436306892906905, + "nauc_recall_at_1_max": 0.2561348630699028, + "nauc_recall_at_1_std": -0.08713074613333902, + "nauc_recall_at_20_diff1": 0.2043424504746087, + "nauc_recall_at_20_max": 0.3396010554649377, + "nauc_recall_at_20_std": 0.06900984030301936, + "nauc_recall_at_3_diff1": 0.33865318587934923, + "nauc_recall_at_3_max": 0.2772569225671119, + "nauc_recall_at_3_std": -0.08533124289305709, + "nauc_recall_at_5_diff1": 0.32006964557701684, + "nauc_recall_at_5_max": 0.31493370659289804, + "nauc_recall_at_5_std": -0.048639793547793256, + "ndcg_at_1": 0.60461, + "ndcg_at_10": 0.68529, + "ndcg_at_100": 0.71664, + "ndcg_at_1000": 0.72396, + "ndcg_at_20": 0.70344, + "ndcg_at_3": 0.6155, + "ndcg_at_5": 0.64948, + "precision_at_1": 0.60461, + "precision_at_10": 0.1328, + "precision_at_100": 0.01555, + "precision_at_1000": 0.00164, + "precision_at_20": 0.07216, + "precision_at_3": 0.33077, + "precision_at_5": 0.23014, + "recall_at_1": 0.42529, + "recall_at_10": 0.81169, + "recall_at_100": 0.93154, + "recall_at_1000": 0.98183, + "recall_at_20": 0.87132, + "recall_at_3": 0.63905, + "recall_at_5": 0.71967 + } + ] + }, + "task_name": "RuBQRetrieval" +} \ No newline at end of file diff --git a/results/intfloat__multilingual-e5-small/fd1525a9fd15316a2d503bf26ab031a61d056e98/RuReviewsClassification.json b/results/intfloat__multilingual-e5-small/fd1525a9fd15316a2d503bf26ab031a61d056e98/RuReviewsClassification.json new file mode 100644 index 000000000..3bc1d4c1f --- /dev/null +++ b/results/intfloat__multilingual-e5-small/fd1525a9fd15316a2d503bf26ab031a61d056e98/RuReviewsClassification.json @@ -0,0 +1,73 @@ +{ + "dataset_revision": "f6d2c31f4dc6b88f468552750bfec05b4b41b05a", + "evaluation_time": 13.412413120269775, + "kg_co2_emissions": null, + "mteb_version": "1.19.4", + "scores": { + "test": [ + { + "accuracy": 0.6064453125, + "f1": 0.5959307702980192, + "f1_weighted": 0.5959354041959388, + "hf_subset": "default", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.6064453125, + "scores_per_experiment": [ + { + "accuracy": 0.6318359375, + "f1": 0.6292213707533455, + "f1_weighted": 0.6292289219042227 + }, + { + "accuracy": 0.5224609375, + "f1": 0.4947457821550861, + "f1_weighted": 0.49481337197104047 + }, + { + "accuracy": 0.61572265625, + "f1": 0.6168475257585656, + "f1_weighted": 0.6168302606674755 + }, + { + "accuracy": 0.64501953125, + "f1": 0.6442740771488549, + "f1_weighted": 0.6442734401160441 + }, + { + "accuracy": 0.64599609375, + "f1": 0.6430137891119658, + "f1_weighted": 0.6430072619158896 + }, + { + "accuracy": 0.5537109375, + "f1": 0.5380866809329515, + "f1_weighted": 0.5381276558223966 + }, + { + "accuracy": 0.62060546875, + "f1": 0.6048385654790182, + "f1_weighted": 0.6048360165436525 + }, + { + "accuracy": 0.59033203125, + "f1": 0.5721374912588382, + "f1_weighted": 0.5721349951239295 + }, + { + "accuracy": 0.59423828125, + "f1": 0.5792890149320954, + "f1_weighted": 0.5792452134679961 + }, + { + "accuracy": 0.64453125, + "f1": 0.6368534054494702, + "f1_weighted": 0.6368569044267405 + } + ] + } + ] + }, + "task_name": "RuReviewsClassification" +} \ No newline at end of file diff --git a/results/intfloat__multilingual-e5-small/fd1525a9fd15316a2d503bf26ab031a61d056e98/RuSTSBenchmarkSTS.json b/results/intfloat__multilingual-e5-small/fd1525a9fd15316a2d503bf26ab031a61d056e98/RuSTSBenchmarkSTS.json new file mode 100644 index 000000000..4482c550f --- /dev/null +++ b/results/intfloat__multilingual-e5-small/fd1525a9fd15316a2d503bf26ab031a61d056e98/RuSTSBenchmarkSTS.json @@ -0,0 +1,26 @@ +{ + "dataset_revision": "7cf24f325c6da6195df55bef3d86b5e0616f3018", + "evaluation_time": 1.5642039775848389, + "kg_co2_emissions": null, + "mteb_version": "1.19.4", + "scores": { + "test": [ + { + "cosine_pearson": 0.780489366956755, + "cosine_spearman": 0.7772461027539413, + "euclidean_pearson": 0.7688724676178713, + "euclidean_spearman": 0.7772428793680338, + "hf_subset": "default", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.7772461027539413, + "manhattan_pearson": 0.7672934108893003, + "manhattan_spearman": 0.7758812652457538, + "pearson": 0.780489366956755, + "spearman": 0.7772461027539413 + } + ] + }, + "task_name": "RuSTSBenchmarkSTS" +} \ No newline at end of file diff --git a/results/intfloat__multilingual-e5-small/fd1525a9fd15316a2d503bf26ab031a61d056e98/RuSciBenchGRNTIClassification.json b/results/intfloat__multilingual-e5-small/fd1525a9fd15316a2d503bf26ab031a61d056e98/RuSciBenchGRNTIClassification.json new file mode 100644 index 000000000..d51ae7d05 --- /dev/null +++ b/results/intfloat__multilingual-e5-small/fd1525a9fd15316a2d503bf26ab031a61d056e98/RuSciBenchGRNTIClassification.json @@ -0,0 +1,73 @@ +{ + "dataset_revision": "673a610d6d3dd91a547a0d57ae1b56f37ebbf6a1", + "evaluation_time": 19.240819692611694, + "kg_co2_emissions": null, + "mteb_version": "1.19.4", + "scores": { + "test": [ + { + "accuracy": 0.535888671875, + "f1": 0.5210574042694371, + "f1_weighted": 0.521221743411326, + "hf_subset": "default", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.535888671875, + "scores_per_experiment": [ + { + "accuracy": 0.5439453125, + "f1": 0.5263606964925389, + "f1_weighted": 0.5265402459980028 + }, + { + "accuracy": 0.5361328125, + "f1": 0.5239255540910781, + "f1_weighted": 0.524101747694495 + }, + { + "accuracy": 0.52587890625, + "f1": 0.5147638083564116, + "f1_weighted": 0.5149641877026432 + }, + { + "accuracy": 0.56005859375, + "f1": 0.5487350557415427, + "f1_weighted": 0.548894806822972 + }, + { + "accuracy": 0.541015625, + "f1": 0.5304532553514425, + "f1_weighted": 0.5305331344975358 + }, + { + "accuracy": 0.50048828125, + "f1": 0.480416441703842, + "f1_weighted": 0.4805965220942894 + }, + { + "accuracy": 0.54248046875, + "f1": 0.522217091817123, + "f1_weighted": 0.5224468347560739 + }, + { + "accuracy": 0.52783203125, + "f1": 0.5113532709549421, + "f1_weighted": 0.5115086385403788 + }, + { + "accuracy": 0.54296875, + "f1": 0.5294004464115111, + "f1_weighted": 0.5295651326488123 + }, + { + "accuracy": 0.5380859375, + "f1": 0.5229484217739401, + "f1_weighted": 0.5230661833580562 + } + ] + } + ] + }, + "task_name": "RuSciBenchGRNTIClassification" +} \ No newline at end of file diff --git a/results/intfloat__multilingual-e5-small/fd1525a9fd15316a2d503bf26ab031a61d056e98/RuSciBenchGRNTIClusteringP2P.json b/results/intfloat__multilingual-e5-small/fd1525a9fd15316a2d503bf26ab031a61d056e98/RuSciBenchGRNTIClusteringP2P.json new file mode 100644 index 000000000..996482200 --- /dev/null +++ b/results/intfloat__multilingual-e5-small/fd1525a9fd15316a2d503bf26ab031a61d056e98/RuSciBenchGRNTIClusteringP2P.json @@ -0,0 +1,34 @@ +{ + "dataset_revision": "673a610d6d3dd91a547a0d57ae1b56f37ebbf6a1", + "evaluation_time": 77.4835524559021, + "kg_co2_emissions": null, + "mteb_version": "1.19.4", + "scores": { + "test": [ + { + "hf_subset": "default", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.4934482704545653, + "v_measure": 0.4934482704545653, + "v_measure_std": 0.007395830548234845, + "v_measures": { + "Level 0": [ + 0.49375026677205286, + 0.48371676343170933, + 0.49573238720822055, + 0.4865622604515397, + 0.49319459886229944, + 0.5039326393126564, + 0.4856073466289032, + 0.49507760956529057, + 0.48909195975927144, + 0.5078168725537094 + ] + } + } + ] + }, + "task_name": "RuSciBenchGRNTIClusteringP2P" +} \ No newline at end of file diff --git a/results/intfloat__multilingual-e5-small/fd1525a9fd15316a2d503bf26ab031a61d056e98/RuSciBenchOECDClassification.json b/results/intfloat__multilingual-e5-small/fd1525a9fd15316a2d503bf26ab031a61d056e98/RuSciBenchOECDClassification.json new file mode 100644 index 000000000..6b0a654bb --- /dev/null +++ b/results/intfloat__multilingual-e5-small/fd1525a9fd15316a2d503bf26ab031a61d056e98/RuSciBenchOECDClassification.json @@ -0,0 +1,73 @@ +{ + "dataset_revision": "26c88e99dcaba32bb45d0e1bfc21902337f6d471", + "evaluation_time": 19.014281511306763, + "kg_co2_emissions": null, + "mteb_version": "1.19.4", + "scores": { + "test": [ + { + "accuracy": 0.403466796875, + "f1": 0.3804475350091884, + "f1_weighted": 0.38048997410452823, + "hf_subset": "default", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.403466796875, + "scores_per_experiment": [ + { + "accuracy": 0.40771484375, + "f1": 0.38158231676574966, + "f1_weighted": 0.3816972560818181 + }, + { + "accuracy": 0.3974609375, + "f1": 0.3770214282257016, + "f1_weighted": 0.37694135402644896 + }, + { + "accuracy": 0.4033203125, + "f1": 0.38432802673810523, + "f1_weighted": 0.384391687329413 + }, + { + "accuracy": 0.40087890625, + "f1": 0.36754207835016633, + "f1_weighted": 0.36759473868921205 + }, + { + "accuracy": 0.39892578125, + "f1": 0.3613803385395889, + "f1_weighted": 0.36144759561899165 + }, + { + "accuracy": 0.40869140625, + "f1": 0.3915561625326378, + "f1_weighted": 0.39156310925368465 + }, + { + "accuracy": 0.4052734375, + "f1": 0.3858284706220626, + "f1_weighted": 0.3859530895742437 + }, + { + "accuracy": 0.41845703125, + "f1": 0.4065895008800672, + "f1_weighted": 0.40670590397463285 + }, + { + "accuracy": 0.38720703125, + "f1": 0.36261347966042223, + "f1_weighted": 0.3626347083087566 + }, + { + "accuracy": 0.40673828125, + "f1": 0.38603354777738197, + "f1_weighted": 0.3859702981880807 + } + ] + } + ] + }, + "task_name": "RuSciBenchOECDClassification" +} \ No newline at end of file diff --git a/results/intfloat__multilingual-e5-small/fd1525a9fd15316a2d503bf26ab031a61d056e98/RuSciBenchOECDClusteringP2P.json b/results/intfloat__multilingual-e5-small/fd1525a9fd15316a2d503bf26ab031a61d056e98/RuSciBenchOECDClusteringP2P.json new file mode 100644 index 000000000..5cf95c02e --- /dev/null +++ b/results/intfloat__multilingual-e5-small/fd1525a9fd15316a2d503bf26ab031a61d056e98/RuSciBenchOECDClusteringP2P.json @@ -0,0 +1,34 @@ +{ + "dataset_revision": "26c88e99dcaba32bb45d0e1bfc21902337f6d471", + "evaluation_time": 76.14725375175476, + "kg_co2_emissions": null, + "mteb_version": "1.19.4", + "scores": { + "test": [ + { + "hf_subset": "default", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.4262261747061461, + "v_measure": 0.4262261747061461, + "v_measure_std": 0.012558957656978547, + "v_measures": { + "Level 0": [ + 0.41556462431623975, + 0.4330370152650926, + 0.45204956416522557, + 0.43028042767788144, + 0.4122269794811792, + 0.40437046909695973, + 0.433389100517328, + 0.4277109077977804, + 0.4285592751451025, + 0.4250733835986719 + ] + } + } + ] + }, + "task_name": "RuSciBenchOECDClusteringP2P" +} \ No newline at end of file diff --git a/results/intfloat__multilingual-e5-small/fd1525a9fd15316a2d503bf26ab031a61d056e98/STS22.json b/results/intfloat__multilingual-e5-small/fd1525a9fd15316a2d503bf26ab031a61d056e98/STS22.json new file mode 100644 index 000000000..b2e130077 --- /dev/null +++ b/results/intfloat__multilingual-e5-small/fd1525a9fd15316a2d503bf26ab031a61d056e98/STS22.json @@ -0,0 +1,26 @@ +{ + "dataset_revision": "d31f33a128469b20e357535c39b82fb3c3f6f2bd", + "evaluation_time": 23.01659321784973, + "kg_co2_emissions": null, + "mteb_version": "1.19.4", + "scores": { + "test": [ + { + "cosine_pearson": 0.6219585494124322, + "cosine_spearman": 0.6604554744448261, + "euclidean_pearson": 0.620602982930849, + "euclidean_spearman": 0.6604554744448261, + "hf_subset": "ru", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.6604554744448261, + "manhattan_pearson": 0.6203177993943501, + "manhattan_spearman": 0.6586398884061588, + "pearson": 0.6219585494124322, + "spearman": 0.6604554744448261 + } + ] + }, + "task_name": "STS22" +} \ No newline at end of file diff --git a/results/intfloat__multilingual-e5-small/fd1525a9fd15316a2d503bf26ab031a61d056e98/SensitiveTopicsClassification.json b/results/intfloat__multilingual-e5-small/fd1525a9fd15316a2d503bf26ab031a61d056e98/SensitiveTopicsClassification.json new file mode 100644 index 000000000..b11499e9e --- /dev/null +++ b/results/intfloat__multilingual-e5-small/fd1525a9fd15316a2d503bf26ab031a61d056e98/SensitiveTopicsClassification.json @@ -0,0 +1,73 @@ +{ + "dataset_revision": "416b34a802308eac30e4192afc0ff99bb8dcc7f2", + "evaluation_time": 8.591712713241577, + "kg_co2_emissions": null, + "mteb_version": "1.19.4", + "scores": { + "test": [ + { + "accuracy": 0.24384765625, + "f1": 0.20448955314534628, + "hf_subset": "default", + "languages": [ + "rus-Cyrl" + ], + "lrap": 0.36029663085936736, + "main_score": 0.24384765625, + "scores_per_experiment": [ + { + "accuracy": 0.2294921875, + "f1": 0.1626592753065714, + "lrap": 0.3289794921874926 + }, + { + "accuracy": 0.22119140625, + "f1": 0.20101999560421557, + "lrap": 0.3626098632812421 + }, + { + "accuracy": 0.2294921875, + "f1": 0.1886542622933147, + "lrap": 0.34320746527777035 + }, + { + "accuracy": 0.2412109375, + "f1": 0.18789099209607027, + "lrap": 0.34902615017360356 + }, + { + "accuracy": 0.2392578125, + "f1": 0.1925921016031841, + "lrap": 0.34366183810763146 + }, + { + "accuracy": 0.26318359375, + "f1": 0.22608816938493714, + "lrap": 0.3791978624131866 + }, + { + "accuracy": 0.26708984375, + "f1": 0.2223412045356411, + "lrap": 0.39020453559027 + }, + { + "accuracy": 0.23876953125, + "f1": 0.20717093780341722, + "lrap": 0.3591308593749926 + }, + { + "accuracy": 0.2607421875, + "f1": 0.23010247058659386, + "lrap": 0.3706461588541591 + }, + { + "accuracy": 0.248046875, + "f1": 0.2263761222395177, + "lrap": 0.3763020833333257 + } + ] + } + ] + }, + "task_name": "SensitiveTopicsClassification" +} \ No newline at end of file diff --git a/results/intfloat__multilingual-e5-small/fd1525a9fd15316a2d503bf26ab031a61d056e98/TERRa.json b/results/intfloat__multilingual-e5-small/fd1525a9fd15316a2d503bf26ab031a61d056e98/TERRa.json new file mode 100644 index 000000000..9080c89a5 --- /dev/null +++ b/results/intfloat__multilingual-e5-small/fd1525a9fd15316a2d503bf26ab031a61d056e98/TERRa.json @@ -0,0 +1,58 @@ +{ + "dataset_revision": "7b58f24536063837d644aab9a023c62199b2a612", + "evaluation_time": 0.8893520832061768, + "kg_co2_emissions": null, + "mteb_version": "1.19.4", + "scores": { + "dev": [ + { + "cosine_accuracy": 0.5830618892508144, + "cosine_accuracy_threshold": 0.8752437829971313, + "cosine_ap": 0.5751046579848809, + "cosine_f1": 0.6728971962616822, + "cosine_f1_threshold": 0.8406317234039307, + "cosine_precision": 0.5236363636363637, + "cosine_recall": 0.9411764705882353, + "dot_accuracy": 0.5830618892508144, + "dot_accuracy_threshold": 0.8752437829971313, + "dot_ap": 0.5751046579848809, + "dot_f1": 0.6728971962616822, + "dot_f1_threshold": 0.8406317234039307, + "dot_precision": 0.5236363636363637, + "dot_recall": 0.9411764705882353, + "euclidean_accuracy": 0.5830618892508144, + "euclidean_accuracy_threshold": 0.49951231479644775, + "euclidean_ap": 0.5751046579848809, + "euclidean_f1": 0.6728971962616822, + "euclidean_f1_threshold": 0.5645672082901001, + "euclidean_precision": 0.5236363636363637, + "euclidean_recall": 0.9411764705882353, + "hf_subset": "default", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.5751046579848809, + "manhattan_accuracy": 0.5765472312703583, + "manhattan_accuracy_threshold": 8.063196182250977, + "manhattan_ap": 0.5728843427691928, + "manhattan_f1": 0.6699507389162561, + "manhattan_f1_threshold": 8.409059524536133, + "manhattan_precision": 0.5375494071146245, + "manhattan_recall": 0.8888888888888888, + "max_accuracy": 0.5830618892508144, + "max_ap": 0.5751046579848809, + "max_f1": 0.6728971962616822, + "max_precision": 0.5375494071146245, + "max_recall": 0.9411764705882353, + "similarity_accuracy": 0.5830618892508144, + "similarity_accuracy_threshold": 0.8752437829971313, + "similarity_ap": 0.5751046579848809, + "similarity_f1": 0.6728971962616822, + "similarity_f1_threshold": 0.8406317234039307, + "similarity_precision": 0.5236363636363637, + "similarity_recall": 0.9411764705882353 + } + ] + }, + "task_name": "TERRa" +} \ No newline at end of file diff --git a/results/intfloat__multilingual-e5-small/fd1525a9fd15316a2d503bf26ab031a61d056e98/model_meta.json b/results/intfloat__multilingual-e5-small/fd1525a9fd15316a2d503bf26ab031a61d056e98/model_meta.json index 7adcba80f..61304469d 100644 --- a/results/intfloat__multilingual-e5-small/fd1525a9fd15316a2d503bf26ab031a61d056e98/model_meta.json +++ b/results/intfloat__multilingual-e5-small/fd1525a9fd15316a2d503bf26ab031a61d056e98/model_meta.json @@ -1 +1,123 @@ -{"name": "intfloat/multilingual-e5-small", "revision": "fd1525a9fd15316a2d503bf26ab031a61d056e98", "release_date": "2024-02-08", "languages": ["afr_Latn", "amh_Latn", "ara_Latn", "asm_Latn", "aze_Latn", "bel_Latn", "bul_Latn", "ben_Latn", "ben_Beng", "bre_Latn", "bos_Latn", "cat_Latn", "ces_Latn", "cym_Latn", "dan_Latn", "deu_Latn", "ell_Latn", "eng_Latn", "epo_Latn", "spa_Latn", "est_Latn", "eus_Latn", "fas_Latn", "fin_Latn", "fra_Latn", "fry_Latn", "gle_Latn", "gla_Latn", "glg_Latn", "guj_Latn", "hau_Latn", "heb_Latn", "hin_Latn", "hin_Deva", "hrv_Latn", "hun_Latn", "hye_Latn", "ind_Latn", "isl_Latn", "ita_Latn", "jpn_Latn", "jav_Latn", "kat_Latn", "kaz_Latn", "khm_Latn", "kan_Latn", "kor_Latn", "kur_Latn", "kir_Latn", "lat_Latn", "lao_Latn", "lit_Latn", "lav_Latn", "mlg_Latn", "mkd_Latn", "mal_Latn", "mon_Latn", "mar_Latn", "msa_Latn", "mya_Latn", "nep_Latn", "nld_Latn", "nob_Latn", "orm_Latn", "ori_Latn", "pan_Latn", "pol_Latn", "pus_Latn", "por_Latn", "ron_Latn", "rus_Latn", "san_Latn", "snd_Latn", "sin_Latn", "slk_Latn", "slv_Latn", "som_Latn", "sqi_Latn", "srp_Latn", "sun_Latn", "swe_Latn", "swa_Latn", "tam_Latn", "tam_Taml", "tel_Latn", "tel_Telu", "tha_Latn", "tgl_Latn", "tur_Latn", "uig_Latn", "ukr_Latn", "urd_Latn", "urd_Arab", "uzb_Latn", "vie_Latn", "xho_Latn", "yid_Latn", "zho_Hant", "zho_Hans"], "n_parameters": 118000000, "memory_usage": null, "max_tokens": 512, "embed_dim": 384, "license": "mit", "open_weights": true, "public_training_data": null, "public_training_code": null, "framework": ["Sentence Transformers", "PyTorch"], "reference": "https://huggingface.co/intfloat/multilingual-e5-small", "similarity_fn_name": "cosine", "use_instructions": true, "zero_shot_benchmarks": null, "loader": "sentence_transformers_loader"} \ No newline at end of file +{ + "name": "intfloat/multilingual-e5-small", + "revision": "fd1525a9fd15316a2d503bf26ab031a61d056e98", + "release_date": "2024-02-08", + "languages": [ + "afr_Latn", + "amh_Latn", + "ara_Latn", + "asm_Latn", + "aze_Latn", + "bel_Latn", + "bul_Latn", + "ben_Latn", + "ben_Beng", + "bre_Latn", + "bos_Latn", + "cat_Latn", + "ces_Latn", + "cym_Latn", + "dan_Latn", + "deu_Latn", + "ell_Latn", + "eng_Latn", + "epo_Latn", + "spa_Latn", + "est_Latn", + "eus_Latn", + "fas_Latn", + "fin_Latn", + "fra_Latn", + "fry_Latn", + "gle_Latn", + "gla_Latn", + "glg_Latn", + "guj_Latn", + "hau_Latn", + "heb_Latn", + "hin_Latn", + "hin_Deva", + "hrv_Latn", + "hun_Latn", + "hye_Latn", + "ind_Latn", + "isl_Latn", + "ita_Latn", + "jpn_Latn", + "jav_Latn", + "kat_Latn", + "kaz_Latn", + "khm_Latn", + "kan_Latn", + "kor_Latn", + "kur_Latn", + "kir_Latn", + "lat_Latn", + "lao_Latn", + "lit_Latn", + "lav_Latn", + "mlg_Latn", + "mkd_Latn", + "mal_Latn", + "mon_Latn", + "mar_Latn", + "msa_Latn", + "mya_Latn", + "nep_Latn", + "nld_Latn", + "nob_Latn", + "orm_Latn", + "ori_Latn", + "pan_Latn", + "pol_Latn", + "pus_Latn", + "por_Latn", + "ron_Latn", + "rus_Latn", + "san_Latn", + "snd_Latn", + "sin_Latn", + "slk_Latn", + "slv_Latn", + "som_Latn", + "sqi_Latn", + "srp_Latn", + "sun_Latn", + "swe_Latn", + "swa_Latn", + "tam_Latn", + "tam_Taml", + "tel_Latn", + "tel_Telu", + "tha_Latn", + "tgl_Latn", + "tur_Latn", + "uig_Latn", + "ukr_Latn", + "urd_Latn", + "urd_Arab", + "uzb_Latn", + "vie_Latn", + "xho_Latn", + "yid_Latn", + "zho_Hant", + "zho_Hans" + ], + "n_parameters": 118000000, + "memory_usage": null, + "max_tokens": 512, + "embed_dim": 384, + "license": "mit", + "open_weights": true, + "public_training_data": null, + "public_training_code": null, + "framework": [ + "Sentence Transformers", + "PyTorch" + ], + "reference": "https://huggingface.co/intfloat/multilingual-e5-small", + "similarity_fn_name": "cosine", + "use_instructions": true, + "zero_shot_benchmarks": null, + "loader": "sentence_transformers_loader" +} \ No newline at end of file diff --git a/results/jinaai__jina-embeddings-v3/215a6e121fa0183376388ac6b1ae230326bfeaed/CEDRClassification.json b/results/jinaai__jina-embeddings-v3/215a6e121fa0183376388ac6b1ae230326bfeaed/CEDRClassification.json index ff65bf5e4..4693e39ff 100644 --- a/results/jinaai__jina-embeddings-v3/215a6e121fa0183376388ac6b1ae230326bfeaed/CEDRClassification.json +++ b/results/jinaai__jina-embeddings-v3/215a6e121fa0183376388ac6b1ae230326bfeaed/CEDRClassification.json @@ -6,64 +6,64 @@ "scores": { "test": [ { - "accuracy": 0.4719978746014878, - "f1": 0.41180935677980235, + "accuracy": 0.4741764080765143, + "f1": 0.41806108416213317, "hf_subset": "default", "languages": [ "rus-Cyrl" ], - "lrap": 0.7282438894792875, - "main_score": 0.4719978746014878, + "lrap": 0.7317507970244523, + "main_score": 0.4741764080765143, "scores_per_experiment": [ { - "accuracy": 0.4165781083953241, - "f1": 0.38158819579465797, - "lrap": 0.6747077577045816 + "accuracy": 0.49256110520722635, + "f1": 0.44964484043965314, + "lrap": 0.7410733262486816 }, { - "accuracy": 0.4022316684378321, - "f1": 0.3775677793754999, - "lrap": 0.7366365568544203 + "accuracy": 0.4128586609989373, + "f1": 0.32098045952780757, + "lrap": 0.6680658873538904 }, { - "accuracy": 0.4798087141339001, - "f1": 0.3991618679588411, - "lrap": 0.6956429330499576 + "accuracy": 0.47449521785334753, + "f1": 0.429388263436818, + "lrap": 0.7442082890542079 }, { - "accuracy": 0.5069075451647184, - "f1": 0.4400502026942411, - "lrap": 0.731668437832104 + "accuracy": 0.5217853347502657, + "f1": 0.43561910220138067, + "lrap": 0.7590860786397542 }, { - "accuracy": 0.485653560042508, - "f1": 0.40574585729201, - "lrap": 0.7530818278427301 + "accuracy": 0.4112646121147715, + "f1": 0.3488720691457515, + "lrap": 0.6907013815090437 }, { - "accuracy": 0.46174282678002126, - "f1": 0.3955828173008574, - "lrap": 0.6903825717322111 + "accuracy": 0.41073326248671627, + "f1": 0.418191679232064, + "lrap": 0.7206429330499569 }, { - "accuracy": 0.48831030818278426, - "f1": 0.4363110301027425, - "lrap": 0.7408607863974597 + "accuracy": 0.48459086078639746, + "f1": 0.35325759968000014, + "lrap": 0.7235919234856639 }, { - "accuracy": 0.5377258235919234, - "f1": 0.42224757911693755, - "lrap": 0.7590329436769491 + "accuracy": 0.4973432518597237, + "f1": 0.4926812536102097, + "lrap": 0.7740170031881062 }, { - "accuracy": 0.4718384697130712, - "f1": 0.3848252003994439, - "lrap": 0.7396386822529322 + "accuracy": 0.5297555791710946, + "f1": 0.4606394716730833, + "lrap": 0.7547821466525072 }, { - "accuracy": 0.4691817215727949, - "f1": 0.47501303776279225, - "lrap": 0.7607863974495306 + "accuracy": 0.5063761955366631, + "f1": 0.4713361026745628, + "lrap": 0.7413390010627098 } ] } diff --git a/results/jinaai__jina-embeddings-v3/215a6e121fa0183376388ac6b1ae230326bfeaed/GeoreviewClassification.json b/results/jinaai__jina-embeddings-v3/215a6e121fa0183376388ac6b1ae230326bfeaed/GeoreviewClassification.json new file mode 100644 index 000000000..d5c7ea02f --- /dev/null +++ b/results/jinaai__jina-embeddings-v3/215a6e121fa0183376388ac6b1ae230326bfeaed/GeoreviewClassification.json @@ -0,0 +1,73 @@ +{ + "dataset_revision": "3765c0d1de6b7d264bc459433c45e5a75513839c", + "evaluation_time": 76.62774014472961, + "kg_co2_emissions": null, + "mteb_version": "1.19.0", + "scores": { + "test": [ + { + "accuracy": 0.515673828125, + "f1": 0.49600379463829547, + "f1_weighted": 0.49598495403292275, + "hf_subset": "default", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.515673828125, + "scores_per_experiment": [ + { + "accuracy": 0.53125, + "f1": 0.5008145001546462, + "f1_weighted": 0.5007698182946703 + }, + { + "accuracy": 0.5078125, + "f1": 0.4848728281814469, + "f1_weighted": 0.4848107101964429 + }, + { + "accuracy": 0.5048828125, + "f1": 0.47923851407248375, + "f1_weighted": 0.4792308844739991 + }, + { + "accuracy": 0.5068359375, + "f1": 0.4843729045001412, + "f1_weighted": 0.48440408834686555 + }, + { + "accuracy": 0.5205078125, + "f1": 0.5036486541101655, + "f1_weighted": 0.503627008736351 + }, + { + "accuracy": 0.49951171875, + "f1": 0.4802312765032502, + "f1_weighted": 0.4801995435897112 + }, + { + "accuracy": 0.53515625, + "f1": 0.5282576992108674, + "f1_weighted": 0.5282439390246128 + }, + { + "accuracy": 0.5302734375, + "f1": 0.5146911694507097, + "f1_weighted": 0.514651235898854 + }, + { + "accuracy": 0.49560546875, + "f1": 0.4825700436365654, + "f1_weighted": 0.482577389147685 + }, + { + "accuracy": 0.52490234375, + "f1": 0.5013403565626778, + "f1_weighted": 0.5013349226200365 + } + ] + } + ] + }, + "task_name": "GeoreviewClassification" +} \ No newline at end of file diff --git a/results/jinaai__jina-embeddings-v3/215a6e121fa0183376388ac6b1ae230326bfeaed/GeoreviewClusteringP2P.json b/results/jinaai__jina-embeddings-v3/215a6e121fa0183376388ac6b1ae230326bfeaed/GeoreviewClusteringP2P.json new file mode 100644 index 000000000..b4a566658 --- /dev/null +++ b/results/jinaai__jina-embeddings-v3/215a6e121fa0183376388ac6b1ae230326bfeaed/GeoreviewClusteringP2P.json @@ -0,0 +1,34 @@ +{ + "dataset_revision": "97a313c8fc85b47f13f33e7e9a95c1ad888c7fec", + "evaluation_time": 121.9709095954895, + "kg_co2_emissions": null, + "mteb_version": "1.19.0", + "scores": { + "test": [ + { + "hf_subset": "default", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.7372652883280046, + "v_measure": 0.7372652883280046, + "v_measure_std": 0.006154577341072406, + "v_measures": { + "Level 0": [ + 0.7427298217225297, + 0.7263100081108423, + 0.7349456221871996, + 0.7417999310405692, + 0.7410106685089711, + 0.7289840951466916, + 0.7396283216879135, + 0.7330505064568388, + 0.747112966600425, + 0.7370809418180649 + ] + } + } + ] + }, + "task_name": "GeoreviewClusteringP2P" +} \ No newline at end of file diff --git a/results/jinaai__jina-embeddings-v3/215a6e121fa0183376388ac6b1ae230326bfeaed/HeadlineClassification.json b/results/jinaai__jina-embeddings-v3/215a6e121fa0183376388ac6b1ae230326bfeaed/HeadlineClassification.json new file mode 100644 index 000000000..d2a5e818f --- /dev/null +++ b/results/jinaai__jina-embeddings-v3/215a6e121fa0183376388ac6b1ae230326bfeaed/HeadlineClassification.json @@ -0,0 +1,73 @@ +{ + "dataset_revision": "2fe05ee6b5832cda29f2ef7aaad7b7fe6a3609eb", + "evaluation_time": 65.71465730667114, + "kg_co2_emissions": null, + "mteb_version": "1.19.0", + "scores": { + "test": [ + { + "accuracy": 0.74501953125, + "f1": 0.7444099729974651, + "f1_weighted": 0.7444170001954504, + "hf_subset": "default", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.74501953125, + "scores_per_experiment": [ + { + "accuracy": 0.72021484375, + "f1": 0.7205880755592998, + "f1_weighted": 0.7206102909266022 + }, + { + "accuracy": 0.76416015625, + "f1": 0.7624334523463435, + "f1_weighted": 0.762433015922326 + }, + { + "accuracy": 0.73974609375, + "f1": 0.7401114012622259, + "f1_weighted": 0.7401165571725079 + }, + { + "accuracy": 0.744140625, + "f1": 0.7448826024263252, + "f1_weighted": 0.7448858246199904 + }, + { + "accuracy": 0.751953125, + "f1": 0.75254029126726, + "f1_weighted": 0.7525403489873008 + }, + { + "accuracy": 0.75146484375, + "f1": 0.7512317574434034, + "f1_weighted": 0.7512402162407303 + }, + { + "accuracy": 0.73291015625, + "f1": 0.7319206834515022, + "f1_weighted": 0.7319099500307488 + }, + { + "accuracy": 0.76025390625, + "f1": 0.7594430896681049, + "f1_weighted": 0.7594422384365651 + }, + { + "accuracy": 0.72607421875, + "f1": 0.7225577440776784, + "f1_weighted": 0.7225961004282714 + }, + { + "accuracy": 0.75927734375, + "f1": 0.7583906324725066, + "f1_weighted": 0.7583954591894604 + } + ] + } + ] + }, + "task_name": "HeadlineClassification" +} \ No newline at end of file diff --git a/results/jinaai__jina-embeddings-v3/215a6e121fa0183376388ac6b1ae230326bfeaed/InappropriatenessClassification.json b/results/jinaai__jina-embeddings-v3/215a6e121fa0183376388ac6b1ae230326bfeaed/InappropriatenessClassification.json new file mode 100644 index 000000000..bc605bf15 --- /dev/null +++ b/results/jinaai__jina-embeddings-v3/215a6e121fa0183376388ac6b1ae230326bfeaed/InappropriatenessClassification.json @@ -0,0 +1,95 @@ +{ + "dataset_revision": "601651fdc45ef243751676e62dd7a19f491c0285", + "evaluation_time": 57.6518759727478, + "kg_co2_emissions": null, + "mteb_version": "1.19.0", + "scores": { + "test": [ + { + "accuracy": 0.61123046875, + "ap": 0.5725051900728242, + "ap_weighted": 0.5725051900728242, + "f1": 0.6066466430418702, + "f1_weighted": 0.6066466430418702, + "hf_subset": "default", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.61123046875, + "scores_per_experiment": [ + { + "accuracy": 0.62548828125, + "ap": 0.5821955810841375, + "ap_weighted": 0.5821955810841375, + "f1": 0.622061945312141, + "f1_weighted": 0.622061945312141 + }, + { + "accuracy": 0.6494140625, + "ap": 0.599341030778556, + "ap_weighted": 0.599341030778556, + "f1": 0.6486420357230243, + "f1_weighted": 0.6486420357230243 + }, + { + "accuracy": 0.6357421875, + "ap": 0.5940768771701389, + "ap_weighted": 0.5940768771701389, + "f1": 0.6275354178368595, + "f1_weighted": 0.6275354178368595 + }, + { + "accuracy": 0.556640625, + "ap": 0.5315159897859922, + "ap_weighted": 0.5315159897859922, + "f1": 0.5566389337117528, + "f1_weighted": 0.5566389337117528 + }, + { + "accuracy": 0.626953125, + "ap": 0.5838015701970443, + "ap_weighted": 0.5838015701970443, + "f1": 0.6229124491487843, + "f1_weighted": 0.6229124491487843 + }, + { + "accuracy": 0.515625, + "ap": 0.5080397727272727, + "ap_weighted": 0.5080397727272727, + "f1": 0.5149570445750871, + "f1_weighted": 0.5149570445750871 + }, + { + "accuracy": 0.65771484375, + "ap": 0.6108159504391468, + "ap_weighted": 0.6108159504391468, + "f1": 0.6534574047588875, + "f1_weighted": 0.6534574047588875 + }, + { + "accuracy": 0.63330078125, + "ap": 0.5800000286592075, + "ap_weighted": 0.5800000286592075, + "f1": 0.6229704344995309, + "f1_weighted": 0.6229704344995309 + }, + { + "accuracy": 0.5869140625, + "ap": 0.5548661043510325, + "ap_weighted": 0.5548661043510325, + "f1": 0.5747771308412041, + "f1_weighted": 0.5747771308412041 + }, + { + "accuracy": 0.62451171875, + "ap": 0.5803989955357143, + "ap_weighted": 0.5803989955357143, + "f1": 0.6225136340114326, + "f1_weighted": 0.6225136340114326 + } + ] + } + ] + }, + "task_name": "InappropriatenessClassification" +} \ No newline at end of file diff --git a/results/jinaai__jina-embeddings-v3/215a6e121fa0183376388ac6b1ae230326bfeaed/KinopoiskClassification.json b/results/jinaai__jina-embeddings-v3/215a6e121fa0183376388ac6b1ae230326bfeaed/KinopoiskClassification.json new file mode 100644 index 000000000..0cf0f36fa --- /dev/null +++ b/results/jinaai__jina-embeddings-v3/215a6e121fa0183376388ac6b1ae230326bfeaed/KinopoiskClassification.json @@ -0,0 +1,73 @@ +{ + "dataset_revision": "5911f26666ac11af46cb9c6849d0dc80a378af24", + "evaluation_time": 51.05471587181091, + "kg_co2_emissions": null, + "mteb_version": "1.19.0", + "scores": { + "test": [ + { + "accuracy": 0.6509333333333334, + "f1": 0.6218714410394597, + "f1_weighted": 0.6218714410394597, + "hf_subset": "default", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.6509333333333334, + "scores_per_experiment": [ + { + "accuracy": 0.6286666666666667, + "f1": 0.6074218092411123, + "f1_weighted": 0.6074218092411123 + }, + { + "accuracy": 0.6506666666666666, + "f1": 0.6103655111149383, + "f1_weighted": 0.6103655111149384 + }, + { + "accuracy": 0.664, + "f1": 0.6247975108373537, + "f1_weighted": 0.6247975108373538 + }, + { + "accuracy": 0.648, + "f1": 0.6293566326415135, + "f1_weighted": 0.6293566326415136 + }, + { + "accuracy": 0.6106666666666667, + "f1": 0.5856756556834276, + "f1_weighted": 0.5856756556834275 + }, + { + "accuracy": 0.674, + "f1": 0.6461465572843667, + "f1_weighted": 0.6461465572843665 + }, + { + "accuracy": 0.6593333333333333, + "f1": 0.6327397582325057, + "f1_weighted": 0.6327397582325057 + }, + { + "accuracy": 0.6766666666666666, + "f1": 0.6476302782812512, + "f1_weighted": 0.6476302782812512 + }, + { + "accuracy": 0.6573333333333333, + "f1": 0.6214681462893633, + "f1_weighted": 0.6214681462893633 + }, + { + "accuracy": 0.64, + "f1": 0.6131125507887649, + "f1_weighted": 0.6131125507887649 + } + ] + } + ] + }, + "task_name": "KinopoiskClassification" +} \ No newline at end of file diff --git a/results/jinaai__jina-embeddings-v3/215a6e121fa0183376388ac6b1ae230326bfeaed/MIRACLReranking.json b/results/jinaai__jina-embeddings-v3/215a6e121fa0183376388ac6b1ae230326bfeaed/MIRACLReranking.json new file mode 100644 index 000000000..b429a8f3f --- /dev/null +++ b/results/jinaai__jina-embeddings-v3/215a6e121fa0183376388ac6b1ae230326bfeaed/MIRACLReranking.json @@ -0,0 +1,130 @@ +{ + "dataset_revision": "6d1962c527217f8927fca80f890f14f36b2802af", + "evaluation_time": 3086.6812217235565, + "kg_co2_emissions": null, + "mteb_version": "1.19.0", + "scores": { + "dev": [ + { + "MAP@1(MIRACL)": 0.20807, + "MAP@10(MIRACL)": 0.33968, + "MAP@100(MIRACL)": 0.36855, + "MAP@1000(MIRACL)": 0.36855, + "MAP@20(MIRACL)": 0.35728, + "MAP@3(MIRACL)": 0.28192, + "MAP@5(MIRACL)": 0.31097, + "NDCG@1(MIRACL)": 0.32959, + "NDCG@10(MIRACL)": 0.42159, + "NDCG@100(MIRACL)": 0.49762, + "NDCG@1000(MIRACL)": 0.49762, + "NDCG@20(MIRACL)": 0.4604, + "NDCG@3(MIRACL)": 0.34527, + "NDCG@5(MIRACL)": 0.37458, + "P@1(MIRACL)": 0.32959, + "P@10(MIRACL)": 0.11716, + "P@100(MIRACL)": 0.01953, + "P@1000(MIRACL)": 0.00195, + "P@20(MIRACL)": 0.07518, + "P@3(MIRACL)": 0.2085, + "P@5(MIRACL)": 0.16712, + "Recall@1(MIRACL)": 0.20807, + "Recall@10(MIRACL)": 0.54947, + "Recall@100(MIRACL)": 0.79952, + "Recall@1000(MIRACL)": 0.79952, + "Recall@20(MIRACL)": 0.66287, + "Recall@3(MIRACL)": 0.34404, + "Recall@5(MIRACL)": 0.42626, + "hf_subset": "ru", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.42159, + "nAUC_MAP@1000_diff1(MIRACL)": 0.14260762070929411, + "nAUC_MAP@1000_max(MIRACL)": 0.12105309324769366, + "nAUC_MAP@1000_std(MIRACL)": 0.0448370036460943, + "nAUC_MAP@100_diff1(MIRACL)": 0.14260762070929411, + "nAUC_MAP@100_max(MIRACL)": 0.12105309324769366, + "nAUC_MAP@100_std(MIRACL)": 0.0448370036460943, + "nAUC_MAP@10_diff1(MIRACL)": 0.168312209380936, + "nAUC_MAP@10_max(MIRACL)": 0.09091171569635538, + "nAUC_MAP@10_std(MIRACL)": 0.030613758749965848, + "nAUC_MAP@1_diff1(MIRACL)": 0.2827415004168066, + "nAUC_MAP@1_max(MIRACL)": 0.023999378979500444, + "nAUC_MAP@1_std(MIRACL)": -0.011292500889229883, + "nAUC_MAP@20_diff1(MIRACL)": 0.15297336380857876, + "nAUC_MAP@20_max(MIRACL)": 0.11048151897482703, + "nAUC_MAP@20_std(MIRACL)": 0.043077353153769087, + "nAUC_MAP@3_diff1(MIRACL)": 0.21953740507848749, + "nAUC_MAP@3_max(MIRACL)": 0.06175660806131447, + "nAUC_MAP@3_std(MIRACL)": 0.009991842060363602, + "nAUC_MAP@5_diff1(MIRACL)": 0.19091790814578738, + "nAUC_MAP@5_max(MIRACL)": 0.07652029008424796, + "nAUC_MAP@5_std(MIRACL)": 0.01620343257719643, + "nAUC_NDCG@1000_diff1(MIRACL)": 0.037716401543400925, + "nAUC_NDCG@1000_max(MIRACL)": 0.2053522360789713, + "nAUC_NDCG@1000_std(MIRACL)": 0.08424838051482676, + "nAUC_NDCG@100_diff1(MIRACL)": 0.037716401543400925, + "nAUC_NDCG@100_max(MIRACL)": 0.2053522360789713, + "nAUC_NDCG@100_std(MIRACL)": 0.08424838051482676, + "nAUC_NDCG@10_diff1(MIRACL)": 0.10858854095040585, + "nAUC_NDCG@10_max(MIRACL)": 0.128385070842372, + "nAUC_NDCG@10_std(MIRACL)": 0.05890048583628576, + "nAUC_NDCG@1_diff1(MIRACL)": 0.18858706826970417, + "nAUC_NDCG@1_max(MIRACL)": 0.10106315844110354, + "nAUC_NDCG@1_std(MIRACL)": 0.03788736522946249, + "nAUC_NDCG@20_diff1(MIRACL)": 0.07190062406328229, + "nAUC_NDCG@20_max(MIRACL)": 0.167411064791661, + "nAUC_NDCG@20_std(MIRACL)": 0.08218222289791627, + "nAUC_NDCG@3_diff1(MIRACL)": 0.16872047459842088, + "nAUC_NDCG@3_max(MIRACL)": 0.09993573921901273, + "nAUC_NDCG@3_std(MIRACL)": 0.03349581856244248, + "nAUC_NDCG@5_diff1(MIRACL)": 0.1472613802557592, + "nAUC_NDCG@5_max(MIRACL)": 0.10767338482393646, + "nAUC_NDCG@5_std(MIRACL)": 0.033918948912733664, + "nAUC_P@1000_diff1(MIRACL)": -0.260540960341792, + "nAUC_P@1000_max(MIRACL)": 0.28269866131974014, + "nAUC_P@1000_std(MIRACL)": 0.09949906918858291, + "nAUC_P@100_diff1(MIRACL)": -0.26054096034179164, + "nAUC_P@100_max(MIRACL)": 0.28269866131974036, + "nAUC_P@100_std(MIRACL)": 0.09949906918858334, + "nAUC_P@10_diff1(MIRACL)": -0.1698514064631271, + "nAUC_P@10_max(MIRACL)": 0.2414052605961978, + "nAUC_P@10_std(MIRACL)": 0.11397080118522872, + "nAUC_P@1_diff1(MIRACL)": 0.18858706826970417, + "nAUC_P@1_max(MIRACL)": 0.10106315844110354, + "nAUC_P@1_std(MIRACL)": 0.03788736522946249, + "nAUC_P@20_diff1(MIRACL)": -0.22329770057888534, + "nAUC_P@20_max(MIRACL)": 0.2762693273491888, + "nAUC_P@20_std(MIRACL)": 0.1470723546908647, + "nAUC_P@3_diff1(MIRACL)": 0.012475979587310292, + "nAUC_P@3_max(MIRACL)": 0.1710746585773252, + "nAUC_P@3_std(MIRACL)": 0.07991748352587799, + "nAUC_P@5_diff1(MIRACL)": -0.07640664844501244, + "nAUC_P@5_max(MIRACL)": 0.2150805857203576, + "nAUC_P@5_std(MIRACL)": 0.08370788251311168, + "nAUC_Recall@1000_diff1(MIRACL)": -0.38470930088303174, + "nAUC_Recall@1000_max(MIRACL)": 0.5105536072898897, + "nAUC_Recall@1000_std(MIRACL)": 0.20505661490312715, + "nAUC_Recall@100_diff1(MIRACL)": -0.38470930088303174, + "nAUC_Recall@100_max(MIRACL)": 0.5105536072898897, + "nAUC_Recall@100_std(MIRACL)": 0.20505661490312715, + "nAUC_Recall@10_diff1(MIRACL)": 0.0017814528069125346, + "nAUC_Recall@10_max(MIRACL)": 0.12755891716376808, + "nAUC_Recall@10_std(MIRACL)": 0.08422314760884957, + "nAUC_Recall@1_diff1(MIRACL)": 0.2827415004168066, + "nAUC_Recall@1_max(MIRACL)": 0.023999378979500444, + "nAUC_Recall@1_std(MIRACL)": -0.011292500889229883, + "nAUC_Recall@20_diff1(MIRACL)": -0.12861810587222106, + "nAUC_Recall@20_max(MIRACL)": 0.24343063185840955, + "nAUC_Recall@20_std(MIRACL)": 0.1554572534898994, + "nAUC_Recall@3_diff1(MIRACL)": 0.16515553008199527, + "nAUC_Recall@3_max(MIRACL)": 0.06640800649118167, + "nAUC_Recall@3_std(MIRACL)": 0.022012757396462777, + "nAUC_Recall@5_diff1(MIRACL)": 0.10134471366525048, + "nAUC_Recall@5_max(MIRACL)": 0.09102717562643031, + "nAUC_Recall@5_std(MIRACL)": 0.02752608362457157 + } + ] + }, + "task_name": "MIRACLReranking" +} \ No newline at end of file diff --git a/results/jinaai__jina-embeddings-v3/215a6e121fa0183376388ac6b1ae230326bfeaed/MassiveScenarioClassification.json b/results/jinaai__jina-embeddings-v3/215a6e121fa0183376388ac6b1ae230326bfeaed/MassiveScenarioClassification.json new file mode 100644 index 000000000..bf815ada0 --- /dev/null +++ b/results/jinaai__jina-embeddings-v3/215a6e121fa0183376388ac6b1ae230326bfeaed/MassiveScenarioClassification.json @@ -0,0 +1,137 @@ +{ + "dataset_revision": "fad2c6e8459f9e1c45d9315f4953d921437d70f8", + "evaluation_time": 223.14069080352783, + "kg_co2_emissions": null, + "mteb_version": "1.19.0", + "scores": { + "test": [ + { + "accuracy": 0.8393073301950237, + "f1": 0.8270612568214959, + "f1_weighted": 0.836458554157962, + "hf_subset": "ru", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.8393073301950237, + "scores_per_experiment": [ + { + "accuracy": 0.8446536650975117, + "f1": 0.8343401564938051, + "f1_weighted": 0.8416522565082213 + }, + { + "accuracy": 0.8389374579690653, + "f1": 0.8272335710872893, + "f1_weighted": 0.8365457478305465 + }, + { + "accuracy": 0.855749831876261, + "f1": 0.8376154038060196, + "f1_weighted": 0.8502882311876104 + }, + { + "accuracy": 0.84364492266308, + "f1": 0.8310131588042805, + "f1_weighted": 0.840132036947145 + }, + { + "accuracy": 0.8312037659717552, + "f1": 0.8146849712213247, + "f1_weighted": 0.8252011148113731 + }, + { + "accuracy": 0.832212508406187, + "f1": 0.8180613549308879, + "f1_weighted": 0.8286765993077079 + }, + { + "accuracy": 0.8167451244115669, + "f1": 0.8074328041317558, + "f1_weighted": 0.8152731387609468 + }, + { + "accuracy": 0.8312037659717552, + "f1": 0.8208901443290727, + "f1_weighted": 0.8311261091003769 + }, + { + "accuracy": 0.8544048419636853, + "f1": 0.8410088012823086, + "f1_weighted": 0.8513623650334743 + }, + { + "accuracy": 0.8443174176193678, + "f1": 0.8383322021282162, + "f1_weighted": 0.8443279420922176 + } + ] + } + ], + "validation": [ + { + "accuracy": 0.8373339891785537, + "f1": 0.822707585935388, + "f1_weighted": 0.8348319882576032, + "hf_subset": "ru", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.8373339891785537, + "scores_per_experiment": [ + { + "accuracy": 0.853418593212002, + "f1": 0.8409798360229135, + "f1_weighted": 0.8508816786505028 + }, + { + "accuracy": 0.8342351205115592, + "f1": 0.8228723619189575, + "f1_weighted": 0.8332449413029882 + }, + { + "accuracy": 0.853418593212002, + "f1": 0.8334023659493713, + "f1_weighted": 0.8489185401824909 + }, + { + "accuracy": 0.8425971470732907, + "f1": 0.8300890295106647, + "f1_weighted": 0.8384190364514511 + }, + { + "accuracy": 0.8293162813575996, + "f1": 0.8127407838091611, + "f1_weighted": 0.8247161178117364 + }, + { + "accuracy": 0.8357107722577471, + "f1": 0.8211888273279606, + "f1_weighted": 0.833011470520948 + }, + { + "accuracy": 0.8214461387112642, + "f1": 0.8052039526396647, + "f1_weighted": 0.8205171529928538 + }, + { + "accuracy": 0.8184948352188883, + "f1": 0.8042460395443892, + "f1_weighted": 0.8185628752396399 + }, + { + "accuracy": 0.8455484505656665, + "f1": 0.8303422578125932, + "f1_weighted": 0.8412225323560316 + }, + { + "accuracy": 0.839153959665519, + "f1": 0.8260104048182032, + "f1_weighted": 0.8388255370673898 + } + ] + } + ] + }, + "task_name": "MassiveScenarioClassification" +} \ No newline at end of file diff --git a/results/jinaai__jina-embeddings-v3/215a6e121fa0183376388ac6b1ae230326bfeaed/RUParaPhraserSTS.json b/results/jinaai__jina-embeddings-v3/215a6e121fa0183376388ac6b1ae230326bfeaed/RUParaPhraserSTS.json new file mode 100644 index 000000000..e332b2f81 --- /dev/null +++ b/results/jinaai__jina-embeddings-v3/215a6e121fa0183376388ac6b1ae230326bfeaed/RUParaPhraserSTS.json @@ -0,0 +1,26 @@ +{ + "dataset_revision": "43265056790b8f7c59e0139acb4be0a8dad2c8f4", + "evaluation_time": 89.01318645477295, + "kg_co2_emissions": null, + "mteb_version": "1.19.0", + "scores": { + "test": [ + { + "cosine_pearson": 0.6793041611909647, + "cosine_spearman": 0.7678832912406242, + "euclidean_pearson": 0.7349818900779371, + "euclidean_spearman": 0.767773301679737, + "hf_subset": "default", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.7678832912406242, + "manhattan_pearson": 0.7352278231796325, + "manhattan_spearman": 0.7671624818662716, + "pearson": 0.6793041611909647, + "spearman": 0.7678832912406242 + } + ] + }, + "task_name": "RUParaPhraserSTS" +} \ No newline at end of file diff --git a/results/jinaai__jina-embeddings-v3/215a6e121fa0183376388ac6b1ae230326bfeaed/RiaNewsRetrieval.json b/results/jinaai__jina-embeddings-v3/215a6e121fa0183376388ac6b1ae230326bfeaed/RiaNewsRetrieval.json new file mode 100644 index 000000000..20f93c249 --- /dev/null +++ b/results/jinaai__jina-embeddings-v3/215a6e121fa0183376388ac6b1ae230326bfeaed/RiaNewsRetrieval.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "82374b0bbacda6114f39ff9c5b925fa1512ca5d7", + "evaluation_time": 17641.05614733696, + "kg_co2_emissions": null, + "mteb_version": "1.19.0", + "scores": { + "test": [ + { + "hf_subset": "default", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.79211, + "map_at_1": 0.6736, + "map_at_10": 0.75646, + "map_at_100": 0.75912, + "map_at_1000": 0.75922, + "map_at_20": 0.75819, + "map_at_3": 0.74215, + "map_at_5": 0.75155, + "mrr_at_1": 0.674, + "mrr_at_10": 0.7567393253968254, + "mrr_at_100": 0.7594021640649564, + "mrr_at_1000": 0.7595032827462562, + "mrr_at_20": 0.7584680698717348, + "mrr_at_3": 0.7424833333333333, + "mrr_at_5": 0.7518283333333333, + "nauc_map_at_1000_diff1": 0.7426858732202793, + "nauc_map_at_1000_max": 0.3629306757055853, + "nauc_map_at_1000_std": -0.20816435550490076, + "nauc_map_at_100_diff1": 0.7426388505570338, + "nauc_map_at_100_max": 0.3629958472744041, + "nauc_map_at_100_std": -0.2080999221194388, + "nauc_map_at_10_diff1": 0.741711669121345, + "nauc_map_at_10_max": 0.36368672428997384, + "nauc_map_at_10_std": -0.2095588623796493, + "nauc_map_at_1_diff1": 0.77344037754793, + "nauc_map_at_1_max": 0.32238716942729395, + "nauc_map_at_1_std": -0.22043961088390046, + "nauc_map_at_20_diff1": 0.7421623926307351, + "nauc_map_at_20_max": 0.3632699217130226, + "nauc_map_at_20_std": -0.20821067101788657, + "nauc_map_at_3_diff1": 0.7427269710081869, + "nauc_map_at_3_max": 0.3628450363334117, + "nauc_map_at_3_std": -0.2187789927541797, + "nauc_map_at_5_diff1": 0.7416967186320205, + "nauc_map_at_5_max": 0.3634331973334331, + "nauc_map_at_5_std": -0.2147195811485785, + "nauc_mrr_at_1000_diff1": 0.7420861276176091, + "nauc_mrr_at_1000_max": 0.36435667533980387, + "nauc_mrr_at_1000_std": -0.2069368138644796, + "nauc_mrr_at_100_diff1": 0.7420393551416841, + "nauc_mrr_at_100_max": 0.3644212137348422, + "nauc_mrr_at_100_std": -0.20687291868854368, + "nauc_mrr_at_10_diff1": 0.7411189288274252, + "nauc_mrr_at_10_max": 0.36509480182669896, + "nauc_mrr_at_10_std": -0.20834840783615344, + "nauc_mrr_at_1_diff1": 0.7726048788146668, + "nauc_mrr_at_1_max": 0.3246990419173934, + "nauc_mrr_at_1_std": -0.21837132783354565, + "nauc_mrr_at_20_diff1": 0.741565162585417, + "nauc_mrr_at_20_max": 0.3646891996466839, + "nauc_mrr_at_20_std": -0.2069891923394524, + "nauc_mrr_at_3_diff1": 0.7421299396492173, + "nauc_mrr_at_3_max": 0.364379419906317, + "nauc_mrr_at_3_std": -0.21753390399115172, + "nauc_mrr_at_5_diff1": 0.7411183686577282, + "nauc_mrr_at_5_max": 0.3648207282272571, + "nauc_mrr_at_5_std": -0.21353223903714091, + "nauc_ndcg_at_1000_diff1": 0.7345130664542262, + "nauc_ndcg_at_1000_max": 0.37688663755338, + "nauc_ndcg_at_1000_std": -0.18979970516778993, + "nauc_ndcg_at_100_diff1": 0.7332068614868412, + "nauc_ndcg_at_100_max": 0.37926263199085253, + "nauc_ndcg_at_100_std": -0.186704938783186, + "nauc_ndcg_at_10_diff1": 0.7279312096604885, + "nauc_ndcg_at_10_max": 0.3820522341739044, + "nauc_ndcg_at_10_std": -0.19415694111085002, + "nauc_ndcg_at_1_diff1": 0.77344037754793, + "nauc_ndcg_at_1_max": 0.32238716942729395, + "nauc_ndcg_at_1_std": -0.22043961088390046, + "nauc_ndcg_at_20_diff1": 0.729348529218933, + "nauc_ndcg_at_20_max": 0.38072823718240323, + "nauc_ndcg_at_20_std": -0.18811338738058794, + "nauc_ndcg_at_3_diff1": 0.7315356304441797, + "nauc_ndcg_at_3_max": 0.3779829016482195, + "nauc_ndcg_at_3_std": -0.21701818649309954, + "nauc_ndcg_at_5_diff1": 0.7285887290621754, + "nauc_ndcg_at_5_max": 0.38045769365930754, + "nauc_ndcg_at_5_std": -0.2087196640269817, + "nauc_precision_at_1000_diff1": 0.5939651054220153, + "nauc_precision_at_1000_max": 0.7200263826460814, + "nauc_precision_at_1000_std": 0.4564550626452062, + "nauc_precision_at_100_diff1": 0.6294201891282823, + "nauc_precision_at_100_max": 0.6106372373510796, + "nauc_precision_at_100_std": 0.20637962787454145, + "nauc_precision_at_10_diff1": 0.6413700578670036, + "nauc_precision_at_10_max": 0.49798853689473094, + "nauc_precision_at_10_std": -0.08303049665215775, + "nauc_precision_at_1_diff1": 0.77344037754793, + "nauc_precision_at_1_max": 0.32238716942729395, + "nauc_precision_at_1_std": -0.22043961088390046, + "nauc_precision_at_20_diff1": 0.626558855661025, + "nauc_precision_at_20_max": 0.5221657713917798, + "nauc_precision_at_20_std": 0.00924949376961922, + "nauc_precision_at_3_diff1": 0.6881541088310619, + "nauc_precision_at_3_max": 0.4368416565412023, + "nauc_precision_at_3_std": -0.20966968774419564, + "nauc_precision_at_5_diff1": 0.6653851990853995, + "nauc_precision_at_5_max": 0.46252876109536256, + "nauc_precision_at_5_std": -0.17726688498366977, + "nauc_recall_at_1000_diff1": 0.5939651054220181, + "nauc_recall_at_1000_max": 0.7200263826460824, + "nauc_recall_at_1000_std": 0.45645506264521013, + "nauc_recall_at_100_diff1": 0.6294201891282826, + "nauc_recall_at_100_max": 0.6106372373510809, + "nauc_recall_at_100_std": 0.20637962787454386, + "nauc_recall_at_10_diff1": 0.641370057867002, + "nauc_recall_at_10_max": 0.4979885368947296, + "nauc_recall_at_10_std": -0.08303049665215836, + "nauc_recall_at_1_diff1": 0.77344037754793, + "nauc_recall_at_1_max": 0.32238716942729395, + "nauc_recall_at_1_std": -0.22043961088390046, + "nauc_recall_at_20_diff1": 0.626558855661024, + "nauc_recall_at_20_max": 0.5221657713917774, + "nauc_recall_at_20_std": 0.00924949376961702, + "nauc_recall_at_3_diff1": 0.6881541088310625, + "nauc_recall_at_3_max": 0.4368416565412013, + "nauc_recall_at_3_std": -0.20966968774419653, + "nauc_recall_at_5_diff1": 0.6653851990853986, + "nauc_recall_at_5_max": 0.462528761095364, + "nauc_recall_at_5_std": -0.17726688498366872, + "ndcg_at_1": 0.6736, + "ndcg_at_10": 0.79211, + "ndcg_at_100": 0.80473, + "ndcg_at_1000": 0.80757, + "ndcg_at_20": 0.79828, + "ndcg_at_3": 0.76346, + "ndcg_at_5": 0.78035, + "precision_at_1": 0.6736, + "precision_at_10": 0.09018, + "precision_at_100": 0.0096, + "precision_at_1000": 0.00098, + "precision_at_20": 0.0463, + "precision_at_3": 0.27493, + "precision_at_5": 0.17314, + "recall_at_1": 0.6736, + "recall_at_10": 0.9018, + "recall_at_100": 0.9601, + "recall_at_1000": 0.9828, + "recall_at_20": 0.9259, + "recall_at_3": 0.8248, + "recall_at_5": 0.8657 + } + ] + }, + "task_name": "RiaNewsRetrieval" +} \ No newline at end of file diff --git a/results/jinaai__jina-embeddings-v3/215a6e121fa0183376388ac6b1ae230326bfeaed/RuBQRetrieval.json b/results/jinaai__jina-embeddings-v3/215a6e121fa0183376388ac6b1ae230326bfeaed/RuBQRetrieval.json new file mode 100644 index 000000000..d8371b5f3 --- /dev/null +++ b/results/jinaai__jina-embeddings-v3/215a6e121fa0183376388ac6b1ae230326bfeaed/RuBQRetrieval.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "e19b6ffa60b3bc248e0b41f4cc37c26a55c2a67b", + "evaluation_time": 1412.2502541542053, + "kg_co2_emissions": null, + "mteb_version": "1.19.0", + "scores": { + "test": [ + { + "hf_subset": "default", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.72276, + "map_at_1": 0.44039, + "map_at_10": 0.64783, + "map_at_100": 0.65705, + "map_at_1000": 0.65733, + "map_at_20": 0.65427, + "map_at_3": 0.59669, + "map_at_5": 0.63034, + "mrr_at_1": 0.6199763593380615, + "mrr_at_10": 0.7329020319711809, + "mrr_at_100": 0.7356072432792593, + "mrr_at_1000": 0.7356830104744264, + "mrr_at_20": 0.7348586534176939, + "mrr_at_3": 0.7134554767533492, + "mrr_at_5": 0.7274034672970843, + "nauc_map_at_1000_diff1": 0.3995722380507381, + "nauc_map_at_1000_max": 0.3246912640119044, + "nauc_map_at_1000_std": -0.06043224598860364, + "nauc_map_at_100_diff1": 0.39939859988074056, + "nauc_map_at_100_max": 0.32487260836708365, + "nauc_map_at_100_std": -0.06016745517207013, + "nauc_map_at_10_diff1": 0.3963006642703574, + "nauc_map_at_10_max": 0.31932275605664134, + "nauc_map_at_10_std": -0.06533361292252163, + "nauc_map_at_1_diff1": 0.47896333178869865, + "nauc_map_at_1_max": 0.21932114575063383, + "nauc_map_at_1_std": -0.07653920774626236, + "nauc_map_at_20_diff1": 0.39886393525749075, + "nauc_map_at_20_max": 0.3254598149797907, + "nauc_map_at_20_std": -0.06082881800527665, + "nauc_map_at_3_diff1": 0.3975900611067023, + "nauc_map_at_3_max": 0.2901018368558066, + "nauc_map_at_3_std": -0.06852623151698276, + "nauc_map_at_5_diff1": 0.39405628578105617, + "nauc_map_at_5_max": 0.3115995669919187, + "nauc_map_at_5_std": -0.06981917271501435, + "nauc_mrr_at_1000_diff1": 0.49626250598127847, + "nauc_mrr_at_1000_max": 0.3784447980855714, + "nauc_mrr_at_1000_std": -0.0990383682095993, + "nauc_mrr_at_100_diff1": 0.49622766989224665, + "nauc_mrr_at_100_max": 0.3785697361359845, + "nauc_mrr_at_100_std": -0.09894320982279657, + "nauc_mrr_at_10_diff1": 0.49368831875482866, + "nauc_mrr_at_10_max": 0.3802692815372114, + "nauc_mrr_at_10_std": -0.09866122829051989, + "nauc_mrr_at_1_diff1": 0.5365460289983304, + "nauc_mrr_at_1_max": 0.331208222241617, + "nauc_mrr_at_1_std": -0.11954715429202196, + "nauc_mrr_at_20_diff1": 0.4957717923061422, + "nauc_mrr_at_20_max": 0.37948549849299607, + "nauc_mrr_at_20_std": -0.09902292372253127, + "nauc_mrr_at_3_diff1": 0.49216457752673787, + "nauc_mrr_at_3_max": 0.37629471273386284, + "nauc_mrr_at_3_std": -0.10035712310051902, + "nauc_mrr_at_5_diff1": 0.49072704745484025, + "nauc_mrr_at_5_max": 0.3791888051221967, + "nauc_mrr_at_5_std": -0.09922478399582084, + "nauc_ndcg_at_1000_diff1": 0.4136157342051366, + "nauc_ndcg_at_1000_max": 0.35829523104943156, + "nauc_ndcg_at_1000_std": -0.05143428232902687, + "nauc_ndcg_at_100_diff1": 0.40955118571179067, + "nauc_ndcg_at_100_max": 0.36376681960165935, + "nauc_ndcg_at_100_std": -0.04412222209228563, + "nauc_ndcg_at_10_diff1": 0.39672137759640114, + "nauc_ndcg_at_10_max": 0.3556344002630545, + "nauc_ndcg_at_10_std": -0.05905241912629651, + "nauc_ndcg_at_1_diff1": 0.5365460289983304, + "nauc_ndcg_at_1_max": 0.331208222241617, + "nauc_ndcg_at_1_std": -0.11954715429202196, + "nauc_ndcg_at_20_diff1": 0.4061189178410458, + "nauc_ndcg_at_20_max": 0.36865598355561946, + "nauc_ndcg_at_20_std": -0.0487851356456261, + "nauc_ndcg_at_3_diff1": 0.39755983556330776, + "nauc_ndcg_at_3_max": 0.3202147307018135, + "nauc_ndcg_at_3_std": -0.07247400952120632, + "nauc_ndcg_at_5_diff1": 0.3908762663296387, + "nauc_ndcg_at_5_max": 0.3408953447016877, + "nauc_ndcg_at_5_std": -0.06977957904720415, + "nauc_precision_at_1000_diff1": -0.14422288418600493, + "nauc_precision_at_1000_max": 0.0635167037355057, + "nauc_precision_at_1000_std": 0.04823673584806904, + "nauc_precision_at_100_diff1": -0.13513158049526, + "nauc_precision_at_100_max": 0.10814530296047509, + "nauc_precision_at_100_std": 0.07430380426735111, + "nauc_precision_at_10_diff1": -0.07511775305361726, + "nauc_precision_at_10_max": 0.1732477389617535, + "nauc_precision_at_10_std": 0.03093000893140734, + "nauc_precision_at_1_diff1": 0.5365460289983304, + "nauc_precision_at_1_max": 0.331208222241617, + "nauc_precision_at_1_std": -0.11954715429202196, + "nauc_precision_at_20_diff1": -0.09787941113038703, + "nauc_precision_at_20_max": 0.1660490367781591, + "nauc_precision_at_20_std": 0.06472885954374098, + "nauc_precision_at_3_diff1": 0.05567034694661691, + "nauc_precision_at_3_max": 0.2379883439640051, + "nauc_precision_at_3_std": -0.015548085238501774, + "nauc_precision_at_5_diff1": -0.02701060755118351, + "nauc_precision_at_5_max": 0.21275839618026404, + "nauc_precision_at_5_std": 0.0011169701304658169, + "nauc_recall_at_1000_diff1": -0.01880165659128809, + "nauc_recall_at_1000_max": 0.46598273641528737, + "nauc_recall_at_1000_std": 0.6258284070558879, + "nauc_recall_at_100_diff1": 0.15055879459239843, + "nauc_recall_at_100_max": 0.5248101119061757, + "nauc_recall_at_100_std": 0.34678262893857587, + "nauc_recall_at_10_diff1": 0.2267217500877515, + "nauc_recall_at_10_max": 0.38711640876281284, + "nauc_recall_at_10_std": 0.012578842238776741, + "nauc_recall_at_1_diff1": 0.47896333178869865, + "nauc_recall_at_1_max": 0.21932114575063383, + "nauc_recall_at_1_std": -0.07653920774626236, + "nauc_recall_at_20_diff1": 0.23245473170059588, + "nauc_recall_at_20_max": 0.478740370790808, + "nauc_recall_at_20_std": 0.09541810336512961, + "nauc_recall_at_3_diff1": 0.3005692317837611, + "nauc_recall_at_3_max": 0.28981219220932686, + "nauc_recall_at_3_std": -0.04754315090107973, + "nauc_recall_at_5_diff1": 0.2506673474876686, + "nauc_recall_at_5_max": 0.33007573705930493, + "nauc_recall_at_5_std": -0.04126609718514233, + "ndcg_at_1": 0.61998, + "ndcg_at_10": 0.72276, + "ndcg_at_100": 0.75028, + "ndcg_at_1000": 0.75526, + "ndcg_at_20": 0.73813, + "ndcg_at_3": 0.65622, + "ndcg_at_5": 0.69437, + "precision_at_1": 0.61998, + "precision_at_10": 0.13924, + "precision_at_100": 0.01592, + "precision_at_1000": 0.00166, + "precision_at_20": 0.07444, + "precision_at_3": 0.35678, + "precision_at_5": 0.24917, + "recall_at_1": 0.44039, + "recall_at_10": 0.85162, + "recall_at_100": 0.95527, + "recall_at_1000": 0.98831, + "recall_at_20": 0.90192, + "recall_at_3": 0.68824, + "recall_at_5": 0.77885 + } + ] + }, + "task_name": "RuBQRetrieval" +} \ No newline at end of file diff --git a/results/jinaai__jina-embeddings-v3/215a6e121fa0183376388ac6b1ae230326bfeaed/RuReviewsClassification.json b/results/jinaai__jina-embeddings-v3/215a6e121fa0183376388ac6b1ae230326bfeaed/RuReviewsClassification.json new file mode 100644 index 000000000..19897b3a7 --- /dev/null +++ b/results/jinaai__jina-embeddings-v3/215a6e121fa0183376388ac6b1ae230326bfeaed/RuReviewsClassification.json @@ -0,0 +1,73 @@ +{ + "dataset_revision": "f6d2c31f4dc6b88f468552750bfec05b4b41b05a", + "evaluation_time": 61.013256311416626, + "kg_co2_emissions": null, + "mteb_version": "1.19.0", + "scores": { + "test": [ + { + "accuracy": 0.697021484375, + "f1": 0.6911599239475726, + "f1_weighted": 0.6911660321330073, + "hf_subset": "default", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.697021484375, + "scores_per_experiment": [ + { + "accuracy": 0.73291015625, + "f1": 0.7290164264693809, + "f1_weighted": 0.7290120805104762 + }, + { + "accuracy": 0.65771484375, + "f1": 0.6437655614289288, + "f1_weighted": 0.6437634798310743 + }, + { + "accuracy": 0.69384765625, + "f1": 0.6927153850703961, + "f1_weighted": 0.6927247308982949 + }, + { + "accuracy": 0.7275390625, + "f1": 0.7273808292316185, + "f1_weighted": 0.7273945250229659 + }, + { + "accuracy": 0.70751953125, + "f1": 0.7128382628243878, + "f1_weighted": 0.7128495584490022 + }, + { + "accuracy": 0.6337890625, + "f1": 0.6238938912686618, + "f1_weighted": 0.6239156811117212 + }, + { + "accuracy": 0.69189453125, + "f1": 0.677768984083424, + "f1_weighted": 0.6777672089675827 + }, + { + "accuracy": 0.689453125, + "f1": 0.6822322707709277, + "f1_weighted": 0.6822347448512209 + }, + { + "accuracy": 0.70166015625, + "f1": 0.6892500369750835, + "f1_weighted": 0.6892298177422647 + }, + { + "accuracy": 0.73388671875, + "f1": 0.7327375913529174, + "f1_weighted": 0.7327684939454711 + } + ] + } + ] + }, + "task_name": "RuReviewsClassification" +} \ No newline at end of file diff --git a/results/jinaai__jina-embeddings-v3/215a6e121fa0183376388ac6b1ae230326bfeaed/RuSTSBenchmarkSTS.json b/results/jinaai__jina-embeddings-v3/215a6e121fa0183376388ac6b1ae230326bfeaed/RuSTSBenchmarkSTS.json new file mode 100644 index 000000000..da4caf6b7 --- /dev/null +++ b/results/jinaai__jina-embeddings-v3/215a6e121fa0183376388ac6b1ae230326bfeaed/RuSTSBenchmarkSTS.json @@ -0,0 +1,26 @@ +{ + "dataset_revision": "7cf24f325c6da6195df55bef3d86b5e0616f3018", + "evaluation_time": 58.95779466629028, + "kg_co2_emissions": null, + "mteb_version": "1.19.0", + "scores": { + "test": [ + { + "cosine_pearson": 0.8589411382130845, + "cosine_spearman": 0.8620670246522056, + "euclidean_pearson": 0.8573954677064619, + "euclidean_spearman": 0.8621946367240504, + "hf_subset": "default", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.8620670246522056, + "manhattan_pearson": 0.8563649623892502, + "manhattan_spearman": 0.8614149504355393, + "pearson": 0.8589411382130845, + "spearman": 0.8620670246522056 + } + ] + }, + "task_name": "RuSTSBenchmarkSTS" +} \ No newline at end of file diff --git a/results/jinaai__jina-embeddings-v3/215a6e121fa0183376388ac6b1ae230326bfeaed/RuSciBenchGRNTIClassification.json b/results/jinaai__jina-embeddings-v3/215a6e121fa0183376388ac6b1ae230326bfeaed/RuSciBenchGRNTIClassification.json new file mode 100644 index 000000000..de8012033 --- /dev/null +++ b/results/jinaai__jina-embeddings-v3/215a6e121fa0183376388ac6b1ae230326bfeaed/RuSciBenchGRNTIClassification.json @@ -0,0 +1,73 @@ +{ + "dataset_revision": "673a610d6d3dd91a547a0d57ae1b56f37ebbf6a1", + "evaluation_time": 129.65155601501465, + "kg_co2_emissions": null, + "mteb_version": "1.19.0", + "scores": { + "test": [ + { + "accuracy": 0.599267578125, + "f1": 0.5856482549031995, + "f1_weighted": 0.5857311704118908, + "hf_subset": "default", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.599267578125, + "scores_per_experiment": [ + { + "accuracy": 0.6083984375, + "f1": 0.5935987042627385, + "f1_weighted": 0.5936956064360758 + }, + { + "accuracy": 0.59423828125, + "f1": 0.5776711983468248, + "f1_weighted": 0.5777629139270739 + }, + { + "accuracy": 0.57568359375, + "f1": 0.557597317321844, + "f1_weighted": 0.5577911307203668 + }, + { + "accuracy": 0.6181640625, + "f1": 0.6113084919380553, + "f1_weighted": 0.6113479046322874 + }, + { + "accuracy": 0.61474609375, + "f1": 0.6030225706345939, + "f1_weighted": 0.6030545132525452 + }, + { + "accuracy": 0.5771484375, + "f1": 0.5608156244527799, + "f1_weighted": 0.5608971247481399 + }, + { + "accuracy": 0.6162109375, + "f1": 0.602488561293025, + "f1_weighted": 0.6026622175478387 + }, + { + "accuracy": 0.5966796875, + "f1": 0.5853404461004884, + "f1_weighted": 0.5854265393807828 + }, + { + "accuracy": 0.6025390625, + "f1": 0.5916534198672841, + "f1_weighted": 0.59167126609686 + }, + { + "accuracy": 0.5888671875, + "f1": 0.5729862148143614, + "f1_weighted": 0.573002487376938 + } + ] + } + ] + }, + "task_name": "RuSciBenchGRNTIClassification" +} \ No newline at end of file diff --git a/results/jinaai__jina-embeddings-v3/215a6e121fa0183376388ac6b1ae230326bfeaed/RuSciBenchGRNTIClusteringP2P.json b/results/jinaai__jina-embeddings-v3/215a6e121fa0183376388ac6b1ae230326bfeaed/RuSciBenchGRNTIClusteringP2P.json new file mode 100644 index 000000000..fe6f7f823 --- /dev/null +++ b/results/jinaai__jina-embeddings-v3/215a6e121fa0183376388ac6b1ae230326bfeaed/RuSciBenchGRNTIClusteringP2P.json @@ -0,0 +1,34 @@ +{ + "dataset_revision": "673a610d6d3dd91a547a0d57ae1b56f37ebbf6a1", + "evaluation_time": 126.98100399971008, + "kg_co2_emissions": null, + "mteb_version": "1.19.0", + "scores": { + "test": [ + { + "hf_subset": "default", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.5838331323966658, + "v_measure": 0.5838331323966658, + "v_measure_std": 0.010813773056850065, + "v_measures": { + "Level 0": [ + 0.5907367624441764, + 0.586781014841984, + 0.5670158217567329, + 0.5939238640738688, + 0.5654648972706554, + 0.5917341963127901, + 0.572040227942462, + 0.596147819698565, + 0.5845433911049284, + 0.5899433285204958 + ] + } + } + ] + }, + "task_name": "RuSciBenchGRNTIClusteringP2P" +} \ No newline at end of file diff --git a/results/jinaai__jina-embeddings-v3/215a6e121fa0183376388ac6b1ae230326bfeaed/RuSciBenchOECDClassification.json b/results/jinaai__jina-embeddings-v3/215a6e121fa0183376388ac6b1ae230326bfeaed/RuSciBenchOECDClassification.json new file mode 100644 index 000000000..694cbf846 --- /dev/null +++ b/results/jinaai__jina-embeddings-v3/215a6e121fa0183376388ac6b1ae230326bfeaed/RuSciBenchOECDClassification.json @@ -0,0 +1,73 @@ +{ + "dataset_revision": "26c88e99dcaba32bb45d0e1bfc21902337f6d471", + "evaluation_time": 128.4568018913269, + "kg_co2_emissions": null, + "mteb_version": "1.19.0", + "scores": { + "test": [ + { + "accuracy": 0.46806640625, + "f1": 0.4461280047840632, + "f1_weighted": 0.4461224412623034, + "hf_subset": "default", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.46806640625, + "scores_per_experiment": [ + { + "accuracy": 0.47021484375, + "f1": 0.4489239267139885, + "f1_weighted": 0.4489986799775514 + }, + { + "accuracy": 0.474609375, + "f1": 0.45241469956817776, + "f1_weighted": 0.4523703434681279 + }, + { + "accuracy": 0.46533203125, + "f1": 0.4459490470804855, + "f1_weighted": 0.4460586448217685 + }, + { + "accuracy": 0.4619140625, + "f1": 0.43016512904357523, + "f1_weighted": 0.43004913465965605 + }, + { + "accuracy": 0.462890625, + "f1": 0.43371395393823275, + "f1_weighted": 0.43365905061739674 + }, + { + "accuracy": 0.4765625, + "f1": 0.46387693456852597, + "f1_weighted": 0.4639685880805891 + }, + { + "accuracy": 0.45458984375, + "f1": 0.4271010456866485, + "f1_weighted": 0.42713351136816413 + }, + { + "accuracy": 0.484375, + "f1": 0.4739990146869187, + "f1_weighted": 0.4740670239187568 + }, + { + "accuracy": 0.45703125, + "f1": 0.43240175971757894, + "f1_weighted": 0.4322425408516514 + }, + { + "accuracy": 0.47314453125, + "f1": 0.4527345368364999, + "f1_weighted": 0.4526768948593723 + } + ] + } + ] + }, + "task_name": "RuSciBenchOECDClassification" +} \ No newline at end of file diff --git a/results/jinaai__jina-embeddings-v3/215a6e121fa0183376388ac6b1ae230326bfeaed/RuSciBenchOECDClusteringP2P.json b/results/jinaai__jina-embeddings-v3/215a6e121fa0183376388ac6b1ae230326bfeaed/RuSciBenchOECDClusteringP2P.json new file mode 100644 index 000000000..7327953fd --- /dev/null +++ b/results/jinaai__jina-embeddings-v3/215a6e121fa0183376388ac6b1ae230326bfeaed/RuSciBenchOECDClusteringP2P.json @@ -0,0 +1,34 @@ +{ + "dataset_revision": "26c88e99dcaba32bb45d0e1bfc21902337f6d471", + "evaluation_time": 179.36029362678528, + "kg_co2_emissions": null, + "mteb_version": "1.19.0", + "scores": { + "test": [ + { + "hf_subset": "default", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.505946965909812, + "v_measure": 0.505946965909812, + "v_measure_std": 0.007614398694795295, + "v_measures": { + "Level 0": [ + 0.5008946351993209, + 0.504287231714026, + 0.4961979250503179, + 0.5023595008914274, + 0.5241285525270556, + 0.511591001552598, + 0.5032683169919246, + 0.5009764858577169, + 0.5033116998981126, + 0.5124543094156206 + ] + } + } + ] + }, + "task_name": "RuSciBenchOECDClusteringP2P" +} \ No newline at end of file diff --git a/results/jinaai__jina-embeddings-v3/215a6e121fa0183376388ac6b1ae230326bfeaed/STS22.json b/results/jinaai__jina-embeddings-v3/215a6e121fa0183376388ac6b1ae230326bfeaed/STS22.json new file mode 100644 index 000000000..320b53a2f --- /dev/null +++ b/results/jinaai__jina-embeddings-v3/215a6e121fa0183376388ac6b1ae230326bfeaed/STS22.json @@ -0,0 +1,26 @@ +{ + "dataset_revision": "de9d86b3b84231dc21f76c7b7af1f28e2f57f6e3", + "evaluation_time": 13.758903503417969, + "kg_co2_emissions": null, + "mteb_version": "1.19.0", + "scores": { + "test": [ + { + "cosine_pearson": 0.6181706226333195, + "cosine_spearman": 0.6511349651683094, + "euclidean_pearson": 0.6166831340112071, + "euclidean_spearman": 0.6508099442332599, + "hf_subset": "ru", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.6511349651683094, + "manhattan_pearson": 0.6168847979693892, + "manhattan_spearman": 0.6513590850200408, + "pearson": 0.6181706226333195, + "spearman": 0.6511349651683094 + } + ] + }, + "task_name": "STS22" +} \ No newline at end of file diff --git a/results/jinaai__jina-embeddings-v3/215a6e121fa0183376388ac6b1ae230326bfeaed/SensitiveTopicsClassification.json b/results/jinaai__jina-embeddings-v3/215a6e121fa0183376388ac6b1ae230326bfeaed/SensitiveTopicsClassification.json new file mode 100644 index 000000000..ac8ece2c9 --- /dev/null +++ b/results/jinaai__jina-embeddings-v3/215a6e121fa0183376388ac6b1ae230326bfeaed/SensitiveTopicsClassification.json @@ -0,0 +1,73 @@ +{ + "dataset_revision": "416b34a802308eac30e4192afc0ff99bb8dcc7f2", + "evaluation_time": 89.13181233406067, + "kg_co2_emissions": null, + "mteb_version": "1.19.0", + "scores": { + "test": [ + { + "accuracy": 0.31064453125, + "f1": 0.3382942448096327, + "hf_subset": "default", + "languages": [ + "rus-Cyrl" + ], + "lrap": 0.4701158311631871, + "main_score": 0.31064453125, + "scores_per_experiment": [ + { + "accuracy": 0.298828125, + "f1": 0.2953423052907327, + "lrap": 0.4468315972222146 + }, + { + "accuracy": 0.28173828125, + "f1": 0.32755647941218186, + "lrap": 0.43085394965277013 + }, + { + "accuracy": 0.30615234375, + "f1": 0.30863038796263065, + "lrap": 0.44984266493054803 + }, + { + "accuracy": 0.30126953125, + "f1": 0.32801367648071617, + "lrap": 0.45836046006943687 + }, + { + "accuracy": 0.30419921875, + "f1": 0.3280726463091239, + "lrap": 0.45735677083332554 + }, + { + "accuracy": 0.32177734375, + "f1": 0.3505765746132131, + "lrap": 0.49005805121527046 + }, + { + "accuracy": 0.32763671875, + "f1": 0.34949212111333483, + "lrap": 0.491604275173604 + }, + { + "accuracy": 0.31982421875, + "f1": 0.3519807922174081, + "lrap": 0.4816216362847149 + }, + { + "accuracy": 0.33251953125, + "f1": 0.37619433250453516, + "lrap": 0.5055474175347159 + }, + { + "accuracy": 0.3125, + "f1": 0.36708313219245065, + "lrap": 0.48908148871527074 + } + ] + } + ] + }, + "task_name": "SensitiveTopicsClassification" +} \ No newline at end of file diff --git a/results/jinaai__jina-embeddings-v3/215a6e121fa0183376388ac6b1ae230326bfeaed/model_meta.json b/results/jinaai__jina-embeddings-v3/215a6e121fa0183376388ac6b1ae230326bfeaed/model_meta.json index 342994aea..331e02511 100644 --- a/results/jinaai__jina-embeddings-v3/215a6e121fa0183376388ac6b1ae230326bfeaed/model_meta.json +++ b/results/jinaai__jina-embeddings-v3/215a6e121fa0183376388ac6b1ae230326bfeaed/model_meta.json @@ -1 +1,123 @@ -{"name": "jinaai/jina-embeddings-v3", "revision": "215a6e121fa0183376388ac6b1ae230326bfeaed", "release_date": "2024-09-18", "languages": ["afr_Latn", "amh_Latn", "ara_Latn", "asm_Latn", "aze_Latn", "bel_Latn", "bul_Latn", "ben_Latn", "ben_Beng", "bre_Latn", "bos_Latn", "cat_Latn", "ces_Latn", "cym_Latn", "dan_Latn", "deu_Latn", "ell_Latn", "eng_Latn", "epo_Latn", "spa_Latn", "est_Latn", "eus_Latn", "fas_Latn", "fin_Latn", "fra_Latn", "fry_Latn", "gle_Latn", "gla_Latn", "glg_Latn", "guj_Latn", "hau_Latn", "heb_Latn", "hin_Latn", "hin_Deva", "hrv_Latn", "hun_Latn", "hye_Latn", "ind_Latn", "isl_Latn", "ita_Latn", "jpn_Latn", "jav_Latn", "kat_Latn", "kaz_Latn", "khm_Latn", "kan_Latn", "kor_Latn", "kur_Latn", "kir_Latn", "lat_Latn", "lao_Latn", "lit_Latn", "lav_Latn", "mlg_Latn", "mkd_Latn", "mal_Latn", "mon_Latn", "mar_Latn", "msa_Latn", "mya_Latn", "nep_Latn", "nld_Latn", "nob_Latn", "orm_Latn", "ori_Latn", "pan_Latn", "pol_Latn", "pus_Latn", "por_Latn", "ron_Latn", "rus_Latn", "san_Latn", "snd_Latn", "sin_Latn", "slk_Latn", "slv_Latn", "som_Latn", "sqi_Latn", "srp_Latn", "sun_Latn", "swe_Latn", "swa_Latn", "tam_Latn", "tam_Taml", "tel_Latn", "tel_Telu", "tha_Latn", "tgl_Latn", "tur_Latn", "uig_Latn", "ukr_Latn", "urd_Latn", "urd_Arab", "uzb_Latn", "vie_Latn", "xho_Latn", "yid_Latn", "zho_Hant", "zho_Hans"], "n_parameters": 572000000, "memory_usage": null, "max_tokens": 8194, "embed_dim": 4096, "license": "cc-by-nc-4.0", "open_weights": true, "public_training_data": null, "public_training_code": null, "framework": ["Sentence Transformers", "PyTorch"], "reference": "https://huggingface.co/jinaai/jina-embeddings-v3", "similarity_fn_name": "cosine", "use_instructions": false, "zero_shot_benchmarks": null, "loader": "JinaWrapper"} \ No newline at end of file +{ + "name": "jinaai/jina-embeddings-v3", + "revision": "215a6e121fa0183376388ac6b1ae230326bfeaed", + "release_date": "2024-09-18", + "languages": [ + "afr_Latn", + "amh_Latn", + "ara_Latn", + "asm_Latn", + "aze_Latn", + "bel_Latn", + "bul_Latn", + "ben_Latn", + "ben_Beng", + "bre_Latn", + "bos_Latn", + "cat_Latn", + "ces_Latn", + "cym_Latn", + "dan_Latn", + "deu_Latn", + "ell_Latn", + "eng_Latn", + "epo_Latn", + "spa_Latn", + "est_Latn", + "eus_Latn", + "fas_Latn", + "fin_Latn", + "fra_Latn", + "fry_Latn", + "gle_Latn", + "gla_Latn", + "glg_Latn", + "guj_Latn", + "hau_Latn", + "heb_Latn", + "hin_Latn", + "hin_Deva", + "hrv_Latn", + "hun_Latn", + "hye_Latn", + "ind_Latn", + "isl_Latn", + "ita_Latn", + "jpn_Latn", + "jav_Latn", + "kat_Latn", + "kaz_Latn", + "khm_Latn", + "kan_Latn", + "kor_Latn", + "kur_Latn", + "kir_Latn", + "lat_Latn", + "lao_Latn", + "lit_Latn", + "lav_Latn", + "mlg_Latn", + "mkd_Latn", + "mal_Latn", + "mon_Latn", + "mar_Latn", + "msa_Latn", + "mya_Latn", + "nep_Latn", + "nld_Latn", + "nob_Latn", + "orm_Latn", + "ori_Latn", + "pan_Latn", + "pol_Latn", + "pus_Latn", + "por_Latn", + "ron_Latn", + "rus_Latn", + "san_Latn", + "snd_Latn", + "sin_Latn", + "slk_Latn", + "slv_Latn", + "som_Latn", + "sqi_Latn", + "srp_Latn", + "sun_Latn", + "swe_Latn", + "swa_Latn", + "tam_Latn", + "tam_Taml", + "tel_Latn", + "tel_Telu", + "tha_Latn", + "tgl_Latn", + "tur_Latn", + "uig_Latn", + "ukr_Latn", + "urd_Latn", + "urd_Arab", + "uzb_Latn", + "vie_Latn", + "xho_Latn", + "yid_Latn", + "zho_Hant", + "zho_Hans" + ], + "n_parameters": 572000000, + "memory_usage": null, + "max_tokens": 8194, + "embed_dim": 4096, + "license": "cc-by-nc-4.0", + "open_weights": true, + "public_training_data": null, + "public_training_code": null, + "framework": [ + "Sentence Transformers", + "PyTorch" + ], + "reference": "https://huggingface.co/jinaai/jina-embeddings-v3", + "similarity_fn_name": "cosine", + "use_instructions": false, + "zero_shot_benchmarks": null, + "loader": "JinaWrapper" +} \ No newline at end of file diff --git a/results/mixedbread-ai__mxbai-embed-large-v1/990580e27d329c7408b3741ecff85876e128e203/CEDRClassification.json b/results/mixedbread-ai__mxbai-embed-large-v1/990580e27d329c7408b3741ecff85876e128e203/CEDRClassification.json new file mode 100644 index 000000000..4bd1ece92 --- /dev/null +++ b/results/mixedbread-ai__mxbai-embed-large-v1/990580e27d329c7408b3741ecff85876e128e203/CEDRClassification.json @@ -0,0 +1,73 @@ +{ + "dataset_revision": "c0ba03d058e3e1b2f3fd20518875a4563dd12db4", + "evaluation_time": 22.37542462348938, + "kg_co2_emissions": null, + "mteb_version": "1.19.0", + "scores": { + "test": [ + { + "accuracy": 0.38241232731137087, + "f1": 0.17009622821680803, + "hf_subset": "default", + "languages": [ + "rus-Cyrl" + ], + "lrap": 0.5897130712008621, + "main_score": 0.38241232731137087, + "scores_per_experiment": [ + { + "accuracy": 0.4022316684378321, + "f1": 0.16695421740708172, + "lrap": 0.5852816153028811 + }, + { + "accuracy": 0.3575982996811902, + "f1": 0.19004336934903948, + "lrap": 0.6020722635494271 + }, + { + "accuracy": 0.3937300743889479, + "f1": 0.17452150999842841, + "lrap": 0.5987778958554844 + }, + { + "accuracy": 0.3900106269925611, + "f1": 0.18123793072727848, + "lrap": 0.5939426142401816 + }, + { + "accuracy": 0.4213602550478215, + "f1": 0.18953893855494874, + "lrap": 0.6021785334750381 + }, + { + "accuracy": 0.3751328374070138, + "f1": 0.19691126645172763, + "lrap": 0.6027630180658995 + }, + { + "accuracy": 0.3575982996811902, + "f1": 0.17646885294682962, + "lrap": 0.5804994686503845 + }, + { + "accuracy": 0.3783209351753454, + "f1": 0.1410394383032652, + "lrap": 0.5819872476089387 + }, + { + "accuracy": 0.3671625929861849, + "f1": 0.1118749065735074, + "lrap": 0.5569606801275366 + }, + { + "accuracy": 0.38097768331562165, + "f1": 0.17237185185597378, + "lrap": 0.5926673751328495 + } + ] + } + ] + }, + "task_name": "CEDRClassification" +} \ No newline at end of file diff --git a/results/mixedbread-ai__mxbai-embed-large-v1/990580e27d329c7408b3741ecff85876e128e203/GeoreviewClassification.json b/results/mixedbread-ai__mxbai-embed-large-v1/990580e27d329c7408b3741ecff85876e128e203/GeoreviewClassification.json new file mode 100644 index 000000000..07ee1da07 --- /dev/null +++ b/results/mixedbread-ai__mxbai-embed-large-v1/990580e27d329c7408b3741ecff85876e128e203/GeoreviewClassification.json @@ -0,0 +1,73 @@ +{ + "dataset_revision": "3765c0d1de6b7d264bc459433c45e5a75513839c", + "evaluation_time": 49.25815224647522, + "kg_co2_emissions": null, + "mteb_version": "1.19.0", + "scores": { + "test": [ + { + "accuracy": 0.278955078125, + "f1": 0.2777666419630948, + "f1_weighted": 0.27778168585556123, + "hf_subset": "default", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.278955078125, + "scores_per_experiment": [ + { + "accuracy": 0.27197265625, + "f1": 0.2705773339982639, + "f1_weighted": 0.2706244535013776 + }, + { + "accuracy": 0.31396484375, + "f1": 0.3157728509428676, + "f1_weighted": 0.31577340035766266 + }, + { + "accuracy": 0.2861328125, + "f1": 0.287593666547863, + "f1_weighted": 0.2876155031911799 + }, + { + "accuracy": 0.27294921875, + "f1": 0.274710271763196, + "f1_weighted": 0.27474814482004467 + }, + { + "accuracy": 0.28564453125, + "f1": 0.27894493123510755, + "f1_weighted": 0.27893920705478525 + }, + { + "accuracy": 0.2685546875, + "f1": 0.2669921684832809, + "f1_weighted": 0.26701463941229064 + }, + { + "accuracy": 0.26708984375, + "f1": 0.2664549726861234, + "f1_weighted": 0.26649843264341955 + }, + { + "accuracy": 0.26904296875, + "f1": 0.2671386339629639, + "f1_weighted": 0.2671215203507122 + }, + { + "accuracy": 0.251953125, + "f1": 0.24991593677178017, + "f1_weighted": 0.24992059479290846 + }, + { + "accuracy": 0.30224609375, + "f1": 0.29956565323950196, + "f1_weighted": 0.2995609624312317 + } + ] + } + ] + }, + "task_name": "GeoreviewClassification" +} \ No newline at end of file diff --git a/results/mixedbread-ai__mxbai-embed-large-v1/990580e27d329c7408b3741ecff85876e128e203/GeoreviewClusteringP2P.json b/results/mixedbread-ai__mxbai-embed-large-v1/990580e27d329c7408b3741ecff85876e128e203/GeoreviewClusteringP2P.json new file mode 100644 index 000000000..d567c3a27 --- /dev/null +++ b/results/mixedbread-ai__mxbai-embed-large-v1/990580e27d329c7408b3741ecff85876e128e203/GeoreviewClusteringP2P.json @@ -0,0 +1,34 @@ +{ + "dataset_revision": "97a313c8fc85b47f13f33e7e9a95c1ad888c7fec", + "evaluation_time": 99.70539593696594, + "kg_co2_emissions": null, + "mteb_version": "1.19.0", + "scores": { + "test": [ + { + "hf_subset": "default", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.19908811014528308, + "v_measure": 0.19908811014528308, + "v_measure_std": 0.009060440887149765, + "v_measures": { + "Level 0": [ + 0.1903534035814409, + 0.2085889472343559, + 0.20186192146295823, + 0.20304684767787776, + 0.1845063094853094, + 0.19092965429391212, + 0.1921587853471286, + 0.21175063966223723, + 0.19683372398972337, + 0.21085086871788739 + ] + } + } + ] + }, + "task_name": "GeoreviewClusteringP2P" +} \ No newline at end of file diff --git a/results/mixedbread-ai__mxbai-embed-large-v1/990580e27d329c7408b3741ecff85876e128e203/HeadlineClassification.json b/results/mixedbread-ai__mxbai-embed-large-v1/990580e27d329c7408b3741ecff85876e128e203/HeadlineClassification.json new file mode 100644 index 000000000..c2a999502 --- /dev/null +++ b/results/mixedbread-ai__mxbai-embed-large-v1/990580e27d329c7408b3741ecff85876e128e203/HeadlineClassification.json @@ -0,0 +1,73 @@ +{ + "dataset_revision": "2fe05ee6b5832cda29f2ef7aaad7b7fe6a3609eb", + "evaluation_time": 20.094862937927246, + "kg_co2_emissions": null, + "mteb_version": "1.19.0", + "scores": { + "test": [ + { + "accuracy": 0.315283203125, + "f1": 0.3145224015402392, + "f1_weighted": 0.3145263023664264, + "hf_subset": "default", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.315283203125, + "scores_per_experiment": [ + { + "accuracy": 0.34765625, + "f1": 0.34617684339447335, + "f1_weighted": 0.34620347106671157 + }, + { + "accuracy": 0.3193359375, + "f1": 0.31882317742682215, + "f1_weighted": 0.31881950365785133 + }, + { + "accuracy": 0.296875, + "f1": 0.29900808747695623, + "f1_weighted": 0.29899629894209 + }, + { + "accuracy": 0.35888671875, + "f1": 0.36104771712960376, + "f1_weighted": 0.361074550492308 + }, + { + "accuracy": 0.330078125, + "f1": 0.3311093824267887, + "f1_weighted": 0.3311137965287544 + }, + { + "accuracy": 0.333984375, + "f1": 0.33095986998472743, + "f1_weighted": 0.33096751233863253 + }, + { + "accuracy": 0.28466796875, + "f1": 0.28063163305392963, + "f1_weighted": 0.28063029166114356 + }, + { + "accuracy": 0.2763671875, + "f1": 0.2710956050118281, + "f1_weighted": 0.2711166845869758 + }, + { + "accuracy": 0.291015625, + "f1": 0.2942977392456814, + "f1_weighted": 0.29428306477396476 + }, + { + "accuracy": 0.31396484375, + "f1": 0.3120739602515817, + "f1_weighted": 0.31205784961583205 + } + ] + } + ] + }, + "task_name": "HeadlineClassification" +} \ No newline at end of file diff --git a/results/mixedbread-ai__mxbai-embed-large-v1/990580e27d329c7408b3741ecff85876e128e203/InappropriatenessClassification.json b/results/mixedbread-ai__mxbai-embed-large-v1/990580e27d329c7408b3741ecff85876e128e203/InappropriatenessClassification.json new file mode 100644 index 000000000..89864dcc5 --- /dev/null +++ b/results/mixedbread-ai__mxbai-embed-large-v1/990580e27d329c7408b3741ecff85876e128e203/InappropriatenessClassification.json @@ -0,0 +1,95 @@ +{ + "dataset_revision": "601651fdc45ef243751676e62dd7a19f491c0285", + "evaluation_time": 20.396709442138672, + "kg_co2_emissions": null, + "mteb_version": "1.19.0", + "scores": { + "test": [ + { + "accuracy": 0.515234375, + "ap": 0.5081697568026293, + "ap_weighted": 0.5081697568026293, + "f1": 0.5123938457720904, + "f1_weighted": 0.5123938457720904, + "hf_subset": "default", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.515234375, + "scores_per_experiment": [ + { + "accuracy": 0.51708984375, + "ap": 0.5089022364097968, + "ap_weighted": 0.5089022364097968, + "f1": 0.5130298473193431, + "f1_weighted": 0.5130298473193431 + }, + { + "accuracy": 0.53076171875, + "ap": 0.5162169198932269, + "ap_weighted": 0.5162169198932269, + "f1": 0.5287138964564607, + "f1_weighted": 0.5287138964564607 + }, + { + "accuracy": 0.509765625, + "ap": 0.5049671443221071, + "ap_weighted": 0.5049671443221071, + "f1": 0.5076578867469856, + "f1_weighted": 0.5076578867469856 + }, + { + "accuracy": 0.48291015625, + "ap": 0.49182567572800495, + "ap_weighted": 0.49182567572800495, + "f1": 0.477038928533968, + "f1_weighted": 0.477038928533968 + }, + { + "accuracy": 0.5322265625, + "ap": 0.5170889478211009, + "ap_weighted": 0.5170889478211009, + "f1": 0.5317402507143287, + "f1_weighted": 0.5317402507143287 + }, + { + "accuracy": 0.53173828125, + "ap": 0.5169016672922924, + "ap_weighted": 0.5169016672922924, + "f1": 0.5316684944174315, + "f1_weighted": 0.5316684944174315 + }, + { + "accuracy": 0.53955078125, + "ap": 0.5214996173641012, + "ap_weighted": 0.5214996173641012, + "f1": 0.5385578834768243, + "f1_weighted": 0.5385578834768243 + }, + { + "accuracy": 0.4951171875, + "ap": 0.49758365952258726, + "ap_weighted": 0.49758365952258726, + "f1": 0.4948160744156931, + "f1_weighted": 0.4948160744156931 + }, + { + "accuracy": 0.51171875, + "ap": 0.5059763675124792, + "ap_weighted": 0.5059763675124792, + "f1": 0.508002171709164, + "f1_weighted": 0.508002171709164 + }, + { + "accuracy": 0.50146484375, + "ap": 0.5007353321605961, + "ap_weighted": 0.5007353321605961, + "f1": 0.4927130239307045, + "f1_weighted": 0.4927130239307045 + } + ] + } + ] + }, + "task_name": "InappropriatenessClassification" +} \ No newline at end of file diff --git a/results/mixedbread-ai__mxbai-embed-large-v1/990580e27d329c7408b3741ecff85876e128e203/KinopoiskClassification.json b/results/mixedbread-ai__mxbai-embed-large-v1/990580e27d329c7408b3741ecff85876e128e203/KinopoiskClassification.json new file mode 100644 index 000000000..2bda5054f --- /dev/null +++ b/results/mixedbread-ai__mxbai-embed-large-v1/990580e27d329c7408b3741ecff85876e128e203/KinopoiskClassification.json @@ -0,0 +1,73 @@ +{ + "dataset_revision": "5911f26666ac11af46cb9c6849d0dc80a378af24", + "evaluation_time": 48.58944487571716, + "kg_co2_emissions": null, + "mteb_version": "1.19.0", + "scores": { + "test": [ + { + "accuracy": 0.3536666666666667, + "f1": 0.3508316185241228, + "f1_weighted": 0.3508316185241228, + "hf_subset": "default", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.3536666666666667, + "scores_per_experiment": [ + { + "accuracy": 0.31866666666666665, + "f1": 0.3183577092977287, + "f1_weighted": 0.3183577092977287 + }, + { + "accuracy": 0.35933333333333334, + "f1": 0.35734093258199323, + "f1_weighted": 0.35734093258199323 + }, + { + "accuracy": 0.356, + "f1": 0.3540970037692517, + "f1_weighted": 0.3540970037692517 + }, + { + "accuracy": 0.3606666666666667, + "f1": 0.3589362565308039, + "f1_weighted": 0.35893625653080397 + }, + { + "accuracy": 0.382, + "f1": 0.380698218359317, + "f1_weighted": 0.38069821835931694 + }, + { + "accuracy": 0.34933333333333333, + "f1": 0.3426582950312039, + "f1_weighted": 0.3426582950312039 + }, + { + "accuracy": 0.356, + "f1": 0.3522820883647026, + "f1_weighted": 0.35228208836470254 + }, + { + "accuracy": 0.364, + "f1": 0.3596498830187847, + "f1_weighted": 0.35964988301878476 + }, + { + "accuracy": 0.3406666666666667, + "f1": 0.33820704328171874, + "f1_weighted": 0.3382070432817187 + }, + { + "accuracy": 0.35, + "f1": 0.34608875500572395, + "f1_weighted": 0.34608875500572395 + } + ] + } + ] + }, + "task_name": "KinopoiskClassification" +} \ No newline at end of file diff --git a/results/mixedbread-ai__mxbai-embed-large-v1/990580e27d329c7408b3741ecff85876e128e203/MIRACLReranking.json b/results/mixedbread-ai__mxbai-embed-large-v1/990580e27d329c7408b3741ecff85876e128e203/MIRACLReranking.json new file mode 100644 index 000000000..921a63bb1 --- /dev/null +++ b/results/mixedbread-ai__mxbai-embed-large-v1/990580e27d329c7408b3741ecff85876e128e203/MIRACLReranking.json @@ -0,0 +1,130 @@ +{ + "dataset_revision": "6d1962c527217f8927fca80f890f14f36b2802af", + "evaluation_time": 2258.2937862873077, + "kg_co2_emissions": null, + "mteb_version": "1.19.0", + "scores": { + "dev": [ + { + "MAP@1(MIRACL)": 0.09872, + "MAP@10(MIRACL)": 0.1719, + "MAP@100(MIRACL)": 0.20514, + "MAP@1000(MIRACL)": 0.20514, + "MAP@20(MIRACL)": 0.18661, + "MAP@3(MIRACL)": 0.13674, + "MAP@5(MIRACL)": 0.15289, + "NDCG@1(MIRACL)": 0.1668, + "NDCG@10(MIRACL)": 0.2328, + "NDCG@100(MIRACL)": 0.35797, + "NDCG@1000(MIRACL)": 0.35797, + "NDCG@20(MIRACL)": 0.27409, + "NDCG@3(MIRACL)": 0.17826, + "NDCG@5(MIRACL)": 0.19685, + "P@1(MIRACL)": 0.1668, + "P@10(MIRACL)": 0.06993, + "P@100(MIRACL)": 0.01953, + "P@1000(MIRACL)": 0.00195, + "P@20(MIRACL)": 0.05188, + "P@3(MIRACL)": 0.11173, + "P@5(MIRACL)": 0.09158, + "Recall@1(MIRACL)": 0.09872, + "Recall@10(MIRACL)": 0.32542, + "Recall@100(MIRACL)": 0.79952, + "Recall@1000(MIRACL)": 0.79952, + "Recall@20(MIRACL)": 0.45131, + "Recall@3(MIRACL)": 0.17564, + "Recall@5(MIRACL)": 0.23105, + "hf_subset": "ru", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.2328, + "nAUC_MAP@1000_diff1(MIRACL)": 0.22943223816097696, + "nAUC_MAP@1000_max(MIRACL)": 0.11056037390240685, + "nAUC_MAP@1000_std(MIRACL)": 0.03144307553346312, + "nAUC_MAP@100_diff1(MIRACL)": 0.22943223816097696, + "nAUC_MAP@100_max(MIRACL)": 0.11056037390240685, + "nAUC_MAP@100_std(MIRACL)": 0.03144307553346312, + "nAUC_MAP@10_diff1(MIRACL)": 0.25083552039333007, + "nAUC_MAP@10_max(MIRACL)": 0.09514375459685312, + "nAUC_MAP@10_std(MIRACL)": 0.023914102277933193, + "nAUC_MAP@1_diff1(MIRACL)": 0.32824526451147285, + "nAUC_MAP@1_max(MIRACL)": 0.043881171010539656, + "nAUC_MAP@1_std(MIRACL)": -0.019021206160934515, + "nAUC_MAP@20_diff1(MIRACL)": 0.2399062719505385, + "nAUC_MAP@20_max(MIRACL)": 0.10313639530442588, + "nAUC_MAP@20_std(MIRACL)": 0.031039589908981206, + "nAUC_MAP@3_diff1(MIRACL)": 0.2741039626356634, + "nAUC_MAP@3_max(MIRACL)": 0.07474997768320613, + "nAUC_MAP@3_std(MIRACL)": 0.0028190560039892715, + "nAUC_MAP@5_diff1(MIRACL)": 0.26625071096706127, + "nAUC_MAP@5_max(MIRACL)": 0.09022053502035311, + "nAUC_MAP@5_std(MIRACL)": 0.017399385611565614, + "nAUC_NDCG@1000_diff1(MIRACL)": 0.13976690417700918, + "nAUC_NDCG@1000_max(MIRACL)": 0.1515296985709361, + "nAUC_NDCG@1000_std(MIRACL)": 0.04712274636787025, + "nAUC_NDCG@100_diff1(MIRACL)": 0.13976690417700918, + "nAUC_NDCG@100_max(MIRACL)": 0.1515296985709361, + "nAUC_NDCG@100_std(MIRACL)": 0.04712274636787025, + "nAUC_NDCG@10_diff1(MIRACL)": 0.20472151541177536, + "nAUC_NDCG@10_max(MIRACL)": 0.10703700554773223, + "nAUC_NDCG@10_std(MIRACL)": 0.03140190935605856, + "nAUC_NDCG@1_diff1(MIRACL)": 0.254561332184422, + "nAUC_NDCG@1_max(MIRACL)": 0.08228078509164971, + "nAUC_NDCG@1_std(MIRACL)": 0.026465218132936705, + "nAUC_NDCG@20_diff1(MIRACL)": 0.18160819920922752, + "nAUC_NDCG@20_max(MIRACL)": 0.12355517463181756, + "nAUC_NDCG@20_std(MIRACL)": 0.046721875444184296, + "nAUC_NDCG@3_diff1(MIRACL)": 0.230387486209137, + "nAUC_NDCG@3_max(MIRACL)": 0.09366230236833628, + "nAUC_NDCG@3_std(MIRACL)": 0.010624920798145547, + "nAUC_NDCG@5_diff1(MIRACL)": 0.22844666860765223, + "nAUC_NDCG@5_max(MIRACL)": 0.109282660424498, + "nAUC_NDCG@5_std(MIRACL)": 0.026625841514727754, + "nAUC_P@1000_diff1(MIRACL)": -0.09562790001742391, + "nAUC_P@1000_max(MIRACL)": 0.13911204089883203, + "nAUC_P@1000_std(MIRACL)": 0.0148478931712007, + "nAUC_P@100_diff1(MIRACL)": -0.09562790001742376, + "nAUC_P@100_max(MIRACL)": 0.13911204089883256, + "nAUC_P@100_std(MIRACL)": 0.014847893171200704, + "nAUC_P@10_diff1(MIRACL)": 0.05620501690447802, + "nAUC_P@10_max(MIRACL)": 0.1503022222662499, + "nAUC_P@10_std(MIRACL)": 0.06840240091153968, + "nAUC_P@1_diff1(MIRACL)": 0.254561332184422, + "nAUC_P@1_max(MIRACL)": 0.08228078509164971, + "nAUC_P@1_std(MIRACL)": 0.026465218132936705, + "nAUC_P@20_diff1(MIRACL)": -0.01228676729128099, + "nAUC_P@20_max(MIRACL)": 0.16129409027127004, + "nAUC_P@20_std(MIRACL)": 0.07004826033030082, + "nAUC_P@3_diff1(MIRACL)": 0.14944994686172197, + "nAUC_P@3_max(MIRACL)": 0.13714006090658531, + "nAUC_P@3_std(MIRACL)": 0.04660662519268613, + "nAUC_P@5_diff1(MIRACL)": 0.1215214038455068, + "nAUC_P@5_max(MIRACL)": 0.16684932240894684, + "nAUC_P@5_std(MIRACL)": 0.06957513406714042, + "nAUC_Recall@1000_diff1(MIRACL)": -0.15329585324031927, + "nAUC_Recall@1000_max(MIRACL)": 0.3210376913457582, + "nAUC_Recall@1000_std(MIRACL)": 0.13862787373761828, + "nAUC_Recall@100_diff1(MIRACL)": -0.15329585324031927, + "nAUC_Recall@100_max(MIRACL)": 0.3210376913457582, + "nAUC_Recall@100_std(MIRACL)": 0.13862787373761828, + "nAUC_Recall@10_diff1(MIRACL)": 0.14102908297460537, + "nAUC_Recall@10_max(MIRACL)": 0.10050212938064954, + "nAUC_Recall@10_std(MIRACL)": 0.04281447984968955, + "nAUC_Recall@1_diff1(MIRACL)": 0.32824526451147285, + "nAUC_Recall@1_max(MIRACL)": 0.043881171010539656, + "nAUC_Recall@1_std(MIRACL)": -0.019021206160934515, + "nAUC_Recall@20_diff1(MIRACL)": 0.0918342521391852, + "nAUC_Recall@20_max(MIRACL)": 0.1342798463264602, + "nAUC_Recall@20_std(MIRACL)": 0.0835167700308801, + "nAUC_Recall@3_diff1(MIRACL)": 0.20956693061477785, + "nAUC_Recall@3_max(MIRACL)": 0.08437089511521746, + "nAUC_Recall@3_std(MIRACL)": -0.0019178054859086962, + "nAUC_Recall@5_diff1(MIRACL)": 0.19371227245561576, + "nAUC_Recall@5_max(MIRACL)": 0.11533486253007798, + "nAUC_Recall@5_std(MIRACL)": 0.041960085083541675 + } + ] + }, + "task_name": "MIRACLReranking" +} \ No newline at end of file diff --git a/results/mixedbread-ai__mxbai-embed-large-v1/990580e27d329c7408b3741ecff85876e128e203/MIRACLRetrieval.json b/results/mixedbread-ai__mxbai-embed-large-v1/990580e27d329c7408b3741ecff85876e128e203/MIRACLRetrieval.json new file mode 100644 index 000000000..e83c73bf8 --- /dev/null +++ b/results/mixedbread-ai__mxbai-embed-large-v1/990580e27d329c7408b3741ecff85876e128e203/MIRACLRetrieval.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "main", + "evaluation_time": 86568.44115161896, + "kg_co2_emissions": null, + "mteb_version": "1.19.4", + "scores": { + "dev": [ + { + "hf_subset": "ru", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.08508, + "map_at_1": 0.03358, + "map_at_10": 0.05768, + "map_at_100": 0.06623, + "map_at_1000": 0.0673, + "map_at_20": 0.06193, + "map_at_3": 0.04601, + "map_at_5": 0.05263, + "mrr_at_1": 0.07028753993610223, + "mrr_at_10": 0.11138337390334195, + "mrr_at_100": 0.11945186503533868, + "mrr_at_1000": 0.12014656424142617, + "mrr_at_20": 0.11557672015935697, + "mrr_at_3": 0.09424920127795527, + "mrr_at_5": 0.10439297124600638, + "nauc_map_at_1000_diff1": 0.2332417630763323, + "nauc_map_at_1000_max": 0.34375410688183183, + "nauc_map_at_1000_std": 0.24257087874005692, + "nauc_map_at_100_diff1": 0.23346869716921098, + "nauc_map_at_100_max": 0.3405305898851264, + "nauc_map_at_100_std": 0.23823475408682834, + "nauc_map_at_10_diff1": 0.24598248460038782, + "nauc_map_at_10_max": 0.3177317953547159, + "nauc_map_at_10_std": 0.19118112140640373, + "nauc_map_at_1_diff1": 0.3303032952166368, + "nauc_map_at_1_max": 0.3116653673439535, + "nauc_map_at_1_std": 0.1252927093411389, + "nauc_map_at_20_diff1": 0.23935079494878647, + "nauc_map_at_20_max": 0.3232801395866603, + "nauc_map_at_20_std": 0.21280840844207724, + "nauc_map_at_3_diff1": 0.26549941709181557, + "nauc_map_at_3_max": 0.3087441373808146, + "nauc_map_at_3_std": 0.159216689727782, + "nauc_map_at_5_diff1": 0.25252602446852485, + "nauc_map_at_5_max": 0.31037962234744754, + "nauc_map_at_5_std": 0.1756185605837292, + "nauc_mrr_at_1000_diff1": 0.22857715845330348, + "nauc_mrr_at_1000_max": 0.2759534820389791, + "nauc_mrr_at_1000_std": 0.18875325049911604, + "nauc_mrr_at_100_diff1": 0.22802112753639242, + "nauc_mrr_at_100_max": 0.2757389885737378, + "nauc_mrr_at_100_std": 0.18841054675382213, + "nauc_mrr_at_10_diff1": 0.2344848009091356, + "nauc_mrr_at_10_max": 0.2655392625520078, + "nauc_mrr_at_10_std": 0.16838877470565358, + "nauc_mrr_at_1_diff1": 0.2868481619167037, + "nauc_mrr_at_1_max": 0.2589905123224229, + "nauc_mrr_at_1_std": 0.1180559605984791, + "nauc_mrr_at_20_diff1": 0.2294935171738862, + "nauc_mrr_at_20_max": 0.2719059319162784, + "nauc_mrr_at_20_std": 0.17951536312193409, + "nauc_mrr_at_3_diff1": 0.25259323659797217, + "nauc_mrr_at_3_max": 0.25948832895734986, + "nauc_mrr_at_3_std": 0.1488540679410989, + "nauc_mrr_at_5_diff1": 0.24308508764441583, + "nauc_mrr_at_5_max": 0.2579377067540972, + "nauc_mrr_at_5_std": 0.1567701372497884, + "nauc_ndcg_at_1000_diff1": 0.19007929762853543, + "nauc_ndcg_at_1000_max": 0.4017380238006145, + "nauc_ndcg_at_1000_std": 0.3723039648776986, + "nauc_ndcg_at_100_diff1": 0.1871744440346825, + "nauc_ndcg_at_100_max": 0.376496913540203, + "nauc_ndcg_at_100_std": 0.3362303919954122, + "nauc_ndcg_at_10_diff1": 0.22068865745534896, + "nauc_ndcg_at_10_max": 0.3072825141320072, + "nauc_ndcg_at_10_std": 0.21151831702533527, + "nauc_ndcg_at_1_diff1": 0.2868481619167037, + "nauc_ndcg_at_1_max": 0.2589905123224229, + "nauc_ndcg_at_1_std": 0.1180559605984791, + "nauc_ndcg_at_20_diff1": 0.2055393275872092, + "nauc_ndcg_at_20_max": 0.3220256568220707, + "nauc_ndcg_at_20_std": 0.25342418941393047, + "nauc_ndcg_at_3_diff1": 0.24697396480590075, + "nauc_ndcg_at_3_max": 0.28337272425286586, + "nauc_ndcg_at_3_std": 0.1590248161769906, + "nauc_ndcg_at_5_diff1": 0.23346457839976836, + "nauc_ndcg_at_5_max": 0.2917992166757424, + "nauc_ndcg_at_5_std": 0.1847786603988893, + "nauc_precision_at_1000_diff1": 0.06995143413701256, + "nauc_precision_at_1000_max": 0.3973077300343736, + "nauc_precision_at_1000_std": 0.4335887012470588, + "nauc_precision_at_100_diff1": 0.07736100276845001, + "nauc_precision_at_100_max": 0.4213162862614371, + "nauc_precision_at_100_std": 0.4558092281872701, + "nauc_precision_at_10_diff1": 0.18270865008310627, + "nauc_precision_at_10_max": 0.3293250382770411, + "nauc_precision_at_10_std": 0.27584811798153647, + "nauc_precision_at_1_diff1": 0.2868481619167037, + "nauc_precision_at_1_max": 0.2589905123224229, + "nauc_precision_at_1_std": 0.1180559605984791, + "nauc_precision_at_20_diff1": 0.14338022090733285, + "nauc_precision_at_20_max": 0.3579929434901598, + "nauc_precision_at_20_std": 0.3526462284581251, + "nauc_precision_at_3_diff1": 0.20862258538278416, + "nauc_precision_at_3_max": 0.28064650657996826, + "nauc_precision_at_3_std": 0.19071839826793766, + "nauc_precision_at_5_diff1": 0.19347876135111597, + "nauc_precision_at_5_max": 0.29162587318748434, + "nauc_precision_at_5_std": 0.22687154012469793, + "nauc_recall_at_1000_diff1": 0.12452323982219853, + "nauc_recall_at_1000_max": 0.46696021214081196, + "nauc_recall_at_1000_std": 0.5420707880503871, + "nauc_recall_at_100_diff1": 0.11054604089456592, + "nauc_recall_at_100_max": 0.40463394213352427, + "nauc_recall_at_100_std": 0.4485611790085342, + "nauc_recall_at_10_diff1": 0.17117595528552904, + "nauc_recall_at_10_max": 0.30024749630714975, + "nauc_recall_at_10_std": 0.2404226183078314, + "nauc_recall_at_1_diff1": 0.3303032952166368, + "nauc_recall_at_1_max": 0.3116653673439535, + "nauc_recall_at_1_std": 0.1252927093411389, + "nauc_recall_at_20_diff1": 0.14444550823816313, + "nauc_recall_at_20_max": 0.310432153616977, + "nauc_recall_at_20_std": 0.300250924482932, + "nauc_recall_at_3_diff1": 0.22313967962619563, + "nauc_recall_at_3_max": 0.29031245725353355, + "nauc_recall_at_3_std": 0.17665706415750718, + "nauc_recall_at_5_diff1": 0.19426374331067578, + "nauc_recall_at_5_max": 0.28127381581699684, + "nauc_recall_at_5_std": 0.2041882524566875, + "ndcg_at_1": 0.07029, + "ndcg_at_10": 0.08508, + "ndcg_at_100": 0.12724, + "ndcg_at_1000": 0.15259, + "ndcg_at_20": 0.09896, + "ndcg_at_3": 0.06585, + "ndcg_at_5": 0.07412, + "precision_at_1": 0.07029, + "precision_at_10": 0.027, + "precision_at_100": 0.00712, + "precision_at_1000": 0.00116, + "precision_at_20": 0.01913, + "precision_at_3": 0.04499, + "precision_at_5": 0.03914, + "recall_at_1": 0.03358, + "recall_at_10": 0.11479, + "recall_at_100": 0.27419, + "recall_at_1000": 0.42951, + "recall_at_20": 0.15706, + "recall_at_3": 0.0594, + "recall_at_5": 0.08445 + } + ] + }, + "task_name": "MIRACLRetrieval" +} \ No newline at end of file diff --git a/results/mixedbread-ai__mxbai-embed-large-v1/990580e27d329c7408b3741ecff85876e128e203/MassiveScenarioClassification.json b/results/mixedbread-ai__mxbai-embed-large-v1/990580e27d329c7408b3741ecff85876e128e203/MassiveScenarioClassification.json new file mode 100644 index 000000000..e66ceb0cf --- /dev/null +++ b/results/mixedbread-ai__mxbai-embed-large-v1/990580e27d329c7408b3741ecff85876e128e203/MassiveScenarioClassification.json @@ -0,0 +1,137 @@ +{ + "dataset_revision": "fad2c6e8459f9e1c45d9315f4953d921437d70f8", + "evaluation_time": 57.44345021247864, + "kg_co2_emissions": null, + "mteb_version": "1.19.0", + "scores": { + "test": [ + { + "accuracy": 0.4203093476798925, + "f1": 0.4134288100957314, + "f1_weighted": 0.42118817795638935, + "hf_subset": "ru", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.4203093476798925, + "scores_per_experiment": [ + { + "accuracy": 0.4428379287155346, + "f1": 0.4397954879703997, + "f1_weighted": 0.4384930897408912 + }, + { + "accuracy": 0.4011432414256893, + "f1": 0.39021688796430076, + "f1_weighted": 0.403618162845751 + }, + { + "accuracy": 0.43308675184936113, + "f1": 0.42653881765330665, + "f1_weighted": 0.43177106101792173 + }, + { + "accuracy": 0.4250168123739072, + "f1": 0.4130199796486607, + "f1_weighted": 0.4237460846865297 + }, + { + "accuracy": 0.4132481506388702, + "f1": 0.4169895309993322, + "f1_weighted": 0.4182838383167752 + }, + { + "accuracy": 0.3964357767316745, + "f1": 0.3933983868428932, + "f1_weighted": 0.3992622200685908 + }, + { + "accuracy": 0.43644922663080027, + "f1": 0.4264914072875832, + "f1_weighted": 0.44529419926680425 + }, + { + "accuracy": 0.4394754539340955, + "f1": 0.42506569915291054, + "f1_weighted": 0.4327197756079876 + }, + { + "accuracy": 0.3853396099529254, + "f1": 0.3829786299060687, + "f1_weighted": 0.3856581440796547 + }, + { + "accuracy": 0.4300605245460659, + "f1": 0.4197932735318587, + "f1_weighted": 0.4330352039329875 + } + ] + } + ], + "validation": [ + { + "accuracy": 0.4221347761928184, + "f1": 0.4211498484655257, + "f1_weighted": 0.42145495676589856, + "hf_subset": "ru", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.4221347761928184, + "scores_per_experiment": [ + { + "accuracy": 0.4618789965568126, + "f1": 0.4624932140982369, + "f1_weighted": 0.46194769932541624 + }, + { + "accuracy": 0.39940973930152485, + "f1": 0.40092348353610674, + "f1_weighted": 0.40039198931262976 + }, + { + "accuracy": 0.4353172651254304, + "f1": 0.4328857815875278, + "f1_weighted": 0.43479148864404904 + }, + { + "accuracy": 0.41269060501721594, + "f1": 0.4113066741984057, + "f1_weighted": 0.4084289115795088 + }, + { + "accuracy": 0.42252828332513526, + "f1": 0.4370842824025748, + "f1_weighted": 0.4235840454143644 + }, + { + "accuracy": 0.3900639449090015, + "f1": 0.3961800858011774, + "f1_weighted": 0.3923545321949321 + }, + { + "accuracy": 0.4466305951795376, + "f1": 0.43992190329883835, + "f1_weighted": 0.4520383336634002 + }, + { + "accuracy": 0.42597147073290703, + "f1": 0.4183951330811652, + "f1_weighted": 0.4121176409241173 + }, + { + "accuracy": 0.3890801770782095, + "f1": 0.38957698601663787, + "f1_weighted": 0.3894786724956017 + }, + { + "accuracy": 0.4377766847024102, + "f1": 0.42273094063458627, + "f1_weighted": 0.439416254104966 + } + ] + } + ] + }, + "task_name": "MassiveScenarioClassification" +} \ No newline at end of file diff --git a/results/mixedbread-ai__mxbai-embed-large-v1/990580e27d329c7408b3741ecff85876e128e203/RUParaPhraserSTS.json b/results/mixedbread-ai__mxbai-embed-large-v1/990580e27d329c7408b3741ecff85876e128e203/RUParaPhraserSTS.json new file mode 100644 index 000000000..2dcd3db13 --- /dev/null +++ b/results/mixedbread-ai__mxbai-embed-large-v1/990580e27d329c7408b3741ecff85876e128e203/RUParaPhraserSTS.json @@ -0,0 +1,26 @@ +{ + "dataset_revision": "43265056790b8f7c59e0139acb4be0a8dad2c8f4", + "evaluation_time": 21.36254644393921, + "kg_co2_emissions": null, + "mteb_version": "1.19.0", + "scores": { + "test": [ + { + "cosine_pearson": 0.46202669478305086, + "cosine_spearman": 0.46776263288657527, + "euclidean_pearson": 0.4797479468981409, + "euclidean_spearman": 0.46696154876598, + "hf_subset": "default", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.46776263288657527, + "manhattan_pearson": 0.47952150504363555, + "manhattan_spearman": 0.4661591085966738, + "pearson": 0.46202669478305086, + "spearman": 0.46776263288657527 + } + ] + }, + "task_name": "RUParaPhraserSTS" +} \ No newline at end of file diff --git a/results/mixedbread-ai__mxbai-embed-large-v1/990580e27d329c7408b3741ecff85876e128e203/RiaNewsRetrieval.json b/results/mixedbread-ai__mxbai-embed-large-v1/990580e27d329c7408b3741ecff85876e128e203/RiaNewsRetrieval.json new file mode 100644 index 000000000..ca87162e2 --- /dev/null +++ b/results/mixedbread-ai__mxbai-embed-large-v1/990580e27d329c7408b3741ecff85876e128e203/RiaNewsRetrieval.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "82374b0bbacda6114f39ff9c5b925fa1512ca5d7", + "evaluation_time": 16215.917149543762, + "kg_co2_emissions": null, + "mteb_version": "1.19.0", + "scores": { + "test": [ + { + "hf_subset": "default", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.22898, + "map_at_1": 0.1584, + "map_at_10": 0.20331, + "map_at_100": 0.20968, + "map_at_1000": 0.21032, + "map_at_20": 0.2069, + "map_at_3": 0.18897, + "map_at_5": 0.19712, + "mrr_at_1": 0.1583, + "mrr_at_10": 0.20328896825396825, + "mrr_at_100": 0.20964547380906892, + "mrr_at_1000": 0.21028737184176594, + "mrr_at_20": 0.20687200072931883, + "mrr_at_3": 0.18891666666666668, + "mrr_at_5": 0.19707666666666668, + "nauc_map_at_1000_diff1": 0.4417870884491414, + "nauc_map_at_1000_max": 0.3009314096383959, + "nauc_map_at_1000_std": 0.1728129922629638, + "nauc_map_at_100_diff1": 0.44182093268933065, + "nauc_map_at_100_max": 0.3009114012766087, + "nauc_map_at_100_std": 0.17257620401123866, + "nauc_map_at_10_diff1": 0.44530926314596264, + "nauc_map_at_10_max": 0.3003224516538063, + "nauc_map_at_10_std": 0.1684483718416887, + "nauc_map_at_1_diff1": 0.5215233851317252, + "nauc_map_at_1_max": 0.3098268716929374, + "nauc_map_at_1_std": 0.14297598480415377, + "nauc_map_at_20_diff1": 0.4436613416652432, + "nauc_map_at_20_max": 0.30116227198292955, + "nauc_map_at_20_std": 0.17093738658348714, + "nauc_map_at_3_diff1": 0.46037983603701105, + "nauc_map_at_3_max": 0.3022672481638709, + "nauc_map_at_3_std": 0.15954801275194896, + "nauc_map_at_5_diff1": 0.4523835326436659, + "nauc_map_at_5_max": 0.30158615598519944, + "nauc_map_at_5_std": 0.16473288266941882, + "nauc_mrr_at_1000_diff1": 0.4421526068015528, + "nauc_mrr_at_1000_max": 0.3001610701094829, + "nauc_mrr_at_1000_std": 0.1728499572670682, + "nauc_mrr_at_100_diff1": 0.44218626365951014, + "nauc_mrr_at_100_max": 0.3001416626664023, + "nauc_mrr_at_100_std": 0.1726131321668136, + "nauc_mrr_at_10_diff1": 0.4457049140955414, + "nauc_mrr_at_10_max": 0.29954144541275457, + "nauc_mrr_at_10_std": 0.16847434393037813, + "nauc_mrr_at_1_diff1": 0.5221871653726692, + "nauc_mrr_at_1_max": 0.30855749593292553, + "nauc_mrr_at_1_std": 0.14336816668121893, + "nauc_mrr_at_20_diff1": 0.4440359872729474, + "nauc_mrr_at_20_max": 0.3004006795469221, + "nauc_mrr_at_20_std": 0.17096774038824336, + "nauc_mrr_at_3_diff1": 0.4607902783621201, + "nauc_mrr_at_3_max": 0.30145558074674833, + "nauc_mrr_at_3_std": 0.15959705143741942, + "nauc_mrr_at_5_diff1": 0.452765024902934, + "nauc_mrr_at_5_max": 0.3007950535504091, + "nauc_mrr_at_5_std": 0.16476233589588365, + "nauc_ndcg_at_1000_diff1": 0.3984697743978017, + "nauc_ndcg_at_1000_max": 0.3002264121316502, + "nauc_ndcg_at_1000_std": 0.20764123479491614, + "nauc_ndcg_at_100_diff1": 0.3984813981298121, + "nauc_ndcg_at_100_max": 0.2984717023999258, + "nauc_ndcg_at_100_std": 0.2014982506311808, + "nauc_ndcg_at_10_diff1": 0.4156509111738234, + "nauc_ndcg_at_10_max": 0.29665211813632103, + "nauc_ndcg_at_10_std": 0.18174428247668833, + "nauc_ndcg_at_1_diff1": 0.5215233851317252, + "nauc_ndcg_at_1_max": 0.3098268716929374, + "nauc_ndcg_at_1_std": 0.14297598480415377, + "nauc_ndcg_at_20_diff1": 0.40993966436683676, + "nauc_ndcg_at_20_max": 0.2996591517712165, + "nauc_ndcg_at_20_std": 0.19018333504909782, + "nauc_ndcg_at_3_diff1": 0.44445280138582316, + "nauc_ndcg_at_3_max": 0.3006296498900482, + "nauc_ndcg_at_3_std": 0.16480731205338434, + "nauc_ndcg_at_5_diff1": 0.43130137480706493, + "nauc_ndcg_at_5_max": 0.29946712412401877, + "nauc_ndcg_at_5_std": 0.17347519635246503, + "nauc_precision_at_1000_diff1": 0.2486044863180585, + "nauc_precision_at_1000_max": 0.3083423917056335, + "nauc_precision_at_1000_std": 0.3628992077642823, + "nauc_precision_at_100_diff1": 0.27212841808067256, + "nauc_precision_at_100_max": 0.29300377634104013, + "nauc_precision_at_100_std": 0.295604797200394, + "nauc_precision_at_10_diff1": 0.3407600702850175, + "nauc_precision_at_10_max": 0.2872237713624827, + "nauc_precision_at_10_std": 0.21628255243497144, + "nauc_precision_at_1_diff1": 0.5215233851317252, + "nauc_precision_at_1_max": 0.3098268716929374, + "nauc_precision_at_1_std": 0.14297598480415377, + "nauc_precision_at_20_diff1": 0.3232708113671474, + "nauc_precision_at_20_max": 0.29759775520428283, + "nauc_precision_at_20_std": 0.24336979283484816, + "nauc_precision_at_3_diff1": 0.4039507105724854, + "nauc_precision_at_3_max": 0.2965654122081556, + "nauc_precision_at_3_std": 0.1784662369744681, + "nauc_precision_at_5_diff1": 0.37855079396626357, + "nauc_precision_at_5_max": 0.29424645834749374, + "nauc_precision_at_5_std": 0.19600508881770728, + "nauc_recall_at_1000_diff1": 0.2486044863180587, + "nauc_recall_at_1000_max": 0.30834239170563393, + "nauc_recall_at_1000_std": 0.3628992077642822, + "nauc_recall_at_100_diff1": 0.27212841808067295, + "nauc_recall_at_100_max": 0.2930037763410402, + "nauc_recall_at_100_std": 0.29560479720039395, + "nauc_recall_at_10_diff1": 0.3407600702850177, + "nauc_recall_at_10_max": 0.28722377136248295, + "nauc_recall_at_10_std": 0.2162825524349719, + "nauc_recall_at_1_diff1": 0.5215233851317252, + "nauc_recall_at_1_max": 0.3098268716929374, + "nauc_recall_at_1_std": 0.14297598480415377, + "nauc_recall_at_20_diff1": 0.32327081136714736, + "nauc_recall_at_20_max": 0.2975977552042829, + "nauc_recall_at_20_std": 0.24336979283484866, + "nauc_recall_at_3_diff1": 0.40395071057248527, + "nauc_recall_at_3_max": 0.2965654122081556, + "nauc_recall_at_3_std": 0.17846623697446803, + "nauc_recall_at_5_diff1": 0.3785507939662636, + "nauc_recall_at_5_max": 0.2942464583474936, + "nauc_recall_at_5_std": 0.1960050888177074, + "ndcg_at_1": 0.1584, + "ndcg_at_10": 0.22898, + "ndcg_at_100": 0.26317, + "ndcg_at_1000": 0.28318, + "ndcg_at_20": 0.24207, + "ndcg_at_3": 0.1992, + "ndcg_at_5": 0.2139, + "precision_at_1": 0.1584, + "precision_at_10": 0.03114, + "precision_at_100": 0.00479, + "precision_at_1000": 0.00064, + "precision_at_20": 0.01816, + "precision_at_3": 0.0763, + "precision_at_5": 0.05292, + "recall_at_1": 0.1584, + "recall_at_10": 0.3114, + "recall_at_100": 0.4789, + "recall_at_1000": 0.6428, + "recall_at_20": 0.3632, + "recall_at_3": 0.2289, + "recall_at_5": 0.2646 + } + ] + }, + "task_name": "RiaNewsRetrieval" +} \ No newline at end of file diff --git a/results/mixedbread-ai__mxbai-embed-large-v1/990580e27d329c7408b3741ecff85876e128e203/RuBQRetrieval.json b/results/mixedbread-ai__mxbai-embed-large-v1/990580e27d329c7408b3741ecff85876e128e203/RuBQRetrieval.json new file mode 100644 index 000000000..05ae7542e --- /dev/null +++ b/results/mixedbread-ai__mxbai-embed-large-v1/990580e27d329c7408b3741ecff85876e128e203/RuBQRetrieval.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "e19b6ffa60b3bc248e0b41f4cc37c26a55c2a67b", + "evaluation_time": 878.2089323997498, + "kg_co2_emissions": null, + "mteb_version": "1.19.0", + "scores": { + "test": [ + { + "hf_subset": "default", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.17076, + "map_at_1": 0.08562, + "map_at_10": 0.13504, + "map_at_100": 0.1408, + "map_at_1000": 0.14169, + "map_at_20": 0.13792, + "map_at_3": 0.11931, + "map_at_5": 0.12822, + "mrr_at_1": 0.13416075650118203, + "mrr_at_10": 0.18635689894555144, + "mrr_at_100": 0.19235396311190525, + "mrr_at_1000": 0.19308073086208968, + "mrr_at_20": 0.18964384893383313, + "mrr_at_3": 0.16991725768321514, + "mrr_at_5": 0.17972813238770688, + "nauc_map_at_1000_diff1": 0.38557620515131624, + "nauc_map_at_1000_max": 0.38797379744003063, + "nauc_map_at_1000_std": 0.24319771396436146, + "nauc_map_at_100_diff1": 0.3859384133175162, + "nauc_map_at_100_max": 0.38807968626741773, + "nauc_map_at_100_std": 0.2432065250886248, + "nauc_map_at_10_diff1": 0.3904499502373525, + "nauc_map_at_10_max": 0.38963676488672805, + "nauc_map_at_10_std": 0.23980123018431246, + "nauc_map_at_1_diff1": 0.45377561479673223, + "nauc_map_at_1_max": 0.37088224429798394, + "nauc_map_at_1_std": 0.180108648475218, + "nauc_map_at_20_diff1": 0.38737449001757085, + "nauc_map_at_20_max": 0.3889631717465292, + "nauc_map_at_20_std": 0.24145453178718923, + "nauc_map_at_3_diff1": 0.3965998378047779, + "nauc_map_at_3_max": 0.3838523025260094, + "nauc_map_at_3_std": 0.2252164287460096, + "nauc_map_at_5_diff1": 0.3910696795906845, + "nauc_map_at_5_max": 0.3882424318411005, + "nauc_map_at_5_std": 0.2358437204866368, + "nauc_mrr_at_1000_diff1": 0.37695503272840475, + "nauc_mrr_at_1000_max": 0.37342806895023223, + "nauc_mrr_at_1000_std": 0.21398176261138507, + "nauc_mrr_at_100_diff1": 0.37690504951579257, + "nauc_mrr_at_100_max": 0.3734160583842091, + "nauc_mrr_at_100_std": 0.21401103711933103, + "nauc_mrr_at_10_diff1": 0.3822659567136559, + "nauc_mrr_at_10_max": 0.37617642820476743, + "nauc_mrr_at_10_std": 0.2127816868657706, + "nauc_mrr_at_1_diff1": 0.4344471743653368, + "nauc_mrr_at_1_max": 0.38029981360479026, + "nauc_mrr_at_1_std": 0.184310898096029, + "nauc_mrr_at_20_diff1": 0.3789034745030878, + "nauc_mrr_at_20_max": 0.37407489051103554, + "nauc_mrr_at_20_std": 0.21282883081016038, + "nauc_mrr_at_3_diff1": 0.3871739144655275, + "nauc_mrr_at_3_max": 0.379274154987768, + "nauc_mrr_at_3_std": 0.20555595388132566, + "nauc_mrr_at_5_diff1": 0.38432091643847, + "nauc_mrr_at_5_max": 0.37861449329032015, + "nauc_mrr_at_5_std": 0.21161700429875382, + "nauc_ndcg_at_1000_diff1": 0.34610573648556575, + "nauc_ndcg_at_1000_max": 0.3752770087656698, + "nauc_ndcg_at_1000_std": 0.2613967974918174, + "nauc_ndcg_at_100_diff1": 0.3495126522395926, + "nauc_ndcg_at_100_max": 0.3744705663655452, + "nauc_ndcg_at_100_std": 0.2594021471948557, + "nauc_ndcg_at_10_diff1": 0.3717739129696632, + "nauc_ndcg_at_10_max": 0.3852067701119578, + "nauc_ndcg_at_10_std": 0.24696980389455495, + "nauc_ndcg_at_1_diff1": 0.43861646817580624, + "nauc_ndcg_at_1_max": 0.3766796943800191, + "nauc_ndcg_at_1_std": 0.1800633982462277, + "nauc_ndcg_at_20_diff1": 0.36184528703440955, + "nauc_ndcg_at_20_max": 0.38106702484750427, + "nauc_ndcg_at_20_std": 0.2504840327554193, + "nauc_ndcg_at_3_diff1": 0.3821750722737472, + "nauc_ndcg_at_3_max": 0.3835779645624937, + "nauc_ndcg_at_3_std": 0.2269989776891949, + "nauc_ndcg_at_5_diff1": 0.3741947694423963, + "nauc_ndcg_at_5_max": 0.3873384607274641, + "nauc_ndcg_at_5_std": 0.24162334184765977, + "nauc_precision_at_1000_diff1": 0.14483224131375697, + "nauc_precision_at_1000_max": 0.2525597370177232, + "nauc_precision_at_1000_std": 0.2336878417220527, + "nauc_precision_at_100_diff1": 0.21430584344277268, + "nauc_precision_at_100_max": 0.30197097089139274, + "nauc_precision_at_100_std": 0.27188173109975744, + "nauc_precision_at_10_diff1": 0.31338181152661476, + "nauc_precision_at_10_max": 0.37687912625377495, + "nauc_precision_at_10_std": 0.27103928300949853, + "nauc_precision_at_1_diff1": 0.43861646817580624, + "nauc_precision_at_1_max": 0.3766796943800191, + "nauc_precision_at_1_std": 0.1800633982462277, + "nauc_precision_at_20_diff1": 0.2845701609742692, + "nauc_precision_at_20_max": 0.36309894772668516, + "nauc_precision_at_20_std": 0.27606082224862893, + "nauc_precision_at_3_diff1": 0.3435094628158413, + "nauc_precision_at_3_max": 0.39384965182707654, + "nauc_precision_at_3_std": 0.24617487829110588, + "nauc_precision_at_5_diff1": 0.3290633326578079, + "nauc_precision_at_5_max": 0.3944105688415801, + "nauc_precision_at_5_std": 0.26796359470866915, + "nauc_recall_at_1000_diff1": 0.20268551648550456, + "nauc_recall_at_1000_max": 0.2981494889697248, + "nauc_recall_at_1000_std": 0.2962374352841488, + "nauc_recall_at_100_diff1": 0.23131661149093763, + "nauc_recall_at_100_max": 0.3069827065734971, + "nauc_recall_at_100_std": 0.28721137725730667, + "nauc_recall_at_10_diff1": 0.31338833226954915, + "nauc_recall_at_10_max": 0.3607120093639793, + "nauc_recall_at_10_std": 0.2668463028067846, + "nauc_recall_at_1_diff1": 0.45377561479673223, + "nauc_recall_at_1_max": 0.37088224429798394, + "nauc_recall_at_1_std": 0.180108648475218, + "nauc_recall_at_20_diff1": 0.2792383424242233, + "nauc_recall_at_20_max": 0.33903877655411796, + "nauc_recall_at_20_std": 0.2678494591744781, + "nauc_recall_at_3_diff1": 0.3375218569308383, + "nauc_recall_at_3_max": 0.3668861217996448, + "nauc_recall_at_3_std": 0.23632991640979045, + "nauc_recall_at_5_diff1": 0.32022712241583595, + "nauc_recall_at_5_max": 0.3724122369338643, + "nauc_recall_at_5_std": 0.26172768429622967, + "ndcg_at_1": 0.13357, + "ndcg_at_10": 0.17076, + "ndcg_at_100": 0.20231, + "ndcg_at_1000": 0.22825, + "ndcg_at_20": 0.18117, + "ndcg_at_3": 0.14092, + "ndcg_at_5": 0.15551, + "precision_at_1": 0.13357, + "precision_at_10": 0.03641, + "precision_at_100": 0.00602, + "precision_at_1000": 0.00092, + "precision_at_20": 0.02128, + "precision_at_3": 0.08038, + "precision_at_5": 0.0591, + "recall_at_1": 0.08562, + "recall_at_10": 0.22795, + "recall_at_100": 0.3665, + "recall_at_1000": 0.55503, + "recall_at_20": 0.26385, + "recall_at_3": 0.15149, + "recall_at_5": 0.18674 + } + ] + }, + "task_name": "RuBQRetrieval" +} \ No newline at end of file diff --git a/results/mixedbread-ai__mxbai-embed-large-v1/990580e27d329c7408b3741ecff85876e128e203/RuReviewsClassification.json b/results/mixedbread-ai__mxbai-embed-large-v1/990580e27d329c7408b3741ecff85876e128e203/RuReviewsClassification.json new file mode 100644 index 000000000..c6b92b120 --- /dev/null +++ b/results/mixedbread-ai__mxbai-embed-large-v1/990580e27d329c7408b3741ecff85876e128e203/RuReviewsClassification.json @@ -0,0 +1,73 @@ +{ + "dataset_revision": "f6d2c31f4dc6b88f468552750bfec05b4b41b05a", + "evaluation_time": 23.95420241355896, + "kg_co2_emissions": null, + "mteb_version": "1.19.0", + "scores": { + "test": [ + { + "accuracy": 0.435107421875, + "f1": 0.4329657011562292, + "f1_weighted": 0.4329589503501623, + "hf_subset": "default", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.435107421875, + "scores_per_experiment": [ + { + "accuracy": 0.44970703125, + "f1": 0.4351702555461274, + "f1_weighted": 0.43514086579941336 + }, + { + "accuracy": 0.44775390625, + "f1": 0.44866717241465476, + "f1_weighted": 0.44866263952519153 + }, + { + "accuracy": 0.44873046875, + "f1": 0.45012981024239956, + "f1_weighted": 0.45009619777985405 + }, + { + "accuracy": 0.48095703125, + "f1": 0.48311813321807584, + "f1_weighted": 0.48310436173383475 + }, + { + "accuracy": 0.46142578125, + "f1": 0.46055153423574474, + "f1_weighted": 0.46053466247750785 + }, + { + "accuracy": 0.3984375, + "f1": 0.4025783589438113, + "f1_weighted": 0.4025720486181139 + }, + { + "accuracy": 0.40087890625, + "f1": 0.38823224065138545, + "f1_weighted": 0.38825835884462545 + }, + { + "accuracy": 0.46875, + "f1": 0.46789908123500723, + "f1_weighted": 0.467885780801292 + }, + { + "accuracy": 0.35205078125, + "f1": 0.35016762144099306, + "f1_weighted": 0.3501913612501647 + }, + { + "accuracy": 0.4423828125, + "f1": 0.44314280363409164, + "f1_weighted": 0.44314322667162576 + } + ] + } + ] + }, + "task_name": "RuReviewsClassification" +} \ No newline at end of file diff --git a/results/mixedbread-ai__mxbai-embed-large-v1/990580e27d329c7408b3741ecff85876e128e203/RuSTSBenchmarkSTS.json b/results/mixedbread-ai__mxbai-embed-large-v1/990580e27d329c7408b3741ecff85876e128e203/RuSTSBenchmarkSTS.json new file mode 100644 index 000000000..00f2ca549 --- /dev/null +++ b/results/mixedbread-ai__mxbai-embed-large-v1/990580e27d329c7408b3741ecff85876e128e203/RuSTSBenchmarkSTS.json @@ -0,0 +1,26 @@ +{ + "dataset_revision": "7cf24f325c6da6195df55bef3d86b5e0616f3018", + "evaluation_time": 14.886790990829468, + "kg_co2_emissions": null, + "mteb_version": "1.19.0", + "scores": { + "test": [ + { + "cosine_pearson": 0.5958093955582096, + "cosine_spearman": 0.5884938218307196, + "euclidean_pearson": 0.5857026301538986, + "euclidean_spearman": 0.5816692677546715, + "hf_subset": "default", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.5884938218307196, + "manhattan_pearson": 0.586009721209628, + "manhattan_spearman": 0.5819798038333853, + "pearson": 0.5958093955582096, + "spearman": 0.5884938218307196 + } + ] + }, + "task_name": "RuSTSBenchmarkSTS" +} \ No newline at end of file diff --git a/results/mixedbread-ai__mxbai-embed-large-v1/990580e27d329c7408b3741ecff85876e128e203/RuSciBenchGRNTIClassification.json b/results/mixedbread-ai__mxbai-embed-large-v1/990580e27d329c7408b3741ecff85876e128e203/RuSciBenchGRNTIClassification.json new file mode 100644 index 000000000..4428d4350 --- /dev/null +++ b/results/mixedbread-ai__mxbai-embed-large-v1/990580e27d329c7408b3741ecff85876e128e203/RuSciBenchGRNTIClassification.json @@ -0,0 +1,73 @@ +{ + "dataset_revision": "673a610d6d3dd91a547a0d57ae1b56f37ebbf6a1", + "evaluation_time": 112.62466406822205, + "kg_co2_emissions": null, + "mteb_version": "1.19.0", + "scores": { + "test": [ + { + "accuracy": 0.221435546875, + "f1": 0.2207432959131881, + "f1_weighted": 0.22083963517991118, + "hf_subset": "default", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.221435546875, + "scores_per_experiment": [ + { + "accuracy": 0.2451171875, + "f1": 0.24550183227605404, + "f1_weighted": 0.24550336555482338 + }, + { + "accuracy": 0.212890625, + "f1": 0.2130583198397948, + "f1_weighted": 0.21319096159719245 + }, + { + "accuracy": 0.1943359375, + "f1": 0.19556101463999373, + "f1_weighted": 0.19556204842922884 + }, + { + "accuracy": 0.23681640625, + "f1": 0.2350295740515891, + "f1_weighted": 0.2351405587495632 + }, + { + "accuracy": 0.2060546875, + "f1": 0.20617887407242483, + "f1_weighted": 0.20624223676400769 + }, + { + "accuracy": 0.21044921875, + "f1": 0.20932198184004627, + "f1_weighted": 0.20944552011235196 + }, + { + "accuracy": 0.23583984375, + "f1": 0.2356888865875344, + "f1_weighted": 0.23589040484944676 + }, + { + "accuracy": 0.21240234375, + "f1": 0.20933046580206968, + "f1_weighted": 0.20945798666161244 + }, + { + "accuracy": 0.22119140625, + "f1": 0.21860146631982819, + "f1_weighted": 0.21870592834801594 + }, + { + "accuracy": 0.2392578125, + "f1": 0.23916054370254564, + "f1_weighted": 0.23925734073286906 + } + ] + } + ] + }, + "task_name": "RuSciBenchGRNTIClassification" +} \ No newline at end of file diff --git a/results/mixedbread-ai__mxbai-embed-large-v1/990580e27d329c7408b3741ecff85876e128e203/RuSciBenchGRNTIClusteringP2P.json b/results/mixedbread-ai__mxbai-embed-large-v1/990580e27d329c7408b3741ecff85876e128e203/RuSciBenchGRNTIClusteringP2P.json new file mode 100644 index 000000000..92cc2efe0 --- /dev/null +++ b/results/mixedbread-ai__mxbai-embed-large-v1/990580e27d329c7408b3741ecff85876e128e203/RuSciBenchGRNTIClusteringP2P.json @@ -0,0 +1,34 @@ +{ + "dataset_revision": "673a610d6d3dd91a547a0d57ae1b56f37ebbf6a1", + "evaluation_time": 130.02561473846436, + "kg_co2_emissions": null, + "mteb_version": "1.19.0", + "scores": { + "test": [ + { + "hf_subset": "default", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.20597081060530337, + "v_measure": 0.20597081060530337, + "v_measure_std": 0.01023133242748098, + "v_measures": { + "Level 0": [ + 0.21568814935780248, + 0.2206360800870867, + 0.2039718439546621, + 0.20068910398124512, + 0.21324853239452746, + 0.21899376991653158, + 0.19836413590608848, + 0.19579229756612987, + 0.20419217926379327, + 0.18813201362516652 + ] + } + } + ] + }, + "task_name": "RuSciBenchGRNTIClusteringP2P" +} \ No newline at end of file diff --git a/results/mixedbread-ai__mxbai-embed-large-v1/990580e27d329c7408b3741ecff85876e128e203/RuSciBenchOECDClassification.json b/results/mixedbread-ai__mxbai-embed-large-v1/990580e27d329c7408b3741ecff85876e128e203/RuSciBenchOECDClassification.json new file mode 100644 index 000000000..748e0b967 --- /dev/null +++ b/results/mixedbread-ai__mxbai-embed-large-v1/990580e27d329c7408b3741ecff85876e128e203/RuSciBenchOECDClassification.json @@ -0,0 +1,73 @@ +{ + "dataset_revision": "26c88e99dcaba32bb45d0e1bfc21902337f6d471", + "evaluation_time": 110.4993965625763, + "kg_co2_emissions": null, + "mteb_version": "1.19.0", + "scores": { + "test": [ + { + "accuracy": 0.161865234375, + "f1": 0.16196586483847594, + "f1_weighted": 0.16197549182238552, + "hf_subset": "default", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.161865234375, + "scores_per_experiment": [ + { + "accuracy": 0.1494140625, + "f1": 0.14751352356859207, + "f1_weighted": 0.14742971725108145 + }, + { + "accuracy": 0.16064453125, + "f1": 0.16021316306216937, + "f1_weighted": 0.160300343800146 + }, + { + "accuracy": 0.15869140625, + "f1": 0.15859304480019362, + "f1_weighted": 0.15851111261205433 + }, + { + "accuracy": 0.1630859375, + "f1": 0.16407539451126046, + "f1_weighted": 0.16404193715990004 + }, + { + "accuracy": 0.15380859375, + "f1": 0.15493250818793852, + "f1_weighted": 0.1550382507017907 + }, + { + "accuracy": 0.1669921875, + "f1": 0.16604178505124725, + "f1_weighted": 0.16615420501567946 + }, + { + "accuracy": 0.17724609375, + "f1": 0.1771040308868535, + "f1_weighted": 0.17705744134229498 + }, + { + "accuracy": 0.1484375, + "f1": 0.1475566521252493, + "f1_weighted": 0.14757183864678228 + }, + { + "accuracy": 0.1630859375, + "f1": 0.16557613521015355, + "f1_weighted": 0.16561837769045565 + }, + { + "accuracy": 0.17724609375, + "f1": 0.17805241098110197, + "f1_weighted": 0.17803169400367028 + } + ] + } + ] + }, + "task_name": "RuSciBenchOECDClassification" +} \ No newline at end of file diff --git a/results/mixedbread-ai__mxbai-embed-large-v1/990580e27d329c7408b3741ecff85876e128e203/RuSciBenchOECDClusteringP2P.json b/results/mixedbread-ai__mxbai-embed-large-v1/990580e27d329c7408b3741ecff85876e128e203/RuSciBenchOECDClusteringP2P.json new file mode 100644 index 000000000..5a33345e0 --- /dev/null +++ b/results/mixedbread-ai__mxbai-embed-large-v1/990580e27d329c7408b3741ecff85876e128e203/RuSciBenchOECDClusteringP2P.json @@ -0,0 +1,34 @@ +{ + "dataset_revision": "26c88e99dcaba32bb45d0e1bfc21902337f6d471", + "evaluation_time": 123.90057277679443, + "kg_co2_emissions": null, + "mteb_version": "1.19.0", + "scores": { + "test": [ + { + "hf_subset": "default", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.17155519924156507, + "v_measure": 0.17155519924156507, + "v_measure_std": 0.008471959564895336, + "v_measures": { + "Level 0": [ + 0.17857172881979727, + 0.16200054357437021, + 0.1735372123946763, + 0.1767242427773198, + 0.18152821072597405, + 0.15687812314241842, + 0.1803819942078775, + 0.15972776724219304, + 0.17096649263046185, + 0.17523567690056205 + ] + } + } + ] + }, + "task_name": "RuSciBenchOECDClusteringP2P" +} \ No newline at end of file diff --git a/results/mixedbread-ai__mxbai-embed-large-v1/990580e27d329c7408b3741ecff85876e128e203/STS22.json b/results/mixedbread-ai__mxbai-embed-large-v1/990580e27d329c7408b3741ecff85876e128e203/STS22.json new file mode 100644 index 000000000..ef9ae6083 --- /dev/null +++ b/results/mixedbread-ai__mxbai-embed-large-v1/990580e27d329c7408b3741ecff85876e128e203/STS22.json @@ -0,0 +1,26 @@ +{ + "dataset_revision": "de9d86b3b84231dc21f76c7b7af1f28e2f57f6e3", + "evaluation_time": 12.704498767852783, + "kg_co2_emissions": null, + "mteb_version": "1.19.0", + "scores": { + "test": [ + { + "cosine_pearson": 0.17059747889484514, + "cosine_spearman": 0.2844060201234706, + "euclidean_pearson": 0.17248325457901784, + "euclidean_spearman": 0.27491800490394286, + "hf_subset": "ru", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.2844060201234706, + "manhattan_pearson": 0.1724318754441268, + "manhattan_spearman": 0.276696504104436, + "pearson": 0.17059747889484514, + "spearman": 0.2844060201234706 + } + ] + }, + "task_name": "STS22" +} \ No newline at end of file diff --git a/results/mixedbread-ai__mxbai-embed-large-v1/990580e27d329c7408b3741ecff85876e128e203/SensitiveTopicsClassification.json b/results/mixedbread-ai__mxbai-embed-large-v1/990580e27d329c7408b3741ecff85876e128e203/SensitiveTopicsClassification.json new file mode 100644 index 000000000..e87305ad1 --- /dev/null +++ b/results/mixedbread-ai__mxbai-embed-large-v1/990580e27d329c7408b3741ecff85876e128e203/SensitiveTopicsClassification.json @@ -0,0 +1,73 @@ +{ + "dataset_revision": "416b34a802308eac30e4192afc0ff99bb8dcc7f2", + "evaluation_time": 32.427332639694214, + "kg_co2_emissions": null, + "mteb_version": "1.19.0", + "scores": { + "test": [ + { + "accuracy": 0.1810546875, + "f1": 0.040591651198480114, + "hf_subset": "default", + "languages": [ + "rus-Cyrl" + ], + "lrap": 0.26594102647568796, + "main_score": 0.1810546875, + "scores_per_experiment": [ + { + "accuracy": 0.17626953125, + "f1": 0.015035087130330391, + "lrap": 0.2574734157986047 + }, + { + "accuracy": 0.1826171875, + "f1": 0.04673775177809726, + "lrap": 0.2660319010416603 + }, + { + "accuracy": 0.181640625, + "f1": 0.049228031137554557, + "lrap": 0.2701822916666601 + }, + { + "accuracy": 0.17822265625, + "f1": 0.029484504293022813, + "lrap": 0.26026746961804914 + }, + { + "accuracy": 0.17578125, + "f1": 0.02900518884611954, + "lrap": 0.2618950737847159 + }, + { + "accuracy": 0.18896484375, + "f1": 0.06189176245217117, + "lrap": 0.27407497829860455 + }, + { + "accuracy": 0.1806640625, + "f1": 0.027386354919965883, + "lrap": 0.26565212673610467 + }, + { + "accuracy": 0.17578125, + "f1": 0.03154293633410062, + "lrap": 0.2617051866319381 + }, + { + "accuracy": 0.1806640625, + "f1": 0.04590970421505351, + "lrap": 0.2652316623263825 + }, + { + "accuracy": 0.18994140625, + "f1": 0.06969519087838541, + "lrap": 0.2768961588541601 + } + ] + } + ] + }, + "task_name": "SensitiveTopicsClassification" +} \ No newline at end of file diff --git a/results/mixedbread-ai__mxbai-embed-large-v1/990580e27d329c7408b3741ecff85876e128e203/model_meta.json b/results/mixedbread-ai__mxbai-embed-large-v1/990580e27d329c7408b3741ecff85876e128e203/model_meta.json index 2f2ac5651..a8c53c4b0 100644 --- a/results/mixedbread-ai__mxbai-embed-large-v1/990580e27d329c7408b3741ecff85876e128e203/model_meta.json +++ b/results/mixedbread-ai__mxbai-embed-large-v1/990580e27d329c7408b3741ecff85876e128e203/model_meta.json @@ -1 +1,25 @@ -{"name": "mixedbread-ai/mxbai-embed-large-v1", "revision": "990580e27d329c7408b3741ecff85876e128e203", "release_date": "2024-03-07", "languages": ["eng_Latn"], "n_parameters": 335000000, "memory_usage": null, "max_tokens": 512, "embed_dim": 1024, "license": "apache-2.0", "open_weights": true, "public_training_data": null, "public_training_code": null, "framework": ["Sentence Transformers", "PyTorch"], "reference": "https://huggingface.co/mixedbread-ai/mxbai-embed-large-v1", "similarity_fn_name": "cosine", "use_instructions": true, "zero_shot_benchmarks": null, "loader": "sentence_transformers_loader"} \ No newline at end of file +{ + "name": "mixedbread-ai/mxbai-embed-large-v1", + "revision": "990580e27d329c7408b3741ecff85876e128e203", + "release_date": "2024-03-07", + "languages": [ + "eng_Latn" + ], + "n_parameters": 335000000, + "memory_usage": null, + "max_tokens": 512, + "embed_dim": 1024, + "license": "apache-2.0", + "open_weights": true, + "public_training_data": null, + "public_training_code": null, + "framework": [ + "Sentence Transformers", + "PyTorch" + ], + "reference": "https://huggingface.co/mixedbread-ai/mxbai-embed-large-v1", + "similarity_fn_name": "cosine", + "use_instructions": true, + "zero_shot_benchmarks": null, + "loader": "sentence_transformers_loader" +} \ No newline at end of file diff --git a/results/nvidia__NV-Embed-v1/570834afd5fef5bf3a3c2311a2b6e0a66f6f4f2c/CEDRClassification.json b/results/nvidia__NV-Embed-v1/570834afd5fef5bf3a3c2311a2b6e0a66f6f4f2c/CEDRClassification.json new file mode 100644 index 000000000..8a9bcbda3 --- /dev/null +++ b/results/nvidia__NV-Embed-v1/570834afd5fef5bf3a3c2311a2b6e0a66f6f4f2c/CEDRClassification.json @@ -0,0 +1,73 @@ +{ + "dataset_revision": "c0ba03d058e3e1b2f3fd20518875a4563dd12db4", + "evaluation_time": 89.23333787918091, + "kg_co2_emissions": null, + "mteb_version": "1.19.2", + "scores": { + "test": [ + { + "accuracy": 0.3871413390010627, + "f1": 0.20862805360039655, + "hf_subset": "default", + "languages": [ + "rus-Cyrl" + ], + "lrap": 0.599346439957504, + "main_score": 0.3871413390010627, + "scores_per_experiment": [ + { + "accuracy": 0.45536663124335813, + "f1": 0.2537053550587935, + "lrap": 0.6303400637619675 + }, + { + "accuracy": 0.3028692879914984, + "f1": 0.1439057882790437, + "lrap": 0.5948459086078752 + }, + { + "accuracy": 0.4351753453772582, + "f1": 0.2775642265564078, + "lrap": 0.6114240170032001 + }, + { + "accuracy": 0.40435706695005313, + "f1": 0.16981367491663374, + "lrap": 0.6004782146652614 + }, + { + "accuracy": 0.42348565356004253, + "f1": 0.21504825103771036, + "lrap": 0.5902763018066006 + }, + { + "accuracy": 0.40116896918172157, + "f1": 0.24941772027712245, + "lrap": 0.6123273113708942 + }, + { + "accuracy": 0.2879914984059511, + "f1": 0.1419221942943736, + "lrap": 0.5744420828905545 + }, + { + "accuracy": 0.3937300743889479, + "f1": 0.2606488694050685, + "lrap": 0.6111052072263666 + }, + { + "accuracy": 0.37619553666312433, + "f1": 0.14791400765450122, + "lrap": 0.5715727948990557 + }, + { + "accuracy": 0.39107332624867164, + "f1": 0.2263404485243108, + "lrap": 0.596652497343264 + } + ] + } + ] + }, + "task_name": "CEDRClassification" +} \ No newline at end of file diff --git a/results/nvidia__NV-Embed-v1/570834afd5fef5bf3a3c2311a2b6e0a66f6f4f2c/GeoreviewClassification.json b/results/nvidia__NV-Embed-v1/570834afd5fef5bf3a3c2311a2b6e0a66f6f4f2c/GeoreviewClassification.json new file mode 100644 index 000000000..3b0fbfe37 --- /dev/null +++ b/results/nvidia__NV-Embed-v1/570834afd5fef5bf3a3c2311a2b6e0a66f6f4f2c/GeoreviewClassification.json @@ -0,0 +1,73 @@ +{ + "dataset_revision": "3765c0d1de6b7d264bc459433c45e5a75513839c", + "evaluation_time": 118.31113409996033, + "kg_co2_emissions": null, + "mteb_version": "1.19.2", + "scores": { + "test": [ + { + "accuracy": 0.4646484375, + "f1": 0.4441295751392896, + "f1_weighted": 0.4440984091810051, + "hf_subset": "default", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.4646484375, + "scores_per_experiment": [ + { + "accuracy": 0.4638671875, + "f1": 0.44413280625591317, + "f1_weighted": 0.4441423683548316 + }, + { + "accuracy": 0.45751953125, + "f1": 0.4354210685201562, + "f1_weighted": 0.43536485922997437 + }, + { + "accuracy": 0.4609375, + "f1": 0.4197302306757268, + "f1_weighted": 0.4196816017094853 + }, + { + "accuracy": 0.47412109375, + "f1": 0.4444612272492126, + "f1_weighted": 0.4444822701586552 + }, + { + "accuracy": 0.474609375, + "f1": 0.4576659762114049, + "f1_weighted": 0.45762944730842203 + }, + { + "accuracy": 0.451171875, + "f1": 0.44149323094970994, + "f1_weighted": 0.4415084063301604 + }, + { + "accuracy": 0.46044921875, + "f1": 0.4362155803539693, + "f1_weighted": 0.4361735561104856 + }, + { + "accuracy": 0.49365234375, + "f1": 0.48309990155972127, + "f1_weighted": 0.48302396070959247 + }, + { + "accuracy": 0.4375, + "f1": 0.4341165559574248, + "f1_weighted": 0.43407095658983763 + }, + { + "accuracy": 0.47265625, + "f1": 0.4449591736596571, + "f1_weighted": 0.444906665308607 + } + ] + } + ] + }, + "task_name": "GeoreviewClassification" +} \ No newline at end of file diff --git a/results/nvidia__NV-Embed-v1/570834afd5fef5bf3a3c2311a2b6e0a66f6f4f2c/GeoreviewClusteringP2P.json b/results/nvidia__NV-Embed-v1/570834afd5fef5bf3a3c2311a2b6e0a66f6f4f2c/GeoreviewClusteringP2P.json new file mode 100644 index 000000000..d8b64eca8 --- /dev/null +++ b/results/nvidia__NV-Embed-v1/570834afd5fef5bf3a3c2311a2b6e0a66f6f4f2c/GeoreviewClusteringP2P.json @@ -0,0 +1,34 @@ +{ + "dataset_revision": "97a313c8fc85b47f13f33e7e9a95c1ad888c7fec", + "evaluation_time": 172.04543828964233, + "kg_co2_emissions": null, + "mteb_version": "1.19.2", + "scores": { + "test": [ + { + "hf_subset": "default", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.6867506740138852, + "v_measure": 0.6867506740138852, + "v_measure_std": 0.009224317308002718, + "v_measures": { + "Level 0": [ + 0.6961681915300013, + 0.6836219190891112, + 0.6752747018124018, + 0.6989435071532972, + 0.682896443957637, + 0.6727987370065233, + 0.6960398345244616, + 0.6889813788929955, + 0.6956377298469909, + 0.677144296325432 + ] + } + } + ] + }, + "task_name": "GeoreviewClusteringP2P" +} \ No newline at end of file diff --git a/results/nvidia__NV-Embed-v1/570834afd5fef5bf3a3c2311a2b6e0a66f6f4f2c/HeadlineClassification.json b/results/nvidia__NV-Embed-v1/570834afd5fef5bf3a3c2311a2b6e0a66f6f4f2c/HeadlineClassification.json new file mode 100644 index 000000000..642e57980 --- /dev/null +++ b/results/nvidia__NV-Embed-v1/570834afd5fef5bf3a3c2311a2b6e0a66f6f4f2c/HeadlineClassification.json @@ -0,0 +1,73 @@ +{ + "dataset_revision": "2fe05ee6b5832cda29f2ef7aaad7b7fe6a3609eb", + "evaluation_time": 83.64405584335327, + "kg_co2_emissions": null, + "mteb_version": "1.19.2", + "scores": { + "test": [ + { + "accuracy": 0.748486328125, + "f1": 0.7466898183273674, + "f1_weighted": 0.7467082986568683, + "hf_subset": "default", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.748486328125, + "scores_per_experiment": [ + { + "accuracy": 0.71875, + "f1": 0.7191045412159892, + "f1_weighted": 0.7191177034689499 + }, + { + "accuracy": 0.7529296875, + "f1": 0.7527358865554752, + "f1_weighted": 0.7527480000676622 + }, + { + "accuracy": 0.73193359375, + "f1": 0.7289363484320929, + "f1_weighted": 0.7289457742791354 + }, + { + "accuracy": 0.7646484375, + "f1": 0.7643018175492848, + "f1_weighted": 0.7643124770124666 + }, + { + "accuracy": 0.77001953125, + "f1": 0.7707095618558135, + "f1_weighted": 0.7707207812925984 + }, + { + "accuracy": 0.7509765625, + "f1": 0.7490085251196779, + "f1_weighted": 0.749029048739996 + }, + { + "accuracy": 0.7470703125, + "f1": 0.7435493835484843, + "f1_weighted": 0.7435676790823603 + }, + { + "accuracy": 0.74951171875, + "f1": 0.747076327965792, + "f1_weighted": 0.7470960661877866 + }, + { + "accuracy": 0.7255859375, + "f1": 0.7192105799032245, + "f1_weighted": 0.719253397507145 + }, + { + "accuracy": 0.7734375, + "f1": 0.7722652111278384, + "f1_weighted": 0.7722920589305833 + } + ] + } + ] + }, + "task_name": "HeadlineClassification" +} \ No newline at end of file diff --git a/results/nvidia__NV-Embed-v1/570834afd5fef5bf3a3c2311a2b6e0a66f6f4f2c/InappropriatenessClassification.json b/results/nvidia__NV-Embed-v1/570834afd5fef5bf3a3c2311a2b6e0a66f6f4f2c/InappropriatenessClassification.json new file mode 100644 index 000000000..84284e386 --- /dev/null +++ b/results/nvidia__NV-Embed-v1/570834afd5fef5bf3a3c2311a2b6e0a66f6f4f2c/InappropriatenessClassification.json @@ -0,0 +1,95 @@ +{ + "dataset_revision": "601651fdc45ef243751676e62dd7a19f491c0285", + "evaluation_time": 75.5907051563263, + "kg_co2_emissions": null, + "mteb_version": "1.19.2", + "scores": { + "test": [ + { + "accuracy": 0.5970703125, + "ap": 0.557315117150581, + "ap_weighted": 0.557315117150581, + "f1": 0.5905441943515085, + "f1_weighted": 0.5905441943515085, + "hf_subset": "default", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.5970703125, + "scores_per_experiment": [ + { + "accuracy": 0.60595703125, + "ap": 0.5627793151907503, + "ap_weighted": 0.5627793151907503, + "f1": 0.6038602115048454, + "f1_weighted": 0.6038602115048454 + }, + { + "accuracy": 0.62353515625, + "ap": 0.5740242034313725, + "ap_weighted": 0.5740242034313725, + "f1": 0.6177941922923591, + "f1_weighted": 0.6177941922923591 + }, + { + "accuracy": 0.6015625, + "ap": 0.5632370283018868, + "ap_weighted": 0.5632370283018868, + "f1": 0.59859804464121, + "f1_weighted": 0.59859804464121 + }, + { + "accuracy": 0.5546875, + "ap": 0.5301899395910781, + "ap_weighted": 0.5301899395910781, + "f1": 0.5544002290294876, + "f1_weighted": 0.5544002290294876 + }, + { + "accuracy": 0.57275390625, + "ap": 0.5430933918060719, + "ap_weighted": 0.5430933918060719, + "f1": 0.567902797419473, + "f1_weighted": 0.567902797419473 + }, + { + "accuracy": 0.58447265625, + "ap": 0.5470657298207204, + "ap_weighted": 0.5470657298207204, + "f1": 0.5593508568377241, + "f1_weighted": 0.5593508568377241 + }, + { + "accuracy": 0.623046875, + "ap": 0.5738086247028527, + "ap_weighted": 0.5738086247028527, + "f1": 0.6178864382283706, + "f1_weighted": 0.6178864382283706 + }, + { + "accuracy": 0.630859375, + "ap": 0.5773909127728513, + "ap_weighted": 0.5773909127728513, + "f1": 0.6128253987306636, + "f1_weighted": 0.6128253987306636 + }, + { + "accuracy": 0.57373046875, + "ap": 0.5423931890826713, + "ap_weighted": 0.5423931890826713, + "f1": 0.5737010954896442, + "f1_weighted": 0.5737010954896442 + }, + { + "accuracy": 0.60009765625, + "ap": 0.5591688368055556, + "ap_weighted": 0.5591688368055556, + "f1": 0.5991226793413069, + "f1_weighted": 0.5991226793413069 + } + ] + } + ] + }, + "task_name": "InappropriatenessClassification" +} \ No newline at end of file diff --git a/results/nvidia__NV-Embed-v1/570834afd5fef5bf3a3c2311a2b6e0a66f6f4f2c/KinopoiskClassification.json b/results/nvidia__NV-Embed-v1/570834afd5fef5bf3a3c2311a2b6e0a66f6f4f2c/KinopoiskClassification.json new file mode 100644 index 000000000..6c23e658f --- /dev/null +++ b/results/nvidia__NV-Embed-v1/570834afd5fef5bf3a3c2311a2b6e0a66f6f4f2c/KinopoiskClassification.json @@ -0,0 +1,73 @@ +{ + "dataset_revision": "5911f26666ac11af46cb9c6849d0dc80a378af24", + "evaluation_time": 182.12329578399658, + "kg_co2_emissions": null, + "mteb_version": "1.19.2", + "scores": { + "test": [ + { + "accuracy": 0.6632666666666667, + "f1": 0.6323889103886595, + "f1_weighted": 0.6323889103886595, + "hf_subset": "default", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.6632666666666667, + "scores_per_experiment": [ + { + "accuracy": 0.6586666666666666, + "f1": 0.5970179684029074, + "f1_weighted": 0.5970179684029074 + }, + { + "accuracy": 0.6886666666666666, + "f1": 0.663002845304172, + "f1_weighted": 0.6630028453041719 + }, + { + "accuracy": 0.6853333333333333, + "f1": 0.6586280555841659, + "f1_weighted": 0.6586280555841658 + }, + { + "accuracy": 0.6646666666666666, + "f1": 0.6330128259262294, + "f1_weighted": 0.6330128259262294 + }, + { + "accuracy": 0.6126666666666667, + "f1": 0.571086336557607, + "f1_weighted": 0.571086336557607 + }, + { + "accuracy": 0.6726666666666666, + "f1": 0.6329877394211296, + "f1_weighted": 0.6329877394211297 + }, + { + "accuracy": 0.662, + "f1": 0.6542642173735581, + "f1_weighted": 0.6542642173735581 + }, + { + "accuracy": 0.686, + "f1": 0.6671053650809032, + "f1_weighted": 0.6671053650809032 + }, + { + "accuracy": 0.69, + "f1": 0.658961605800031, + "f1_weighted": 0.658961605800031 + }, + { + "accuracy": 0.612, + "f1": 0.5878221444358918, + "f1_weighted": 0.5878221444358918 + } + ] + } + ] + }, + "task_name": "KinopoiskClassification" +} \ No newline at end of file diff --git a/results/nvidia__NV-Embed-v1/570834afd5fef5bf3a3c2311a2b6e0a66f6f4f2c/MIRACLReranking.json b/results/nvidia__NV-Embed-v1/570834afd5fef5bf3a3c2311a2b6e0a66f6f4f2c/MIRACLReranking.json new file mode 100644 index 000000000..a10fabb2d --- /dev/null +++ b/results/nvidia__NV-Embed-v1/570834afd5fef5bf3a3c2311a2b6e0a66f6f4f2c/MIRACLReranking.json @@ -0,0 +1,130 @@ +{ + "dataset_revision": "6d1962c527217f8927fca80f890f14f36b2802af", + "evaluation_time": 5656.726373434067, + "kg_co2_emissions": null, + "mteb_version": "1.19.2", + "scores": { + "dev": [ + { + "MAP@1(MIRACL)": 0.35562, + "MAP@10(MIRACL)": 0.54166, + "MAP@100(MIRACL)": 0.55875, + "MAP@1000(MIRACL)": 0.55875, + "MAP@20(MIRACL)": 0.55249, + "MAP@3(MIRACL)": 0.47377, + "MAP@5(MIRACL)": 0.51244, + "NDCG@1(MIRACL)": 0.57979, + "NDCG@10(MIRACL)": 0.61039, + "NDCG@100(MIRACL)": 0.6469, + "NDCG@1000(MIRACL)": 0.6469, + "NDCG@20(MIRACL)": 0.62947, + "NDCG@3(MIRACL)": 0.55782, + "NDCG@5(MIRACL)": 0.57902, + "P@1(MIRACL)": 0.57979, + "P@10(MIRACL)": 0.15325, + "P@100(MIRACL)": 0.01953, + "P@1000(MIRACL)": 0.00195, + "P@20(MIRACL)": 0.08613, + "P@3(MIRACL)": 0.33734, + "P@5(MIRACL)": 0.25004, + "Recall@1(MIRACL)": 0.35562, + "Recall@10(MIRACL)": 0.68927, + "Recall@100(MIRACL)": 0.79952, + "Recall@1000(MIRACL)": 0.79952, + "Recall@20(MIRACL)": 0.74038, + "Recall@3(MIRACL)": 0.52443, + "Recall@5(MIRACL)": 0.60693, + "hf_subset": "ru", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.61039, + "nAUC_MAP@1000_diff1(MIRACL)": 0.24166196041330218, + "nAUC_MAP@1000_max(MIRACL)": 0.35568647741873655, + "nAUC_MAP@1000_std(MIRACL)": 0.21561140424827818, + "nAUC_MAP@100_diff1(MIRACL)": 0.24166196041330218, + "nAUC_MAP@100_max(MIRACL)": 0.35568647741873655, + "nAUC_MAP@100_std(MIRACL)": 0.21561140424827818, + "nAUC_MAP@10_diff1(MIRACL)": 0.25370860368681464, + "nAUC_MAP@10_max(MIRACL)": 0.33483518973199056, + "nAUC_MAP@10_std(MIRACL)": 0.19116925372573793, + "nAUC_MAP@1_diff1(MIRACL)": 0.41237485067403984, + "nAUC_MAP@1_max(MIRACL)": 0.18643188249819215, + "nAUC_MAP@1_std(MIRACL)": 0.02516969078297421, + "nAUC_MAP@20_diff1(MIRACL)": 0.2476671929723017, + "nAUC_MAP@20_max(MIRACL)": 0.34655538682240156, + "nAUC_MAP@20_std(MIRACL)": 0.20607968764162457, + "nAUC_MAP@3_diff1(MIRACL)": 0.30619414889102897, + "nAUC_MAP@3_max(MIRACL)": 0.26658354366403, + "nAUC_MAP@3_std(MIRACL)": 0.10465909243365186, + "nAUC_MAP@5_diff1(MIRACL)": 0.2760801205950606, + "nAUC_MAP@5_max(MIRACL)": 0.302260005312864, + "nAUC_MAP@5_std(MIRACL)": 0.14918208207478914, + "nAUC_NDCG@1000_diff1(MIRACL)": 0.1667702502148141, + "nAUC_NDCG@1000_max(MIRACL)": 0.45110569800775163, + "nAUC_NDCG@1000_std(MIRACL)": 0.31932834288787376, + "nAUC_NDCG@100_diff1(MIRACL)": 0.1667702502148141, + "nAUC_NDCG@100_max(MIRACL)": 0.45110569800775163, + "nAUC_NDCG@100_std(MIRACL)": 0.31932834288787376, + "nAUC_NDCG@10_diff1(MIRACL)": 0.20182037369360364, + "nAUC_NDCG@10_max(MIRACL)": 0.3987898544333689, + "nAUC_NDCG@10_std(MIRACL)": 0.26365104459888955, + "nAUC_NDCG@1_diff1(MIRACL)": 0.2850254882689527, + "nAUC_NDCG@1_max(MIRACL)": 0.39453187308427956, + "nAUC_NDCG@1_std(MIRACL)": 0.22206630996097615, + "nAUC_NDCG@20_diff1(MIRACL)": 0.1869038719700204, + "nAUC_NDCG@20_max(MIRACL)": 0.42163580528358574, + "nAUC_NDCG@20_std(MIRACL)": 0.29278019012132067, + "nAUC_NDCG@3_diff1(MIRACL)": 0.2390423862149443, + "nAUC_NDCG@3_max(MIRACL)": 0.36039294758520696, + "nAUC_NDCG@3_std(MIRACL)": 0.20435172394160647, + "nAUC_NDCG@5_diff1(MIRACL)": 0.2253085775853163, + "nAUC_NDCG@5_max(MIRACL)": 0.36712247003638615, + "nAUC_NDCG@5_std(MIRACL)": 0.22334067893474482, + "nAUC_P@1000_diff1(MIRACL)": -0.2505238265276962, + "nAUC_P@1000_max(MIRACL)": 0.3108382178059431, + "nAUC_P@1000_std(MIRACL)": 0.3590076805919173, + "nAUC_P@100_diff1(MIRACL)": -0.25052382652769606, + "nAUC_P@100_max(MIRACL)": 0.31083821780594323, + "nAUC_P@100_std(MIRACL)": 0.35900768059191723, + "nAUC_P@10_diff1(MIRACL)": -0.2006269415707279, + "nAUC_P@10_max(MIRACL)": 0.33127439868005815, + "nAUC_P@10_std(MIRACL)": 0.36684435162248324, + "nAUC_P@1_diff1(MIRACL)": 0.2850254882689527, + "nAUC_P@1_max(MIRACL)": 0.39453187308427956, + "nAUC_P@1_std(MIRACL)": 0.22206630996097615, + "nAUC_P@20_diff1(MIRACL)": -0.21978003355315004, + "nAUC_P@20_max(MIRACL)": 0.3154914135224064, + "nAUC_P@20_std(MIRACL)": 0.37140439457717805, + "nAUC_P@3_diff1(MIRACL)": -0.0811432635162843, + "nAUC_P@3_max(MIRACL)": 0.3544792884825572, + "nAUC_P@3_std(MIRACL)": 0.30682906496760476, + "nAUC_P@5_diff1(MIRACL)": -0.1603963847443979, + "nAUC_P@5_max(MIRACL)": 0.350934705256882, + "nAUC_P@5_std(MIRACL)": 0.3467924406153275, + "nAUC_Recall@1000_diff1(MIRACL)": -0.07108215088746936, + "nAUC_Recall@1000_max(MIRACL)": 0.6763294600716278, + "nAUC_Recall@1000_std(MIRACL)": 0.5873868036680795, + "nAUC_Recall@100_diff1(MIRACL)": -0.07108215088746936, + "nAUC_Recall@100_max(MIRACL)": 0.6763294600716278, + "nAUC_Recall@100_std(MIRACL)": 0.5873868036680795, + "nAUC_Recall@10_diff1(MIRACL)": 0.09897774528482779, + "nAUC_Recall@10_max(MIRACL)": 0.41905102243444653, + "nAUC_Recall@10_std(MIRACL)": 0.3275225052394844, + "nAUC_Recall@1_diff1(MIRACL)": 0.41237485067403984, + "nAUC_Recall@1_max(MIRACL)": 0.18643188249819215, + "nAUC_Recall@1_std(MIRACL)": 0.02516969078297421, + "nAUC_Recall@20_diff1(MIRACL)": 0.04186033843724706, + "nAUC_Recall@20_max(MIRACL)": 0.5015312232950867, + "nAUC_Recall@20_std(MIRACL)": 0.4301757542566192, + "nAUC_Recall@3_diff1(MIRACL)": 0.2504495577370317, + "nAUC_Recall@3_max(MIRACL)": 0.27247587073856216, + "nAUC_Recall@3_std(MIRACL)": 0.11927344475985446, + "nAUC_Recall@5_diff1(MIRACL)": 0.17354421974056103, + "nAUC_Recall@5_max(MIRACL)": 0.32577238016696397, + "nAUC_Recall@5_std(MIRACL)": 0.2033133815439622 + } + ] + }, + "task_name": "MIRACLReranking" +} \ No newline at end of file diff --git a/results/nvidia__NV-Embed-v1/570834afd5fef5bf3a3c2311a2b6e0a66f6f4f2c/MassiveIntentClassification.json b/results/nvidia__NV-Embed-v1/570834afd5fef5bf3a3c2311a2b6e0a66f6f4f2c/MassiveIntentClassification.json new file mode 100644 index 000000000..4502f853a --- /dev/null +++ b/results/nvidia__NV-Embed-v1/570834afd5fef5bf3a3c2311a2b6e0a66f6f4f2c/MassiveIntentClassification.json @@ -0,0 +1,137 @@ +{ + "dataset_revision": "4672e20407010da34463acc759c162ca9734bca6", + "evaluation_time": 479.7503037452698, + "kg_co2_emissions": null, + "mteb_version": "1.19.2", + "scores": { + "test": [ + { + "accuracy": 0.6699058507061196, + "f1": 0.6485280135662423, + "f1_weighted": 0.6716347128608126, + "hf_subset": "ru", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.6699058507061196, + "scores_per_experiment": [ + { + "accuracy": 0.6919973100201748, + "f1": 0.6584318355602053, + "f1_weighted": 0.6836982894633654 + }, + { + "accuracy": 0.6819098856758574, + "f1": 0.6613405396842658, + "f1_weighted": 0.6905392170018135 + }, + { + "accuracy": 0.6540013449899126, + "f1": 0.6343808788266457, + "f1_weighted": 0.6622272221218418 + }, + { + "accuracy": 0.699394754539341, + "f1": 0.6572861434085577, + "f1_weighted": 0.6978726286881625 + }, + { + "accuracy": 0.6661062542030934, + "f1": 0.644471443408446, + "f1_weighted": 0.6654547316893268 + }, + { + "accuracy": 0.6371889710827169, + "f1": 0.6361456125674796, + "f1_weighted": 0.6464521290940934 + }, + { + "accuracy": 0.6735036987222596, + "f1": 0.656925101314275, + "f1_weighted": 0.6728953080106177 + }, + { + "accuracy": 0.6617350369872226, + "f1": 0.6387988033567452, + "f1_weighted": 0.6675235224685624 + }, + { + "accuracy": 0.6570275722932079, + "f1": 0.6485873320312703, + "f1_weighted": 0.6514586588953614 + }, + { + "accuracy": 0.6761936785474109, + "f1": 0.6489124455045331, + "f1_weighted": 0.6782254211749801 + } + ] + } + ], + "validation": [ + { + "accuracy": 0.6669945892769307, + "f1": 0.6297966321975441, + "f1_weighted": 0.6667070403982958, + "hf_subset": "ru", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.6669945892769307, + "scores_per_experiment": [ + { + "accuracy": 0.6817511067388097, + "f1": 0.639374657133745, + "f1_weighted": 0.6708565481658018 + }, + { + "accuracy": 0.6792916871618299, + "f1": 0.6383954617747153, + "f1_weighted": 0.6851234660228555 + }, + { + "accuracy": 0.663059517953763, + "f1": 0.6323619369795825, + "f1_weighted": 0.6692901098776366 + }, + { + "accuracy": 0.6886374815543532, + "f1": 0.6377291657955656, + "f1_weighted": 0.6857839766202609 + }, + { + "accuracy": 0.6738809640924742, + "f1": 0.6249872669692298, + "f1_weighted": 0.6727735636029885 + }, + { + "accuracy": 0.646827348745696, + "f1": 0.6263668397741052, + "f1_weighted": 0.6560985279690974 + }, + { + "accuracy": 0.6606000983767831, + "f1": 0.6242314774906661, + "f1_weighted": 0.6571848159616163 + }, + { + "accuracy": 0.647319232661092, + "f1": 0.6135922997307482, + "f1_weighted": 0.6443740750603317 + }, + { + "accuracy": 0.6527299557304476, + "f1": 0.6277916636903941, + "f1_weighted": 0.6433455280442218 + }, + { + "accuracy": 0.675848499754058, + "f1": 0.6331355526366887, + "f1_weighted": 0.6822397926581473 + } + ] + } + ] + }, + "task_name": "MassiveIntentClassification" +} \ No newline at end of file diff --git a/results/nvidia__NV-Embed-v1/570834afd5fef5bf3a3c2311a2b6e0a66f6f4f2c/MassiveScenarioClassification.json b/results/nvidia__NV-Embed-v1/570834afd5fef5bf3a3c2311a2b6e0a66f6f4f2c/MassiveScenarioClassification.json new file mode 100644 index 000000000..942e491a7 --- /dev/null +++ b/results/nvidia__NV-Embed-v1/570834afd5fef5bf3a3c2311a2b6e0a66f6f4f2c/MassiveScenarioClassification.json @@ -0,0 +1,137 @@ +{ + "dataset_revision": "fad2c6e8459f9e1c45d9315f4953d921437d70f8", + "evaluation_time": 263.1152024269104, + "kg_co2_emissions": null, + "mteb_version": "1.19.2", + "scores": { + "test": [ + { + "accuracy": 0.7175857431069267, + "f1": 0.7097228177990489, + "f1_weighted": 0.7168780557698059, + "hf_subset": "ru", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.7175857431069267, + "scores_per_experiment": [ + { + "accuracy": 0.7202420981842637, + "f1": 0.7184384255989378, + "f1_weighted": 0.7191872031321452 + }, + { + "accuracy": 0.7468056489576328, + "f1": 0.7346494718347905, + "f1_weighted": 0.7459663136668221 + }, + { + "accuracy": 0.6956960322797578, + "f1": 0.6907400337159871, + "f1_weighted": 0.697362341904245 + }, + { + "accuracy": 0.7622730329522529, + "f1": 0.7495696619918264, + "f1_weighted": 0.7620505979486852 + }, + { + "accuracy": 0.703429724277068, + "f1": 0.6945343369132188, + "f1_weighted": 0.6962676849789758 + }, + { + "accuracy": 0.6859448554135844, + "f1": 0.6777207078180286, + "f1_weighted": 0.6788350877892521 + }, + { + "accuracy": 0.7061197041022192, + "f1": 0.6923648673728134, + "f1_weighted": 0.7033328484276032 + }, + { + "accuracy": 0.7215870880968392, + "f1": 0.717170655494057, + "f1_weighted": 0.7251444282235334 + }, + { + "accuracy": 0.7276395427034297, + "f1": 0.7216268086299155, + "f1_weighted": 0.7309324899226076 + }, + { + "accuracy": 0.7061197041022192, + "f1": 0.7004132086209134, + "f1_weighted": 0.7097015617041901 + } + ] + } + ], + "validation": [ + { + "accuracy": 0.7124938514510575, + "f1": 0.7029306387885386, + "f1_weighted": 0.7140464383446744, + "hf_subset": "ru", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.7124938514510575, + "scores_per_experiment": [ + { + "accuracy": 0.7191342843089031, + "f1": 0.7107570411148275, + "f1_weighted": 0.7210830082731564 + }, + { + "accuracy": 0.7289719626168224, + "f1": 0.7174764946796608, + "f1_weighted": 0.7313522045191215 + }, + { + "accuracy": 0.6945400885391048, + "f1": 0.6905883886978903, + "f1_weighted": 0.6990321774681998 + }, + { + "accuracy": 0.7422528283325135, + "f1": 0.7304652715228094, + "f1_weighted": 0.7440514107721333 + }, + { + "accuracy": 0.7166748647319232, + "f1": 0.7026265464565243, + "f1_weighted": 0.7134887317484762 + }, + { + "accuracy": 0.6807673389080177, + "f1": 0.6733916005263093, + "f1_weighted": 0.6756664895289141 + }, + { + "accuracy": 0.7088047220855878, + "f1": 0.6929017624514268, + "f1_weighted": 0.7082840011855243 + }, + { + "accuracy": 0.70486965076242, + "f1": 0.7013007870904535, + "f1_weighted": 0.7096176369068345 + }, + { + "accuracy": 0.7299557304476144, + "f1": 0.7192572356063559, + "f1_weighted": 0.7337323491525061 + }, + { + "accuracy": 0.6989670437776685, + "f1": 0.6905412597391274, + "f1_weighted": 0.704156373891879 + } + ] + } + ] + }, + "task_name": "MassiveScenarioClassification" +} \ No newline at end of file diff --git a/results/nvidia__NV-Embed-v1/570834afd5fef5bf3a3c2311a2b6e0a66f6f4f2c/RUParaPhraserSTS.json b/results/nvidia__NV-Embed-v1/570834afd5fef5bf3a3c2311a2b6e0a66f6f4f2c/RUParaPhraserSTS.json new file mode 100644 index 000000000..534c3fb97 --- /dev/null +++ b/results/nvidia__NV-Embed-v1/570834afd5fef5bf3a3c2311a2b6e0a66f6f4f2c/RUParaPhraserSTS.json @@ -0,0 +1,26 @@ +{ + "dataset_revision": "43265056790b8f7c59e0139acb4be0a8dad2c8f4", + "evaluation_time": 116.41828107833862, + "kg_co2_emissions": null, + "mteb_version": "1.19.2", + "scores": { + "test": [ + { + "cosine_pearson": 0.6184420447292884, + "cosine_spearman": 0.6863484334803941, + "euclidean_pearson": 0.6613717909450216, + "euclidean_spearman": 0.6863484334803941, + "hf_subset": "default", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.6863484334803941, + "manhattan_pearson": 0.6615557717395811, + "manhattan_spearman": 0.6855432413692231, + "pearson": 0.6184420447292884, + "spearman": 0.6863484334803941 + } + ] + }, + "task_name": "RUParaPhraserSTS" +} \ No newline at end of file diff --git a/results/nvidia__NV-Embed-v1/570834afd5fef5bf3a3c2311a2b6e0a66f6f4f2c/RiaNewsRetrieval.json b/results/nvidia__NV-Embed-v1/570834afd5fef5bf3a3c2311a2b6e0a66f6f4f2c/RiaNewsRetrieval.json new file mode 100644 index 000000000..2fa8fe1af --- /dev/null +++ b/results/nvidia__NV-Embed-v1/570834afd5fef5bf3a3c2311a2b6e0a66f6f4f2c/RiaNewsRetrieval.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "82374b0bbacda6114f39ff9c5b925fa1512ca5d7", + "evaluation_time": 48124.45526242256, + "kg_co2_emissions": null, + "mteb_version": "1.19.2", + "scores": { + "test": [ + { + "hf_subset": "default", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.69704, + "map_at_1": 0.5571, + "map_at_10": 0.6522, + "map_at_100": 0.65677, + "map_at_1000": 0.65695, + "map_at_20": 0.65509, + "map_at_3": 0.63188, + "map_at_5": 0.64404, + "mrr_at_1": 0.5571, + "mrr_at_10": 0.6521985317460318, + "mrr_at_100": 0.6567739177422577, + "mrr_at_1000": 0.6569454647020231, + "mrr_at_20": 0.6550921350007921, + "mrr_at_3": 0.6318833333333334, + "mrr_at_5": 0.6440383333333333, + "nauc_map_at_1000_diff1": 0.666289404123591, + "nauc_map_at_1000_max": 0.22920787089371555, + "nauc_map_at_1000_std": -0.14445828100128477, + "nauc_map_at_100_diff1": 0.6662341314522475, + "nauc_map_at_100_max": 0.22927689204704899, + "nauc_map_at_100_std": -0.1443907808296551, + "nauc_map_at_10_diff1": 0.6656310917951208, + "nauc_map_at_10_max": 0.2287686085612718, + "nauc_map_at_10_std": -0.14577087300392091, + "nauc_map_at_1_diff1": 0.6983363480312772, + "nauc_map_at_1_max": 0.2088575094279177, + "nauc_map_at_1_std": -0.1584028850539041, + "nauc_map_at_20_diff1": 0.665919277281886, + "nauc_map_at_20_max": 0.2291712874778309, + "nauc_map_at_20_std": -0.1449596244804602, + "nauc_map_at_3_diff1": 0.6667598388802628, + "nauc_map_at_3_max": 0.2263334053446113, + "nauc_map_at_3_std": -0.15344029407308593, + "nauc_map_at_5_diff1": 0.6658585970546774, + "nauc_map_at_5_max": 0.22882890380843085, + "nauc_map_at_5_std": -0.14901776944022088, + "nauc_mrr_at_1000_diff1": 0.666289404123591, + "nauc_mrr_at_1000_max": 0.22920787089371555, + "nauc_mrr_at_1000_std": -0.14445828100128477, + "nauc_mrr_at_100_diff1": 0.6662341314522475, + "nauc_mrr_at_100_max": 0.22927689204704899, + "nauc_mrr_at_100_std": -0.1443907808296551, + "nauc_mrr_at_10_diff1": 0.6656310917951208, + "nauc_mrr_at_10_max": 0.2287686085612718, + "nauc_mrr_at_10_std": -0.14577087300392091, + "nauc_mrr_at_1_diff1": 0.6983363480312772, + "nauc_mrr_at_1_max": 0.2088575094279177, + "nauc_mrr_at_1_std": -0.1584028850539041, + "nauc_mrr_at_20_diff1": 0.665919277281886, + "nauc_mrr_at_20_max": 0.2291712874778309, + "nauc_mrr_at_20_std": -0.1449596244804602, + "nauc_mrr_at_3_diff1": 0.6667598388802628, + "nauc_mrr_at_3_max": 0.2263334053446113, + "nauc_mrr_at_3_std": -0.15344029407308593, + "nauc_mrr_at_5_diff1": 0.6658585970546774, + "nauc_mrr_at_5_max": 0.22882890380843085, + "nauc_mrr_at_5_std": -0.14901776944022088, + "nauc_ndcg_at_1000_diff1": 0.6582467447247684, + "nauc_ndcg_at_1000_max": 0.23855126030702195, + "nauc_ndcg_at_1000_std": -0.1287350138682392, + "nauc_ndcg_at_100_diff1": 0.6563943515263573, + "nauc_ndcg_at_100_max": 0.24059983904552912, + "nauc_ndcg_at_100_std": -0.12530023270452376, + "nauc_ndcg_at_10_diff1": 0.6527619921781402, + "nauc_ndcg_at_10_max": 0.2371384488022888, + "nauc_ndcg_at_10_std": -0.13330978767681867, + "nauc_ndcg_at_1_diff1": 0.6983363480312772, + "nauc_ndcg_at_1_max": 0.2088575094279177, + "nauc_ndcg_at_1_std": -0.1584028850539041, + "nauc_ndcg_at_20_diff1": 0.6532277449614098, + "nauc_ndcg_at_20_max": 0.23886029055800456, + "nauc_ndcg_at_20_std": -0.12973600342836217, + "nauc_ndcg_at_3_diff1": 0.6560345496891724, + "nauc_ndcg_at_3_max": 0.2315023783726975, + "nauc_ndcg_at_3_std": -0.15136777735515922, + "nauc_ndcg_at_5_diff1": 0.653766542964688, + "nauc_ndcg_at_5_max": 0.23676285233779837, + "nauc_ndcg_at_5_std": -0.1421659978219634, + "nauc_precision_at_1000_diff1": 0.5420493778906961, + "nauc_precision_at_1000_max": 0.5491433782815478, + "nauc_precision_at_1000_std": 0.42354678305430804, + "nauc_precision_at_100_diff1": 0.555226728372508, + "nauc_precision_at_100_max": 0.41944951893800964, + "nauc_precision_at_100_std": 0.19758860065765155, + "nauc_precision_at_10_diff1": 0.5865825103117884, + "nauc_precision_at_10_max": 0.28127205195705074, + "nauc_precision_at_10_std": -0.06057350028582767, + "nauc_precision_at_1_diff1": 0.6983363480312772, + "nauc_precision_at_1_max": 0.2088575094279177, + "nauc_precision_at_1_std": -0.1584028850539041, + "nauc_precision_at_20_diff1": 0.5697224257848147, + "nauc_precision_at_20_max": 0.3067804722773795, + "nauc_precision_at_20_std": -0.012105304732550332, + "nauc_precision_at_3_diff1": 0.6191454012834254, + "nauc_precision_at_3_max": 0.24897391415852654, + "nauc_precision_at_3_std": -0.14408460281261287, + "nauc_precision_at_5_diff1": 0.6054600192783229, + "nauc_precision_at_5_max": 0.2690982623123013, + "nauc_precision_at_5_std": -0.11234590477903747, + "nauc_recall_at_1000_diff1": 0.5420493778906897, + "nauc_recall_at_1000_max": 0.5491433782815398, + "nauc_recall_at_1000_std": 0.42354678305429927, + "nauc_recall_at_100_diff1": 0.5552267283725049, + "nauc_recall_at_100_max": 0.41944951893800964, + "nauc_recall_at_100_std": 0.1975886006576511, + "nauc_recall_at_10_diff1": 0.5865825103117889, + "nauc_recall_at_10_max": 0.281272051957051, + "nauc_recall_at_10_std": -0.06057350028582675, + "nauc_recall_at_1_diff1": 0.6983363480312772, + "nauc_recall_at_1_max": 0.2088575094279177, + "nauc_recall_at_1_std": -0.1584028850539041, + "nauc_recall_at_20_diff1": 0.5697224257848146, + "nauc_recall_at_20_max": 0.3067804722773823, + "nauc_recall_at_20_std": -0.012105304732550882, + "nauc_recall_at_3_diff1": 0.6191454012834258, + "nauc_recall_at_3_max": 0.24897391415852685, + "nauc_recall_at_3_std": -0.14408460281261365, + "nauc_recall_at_5_diff1": 0.6054600192783223, + "nauc_recall_at_5_max": 0.2690982623123014, + "nauc_recall_at_5_std": -0.11234590477903701, + "ndcg_at_1": 0.5571, + "ndcg_at_10": 0.69704, + "ndcg_at_100": 0.71946, + "ndcg_at_1000": 0.72403, + "ndcg_at_20": 0.7074, + "ndcg_at_3": 0.65562, + "ndcg_at_5": 0.67748, + "precision_at_1": 0.5571, + "precision_at_10": 0.08371, + "precision_at_100": 0.00942, + "precision_at_1000": 0.00098, + "precision_at_20": 0.04389, + "precision_at_3": 0.24137, + "precision_at_5": 0.15542, + "recall_at_1": 0.5571, + "recall_at_10": 0.8371, + "recall_at_100": 0.9425, + "recall_at_1000": 0.9785, + "recall_at_20": 0.8777, + "recall_at_3": 0.7241, + "recall_at_5": 0.7771 + } + ] + }, + "task_name": "RiaNewsRetrieval" +} \ No newline at end of file diff --git a/results/nvidia__NV-Embed-v1/570834afd5fef5bf3a3c2311a2b6e0a66f6f4f2c/RuBQReranking.json b/results/nvidia__NV-Embed-v1/570834afd5fef5bf3a3c2311a2b6e0a66f6f4f2c/RuBQReranking.json new file mode 100644 index 000000000..8151ae787 --- /dev/null +++ b/results/nvidia__NV-Embed-v1/570834afd5fef5bf3a3c2311a2b6e0a66f6f4f2c/RuBQReranking.json @@ -0,0 +1,26 @@ +{ + "dataset_revision": "2e96b8f098fa4b0950fc58eacadeb31c0d0c7fa2", + "evaluation_time": 1688.2114102840424, + "kg_co2_emissions": null, + "mteb_version": "1.19.2", + "scores": { + "test": [ + { + "hf_subset": "default", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.7567224574214443, + "map": 0.7567224574214443, + "mrr": 0.8055524853397193, + "nAUC_map_diff1": 0.4496213556058314, + "nAUC_map_max": 0.1691317908522992, + "nAUC_map_std": 0.05646975938784306, + "nAUC_mrr_diff1": 0.49743238945021395, + "nAUC_mrr_max": 0.2352557461893819, + "nAUC_mrr_std": 0.11840744830479893 + } + ] + }, + "task_name": "RuBQReranking" +} \ No newline at end of file diff --git a/results/nvidia__NV-Embed-v1/570834afd5fef5bf3a3c2311a2b6e0a66f6f4f2c/RuBQRetrieval.json b/results/nvidia__NV-Embed-v1/570834afd5fef5bf3a3c2311a2b6e0a66f6f4f2c/RuBQRetrieval.json new file mode 100644 index 000000000..92453ab8e --- /dev/null +++ b/results/nvidia__NV-Embed-v1/570834afd5fef5bf3a3c2311a2b6e0a66f6f4f2c/RuBQRetrieval.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "e19b6ffa60b3bc248e0b41f4cc37c26a55c2a67b", + "evaluation_time": 2491.1460316181183, + "kg_co2_emissions": null, + "mteb_version": "1.19.2", + "scores": { + "test": [ + { + "hf_subset": "default", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.75722, + "map_at_1": 0.46316, + "map_at_10": 0.68229, + "map_at_100": 0.69028, + "map_at_1000": 0.69041, + "map_at_20": 0.68824, + "map_at_3": 0.62334, + "map_at_5": 0.66144, + "mrr_at_1": 0.6554373522458629, + "mrr_at_10": 0.7588495346917332, + "mrr_at_100": 0.7610024313546563, + "mrr_at_1000": 0.7610435428991298, + "mrr_at_20": 0.760432124228701, + "mrr_at_3": 0.7393617021276596, + "mrr_at_5": 0.7525413711583924, + "nauc_map_at_1000_diff1": 0.4236732566195218, + "nauc_map_at_1000_max": 0.22947771692290356, + "nauc_map_at_1000_std": -0.11832902325666075, + "nauc_map_at_100_diff1": 0.42360305281619737, + "nauc_map_at_100_max": 0.22964250247399448, + "nauc_map_at_100_std": -0.11811334719815919, + "nauc_map_at_10_diff1": 0.4213844495162417, + "nauc_map_at_10_max": 0.2250638748125005, + "nauc_map_at_10_std": -0.12153958065976897, + "nauc_map_at_1_diff1": 0.45653656986713875, + "nauc_map_at_1_max": 0.1611044875792229, + "nauc_map_at_1_std": -0.12003220800587164, + "nauc_map_at_20_diff1": 0.4231406586290676, + "nauc_map_at_20_max": 0.22962401882005898, + "nauc_map_at_20_std": -0.11795443881614621, + "nauc_map_at_3_diff1": 0.423301050001386, + "nauc_map_at_3_max": 0.20773454761392643, + "nauc_map_at_3_std": -0.1319166015288595, + "nauc_map_at_5_diff1": 0.41718251816188817, + "nauc_map_at_5_max": 0.21582134831635041, + "nauc_map_at_5_std": -0.13142047918800565, + "nauc_mrr_at_1000_diff1": 0.4862321806587656, + "nauc_mrr_at_1000_max": 0.2861166657202891, + "nauc_mrr_at_1000_std": -0.10632801925353655, + "nauc_mrr_at_100_diff1": 0.4862476311641955, + "nauc_mrr_at_100_max": 0.2862241480545193, + "nauc_mrr_at_100_std": -0.10620319217939522, + "nauc_mrr_at_10_diff1": 0.4865616026066755, + "nauc_mrr_at_10_max": 0.287330546450005, + "nauc_mrr_at_10_std": -0.10591278500358396, + "nauc_mrr_at_1_diff1": 0.5089115514577714, + "nauc_mrr_at_1_max": 0.26070577642657256, + "nauc_mrr_at_1_std": -0.11884882754872553, + "nauc_mrr_at_20_diff1": 0.48640399045211324, + "nauc_mrr_at_20_max": 0.2874113568612935, + "nauc_mrr_at_20_std": -0.10565998577923047, + "nauc_mrr_at_3_diff1": 0.48398581249372774, + "nauc_mrr_at_3_max": 0.28323223755021265, + "nauc_mrr_at_3_std": -0.11266817916011874, + "nauc_mrr_at_5_diff1": 0.48131668792901444, + "nauc_mrr_at_5_max": 0.2870350481091371, + "nauc_mrr_at_5_std": -0.10876261970018632, + "nauc_ndcg_at_1000_diff1": 0.43642438318516635, + "nauc_ndcg_at_1000_max": 0.2589643236583601, + "nauc_ndcg_at_1000_std": -0.09918713175041675, + "nauc_ndcg_at_100_diff1": 0.4347471975104966, + "nauc_ndcg_at_100_max": 0.26263262390252906, + "nauc_ndcg_at_100_std": -0.09340030760077238, + "nauc_ndcg_at_10_diff1": 0.4278389507011489, + "nauc_ndcg_at_10_max": 0.25383569765294245, + "nauc_ndcg_at_10_std": -0.10113453928700722, + "nauc_ndcg_at_1_diff1": 0.5089115514577714, + "nauc_ndcg_at_1_max": 0.26070577642657256, + "nauc_ndcg_at_1_std": -0.11884882754872553, + "nauc_ndcg_at_20_diff1": 0.4327347124897482, + "nauc_ndcg_at_20_max": 0.2664433068709857, + "nauc_ndcg_at_20_std": -0.08995655451511293, + "nauc_ndcg_at_3_diff1": 0.4293039206198754, + "nauc_ndcg_at_3_max": 0.2295387645073092, + "nauc_ndcg_at_3_std": -0.13199137561538513, + "nauc_ndcg_at_5_diff1": 0.4184041560122243, + "nauc_ndcg_at_5_max": 0.23694527076217636, + "nauc_ndcg_at_5_std": -0.12319564412332111, + "nauc_precision_at_1000_diff1": -0.1439322608367948, + "nauc_precision_at_1000_max": 0.04909872652131748, + "nauc_precision_at_1000_std": 0.07649483307003255, + "nauc_precision_at_100_diff1": -0.13722643118696673, + "nauc_precision_at_100_max": 0.0667250531672719, + "nauc_precision_at_100_std": 0.09143700141030399, + "nauc_precision_at_10_diff1": -0.0797700905772425, + "nauc_precision_at_10_max": 0.09927252518856493, + "nauc_precision_at_10_std": 0.06088268673393873, + "nauc_precision_at_1_diff1": 0.5089115514577714, + "nauc_precision_at_1_max": 0.26070577642657256, + "nauc_precision_at_1_std": -0.11884882754872553, + "nauc_precision_at_20_diff1": -0.10840153789378938, + "nauc_precision_at_20_max": 0.09401983888236805, + "nauc_precision_at_20_std": 0.08922231444961612, + "nauc_precision_at_3_diff1": 0.0844016989497562, + "nauc_precision_at_3_max": 0.1627764719588861, + "nauc_precision_at_3_std": -0.02977237541195357, + "nauc_precision_at_5_diff1": -0.019775373175307646, + "nauc_precision_at_5_max": 0.11695845130827487, + "nauc_precision_at_5_std": 0.004752759635574466, + "nauc_recall_at_1000_diff1": 0.3422299148792753, + "nauc_recall_at_1000_max": 0.6658911302615654, + "nauc_recall_at_1000_std": 0.5967832836625414, + "nauc_recall_at_100_diff1": 0.2696854925573016, + "nauc_recall_at_100_max": 0.5625341347350288, + "nauc_recall_at_100_std": 0.4454132986828331, + "nauc_recall_at_10_diff1": 0.319916179028863, + "nauc_recall_at_10_max": 0.2790887269212939, + "nauc_recall_at_10_std": -0.0013726365253641953, + "nauc_recall_at_1_diff1": 0.45653656986713875, + "nauc_recall_at_1_max": 0.1611044875792229, + "nauc_recall_at_1_std": -0.12003220800587164, + "nauc_recall_at_20_diff1": 0.3188082397685211, + "nauc_recall_at_20_max": 0.4188333827247561, + "nauc_recall_at_20_std": 0.15758428226923943, + "nauc_recall_at_3_diff1": 0.3611468643259357, + "nauc_recall_at_3_max": 0.19543606306666605, + "nauc_recall_at_3_std": -0.12383717876720095, + "nauc_recall_at_5_diff1": 0.3180033121002249, + "nauc_recall_at_5_max": 0.21616613312257071, + "nauc_recall_at_5_std": -0.1043697618424628, + "ndcg_at_1": 0.65544, + "ndcg_at_10": 0.75722, + "ndcg_at_100": 0.77987, + "ndcg_at_1000": 0.78212, + "ndcg_at_20": 0.77079, + "ndcg_at_3": 0.683, + "ndcg_at_5": 0.72431, + "precision_at_1": 0.65544, + "precision_at_10": 0.1474, + "precision_at_100": 0.0164, + "precision_at_1000": 0.00167, + "precision_at_20": 0.0781, + "precision_at_3": 0.37234, + "precision_at_5": 0.26123, + "recall_at_1": 0.46316, + "recall_at_10": 0.89508, + "recall_at_100": 0.97929, + "recall_at_1000": 0.99397, + "recall_at_20": 0.93862, + "recall_at_3": 0.71526, + "recall_at_5": 0.81179 + } + ] + }, + "task_name": "RuBQRetrieval" +} \ No newline at end of file diff --git a/results/nvidia__NV-Embed-v1/570834afd5fef5bf3a3c2311a2b6e0a66f6f4f2c/RuReviewsClassification.json b/results/nvidia__NV-Embed-v1/570834afd5fef5bf3a3c2311a2b6e0a66f6f4f2c/RuReviewsClassification.json new file mode 100644 index 000000000..1219a1bb0 --- /dev/null +++ b/results/nvidia__NV-Embed-v1/570834afd5fef5bf3a3c2311a2b6e0a66f6f4f2c/RuReviewsClassification.json @@ -0,0 +1,73 @@ +{ + "dataset_revision": "f6d2c31f4dc6b88f468552750bfec05b4b41b05a", + "evaluation_time": 78.14594507217407, + "kg_co2_emissions": null, + "mteb_version": "1.19.2", + "scores": { + "test": [ + { + "accuracy": 0.62529296875, + "f1": 0.6167672496731613, + "f1_weighted": 0.6167681451340391, + "hf_subset": "default", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.62529296875, + "scores_per_experiment": [ + { + "accuracy": 0.65625, + "f1": 0.6497163146483244, + "f1_weighted": 0.6497193465252993 + }, + { + "accuracy": 0.6044921875, + "f1": 0.5866212731441977, + "f1_weighted": 0.5865952400222377 + }, + { + "accuracy": 0.6123046875, + "f1": 0.6074455877047897, + "f1_weighted": 0.6074475464008124 + }, + { + "accuracy": 0.64306640625, + "f1": 0.6421343552870485, + "f1_weighted": 0.6421550808152551 + }, + { + "accuracy": 0.66455078125, + "f1": 0.6673582735217226, + "f1_weighted": 0.6673609632495248 + }, + { + "accuracy": 0.6083984375, + "f1": 0.5906677915359994, + "f1_weighted": 0.5906797612358929 + }, + { + "accuracy": 0.603515625, + "f1": 0.5913842360589063, + "f1_weighted": 0.5913952265786976 + }, + { + "accuracy": 0.58740234375, + "f1": 0.5835766437687119, + "f1_weighted": 0.5835720031041085 + }, + { + "accuracy": 0.6015625, + "f1": 0.5820806374461871, + "f1_weighted": 0.5820429382762148 + }, + { + "accuracy": 0.67138671875, + "f1": 0.6666873836157258, + "f1_weighted": 0.6667133451323478 + } + ] + } + ] + }, + "task_name": "RuReviewsClassification" +} \ No newline at end of file diff --git a/results/nvidia__NV-Embed-v1/570834afd5fef5bf3a3c2311a2b6e0a66f6f4f2c/RuSTSBenchmarkSTS.json b/results/nvidia__NV-Embed-v1/570834afd5fef5bf3a3c2311a2b6e0a66f6f4f2c/RuSTSBenchmarkSTS.json new file mode 100644 index 000000000..104ecad0f --- /dev/null +++ b/results/nvidia__NV-Embed-v1/570834afd5fef5bf3a3c2311a2b6e0a66f6f4f2c/RuSTSBenchmarkSTS.json @@ -0,0 +1,26 @@ +{ + "dataset_revision": "7cf24f325c6da6195df55bef3d86b5e0616f3018", + "evaluation_time": 76.91748118400574, + "kg_co2_emissions": null, + "mteb_version": "1.19.2", + "scores": { + "test": [ + { + "cosine_pearson": 0.7178385239398417, + "cosine_spearman": 0.7196854086154546, + "euclidean_pearson": 0.7279545912073366, + "euclidean_spearman": 0.7196863475546109, + "hf_subset": "default", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.7196854086154546, + "manhattan_pearson": 0.7224009622538086, + "manhattan_spearman": 0.713117811786393, + "pearson": 0.7178385239398417, + "spearman": 0.7196854086154546 + } + ] + }, + "task_name": "RuSTSBenchmarkSTS" +} \ No newline at end of file diff --git a/results/nvidia__NV-Embed-v1/570834afd5fef5bf3a3c2311a2b6e0a66f6f4f2c/RuSciBenchGRNTIClassification.json b/results/nvidia__NV-Embed-v1/570834afd5fef5bf3a3c2311a2b6e0a66f6f4f2c/RuSciBenchGRNTIClassification.json new file mode 100644 index 000000000..713a42ca5 --- /dev/null +++ b/results/nvidia__NV-Embed-v1/570834afd5fef5bf3a3c2311a2b6e0a66f6f4f2c/RuSciBenchGRNTIClassification.json @@ -0,0 +1,73 @@ +{ + "dataset_revision": "673a610d6d3dd91a547a0d57ae1b56f37ebbf6a1", + "evaluation_time": 272.66169142723083, + "kg_co2_emissions": null, + "mteb_version": "1.19.2", + "scores": { + "test": [ + { + "accuracy": 0.58779296875, + "f1": 0.5789042984882291, + "f1_weighted": 0.5790140242998455, + "hf_subset": "default", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.58779296875, + "scores_per_experiment": [ + { + "accuracy": 0.60302734375, + "f1": 0.5950025005769458, + "f1_weighted": 0.5950857136635225 + }, + { + "accuracy": 0.5908203125, + "f1": 0.5839508656773166, + "f1_weighted": 0.5840157507449846 + }, + { + "accuracy": 0.58447265625, + "f1": 0.5755836316191425, + "f1_weighted": 0.5757199710299893 + }, + { + "accuracy": 0.609375, + "f1": 0.6029495526228784, + "f1_weighted": 0.6030428564143574 + }, + { + "accuracy": 0.5927734375, + "f1": 0.5914432105663955, + "f1_weighted": 0.5915239969512561 + }, + { + "accuracy": 0.56640625, + "f1": 0.554191659084123, + "f1_weighted": 0.5542920706476402 + }, + { + "accuracy": 0.59228515625, + "f1": 0.5796296935755552, + "f1_weighted": 0.5798100486072378 + }, + { + "accuracy": 0.56884765625, + "f1": 0.556608358241599, + "f1_weighted": 0.5567119457248467 + }, + { + "accuracy": 0.58642578125, + "f1": 0.5772105188299016, + "f1_weighted": 0.5773440547288933 + }, + { + "accuracy": 0.58349609375, + "f1": 0.5724729940884321, + "f1_weighted": 0.5725938344857269 + } + ] + } + ] + }, + "task_name": "RuSciBenchGRNTIClassification" +} \ No newline at end of file diff --git a/results/nvidia__NV-Embed-v1/570834afd5fef5bf3a3c2311a2b6e0a66f6f4f2c/RuSciBenchGRNTIClusteringP2P.json b/results/nvidia__NV-Embed-v1/570834afd5fef5bf3a3c2311a2b6e0a66f6f4f2c/RuSciBenchGRNTIClusteringP2P.json new file mode 100644 index 000000000..0039fad31 --- /dev/null +++ b/results/nvidia__NV-Embed-v1/570834afd5fef5bf3a3c2311a2b6e0a66f6f4f2c/RuSciBenchGRNTIClusteringP2P.json @@ -0,0 +1,34 @@ +{ + "dataset_revision": "673a610d6d3dd91a547a0d57ae1b56f37ebbf6a1", + "evaluation_time": 208.3949372768402, + "kg_co2_emissions": null, + "mteb_version": "1.19.2", + "scores": { + "test": [ + { + "hf_subset": "default", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.5000312879338047, + "v_measure": 0.5000312879338047, + "v_measure_std": 0.012378314604940233, + "v_measures": { + "Level 0": [ + 0.519615625142969, + 0.516054923010248, + 0.4796236958172153, + 0.4900926653409032, + 0.5081322181044112, + 0.49640854366585974, + 0.4835996608941322, + 0.4993176490040661, + 0.5037099636185596, + 0.5037579347396823 + ] + } + } + ] + }, + "task_name": "RuSciBenchGRNTIClusteringP2P" +} \ No newline at end of file diff --git a/results/nvidia__NV-Embed-v1/570834afd5fef5bf3a3c2311a2b6e0a66f6f4f2c/RuSciBenchOECDClassification.json b/results/nvidia__NV-Embed-v1/570834afd5fef5bf3a3c2311a2b6e0a66f6f4f2c/RuSciBenchOECDClassification.json new file mode 100644 index 000000000..61d18ae45 --- /dev/null +++ b/results/nvidia__NV-Embed-v1/570834afd5fef5bf3a3c2311a2b6e0a66f6f4f2c/RuSciBenchOECDClassification.json @@ -0,0 +1,73 @@ +{ + "dataset_revision": "26c88e99dcaba32bb45d0e1bfc21902337f6d471", + "evaluation_time": 267.288774728775, + "kg_co2_emissions": null, + "mteb_version": "1.19.2", + "scores": { + "test": [ + { + "accuracy": 0.452685546875, + "f1": 0.4356759120528972, + "f1_weighted": 0.4357387641663437, + "hf_subset": "default", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.452685546875, + "scores_per_experiment": [ + { + "accuracy": 0.4453125, + "f1": 0.426066352105914, + "f1_weighted": 0.4263857193869032 + }, + { + "accuracy": 0.4609375, + "f1": 0.4470972443100088, + "f1_weighted": 0.4470716585116127 + }, + { + "accuracy": 0.45361328125, + "f1": 0.44017786797963604, + "f1_weighted": 0.4402696624948551 + }, + { + "accuracy": 0.43798828125, + "f1": 0.4109451857756568, + "f1_weighted": 0.4108271687284259 + }, + { + "accuracy": 0.4541015625, + "f1": 0.4335082102263128, + "f1_weighted": 0.43358576275793786 + }, + { + "accuracy": 0.4521484375, + "f1": 0.4401179851511892, + "f1_weighted": 0.44027485813531686 + }, + { + "accuracy": 0.45703125, + "f1": 0.43560020909340363, + "f1_weighted": 0.43561698747629707 + }, + { + "accuracy": 0.4765625, + "f1": 0.4678767459068051, + "f1_weighted": 0.4680243006026285 + }, + { + "accuracy": 0.439453125, + "f1": 0.4190490169549671, + "f1_weighted": 0.41900069509449633 + }, + { + "accuracy": 0.44970703125, + "f1": 0.43632030302507907, + "f1_weighted": 0.4363308284749643 + } + ] + } + ] + }, + "task_name": "RuSciBenchOECDClassification" +} \ No newline at end of file diff --git a/results/nvidia__NV-Embed-v1/570834afd5fef5bf3a3c2311a2b6e0a66f6f4f2c/RuSciBenchOECDClusteringP2P.json b/results/nvidia__NV-Embed-v1/570834afd5fef5bf3a3c2311a2b6e0a66f6f4f2c/RuSciBenchOECDClusteringP2P.json new file mode 100644 index 000000000..97e6fdb60 --- /dev/null +++ b/results/nvidia__NV-Embed-v1/570834afd5fef5bf3a3c2311a2b6e0a66f6f4f2c/RuSciBenchOECDClusteringP2P.json @@ -0,0 +1,34 @@ +{ + "dataset_revision": "26c88e99dcaba32bb45d0e1bfc21902337f6d471", + "evaluation_time": 197.94107103347778, + "kg_co2_emissions": null, + "mteb_version": "1.19.2", + "scores": { + "test": [ + { + "hf_subset": "default", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.4244600483328404, + "v_measure": 0.4244600483328404, + "v_measure_std": 0.01301536261717212, + "v_measures": { + "Level 0": [ + 0.4252872357251034, + 0.42713522568879053, + 0.44383212920422516, + 0.42545275585041603, + 0.4012887895113197, + 0.43979601562933984, + 0.4278341463726104, + 0.4174396892566613, + 0.43241684492879545, + 0.4041176511611423 + ] + } + } + ] + }, + "task_name": "RuSciBenchOECDClusteringP2P" +} \ No newline at end of file diff --git a/results/nvidia__NV-Embed-v1/570834afd5fef5bf3a3c2311a2b6e0a66f6f4f2c/STS22.json b/results/nvidia__NV-Embed-v1/570834afd5fef5bf3a3c2311a2b6e0a66f6f4f2c/STS22.json new file mode 100644 index 000000000..ed7c76d5a --- /dev/null +++ b/results/nvidia__NV-Embed-v1/570834afd5fef5bf3a3c2311a2b6e0a66f6f4f2c/STS22.json @@ -0,0 +1,26 @@ +{ + "dataset_revision": "de9d86b3b84231dc21f76c7b7af1f28e2f57f6e3", + "evaluation_time": 47.48003077507019, + "kg_co2_emissions": null, + "mteb_version": "1.19.2", + "scores": { + "test": [ + { + "cosine_pearson": 0.45534279173336994, + "cosine_spearman": 0.5697741865995578, + "euclidean_pearson": 0.47658009725622785, + "euclidean_spearman": 0.5697741865995578, + "hf_subset": "ru", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.5697741865995578, + "manhattan_pearson": 0.5011840361464873, + "manhattan_spearman": 0.570467606400952, + "pearson": 0.45534279173336994, + "spearman": 0.5697741865995578 + } + ] + }, + "task_name": "STS22" +} \ No newline at end of file diff --git a/results/nvidia__NV-Embed-v1/570834afd5fef5bf3a3c2311a2b6e0a66f6f4f2c/SensitiveTopicsClassification.json b/results/nvidia__NV-Embed-v1/570834afd5fef5bf3a3c2311a2b6e0a66f6f4f2c/SensitiveTopicsClassification.json new file mode 100644 index 000000000..4bea96125 --- /dev/null +++ b/results/nvidia__NV-Embed-v1/570834afd5fef5bf3a3c2311a2b6e0a66f6f4f2c/SensitiveTopicsClassification.json @@ -0,0 +1,73 @@ +{ + "dataset_revision": "416b34a802308eac30e4192afc0ff99bb8dcc7f2", + "evaluation_time": 127.88109421730042, + "kg_co2_emissions": null, + "mteb_version": "1.19.2", + "scores": { + "test": [ + { + "accuracy": 0.228076171875, + "f1": 0.17234813990857445, + "hf_subset": "default", + "languages": [ + "rus-Cyrl" + ], + "lrap": 0.33500162760415925, + "main_score": 0.228076171875, + "scores_per_experiment": [ + { + "accuracy": 0.19970703125, + "f1": 0.12819023768703086, + "lrap": 0.3152262369791594 + }, + { + "accuracy": 0.25439453125, + "f1": 0.1868636606720437, + "lrap": 0.3655870225694369 + }, + { + "accuracy": 0.203125, + "f1": 0.1253919445999908, + "lrap": 0.3029920789930484 + }, + { + "accuracy": 0.25634765625, + "f1": 0.20087480666887114, + "lrap": 0.37571885850693665 + }, + { + "accuracy": 0.23046875, + "f1": 0.23222690619489839, + "lrap": 0.36040581597221466 + }, + { + "accuracy": 0.21923828125, + "f1": 0.15741072926638097, + "lrap": 0.31617567274304825 + }, + { + "accuracy": 0.23388671875, + "f1": 0.170132659038192, + "lrap": 0.337361653645826 + }, + { + "accuracy": 0.22021484375, + "f1": 0.16376889880809606, + "lrap": 0.31994628906249256 + }, + { + "accuracy": 0.24853515625, + "f1": 0.20043936956687872, + "lrap": 0.3488498263888811 + }, + { + "accuracy": 0.21484375, + "f1": 0.15818218658336164, + "lrap": 0.3077528211805482 + } + ] + } + ] + }, + "task_name": "SensitiveTopicsClassification" +} \ No newline at end of file diff --git a/results/nvidia__NV-Embed-v1/570834afd5fef5bf3a3c2311a2b6e0a66f6f4f2c/TERRa.json b/results/nvidia__NV-Embed-v1/570834afd5fef5bf3a3c2311a2b6e0a66f6f4f2c/TERRa.json new file mode 100644 index 000000000..f0af341bf --- /dev/null +++ b/results/nvidia__NV-Embed-v1/570834afd5fef5bf3a3c2311a2b6e0a66f6f4f2c/TERRa.json @@ -0,0 +1,58 @@ +{ + "dataset_revision": "7b58f24536063837d644aab9a023c62199b2a612", + "evaluation_time": 19.495412826538086, + "kg_co2_emissions": null, + "mteb_version": "1.19.2", + "scores": { + "dev": [ + { + "cosine_accuracy": 0.5439739413680782, + "cosine_accuracy_threshold": 0.751105546951294, + "cosine_ap": 0.5270181720546976, + "cosine_f1": 0.6681715575620768, + "cosine_f1_threshold": 0.3606880009174347, + "cosine_precision": 0.5103448275862069, + "cosine_recall": 0.9673202614379085, + "dot_accuracy": 0.5439739413680782, + "dot_accuracy_threshold": 0.7511056661605835, + "dot_ap": 0.5270181720546976, + "dot_f1": 0.6681715575620768, + "dot_f1_threshold": 0.36068838834762573, + "dot_precision": 0.5103448275862069, + "dot_recall": 0.9673202614379085, + "euclidean_accuracy": 0.5439739413680782, + "euclidean_accuracy_threshold": 0.7055414915084839, + "euclidean_ap": 0.5270181720546976, + "euclidean_f1": 0.6681715575620768, + "euclidean_f1_threshold": 1.1307551860809326, + "euclidean_precision": 0.5103448275862069, + "euclidean_recall": 0.9673202614379085, + "hf_subset": "default", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.5351598291946672, + "manhattan_accuracy": 0.5439739413680782, + "manhattan_accuracy_threshold": 32.67607116699219, + "manhattan_ap": 0.5351598291946672, + "manhattan_f1": 0.6696230598669624, + "manhattan_f1_threshold": 55.09137725830078, + "manhattan_precision": 0.5067114093959731, + "manhattan_recall": 0.9869281045751634, + "max_accuracy": 0.5439739413680782, + "max_ap": 0.5351598291946672, + "max_f1": 0.6696230598669624, + "max_precision": 0.5103448275862069, + "max_recall": 0.9869281045751634, + "similarity_accuracy": 0.5439739413680782, + "similarity_accuracy_threshold": 0.751105546951294, + "similarity_ap": 0.5270181720546976, + "similarity_f1": 0.6681715575620768, + "similarity_f1_threshold": 0.3606880009174347, + "similarity_precision": 0.5103448275862069, + "similarity_recall": 0.9673202614379085 + } + ] + }, + "task_name": "TERRa" +} \ No newline at end of file diff --git a/results/nvidia__NV-Embed-v1/570834afd5fef5bf3a3c2311a2b6e0a66f6f4f2c/model_meta.json b/results/nvidia__NV-Embed-v1/570834afd5fef5bf3a3c2311a2b6e0a66f6f4f2c/model_meta.json new file mode 100644 index 000000000..871360f6a --- /dev/null +++ b/results/nvidia__NV-Embed-v1/570834afd5fef5bf3a3c2311a2b6e0a66f6f4f2c/model_meta.json @@ -0,0 +1 @@ +{"name": "nvidia/NV-Embed-v1", "revision": "570834afd5fef5bf3a3c2311a2b6e0a66f6f4f2c", "release_date": "2024-09-13", "languages": ["eng_Latn"], "n_parameters": 7850000000, "memory_usage": null, "max_tokens": 32768, "embed_dim": 4096, "license": "cc-by-nc-4.0", "open_weights": true, "public_training_data": null, "public_training_code": null, "framework": ["Sentence Transformers", "PyTorch"], "reference": "https://huggingface.co/nvidia/NV-Embed-v1", "similarity_fn_name": "cosine", "use_instructions": true, "zero_shot_benchmarks": null, "loader": "NvEmbedWrapper"} \ No newline at end of file diff --git a/results/nvidia__NV-Embed-v2/7604d305b621f14095a1aa23d351674c2859553a/CEDRClassification.json b/results/nvidia__NV-Embed-v2/7604d305b621f14095a1aa23d351674c2859553a/CEDRClassification.json new file mode 100644 index 000000000..d1637f0f4 --- /dev/null +++ b/results/nvidia__NV-Embed-v2/7604d305b621f14095a1aa23d351674c2859553a/CEDRClassification.json @@ -0,0 +1,73 @@ +{ + "dataset_revision": "c0ba03d058e3e1b2f3fd20518875a4563dd12db4", + "evaluation_time": 88.35089159011841, + "kg_co2_emissions": null, + "mteb_version": "1.19.2", + "scores": { + "test": [ + { + "accuracy": 0.39234856535600426, + "f1": 0.2609704555281428, + "hf_subset": "default", + "languages": [ + "rus-Cyrl" + ], + "lrap": 0.6278852284803522, + "main_score": 0.39234856535600426, + "scores_per_experiment": [ + { + "accuracy": 0.4303931987247609, + "f1": 0.2953750300982859, + "lrap": 0.6271519659936355 + }, + { + "accuracy": 0.2917109458023379, + "f1": 0.20606962355305977, + "lrap": 0.5936238044633494 + }, + { + "accuracy": 0.39160467587672687, + "f1": 0.2699311508038231, + "lrap": 0.6176408076514469 + }, + { + "accuracy": 0.4394261424017003, + "f1": 0.31149806093529286, + "lrap": 0.6121147715196729 + }, + { + "accuracy": 0.36822529224229544, + "f1": 0.19437602932707435, + "lrap": 0.5918703506907665 + }, + { + "accuracy": 0.36397449521785336, + "f1": 0.2208462710006382, + "lrap": 0.5936769394261554 + }, + { + "accuracy": 0.41923485653560044, + "f1": 0.2620048988968031, + "lrap": 0.6758235919234965 + }, + { + "accuracy": 0.39160467587672687, + "f1": 0.2477897980952243, + "lrap": 0.642507970244433 + }, + { + "accuracy": 0.3650371944739639, + "f1": 0.2754004706909032, + "lrap": 0.6309245483528286 + }, + { + "accuracy": 0.4622741764080765, + "f1": 0.326413221880323, + "lrap": 0.6935175345377368 + } + ] + } + ] + }, + "task_name": "CEDRClassification" +} \ No newline at end of file diff --git a/results/nvidia__NV-Embed-v2/7604d305b621f14095a1aa23d351674c2859553a/GeoreviewClassification.json b/results/nvidia__NV-Embed-v2/7604d305b621f14095a1aa23d351674c2859553a/GeoreviewClassification.json new file mode 100644 index 000000000..7532fbe95 --- /dev/null +++ b/results/nvidia__NV-Embed-v2/7604d305b621f14095a1aa23d351674c2859553a/GeoreviewClassification.json @@ -0,0 +1,73 @@ +{ + "dataset_revision": "3765c0d1de6b7d264bc459433c45e5a75513839c", + "evaluation_time": 117.85539245605469, + "kg_co2_emissions": null, + "mteb_version": "1.19.2", + "scores": { + "test": [ + { + "accuracy": 0.491259765625, + "f1": 0.4717580256758992, + "f1_weighted": 0.471723930917474, + "hf_subset": "default", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.491259765625, + "scores_per_experiment": [ + { + "accuracy": 0.4873046875, + "f1": 0.4637021768749374, + "f1_weighted": 0.4636949290226906 + }, + { + "accuracy": 0.5, + "f1": 0.4762976166565641, + "f1_weighted": 0.4762424087723962 + }, + { + "accuracy": 0.46875, + "f1": 0.4296736708935394, + "f1_weighted": 0.42964358356108934 + }, + { + "accuracy": 0.4814453125, + "f1": 0.4653418603146976, + "f1_weighted": 0.4653711297103874 + }, + { + "accuracy": 0.5126953125, + "f1": 0.49684561100273117, + "f1_weighted": 0.4968021375409004 + }, + { + "accuracy": 0.48779296875, + "f1": 0.47670156496819127, + "f1_weighted": 0.4766860794812844 + }, + { + "accuracy": 0.4921875, + "f1": 0.46649070880151805, + "f1_weighted": 0.4664277228240942 + }, + { + "accuracy": 0.5185546875, + "f1": 0.5025854408103075, + "f1_weighted": 0.5025207638780578 + }, + { + "accuracy": 0.4638671875, + "f1": 0.46475847459782216, + "f1_weighted": 0.4647195460958866 + }, + { + "accuracy": 0.5, + "f1": 0.4751831318386822, + "f1_weighted": 0.4751310082879533 + } + ] + } + ] + }, + "task_name": "GeoreviewClassification" +} \ No newline at end of file diff --git a/results/nvidia__NV-Embed-v2/7604d305b621f14095a1aa23d351674c2859553a/GeoreviewClusteringP2P.json b/results/nvidia__NV-Embed-v2/7604d305b621f14095a1aa23d351674c2859553a/GeoreviewClusteringP2P.json new file mode 100644 index 000000000..42f0e4374 --- /dev/null +++ b/results/nvidia__NV-Embed-v2/7604d305b621f14095a1aa23d351674c2859553a/GeoreviewClusteringP2P.json @@ -0,0 +1,34 @@ +{ + "dataset_revision": "97a313c8fc85b47f13f33e7e9a95c1ad888c7fec", + "evaluation_time": 174.16882228851318, + "kg_co2_emissions": null, + "mteb_version": "1.19.2", + "scores": { + "test": [ + { + "hf_subset": "default", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.7116838641198839, + "v_measure": 0.7116838641198839, + "v_measure_std": 0.009736906621379802, + "v_measures": { + "Level 0": [ + 0.7169780361572674, + 0.7118053701134585, + 0.7175055208189192, + 0.7126973830234291, + 0.7189090665894059, + 0.7107586261609896, + 0.7145568591629871, + 0.6836623138436597, + 0.7178832079032867, + 0.7120822574254357 + ] + } + } + ] + }, + "task_name": "GeoreviewClusteringP2P" +} \ No newline at end of file diff --git a/results/nvidia__NV-Embed-v2/7604d305b621f14095a1aa23d351674c2859553a/HeadlineClassification.json b/results/nvidia__NV-Embed-v2/7604d305b621f14095a1aa23d351674c2859553a/HeadlineClassification.json new file mode 100644 index 000000000..2eb64f013 --- /dev/null +++ b/results/nvidia__NV-Embed-v2/7604d305b621f14095a1aa23d351674c2859553a/HeadlineClassification.json @@ -0,0 +1,73 @@ +{ + "dataset_revision": "2fe05ee6b5832cda29f2ef7aaad7b7fe6a3609eb", + "evaluation_time": 83.46812438964844, + "kg_co2_emissions": null, + "mteb_version": "1.19.2", + "scores": { + "test": [ + { + "accuracy": 0.76064453125, + "f1": 0.7588830351102249, + "f1_weighted": 0.7588831505827336, + "hf_subset": "default", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.76064453125, + "scores_per_experiment": [ + { + "accuracy": 0.72509765625, + "f1": 0.7235678810170798, + "f1_weighted": 0.7235580957545413 + }, + { + "accuracy": 0.76806640625, + "f1": 0.7663446050823236, + "f1_weighted": 0.7663496627268237 + }, + { + "accuracy": 0.755859375, + "f1": 0.7547521403125125, + "f1_weighted": 0.7547413156876903 + }, + { + "accuracy": 0.7607421875, + "f1": 0.7589565970319835, + "f1_weighted": 0.7589654327521121 + }, + { + "accuracy": 0.77294921875, + "f1": 0.7726321933421468, + "f1_weighted": 0.7726286350177812 + }, + { + "accuracy": 0.7744140625, + "f1": 0.7740455055761636, + "f1_weighted": 0.7740437523191865 + }, + { + "accuracy": 0.7666015625, + "f1": 0.7648900534693704, + "f1_weighted": 0.7648968999273557 + }, + { + "accuracy": 0.76806640625, + "f1": 0.7658322468612123, + "f1_weighted": 0.7658324649681927 + }, + { + "accuracy": 0.728515625, + "f1": 0.722703982601952, + "f1_weighted": 0.7227151771904361 + }, + { + "accuracy": 0.7861328125, + "f1": 0.7851051458075049, + "f1_weighted": 0.785100069483217 + } + ] + } + ] + }, + "task_name": "HeadlineClassification" +} \ No newline at end of file diff --git a/results/nvidia__NV-Embed-v2/7604d305b621f14095a1aa23d351674c2859553a/InappropriatenessClassification.json b/results/nvidia__NV-Embed-v2/7604d305b621f14095a1aa23d351674c2859553a/InappropriatenessClassification.json new file mode 100644 index 000000000..4b8546cb1 --- /dev/null +++ b/results/nvidia__NV-Embed-v2/7604d305b621f14095a1aa23d351674c2859553a/InappropriatenessClassification.json @@ -0,0 +1,95 @@ +{ + "dataset_revision": "601651fdc45ef243751676e62dd7a19f491c0285", + "evaluation_time": 75.39574694633484, + "kg_co2_emissions": null, + "mteb_version": "1.19.2", + "scores": { + "test": [ + { + "accuracy": 0.609912109375, + "ap": 0.5676846020508237, + "ap_weighted": 0.5676846020508237, + "f1": 0.6061637318348655, + "f1_weighted": 0.6061637318348655, + "hf_subset": "default", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.609912109375, + "scores_per_experiment": [ + { + "accuracy": 0.63671875, + "ap": 0.5882560745841996, + "ap_weighted": 0.5882560745841996, + "f1": 0.6363855042167208, + "f1_weighted": 0.6363855042167208 + }, + { + "accuracy": 0.6171875, + "ap": 0.5691195733532934, + "ap_weighted": 0.5691195733532934, + "f1": 0.6080918534718425, + "f1_weighted": 0.6080918534718425 + }, + { + "accuracy": 0.6318359375, + "ap": 0.5856058576465708, + "ap_weighted": 0.5856058576465708, + "f1": 0.6305675919829737, + "f1_weighted": 0.6305675919829737 + }, + { + "accuracy": 0.56494140625, + "ap": 0.5363026494565217, + "ap_weighted": 0.5363026494565217, + "f1": 0.5638381861010422, + "f1_weighted": 0.5638381861010422 + }, + { + "accuracy": 0.61474609375, + "ap": 0.5710056717644085, + "ap_weighted": 0.5710056717644085, + "f1": 0.6146335425590599, + "f1_weighted": 0.6146335425590599 + }, + { + "accuracy": 0.541015625, + "ap": 0.5217579258889695, + "ap_weighted": 0.5217579258889695, + "f1": 0.5268799023246075, + "f1_weighted": 0.5268799023246075 + }, + { + "accuracy": 0.623046875, + "ap": 0.5730078125, + "ap_weighted": 0.5730078125, + "f1": 0.6132472673866226, + "f1_weighted": 0.6132472673866226 + }, + { + "accuracy": 0.66748046875, + "ap": 0.6088696310422135, + "ap_weighted": 0.6088696310422135, + "f1": 0.6663539979373911, + "f1_weighted": 0.6663539979373911 + }, + { + "accuracy": 0.58935546875, + "ap": 0.5529113953927491, + "ap_weighted": 0.5529113953927491, + "f1": 0.5892613602083112, + "f1_weighted": 0.5892613602083112 + }, + { + "accuracy": 0.61279296875, + "ap": 0.5700094288793103, + "ap_weighted": 0.5700094288793103, + "f1": 0.6123781121600834, + "f1_weighted": 0.6123781121600834 + } + ] + } + ] + }, + "task_name": "InappropriatenessClassification" +} \ No newline at end of file diff --git a/results/nvidia__NV-Embed-v2/7604d305b621f14095a1aa23d351674c2859553a/KinopoiskClassification.json b/results/nvidia__NV-Embed-v2/7604d305b621f14095a1aa23d351674c2859553a/KinopoiskClassification.json new file mode 100644 index 000000000..d3a53101f --- /dev/null +++ b/results/nvidia__NV-Embed-v2/7604d305b621f14095a1aa23d351674c2859553a/KinopoiskClassification.json @@ -0,0 +1,73 @@ +{ + "dataset_revision": "5911f26666ac11af46cb9c6849d0dc80a378af24", + "evaluation_time": 180.54421424865723, + "kg_co2_emissions": null, + "mteb_version": "1.19.2", + "scores": { + "test": [ + { + "accuracy": 0.6063333333333334, + "f1": 0.5802843516488296, + "f1_weighted": 0.5802843516488296, + "hf_subset": "default", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.6063333333333334, + "scores_per_experiment": [ + { + "accuracy": 0.61, + "f1": 0.5796957891446487, + "f1_weighted": 0.5796957891446487 + }, + { + "accuracy": 0.606, + "f1": 0.5833010055244799, + "f1_weighted": 0.5833010055244799 + }, + { + "accuracy": 0.6386666666666667, + "f1": 0.5884491282011627, + "f1_weighted": 0.5884491282011627 + }, + { + "accuracy": 0.6266666666666667, + "f1": 0.594763298890922, + "f1_weighted": 0.594763298890922 + }, + { + "accuracy": 0.5593333333333333, + "f1": 0.5590784628569053, + "f1_weighted": 0.5590784628569053 + }, + { + "accuracy": 0.5993333333333334, + "f1": 0.583375715371619, + "f1_weighted": 0.583375715371619 + }, + { + "accuracy": 0.6046666666666667, + "f1": 0.5825569855040665, + "f1_weighted": 0.5825569855040665 + }, + { + "accuracy": 0.572, + "f1": 0.5550000291584115, + "f1_weighted": 0.5550000291584115 + }, + { + "accuracy": 0.622, + "f1": 0.5870583843226836, + "f1_weighted": 0.5870583843226835 + }, + { + "accuracy": 0.6246666666666667, + "f1": 0.5895647175133977, + "f1_weighted": 0.5895647175133976 + } + ] + } + ] + }, + "task_name": "KinopoiskClassification" +} \ No newline at end of file diff --git a/results/nvidia__NV-Embed-v2/7604d305b621f14095a1aa23d351674c2859553a/MIRACLReranking.json b/results/nvidia__NV-Embed-v2/7604d305b621f14095a1aa23d351674c2859553a/MIRACLReranking.json new file mode 100644 index 000000000..9fef5e81b --- /dev/null +++ b/results/nvidia__NV-Embed-v2/7604d305b621f14095a1aa23d351674c2859553a/MIRACLReranking.json @@ -0,0 +1,130 @@ +{ + "dataset_revision": "6d1962c527217f8927fca80f890f14f36b2802af", + "evaluation_time": 5654.395991325378, + "kg_co2_emissions": null, + "mteb_version": "1.19.2", + "scores": { + "dev": [ + { + "MAP@1(MIRACL)": 0.18385, + "MAP@10(MIRACL)": 0.26804, + "MAP@100(MIRACL)": 0.2962, + "MAP@1000(MIRACL)": 0.2962, + "MAP@20(MIRACL)": 0.27803, + "MAP@3(MIRACL)": 0.23526, + "MAP@5(MIRACL)": 0.25285, + "NDCG@1(MIRACL)": 0.32799, + "NDCG@10(MIRACL)": 0.3282, + "NDCG@100(MIRACL)": 0.4366, + "NDCG@1000(MIRACL)": 0.4366, + "NDCG@20(MIRACL)": 0.35643, + "NDCG@3(MIRACL)": 0.29528, + "NDCG@5(MIRACL)": 0.30468, + "P@1(MIRACL)": 0.32799, + "P@10(MIRACL)": 0.08565, + "P@100(MIRACL)": 0.01953, + "P@1000(MIRACL)": 0.00195, + "P@20(MIRACL)": 0.05325, + "P@3(MIRACL)": 0.18444, + "P@5(MIRACL)": 0.13601, + "Recall@1(MIRACL)": 0.18385, + "Recall@10(MIRACL)": 0.38256, + "Recall@100(MIRACL)": 0.79952, + "Recall@1000(MIRACL)": 0.79952, + "Recall@20(MIRACL)": 0.47372, + "Recall@3(MIRACL)": 0.26487, + "Recall@5(MIRACL)": 0.31208, + "hf_subset": "ru", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.3282, + "nAUC_MAP@1000_diff1(MIRACL)": 0.3705358153018016, + "nAUC_MAP@1000_max(MIRACL)": 0.4068485635661681, + "nAUC_MAP@1000_std(MIRACL)": 0.17315409567803305, + "nAUC_MAP@100_diff1(MIRACL)": 0.3705358153018016, + "nAUC_MAP@100_max(MIRACL)": 0.4068485635661681, + "nAUC_MAP@100_std(MIRACL)": 0.17315409567803305, + "nAUC_MAP@10_diff1(MIRACL)": 0.37563758975598716, + "nAUC_MAP@10_max(MIRACL)": 0.38729227453808784, + "nAUC_MAP@10_std(MIRACL)": 0.14782122821901447, + "nAUC_MAP@1_diff1(MIRACL)": 0.4706128853002108, + "nAUC_MAP@1_max(MIRACL)": 0.33974411155999445, + "nAUC_MAP@1_std(MIRACL)": 0.07482014115265016, + "nAUC_MAP@20_diff1(MIRACL)": 0.3731361352181663, + "nAUC_MAP@20_max(MIRACL)": 0.39038350767703583, + "nAUC_MAP@20_std(MIRACL)": 0.15445227496957564, + "nAUC_MAP@3_diff1(MIRACL)": 0.3969221246822535, + "nAUC_MAP@3_max(MIRACL)": 0.3807798899867365, + "nAUC_MAP@3_std(MIRACL)": 0.12077218411716233, + "nAUC_MAP@5_diff1(MIRACL)": 0.38349228185080564, + "nAUC_MAP@5_max(MIRACL)": 0.3848280026973373, + "nAUC_MAP@5_std(MIRACL)": 0.13411346229562954, + "nAUC_NDCG@1000_diff1(MIRACL)": 0.34279527239760565, + "nAUC_NDCG@1000_max(MIRACL)": 0.48171699212976543, + "nAUC_NDCG@1000_std(MIRACL)": 0.2658568003045174, + "nAUC_NDCG@100_diff1(MIRACL)": 0.34279527239760565, + "nAUC_NDCG@100_max(MIRACL)": 0.48171699212976543, + "nAUC_NDCG@100_std(MIRACL)": 0.2658568003045174, + "nAUC_NDCG@10_diff1(MIRACL)": 0.3584288802387542, + "nAUC_NDCG@10_max(MIRACL)": 0.41509280592103676, + "nAUC_NDCG@10_std(MIRACL)": 0.19089337900613176, + "nAUC_NDCG@1_diff1(MIRACL)": 0.4312252120286899, + "nAUC_NDCG@1_max(MIRACL)": 0.5140870392370913, + "nAUC_NDCG@1_std(MIRACL)": 0.2540696362593216, + "nAUC_NDCG@20_diff1(MIRACL)": 0.3510478184679465, + "nAUC_NDCG@20_max(MIRACL)": 0.4169816545427362, + "nAUC_NDCG@20_std(MIRACL)": 0.19760696142669298, + "nAUC_NDCG@3_diff1(MIRACL)": 0.3611192496855181, + "nAUC_NDCG@3_max(MIRACL)": 0.4458067829985373, + "nAUC_NDCG@3_std(MIRACL)": 0.20092499581082138, + "nAUC_NDCG@5_diff1(MIRACL)": 0.3604245648899712, + "nAUC_NDCG@5_max(MIRACL)": 0.4219231060470666, + "nAUC_NDCG@5_std(MIRACL)": 0.18518068841403654, + "nAUC_P@1000_diff1(MIRACL)": -0.011291791936001614, + "nAUC_P@1000_max(MIRACL)": 0.39022171311104203, + "nAUC_P@1000_std(MIRACL)": 0.37125580923751816, + "nAUC_P@100_diff1(MIRACL)": -0.011291791936001385, + "nAUC_P@100_max(MIRACL)": 0.39022171311104215, + "nAUC_P@100_std(MIRACL)": 0.3712558092375184, + "nAUC_P@10_diff1(MIRACL)": 0.10693712353824036, + "nAUC_P@10_max(MIRACL)": 0.42409575828329765, + "nAUC_P@10_std(MIRACL)": 0.3485874678939846, + "nAUC_P@1_diff1(MIRACL)": 0.4312252120286899, + "nAUC_P@1_max(MIRACL)": 0.5140870392370913, + "nAUC_P@1_std(MIRACL)": 0.2540696362593216, + "nAUC_P@20_diff1(MIRACL)": 0.07735456843948212, + "nAUC_P@20_max(MIRACL)": 0.40648689748396954, + "nAUC_P@20_std(MIRACL)": 0.3559827661232224, + "nAUC_P@3_diff1(MIRACL)": 0.19493271515109414, + "nAUC_P@3_max(MIRACL)": 0.4914100293382076, + "nAUC_P@3_std(MIRACL)": 0.3121680667227599, + "nAUC_P@5_diff1(MIRACL)": 0.14694023174984347, + "nAUC_P@5_max(MIRACL)": 0.45979324142375444, + "nAUC_P@5_std(MIRACL)": 0.32405021703904235, + "nAUC_Recall@1000_diff1(MIRACL)": 0.21498370648022272, + "nAUC_Recall@1000_max(MIRACL)": 0.6387029873261129, + "nAUC_Recall@1000_std(MIRACL)": 0.5053936568159575, + "nAUC_Recall@100_diff1(MIRACL)": 0.21498370648022272, + "nAUC_Recall@100_max(MIRACL)": 0.6387029873261129, + "nAUC_Recall@100_std(MIRACL)": 0.5053936568159575, + "nAUC_Recall@10_diff1(MIRACL)": 0.28468424197903397, + "nAUC_Recall@10_max(MIRACL)": 0.3087143500769356, + "nAUC_Recall@10_std(MIRACL)": 0.13640309639191414, + "nAUC_Recall@1_diff1(MIRACL)": 0.4706128853002108, + "nAUC_Recall@1_max(MIRACL)": 0.33974411155999445, + "nAUC_Recall@1_std(MIRACL)": 0.07482014115265016, + "nAUC_Recall@20_diff1(MIRACL)": 0.24642958197944093, + "nAUC_Recall@20_max(MIRACL)": 0.2791725277834431, + "nAUC_Recall@20_std(MIRACL)": 0.1286318587934121, + "nAUC_Recall@3_diff1(MIRACL)": 0.3491925521865891, + "nAUC_Recall@3_max(MIRACL)": 0.36065217524626075, + "nAUC_Recall@3_std(MIRACL)": 0.12352364099425858, + "nAUC_Recall@5_diff1(MIRACL)": 0.3108316633801861, + "nAUC_Recall@5_max(MIRACL)": 0.33677221191622436, + "nAUC_Recall@5_std(MIRACL)": 0.1261625958955805 + } + ] + }, + "task_name": "MIRACLReranking" +} \ No newline at end of file diff --git a/results/nvidia__NV-Embed-v2/7604d305b621f14095a1aa23d351674c2859553a/MassiveIntentClassification.json b/results/nvidia__NV-Embed-v2/7604d305b621f14095a1aa23d351674c2859553a/MassiveIntentClassification.json new file mode 100644 index 000000000..978593fc4 --- /dev/null +++ b/results/nvidia__NV-Embed-v2/7604d305b621f14095a1aa23d351674c2859553a/MassiveIntentClassification.json @@ -0,0 +1,137 @@ +{ + "dataset_revision": "4672e20407010da34463acc759c162ca9734bca6", + "evaluation_time": 471.122656583786, + "kg_co2_emissions": null, + "mteb_version": "1.19.2", + "scores": { + "test": [ + { + "accuracy": 0.7178547410894419, + "f1": 0.6898441940684256, + "f1_weighted": 0.7136645623324196, + "hf_subset": "ru", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.7178547410894419, + "scores_per_experiment": [ + { + "accuracy": 0.7441156691324815, + "f1": 0.7133171277889675, + "f1_weighted": 0.7387104144834933 + }, + { + "accuracy": 0.7256220578345662, + "f1": 0.6933669519501607, + "f1_weighted": 0.727884098368525 + }, + { + "accuracy": 0.7135171486213854, + "f1": 0.6881913353888739, + "f1_weighted": 0.7135193775406943 + }, + { + "accuracy": 0.7387357094821789, + "f1": 0.7003035026152659, + "f1_weighted": 0.7377064355413773 + }, + { + "accuracy": 0.7195696032279758, + "f1": 0.6899397668752968, + "f1_weighted": 0.709380063906068 + }, + { + "accuracy": 0.6886348352387357, + "f1": 0.6669694520307593, + "f1_weighted": 0.6866477598893861 + }, + { + "accuracy": 0.718897108271688, + "f1": 0.6961703486673134, + "f1_weighted": 0.7154700307918725 + }, + { + "accuracy": 0.6967047747141897, + "f1": 0.6667140760203905, + "f1_weighted": 0.6891539365898484 + }, + { + "accuracy": 0.703093476798924, + "f1": 0.6842290362288688, + "f1_weighted": 0.6914601101611271 + }, + { + "accuracy": 0.7296570275722932, + "f1": 0.6992403431183593, + "f1_weighted": 0.7267133960518033 + } + ] + } + ], + "validation": [ + { + "accuracy": 0.7141662567634038, + "f1": 0.6732004745587511, + "f1_weighted": 0.7077613503749703, + "hf_subset": "ru", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.7141662567634038, + "scores_per_experiment": [ + { + "accuracy": 0.7442203639940974, + "f1": 0.7127082157245627, + "f1_weighted": 0.7367187747608305 + }, + { + "accuracy": 0.720118052139695, + "f1": 0.6679715910682619, + "f1_weighted": 0.7189188447367377 + }, + { + "accuracy": 0.7260206591244466, + "f1": 0.6852257248723899, + "f1_weighted": 0.7236805624917242 + }, + { + "accuracy": 0.7289719626168224, + "f1": 0.6776079380888448, + "f1_weighted": 0.7239346578102468 + }, + { + "accuracy": 0.7127397934087555, + "f1": 0.667175647871863, + "f1_weighted": 0.7027077995239003 + }, + { + "accuracy": 0.6960157402852927, + "f1": 0.6641490689139968, + "f1_weighted": 0.6939169784647554 + }, + { + "accuracy": 0.7097884899163798, + "f1": 0.6742290115691805, + "f1_weighted": 0.7017467666210515 + }, + { + "accuracy": 0.6876537137235612, + "f1": 0.6462768028707662, + "f1_weighted": 0.6793417095045327 + }, + { + "accuracy": 0.6871618298081653, + "f1": 0.6566338882738231, + "f1_weighted": 0.6697367530917898 + }, + { + "accuracy": 0.7289719626168224, + "f1": 0.6800268563338224, + "f1_weighted": 0.7269106567441339 + } + ] + } + ] + }, + "task_name": "MassiveIntentClassification" +} \ No newline at end of file diff --git a/results/nvidia__NV-Embed-v2/7604d305b621f14095a1aa23d351674c2859553a/MassiveScenarioClassification.json b/results/nvidia__NV-Embed-v2/7604d305b621f14095a1aa23d351674c2859553a/MassiveScenarioClassification.json new file mode 100644 index 000000000..b42c9f100 --- /dev/null +++ b/results/nvidia__NV-Embed-v2/7604d305b621f14095a1aa23d351674c2859553a/MassiveScenarioClassification.json @@ -0,0 +1,137 @@ +{ + "dataset_revision": "fad2c6e8459f9e1c45d9315f4953d921437d70f8", + "evaluation_time": 260.02703046798706, + "kg_co2_emissions": null, + "mteb_version": "1.19.2", + "scores": { + "test": [ + { + "accuracy": 0.7744115669132481, + "f1": 0.764196629760842, + "f1_weighted": 0.7713290137945937, + "hf_subset": "ru", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.7744115669132481, + "scores_per_experiment": [ + { + "accuracy": 0.7726967047747142, + "f1": 0.7669945395156134, + "f1_weighted": 0.7720271054793457 + }, + { + "accuracy": 0.7723604572965703, + "f1": 0.7606743132712874, + "f1_weighted": 0.7689918576400112 + }, + { + "accuracy": 0.7804303967720242, + "f1": 0.7648004681415664, + "f1_weighted": 0.774707364167859 + }, + { + "accuracy": 0.7992602555480834, + "f1": 0.7822413744969521, + "f1_weighted": 0.7953495024375533 + }, + { + "accuracy": 0.7763954270342972, + "f1": 0.7599806982283944, + "f1_weighted": 0.7692252873375505 + }, + { + "accuracy": 0.7407531943510424, + "f1": 0.7374525855410359, + "f1_weighted": 0.737498432059467 + }, + { + "accuracy": 0.765635507733692, + "f1": 0.7547839902068046, + "f1_weighted": 0.7604861528059819 + }, + { + "accuracy": 0.7723604572965703, + "f1": 0.7696353007388201, + "f1_weighted": 0.7754282367175047 + }, + { + "accuracy": 0.7948890383322125, + "f1": 0.7831461824895359, + "f1_weighted": 0.7925798195947932 + }, + { + "accuracy": 0.769334229993275, + "f1": 0.7622568449784084, + "f1_weighted": 0.7669963797058712 + } + ] + } + ], + "validation": [ + { + "accuracy": 0.769749139203148, + "f1": 0.7558298375150284, + "f1_weighted": 0.7676718588835427, + "hf_subset": "ru", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.769749139203148, + "scores_per_experiment": [ + { + "accuracy": 0.7668470241023119, + "f1": 0.7568565705375698, + "f1_weighted": 0.7669253839197117 + }, + { + "accuracy": 0.7668470241023119, + "f1": 0.756079301369588, + "f1_weighted": 0.766920440971579 + }, + { + "accuracy": 0.7865223807181505, + "f1": 0.7699808718951124, + "f1_weighted": 0.781526953131049 + }, + { + "accuracy": 0.7968519429414658, + "f1": 0.7760660919266599, + "f1_weighted": 0.7925657688723198 + }, + { + "accuracy": 0.7757009345794392, + "f1": 0.7520749126939, + "f1_weighted": 0.7694388847482563 + }, + { + "accuracy": 0.7324151500245942, + "f1": 0.7268934507203584, + "f1_weighted": 0.7286901163718955 + }, + { + "accuracy": 0.750122970978849, + "f1": 0.7329980036543031, + "f1_weighted": 0.7470975243407222 + }, + { + "accuracy": 0.7624200688637481, + "f1": 0.7535502440606934, + "f1_weighted": 0.7667031190635183 + }, + { + "accuracy": 0.794884407279882, + "f1": 0.7837787411206423, + "f1_weighted": 0.7929389443712709 + }, + { + "accuracy": 0.764879488440728, + "f1": 0.7500201871714574, + "f1_weighted": 0.7639114530451045 + } + ] + } + ] + }, + "task_name": "MassiveScenarioClassification" +} \ No newline at end of file diff --git a/results/nvidia__NV-Embed-v2/7604d305b621f14095a1aa23d351674c2859553a/RUParaPhraserSTS.json b/results/nvidia__NV-Embed-v2/7604d305b621f14095a1aa23d351674c2859553a/RUParaPhraserSTS.json new file mode 100644 index 000000000..08dd6177f --- /dev/null +++ b/results/nvidia__NV-Embed-v2/7604d305b621f14095a1aa23d351674c2859553a/RUParaPhraserSTS.json @@ -0,0 +1,26 @@ +{ + "dataset_revision": "43265056790b8f7c59e0139acb4be0a8dad2c8f4", + "evaluation_time": 116.25867652893066, + "kg_co2_emissions": null, + "mteb_version": "1.19.2", + "scores": { + "test": [ + { + "cosine_pearson": 0.6629474648390903, + "cosine_spearman": 0.7325275944955283, + "euclidean_pearson": 0.7054668332273275, + "euclidean_spearman": 0.7325275941869749, + "hf_subset": "default", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.7325275944955283, + "manhattan_pearson": 0.7055022317703187, + "manhattan_spearman": 0.7320201998050069, + "pearson": 0.6629474648390903, + "spearman": 0.7325275944955283 + } + ] + }, + "task_name": "RUParaPhraserSTS" +} \ No newline at end of file diff --git a/results/nvidia__NV-Embed-v2/7604d305b621f14095a1aa23d351674c2859553a/RiaNewsRetrieval.json b/results/nvidia__NV-Embed-v2/7604d305b621f14095a1aa23d351674c2859553a/RiaNewsRetrieval.json new file mode 100644 index 000000000..f658b14b0 --- /dev/null +++ b/results/nvidia__NV-Embed-v2/7604d305b621f14095a1aa23d351674c2859553a/RiaNewsRetrieval.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "82374b0bbacda6114f39ff9c5b925fa1512ca5d7", + "evaluation_time": 48394.96449136734, + "kg_co2_emissions": null, + "mteb_version": "1.19.2", + "scores": { + "test": [ + { + "hf_subset": "default", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.75396, + "map_at_1": 0.6184, + "map_at_10": 0.7126, + "map_at_100": 0.71606, + "map_at_1000": 0.71616, + "map_at_20": 0.71489, + "map_at_3": 0.69558, + "map_at_5": 0.70608, + "mrr_at_1": 0.6184, + "mrr_at_10": 0.7126037698412698, + "mrr_at_100": 0.7160555923660614, + "mrr_at_1000": 0.7161649428903615, + "mrr_at_20": 0.7148947983780065, + "mrr_at_3": 0.6955833333333333, + "mrr_at_5": 0.7060783333333334, + "nauc_map_at_1000_diff1": 0.7022093166006353, + "nauc_map_at_1000_max": 0.15408445620110453, + "nauc_map_at_1000_std": -0.1580739070284931, + "nauc_map_at_100_diff1": 0.7021882173230748, + "nauc_map_at_100_max": 0.15406684653773367, + "nauc_map_at_100_std": -0.1580028173945722, + "nauc_map_at_10_diff1": 0.7015607700644811, + "nauc_map_at_10_max": 0.15330951317891642, + "nauc_map_at_10_std": -0.15991418592951384, + "nauc_map_at_1_diff1": 0.7320122179424725, + "nauc_map_at_1_max": 0.13362032470743646, + "nauc_map_at_1_std": -0.17229728486076992, + "nauc_map_at_20_diff1": 0.7018874472142529, + "nauc_map_at_20_max": 0.15397463940170095, + "nauc_map_at_20_std": -0.15850072963346193, + "nauc_map_at_3_diff1": 0.7005674945769985, + "nauc_map_at_3_max": 0.1530218892070655, + "nauc_map_at_3_std": -0.1673385529170003, + "nauc_map_at_5_diff1": 0.7007967903332945, + "nauc_map_at_5_max": 0.1526173203537391, + "nauc_map_at_5_std": -0.1625481975560051, + "nauc_mrr_at_1000_diff1": 0.7022093166006353, + "nauc_mrr_at_1000_max": 0.15408445620110453, + "nauc_mrr_at_1000_std": -0.1580739070284931, + "nauc_mrr_at_100_diff1": 0.7021882173230748, + "nauc_mrr_at_100_max": 0.15406684653773367, + "nauc_mrr_at_100_std": -0.1580028173945722, + "nauc_mrr_at_10_diff1": 0.7015607700644811, + "nauc_mrr_at_10_max": 0.15330951317891642, + "nauc_mrr_at_10_std": -0.15991418592951384, + "nauc_mrr_at_1_diff1": 0.7320122179424725, + "nauc_mrr_at_1_max": 0.13362032470743646, + "nauc_mrr_at_1_std": -0.17229728486076992, + "nauc_mrr_at_20_diff1": 0.7018874472142529, + "nauc_mrr_at_20_max": 0.15397463940170095, + "nauc_mrr_at_20_std": -0.15850072963346193, + "nauc_mrr_at_3_diff1": 0.7005674945769985, + "nauc_mrr_at_3_max": 0.1530218892070655, + "nauc_mrr_at_3_std": -0.1673385529170003, + "nauc_mrr_at_5_diff1": 0.7007967903332945, + "nauc_mrr_at_5_max": 0.1526173203537391, + "nauc_mrr_at_5_std": -0.1625481975560051, + "nauc_ndcg_at_1000_diff1": 0.6937618243652298, + "nauc_ndcg_at_1000_max": 0.16570936799073743, + "nauc_ndcg_at_1000_std": -0.1403394604306612, + "nauc_ndcg_at_100_diff1": 0.6929943285336365, + "nauc_ndcg_at_100_max": 0.16586797986943694, + "nauc_ndcg_at_100_std": -0.13682578018881245, + "nauc_ndcg_at_10_diff1": 0.6895062020732455, + "nauc_ndcg_at_10_max": 0.16183556910743857, + "nauc_ndcg_at_10_std": -0.14696649723996935, + "nauc_ndcg_at_1_diff1": 0.7320122179424725, + "nauc_ndcg_at_1_max": 0.13362032470743646, + "nauc_ndcg_at_1_std": -0.17229728486076992, + "nauc_ndcg_at_20_diff1": 0.6905217307260997, + "nauc_ndcg_at_20_max": 0.1650296083498101, + "nauc_ndcg_at_20_std": -0.14055365479656015, + "nauc_ndcg_at_3_diff1": 0.6887391721263058, + "nauc_ndcg_at_3_max": 0.1598852254476299, + "nauc_ndcg_at_3_std": -0.1639213682056404, + "nauc_ndcg_at_5_diff1": 0.6884233811648115, + "nauc_ndcg_at_5_max": 0.15959103191661803, + "nauc_ndcg_at_5_std": -0.1545243679409728, + "nauc_precision_at_1000_diff1": 0.5049941344059053, + "nauc_precision_at_1000_max": 0.5944097297038612, + "nauc_precision_at_1000_std": 0.46890038066508694, + "nauc_precision_at_100_diff1": 0.5783943725120166, + "nauc_precision_at_100_max": 0.3739539327774572, + "nauc_precision_at_100_std": 0.27507459860401, + "nauc_precision_at_10_diff1": 0.6181498228171558, + "nauc_precision_at_10_max": 0.2138442675972491, + "nauc_precision_at_10_std": -0.060530154917176344, + "nauc_precision_at_1_diff1": 0.7320122179424725, + "nauc_precision_at_1_max": 0.13362032470743646, + "nauc_precision_at_1_std": -0.17229728486076992, + "nauc_precision_at_20_diff1": 0.6032825054140889, + "nauc_precision_at_20_max": 0.26162714408256244, + "nauc_precision_at_20_std": 0.028482937890599316, + "nauc_precision_at_3_diff1": 0.6443485496727034, + "nauc_precision_at_3_max": 0.18546302077149063, + "nauc_precision_at_3_std": -0.15045565709995787, + "nauc_precision_at_5_diff1": 0.6327330363048663, + "nauc_precision_at_5_max": 0.1907049939173931, + "nauc_precision_at_5_std": -0.11575908958813723, + "nauc_recall_at_1000_diff1": 0.5049941344058917, + "nauc_recall_at_1000_max": 0.5944097297038524, + "nauc_recall_at_1000_std": 0.46890038066508566, + "nauc_recall_at_100_diff1": 0.5783943725120191, + "nauc_recall_at_100_max": 0.37395393277746575, + "nauc_recall_at_100_std": 0.2750745986040076, + "nauc_recall_at_10_diff1": 0.6181498228171549, + "nauc_recall_at_10_max": 0.21384426759724823, + "nauc_recall_at_10_std": -0.06053015491717598, + "nauc_recall_at_1_diff1": 0.7320122179424725, + "nauc_recall_at_1_max": 0.13362032470743646, + "nauc_recall_at_1_std": -0.17229728486076992, + "nauc_recall_at_20_diff1": 0.6032825054140882, + "nauc_recall_at_20_max": 0.261627144082565, + "nauc_recall_at_20_std": 0.028482937890602338, + "nauc_recall_at_3_diff1": 0.6443485496727042, + "nauc_recall_at_3_max": 0.1854630207714909, + "nauc_recall_at_3_std": -0.15045565709995778, + "nauc_recall_at_5_diff1": 0.6327330363048668, + "nauc_recall_at_5_max": 0.1907049939173938, + "nauc_recall_at_5_std": -0.11575908958813577, + "ndcg_at_1": 0.6184, + "ndcg_at_10": 0.75396, + "ndcg_at_100": 0.7703, + "ndcg_at_1000": 0.77325, + "ndcg_at_20": 0.76218, + "ndcg_at_3": 0.71961, + "ndcg_at_5": 0.73844, + "precision_at_1": 0.6184, + "precision_at_10": 0.08817, + "precision_at_100": 0.00957, + "precision_at_1000": 0.00098, + "precision_at_20": 0.0457, + "precision_at_3": 0.26293, + "precision_at_5": 0.16686, + "recall_at_1": 0.6184, + "recall_at_10": 0.8817, + "recall_at_100": 0.9571, + "recall_at_1000": 0.9805, + "recall_at_20": 0.9139, + "recall_at_3": 0.7888, + "recall_at_5": 0.8343 + } + ] + }, + "task_name": "RiaNewsRetrieval" +} \ No newline at end of file diff --git a/results/nvidia__NV-Embed-v2/7604d305b621f14095a1aa23d351674c2859553a/RuBQReranking.json b/results/nvidia__NV-Embed-v2/7604d305b621f14095a1aa23d351674c2859553a/RuBQReranking.json new file mode 100644 index 000000000..07db88943 --- /dev/null +++ b/results/nvidia__NV-Embed-v2/7604d305b621f14095a1aa23d351674c2859553a/RuBQReranking.json @@ -0,0 +1,26 @@ +{ + "dataset_revision": "2e96b8f098fa4b0950fc58eacadeb31c0d0c7fa2", + "evaluation_time": 1687.9647631645203, + "kg_co2_emissions": null, + "mteb_version": "1.19.2", + "scores": { + "test": [ + { + "hf_subset": "default", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.7709527141282384, + "map": 0.7709527141282384, + "mrr": 0.821077543417969, + "nAUC_map_diff1": 0.4235023290028287, + "nAUC_map_max": 0.14407283149170064, + "nAUC_map_std": 0.053568619158399534, + "nAUC_mrr_diff1": 0.4951068145980548, + "nAUC_mrr_max": 0.20248534680587768, + "nAUC_mrr_std": 0.07943495228776307 + } + ] + }, + "task_name": "RuBQReranking" +} \ No newline at end of file diff --git a/results/nvidia__NV-Embed-v2/7604d305b621f14095a1aa23d351674c2859553a/RuBQRetrieval.json b/results/nvidia__NV-Embed-v2/7604d305b621f14095a1aa23d351674c2859553a/RuBQRetrieval.json new file mode 100644 index 000000000..b0ac3c50a --- /dev/null +++ b/results/nvidia__NV-Embed-v2/7604d305b621f14095a1aa23d351674c2859553a/RuBQRetrieval.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "e19b6ffa60b3bc248e0b41f4cc37c26a55c2a67b", + "evaluation_time": 2491.336893796921, + "kg_co2_emissions": null, + "mteb_version": "1.19.2", + "scores": { + "test": [ + { + "hf_subset": "default", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.7475, + "map_at_1": 0.46721, + "map_at_10": 0.67176, + "map_at_100": 0.68054, + "map_at_1000": 0.68074, + "map_at_20": 0.6779, + "map_at_3": 0.61876, + "map_at_5": 0.65069, + "mrr_at_1": 0.6583924349881797, + "mrr_at_10": 0.7597367631055574, + "mrr_at_100": 0.7618198001207078, + "mrr_at_1000": 0.7618584346630259, + "mrr_at_20": 0.7611430666200335, + "mrr_at_3": 0.7398542159180457, + "mrr_at_5": 0.7530338849487787, + "nauc_map_at_1000_diff1": 0.4029894201577751, + "nauc_map_at_1000_max": 0.1961329743785155, + "nauc_map_at_1000_std": -0.10665541981853296, + "nauc_map_at_100_diff1": 0.40282155403682957, + "nauc_map_at_100_max": 0.19640836094185482, + "nauc_map_at_100_std": -0.10643353979479107, + "nauc_map_at_10_diff1": 0.3980463365016822, + "nauc_map_at_10_max": 0.19316986718750512, + "nauc_map_at_10_std": -0.10966915732105587, + "nauc_map_at_1_diff1": 0.4425036433183728, + "nauc_map_at_1_max": 0.11082572906659025, + "nauc_map_at_1_std": -0.11821974025537706, + "nauc_map_at_20_diff1": 0.40229354147779206, + "nauc_map_at_20_max": 0.19671900183334595, + "nauc_map_at_20_std": -0.10733141074240693, + "nauc_map_at_3_diff1": 0.3917164390064412, + "nauc_map_at_3_max": 0.16576332777175726, + "nauc_map_at_3_std": -0.11424529869996497, + "nauc_map_at_5_diff1": 0.39412013171410254, + "nauc_map_at_5_max": 0.17641474063781779, + "nauc_map_at_5_std": -0.11838962197091593, + "nauc_mrr_at_1000_diff1": 0.5081791125651745, + "nauc_mrr_at_1000_max": 0.24376178043203386, + "nauc_mrr_at_1000_std": -0.0975169126696244, + "nauc_mrr_at_100_diff1": 0.5080803257890937, + "nauc_mrr_at_100_max": 0.24380476779247645, + "nauc_mrr_at_100_std": -0.09742519760452371, + "nauc_mrr_at_10_diff1": 0.5076697246744789, + "nauc_mrr_at_10_max": 0.24554901155643805, + "nauc_mrr_at_10_std": -0.09689200921788152, + "nauc_mrr_at_1_diff1": 0.5373546210966907, + "nauc_mrr_at_1_max": 0.20088134567173593, + "nauc_mrr_at_1_std": -0.11026537163488087, + "nauc_mrr_at_20_diff1": 0.5081346360171212, + "nauc_mrr_at_20_max": 0.24471267699504926, + "nauc_mrr_at_20_std": -0.09719692520638197, + "nauc_mrr_at_3_diff1": 0.5031419108124348, + "nauc_mrr_at_3_max": 0.23892386377182928, + "nauc_mrr_at_3_std": -0.11367004932244656, + "nauc_mrr_at_5_diff1": 0.5065741303889483, + "nauc_mrr_at_5_max": 0.24642565299998895, + "nauc_mrr_at_5_std": -0.09888289610186562, + "nauc_ndcg_at_1000_diff1": 0.4246488416664654, + "nauc_ndcg_at_1000_max": 0.23016194214161714, + "nauc_ndcg_at_1000_std": -0.08888268156390586, + "nauc_ndcg_at_100_diff1": 0.41946045802704507, + "nauc_ndcg_at_100_max": 0.23608207132891734, + "nauc_ndcg_at_100_std": -0.08217302757578117, + "nauc_ndcg_at_10_diff1": 0.40830441239334797, + "nauc_ndcg_at_10_max": 0.23465206134902808, + "nauc_ndcg_at_10_std": -0.09193254240587027, + "nauc_ndcg_at_1_diff1": 0.5373546210966907, + "nauc_ndcg_at_1_max": 0.20088134567173593, + "nauc_ndcg_at_1_std": -0.11026537163488087, + "nauc_ndcg_at_20_diff1": 0.4176853343697519, + "nauc_ndcg_at_20_max": 0.24279692831513397, + "nauc_ndcg_at_20_std": -0.08577710043134242, + "nauc_ndcg_at_3_diff1": 0.3995440944062043, + "nauc_ndcg_at_3_max": 0.19029128633998132, + "nauc_ndcg_at_3_std": -0.11889143404204877, + "nauc_ndcg_at_5_diff1": 0.4035260683453763, + "nauc_ndcg_at_5_max": 0.2070409476134944, + "nauc_ndcg_at_5_std": -0.10981111382064025, + "nauc_precision_at_1000_diff1": -0.11423440974207502, + "nauc_precision_at_1000_max": 0.061771999492510554, + "nauc_precision_at_1000_std": 0.06304040160623768, + "nauc_precision_at_100_diff1": -0.11320848148439279, + "nauc_precision_at_100_max": 0.09370286269307644, + "nauc_precision_at_100_std": 0.08263787798877174, + "nauc_precision_at_10_diff1": -0.05526946992017798, + "nauc_precision_at_10_max": 0.15590472136445, + "nauc_precision_at_10_std": 0.044398947416582575, + "nauc_precision_at_1_diff1": 0.5373546210966907, + "nauc_precision_at_1_max": 0.20088134567173593, + "nauc_precision_at_1_std": -0.11026537163488087, + "nauc_precision_at_20_diff1": -0.07580223519798397, + "nauc_precision_at_20_max": 0.1427713371219493, + "nauc_precision_at_20_std": 0.0658942992584397, + "nauc_precision_at_3_diff1": 0.08964160040035904, + "nauc_precision_at_3_max": 0.16962391084270945, + "nauc_precision_at_3_std": -0.02852685044347902, + "nauc_precision_at_5_diff1": 0.012173038386151462, + "nauc_precision_at_5_max": 0.15479204574549926, + "nauc_precision_at_5_std": -0.00793573230425635, + "nauc_recall_at_1000_diff1": 0.3025063683499687, + "nauc_recall_at_1000_max": 0.7388714686317338, + "nauc_recall_at_1000_std": 0.6148673188884917, + "nauc_recall_at_100_diff1": 0.11003099496814539, + "nauc_recall_at_100_max": 0.5721472502810859, + "nauc_recall_at_100_std": 0.39888604806977357, + "nauc_recall_at_10_diff1": 0.24815444932217534, + "nauc_recall_at_10_max": 0.3266405854309893, + "nauc_recall_at_10_std": -0.005029352701552229, + "nauc_recall_at_1_diff1": 0.4425036433183728, + "nauc_recall_at_1_max": 0.11082572906659025, + "nauc_recall_at_1_std": -0.11821974025537706, + "nauc_recall_at_20_diff1": 0.2554662395791254, + "nauc_recall_at_20_max": 0.43569047606496986, + "nauc_recall_at_20_std": 0.07337770395299659, + "nauc_recall_at_3_diff1": 0.3022984377401223, + "nauc_recall_at_3_max": 0.17861247041692405, + "nauc_recall_at_3_std": -0.10560221795922661, + "nauc_recall_at_5_diff1": 0.29237491125723525, + "nauc_recall_at_5_max": 0.21953045344429245, + "nauc_recall_at_5_std": -0.08242522211191923, + "ndcg_at_1": 0.65839, + "ndcg_at_10": 0.7475, + "ndcg_at_100": 0.77284, + "ndcg_at_1000": 0.7761, + "ndcg_at_20": 0.76146, + "ndcg_at_3": 0.67701, + "ndcg_at_5": 0.71327, + "precision_at_1": 0.65839, + "precision_at_10": 0.14261, + "precision_at_100": 0.01622, + "precision_at_1000": 0.00167, + "precision_at_20": 0.07603, + "precision_at_3": 0.36052, + "precision_at_5": 0.24976, + "recall_at_1": 0.46721, + "recall_at_10": 0.87752, + "recall_at_100": 0.97179, + "recall_at_1000": 0.99318, + "recall_at_20": 0.92186, + "recall_at_3": 0.7046, + "recall_at_5": 0.79098 + } + ] + }, + "task_name": "RuBQRetrieval" +} \ No newline at end of file diff --git a/results/nvidia__NV-Embed-v2/7604d305b621f14095a1aa23d351674c2859553a/RuReviewsClassification.json b/results/nvidia__NV-Embed-v2/7604d305b621f14095a1aa23d351674c2859553a/RuReviewsClassification.json new file mode 100644 index 000000000..a5c688992 --- /dev/null +++ b/results/nvidia__NV-Embed-v2/7604d305b621f14095a1aa23d351674c2859553a/RuReviewsClassification.json @@ -0,0 +1,73 @@ +{ + "dataset_revision": "f6d2c31f4dc6b88f468552750bfec05b4b41b05a", + "evaluation_time": 78.65246558189392, + "kg_co2_emissions": null, + "mteb_version": "1.19.2", + "scores": { + "test": [ + { + "accuracy": 0.64072265625, + "f1": 0.6352181660621422, + "f1_weighted": 0.6352234814862108, + "hf_subset": "default", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.64072265625, + "scores_per_experiment": [ + { + "accuracy": 0.64794921875, + "f1": 0.6427967130256643, + "f1_weighted": 0.6427957556426251 + }, + { + "accuracy": 0.61962890625, + "f1": 0.6135087758445888, + "f1_weighted": 0.6135269846809336 + }, + { + "accuracy": 0.64208984375, + "f1": 0.6406603336917795, + "f1_weighted": 0.6406691550743149 + }, + { + "accuracy": 0.66845703125, + "f1": 0.6674097738913966, + "f1_weighted": 0.6674400631966807 + }, + { + "accuracy": 0.673828125, + "f1": 0.678918786377967, + "f1_weighted": 0.6789312219144621 + }, + { + "accuracy": 0.61181640625, + "f1": 0.6007126609313799, + "f1_weighted": 0.600732195237613 + }, + { + "accuracy": 0.650390625, + "f1": 0.6349927553064475, + "f1_weighted": 0.6349949090603138 + }, + { + "accuracy": 0.58447265625, + "f1": 0.5822995543083848, + "f1_weighted": 0.5822835217394473 + }, + { + "accuracy": 0.62353515625, + "f1": 0.610790511564201, + "f1_weighted": 0.6107442474787341 + }, + { + "accuracy": 0.68505859375, + "f1": 0.6800917956796128, + "f1_weighted": 0.6801167608369832 + } + ] + } + ] + }, + "task_name": "RuReviewsClassification" +} \ No newline at end of file diff --git a/results/nvidia__NV-Embed-v2/7604d305b621f14095a1aa23d351674c2859553a/RuSTSBenchmarkSTS.json b/results/nvidia__NV-Embed-v2/7604d305b621f14095a1aa23d351674c2859553a/RuSTSBenchmarkSTS.json new file mode 100644 index 000000000..0f30fd1e3 --- /dev/null +++ b/results/nvidia__NV-Embed-v2/7604d305b621f14095a1aa23d351674c2859553a/RuSTSBenchmarkSTS.json @@ -0,0 +1,26 @@ +{ + "dataset_revision": "7cf24f325c6da6195df55bef3d86b5e0616f3018", + "evaluation_time": 76.84879207611084, + "kg_co2_emissions": null, + "mteb_version": "1.19.2", + "scores": { + "test": [ + { + "cosine_pearson": 0.7937594127595271, + "cosine_spearman": 0.7877072769730338, + "euclidean_pearson": 0.7876571299284666, + "euclidean_spearman": 0.7877084098016619, + "hf_subset": "default", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.7877072769730338, + "manhattan_pearson": 0.7877237863887014, + "manhattan_spearman": 0.787477114222879, + "pearson": 0.7937594127595271, + "spearman": 0.7877072769730338 + } + ] + }, + "task_name": "RuSTSBenchmarkSTS" +} \ No newline at end of file diff --git a/results/nvidia__NV-Embed-v2/7604d305b621f14095a1aa23d351674c2859553a/RuSciBenchGRNTIClassification.json b/results/nvidia__NV-Embed-v2/7604d305b621f14095a1aa23d351674c2859553a/RuSciBenchGRNTIClassification.json new file mode 100644 index 000000000..daf8196e9 --- /dev/null +++ b/results/nvidia__NV-Embed-v2/7604d305b621f14095a1aa23d351674c2859553a/RuSciBenchGRNTIClassification.json @@ -0,0 +1,73 @@ +{ + "dataset_revision": "673a610d6d3dd91a547a0d57ae1b56f37ebbf6a1", + "evaluation_time": 270.3034965991974, + "kg_co2_emissions": null, + "mteb_version": "1.19.2", + "scores": { + "test": [ + { + "accuracy": 0.550537109375, + "f1": 0.5392724697666662, + "f1_weighted": 0.5393986560765207, + "hf_subset": "default", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.550537109375, + "scores_per_experiment": [ + { + "accuracy": 0.55908203125, + "f1": 0.5433095104346162, + "f1_weighted": 0.5434831704770664 + }, + { + "accuracy": 0.546875, + "f1": 0.5373855667041704, + "f1_weighted": 0.5374903215413845 + }, + { + "accuracy": 0.54345703125, + "f1": 0.5341989333687146, + "f1_weighted": 0.5343169288460694 + }, + { + "accuracy": 0.57177734375, + "f1": 0.5607836013034991, + "f1_weighted": 0.5608948234587834 + }, + { + "accuracy": 0.55078125, + "f1": 0.5446757191464259, + "f1_weighted": 0.5447381877393418 + }, + { + "accuracy": 0.53271484375, + "f1": 0.5140392716966641, + "f1_weighted": 0.5141315018561273 + }, + { + "accuracy": 0.54541015625, + "f1": 0.5332776365657586, + "f1_weighted": 0.5334759410455508 + }, + { + "accuracy": 0.5478515625, + "f1": 0.5358493734592435, + "f1_weighted": 0.5359757146911633 + }, + { + "accuracy": 0.552734375, + "f1": 0.5440330388094133, + "f1_weighted": 0.5441551130705831 + }, + { + "accuracy": 0.5546875, + "f1": 0.5451720461781565, + "f1_weighted": 0.5453248580391366 + } + ] + } + ] + }, + "task_name": "RuSciBenchGRNTIClassification" +} \ No newline at end of file diff --git a/results/nvidia__NV-Embed-v2/7604d305b621f14095a1aa23d351674c2859553a/RuSciBenchGRNTIClusteringP2P.json b/results/nvidia__NV-Embed-v2/7604d305b621f14095a1aa23d351674c2859553a/RuSciBenchGRNTIClusteringP2P.json new file mode 100644 index 000000000..5b6cdb494 --- /dev/null +++ b/results/nvidia__NV-Embed-v2/7604d305b621f14095a1aa23d351674c2859553a/RuSciBenchGRNTIClusteringP2P.json @@ -0,0 +1,34 @@ +{ + "dataset_revision": "673a610d6d3dd91a547a0d57ae1b56f37ebbf6a1", + "evaluation_time": 206.68730282783508, + "kg_co2_emissions": null, + "mteb_version": "1.19.2", + "scores": { + "test": [ + { + "hf_subset": "default", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.44620386629574726, + "v_measure": 0.44620386629574726, + "v_measure_std": 0.014724590116090626, + "v_measures": { + "Level 0": [ + 0.4365607115111906, + 0.43154339872484637, + 0.44432780696378116, + 0.43677378414801077, + 0.4543310482499685, + 0.4684008477640952, + 0.4483928542672086, + 0.4669574205946946, + 0.4549484883537797, + 0.41980230237989785 + ] + } + } + ] + }, + "task_name": "RuSciBenchGRNTIClusteringP2P" +} \ No newline at end of file diff --git a/results/nvidia__NV-Embed-v2/7604d305b621f14095a1aa23d351674c2859553a/RuSciBenchOECDClassification.json b/results/nvidia__NV-Embed-v2/7604d305b621f14095a1aa23d351674c2859553a/RuSciBenchOECDClassification.json new file mode 100644 index 000000000..2edccbdc8 --- /dev/null +++ b/results/nvidia__NV-Embed-v2/7604d305b621f14095a1aa23d351674c2859553a/RuSciBenchOECDClassification.json @@ -0,0 +1,73 @@ +{ + "dataset_revision": "26c88e99dcaba32bb45d0e1bfc21902337f6d471", + "evaluation_time": 269.1332268714905, + "kg_co2_emissions": null, + "mteb_version": "1.19.2", + "scores": { + "test": [ + { + "accuracy": 0.425048828125, + "f1": 0.40427671833957135, + "f1_weighted": 0.4043403394829116, + "hf_subset": "default", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.425048828125, + "scores_per_experiment": [ + { + "accuracy": 0.4052734375, + "f1": 0.38075456030644533, + "f1_weighted": 0.3809799098950216 + }, + { + "accuracy": 0.42041015625, + "f1": 0.4022108756202202, + "f1_weighted": 0.4022105943314688 + }, + { + "accuracy": 0.4287109375, + "f1": 0.4124294791767373, + "f1_weighted": 0.41256756703429215 + }, + { + "accuracy": 0.421875, + "f1": 0.38592026384559297, + "f1_weighted": 0.38582032973379937 + }, + { + "accuracy": 0.4267578125, + "f1": 0.40161473560296157, + "f1_weighted": 0.4016559989786852 + }, + { + "accuracy": 0.4306640625, + "f1": 0.41735961130428356, + "f1_weighted": 0.4174925125933628 + }, + { + "accuracy": 0.43017578125, + "f1": 0.40641604964219963, + "f1_weighted": 0.40639416861451955 + }, + { + "accuracy": 0.44384765625, + "f1": 0.4311275325543602, + "f1_weighted": 0.43125570041008476 + }, + { + "accuracy": 0.416015625, + "f1": 0.3985935903989258, + "f1_weighted": 0.3986180453784995 + }, + { + "accuracy": 0.4267578125, + "f1": 0.40634048494398667, + "f1_weighted": 0.4064085678593823 + } + ] + } + ] + }, + "task_name": "RuSciBenchOECDClassification" +} \ No newline at end of file diff --git a/results/nvidia__NV-Embed-v2/7604d305b621f14095a1aa23d351674c2859553a/RuSciBenchOECDClusteringP2P.json b/results/nvidia__NV-Embed-v2/7604d305b621f14095a1aa23d351674c2859553a/RuSciBenchOECDClusteringP2P.json new file mode 100644 index 000000000..af5c7a32a --- /dev/null +++ b/results/nvidia__NV-Embed-v2/7604d305b621f14095a1aa23d351674c2859553a/RuSciBenchOECDClusteringP2P.json @@ -0,0 +1,34 @@ +{ + "dataset_revision": "26c88e99dcaba32bb45d0e1bfc21902337f6d471", + "evaluation_time": 207.09291243553162, + "kg_co2_emissions": null, + "mteb_version": "1.19.2", + "scores": { + "test": [ + { + "hf_subset": "default", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.3936901351136865, + "v_measure": 0.3936901351136865, + "v_measure_std": 0.012721865522601662, + "v_measures": { + "Level 0": [ + 0.38536482154126306, + 0.38937553946517434, + 0.36629877600296573, + 0.4103688926568211, + 0.39839516027602867, + 0.4019748268627427, + 0.3903474470837463, + 0.38581023201624726, + 0.3972350591040048, + 0.411730596127871 + ] + } + } + ] + }, + "task_name": "RuSciBenchOECDClusteringP2P" +} \ No newline at end of file diff --git a/results/nvidia__NV-Embed-v2/7604d305b621f14095a1aa23d351674c2859553a/STS22.json b/results/nvidia__NV-Embed-v2/7604d305b621f14095a1aa23d351674c2859553a/STS22.json new file mode 100644 index 000000000..725bda372 --- /dev/null +++ b/results/nvidia__NV-Embed-v2/7604d305b621f14095a1aa23d351674c2859553a/STS22.json @@ -0,0 +1,26 @@ +{ + "dataset_revision": "de9d86b3b84231dc21f76c7b7af1f28e2f57f6e3", + "evaluation_time": 47.662322759628296, + "kg_co2_emissions": null, + "mteb_version": "1.19.2", + "scores": { + "test": [ + { + "cosine_pearson": 0.3998763516085712, + "cosine_spearman": 0.56249044968301, + "euclidean_pearson": 0.44919848301792925, + "euclidean_spearman": 0.56249044968301, + "hf_subset": "ru", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.56249044968301, + "manhattan_pearson": 0.47631937966724425, + "manhattan_spearman": 0.5653087642520488, + "pearson": 0.3998763516085712, + "spearman": 0.56249044968301 + } + ] + }, + "task_name": "STS22" +} \ No newline at end of file diff --git a/results/nvidia__NV-Embed-v2/7604d305b621f14095a1aa23d351674c2859553a/SensitiveTopicsClassification.json b/results/nvidia__NV-Embed-v2/7604d305b621f14095a1aa23d351674c2859553a/SensitiveTopicsClassification.json new file mode 100644 index 000000000..b1417fefc --- /dev/null +++ b/results/nvidia__NV-Embed-v2/7604d305b621f14095a1aa23d351674c2859553a/SensitiveTopicsClassification.json @@ -0,0 +1,73 @@ +{ + "dataset_revision": "416b34a802308eac30e4192afc0ff99bb8dcc7f2", + "evaluation_time": 127.74076056480408, + "kg_co2_emissions": null, + "mteb_version": "1.19.2", + "scores": { + "test": [ + { + "accuracy": 0.228662109375, + "f1": 0.18450234464134513, + "hf_subset": "default", + "languages": [ + "rus-Cyrl" + ], + "lrap": 0.34300944010415935, + "main_score": 0.228662109375, + "scores_per_experiment": [ + { + "accuracy": 0.21826171875, + "f1": 0.15003914362629694, + "lrap": 0.33213975694443715 + }, + { + "accuracy": 0.25146484375, + "f1": 0.20332822200691228, + "lrap": 0.36671278211804803 + }, + { + "accuracy": 0.232421875, + "f1": 0.20261834717292493, + "lrap": 0.3421223958333261 + }, + { + "accuracy": 0.22998046875, + "f1": 0.17758965702004292, + "lrap": 0.32891167534721494 + }, + { + "accuracy": 0.19921875, + "f1": 0.17580123767623104, + "lrap": 0.3311225043402706 + }, + { + "accuracy": 0.22216796875, + "f1": 0.18230593303775794, + "lrap": 0.3277859157986038 + }, + { + "accuracy": 0.25244140625, + "f1": 0.20743512536454817, + "lrap": 0.36937120225693715 + }, + { + "accuracy": 0.20361328125, + "f1": 0.17347506758639375, + "lrap": 0.31685384114582615 + }, + { + "accuracy": 0.24462890625, + "f1": 0.18509336281142036, + "lrap": 0.3697645399305482 + }, + { + "accuracy": 0.232421875, + "f1": 0.18733735011092298, + "lrap": 0.3453097873263815 + } + ] + } + ] + }, + "task_name": "SensitiveTopicsClassification" +} \ No newline at end of file diff --git a/results/nvidia__NV-Embed-v2/7604d305b621f14095a1aa23d351674c2859553a/TERRa.json b/results/nvidia__NV-Embed-v2/7604d305b621f14095a1aa23d351674c2859553a/TERRa.json new file mode 100644 index 000000000..244d78807 --- /dev/null +++ b/results/nvidia__NV-Embed-v2/7604d305b621f14095a1aa23d351674c2859553a/TERRa.json @@ -0,0 +1,58 @@ +{ + "dataset_revision": "7b58f24536063837d644aab9a023c62199b2a612", + "evaluation_time": 19.509780645370483, + "kg_co2_emissions": null, + "mteb_version": "1.19.2", + "scores": { + "dev": [ + { + "cosine_accuracy": 0.5960912052117264, + "cosine_accuracy_threshold": 0.6675424575805664, + "cosine_ap": 0.6025435858027424, + "cosine_f1": 0.6763285024154588, + "cosine_f1_threshold": 0.5821673274040222, + "cosine_precision": 0.5363984674329502, + "cosine_recall": 0.9150326797385621, + "dot_accuracy": 0.5960912052117264, + "dot_accuracy_threshold": 0.6675424575805664, + "dot_ap": 0.6025435858027424, + "dot_f1": 0.6763285024154588, + "dot_f1_threshold": 0.5821675062179565, + "dot_precision": 0.5363984674329502, + "dot_recall": 0.9150326797385621, + "euclidean_accuracy": 0.5960912052117264, + "euclidean_accuracy_threshold": 0.8154230713844299, + "euclidean_ap": 0.6025435858027424, + "euclidean_f1": 0.6763285024154588, + "euclidean_f1_threshold": 0.9141471982002258, + "euclidean_precision": 0.5363984674329502, + "euclidean_recall": 0.9150326797385621, + "hf_subset": "default", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.6028542256247589, + "manhattan_accuracy": 0.5895765472312704, + "manhattan_accuracy_threshold": 39.35441589355469, + "manhattan_ap": 0.6028542256247589, + "manhattan_f1": 0.6780487804878048, + "manhattan_f1_threshold": 44.59588623046875, + "manhattan_precision": 0.5408560311284046, + "manhattan_recall": 0.9084967320261438, + "max_accuracy": 0.5960912052117264, + "max_ap": 0.6028542256247589, + "max_f1": 0.6780487804878048, + "max_precision": 0.5408560311284046, + "max_recall": 0.9150326797385621, + "similarity_accuracy": 0.5960912052117264, + "similarity_accuracy_threshold": 0.6675424575805664, + "similarity_ap": 0.6025435858027424, + "similarity_f1": 0.6763285024154588, + "similarity_f1_threshold": 0.5821673274040222, + "similarity_precision": 0.5363984674329502, + "similarity_recall": 0.9150326797385621 + } + ] + }, + "task_name": "TERRa" +} \ No newline at end of file diff --git a/results/nvidia__NV-Embed-v2/7604d305b621f14095a1aa23d351674c2859553a/model_meta.json b/results/nvidia__NV-Embed-v2/7604d305b621f14095a1aa23d351674c2859553a/model_meta.json index 6f0adce37..2b1da3fb2 100644 --- a/results/nvidia__NV-Embed-v2/7604d305b621f14095a1aa23d351674c2859553a/model_meta.json +++ b/results/nvidia__NV-Embed-v2/7604d305b621f14095a1aa23d351674c2859553a/model_meta.json @@ -1 +1,25 @@ -{"name": "nvidia/NV-Embed-v2", "revision": "7604d305b621f14095a1aa23d351674c2859553a", "release_date": "2024-09-09", "languages": ["eng_Latn"], "n_parameters": 7850000000, "memory_usage": null, "max_tokens": 32768, "embed_dim": 4096, "license": "cc-by-nc-4.0", "open_weights": true, "public_training_data": null, "public_training_code": null, "framework": ["Sentence Transformers", "PyTorch"], "reference": "https://huggingface.co/nvidia/NV-Embed-v2", "similarity_fn_name": "cosine", "use_instructions": true, "zero_shot_benchmarks": null, "loader": "NvEmbedWrapper"} \ No newline at end of file +{ + "name": "nvidia/NV-Embed-v2", + "revision": "7604d305b621f14095a1aa23d351674c2859553a", + "release_date": "2024-09-09", + "languages": [ + "eng_Latn" + ], + "n_parameters": 7850000000, + "memory_usage": null, + "max_tokens": 32768, + "embed_dim": 4096, + "license": "cc-by-nc-4.0", + "open_weights": true, + "public_training_data": null, + "public_training_code": null, + "framework": [ + "Sentence Transformers", + "PyTorch" + ], + "reference": "https://huggingface.co/nvidia/NV-Embed-v2", + "similarity_fn_name": "cosine", + "use_instructions": true, + "zero_shot_benchmarks": null, + "loader": "NvEmbedWrapper" +} \ No newline at end of file diff --git a/results/sergeyzh__LaBSE-ru-turbo/1940b046c6b5e125df11722b899130329d0a46da/RuSciBenchGRNTIClusteringP2P.json b/results/sergeyzh__LaBSE-ru-turbo/1940b046c6b5e125df11722b899130329d0a46da/RuSciBenchGRNTIClusteringP2P.json index 52f347188..c994f7f4e 100644 --- a/results/sergeyzh__LaBSE-ru-turbo/1940b046c6b5e125df11722b899130329d0a46da/RuSciBenchGRNTIClusteringP2P.json +++ b/results/sergeyzh__LaBSE-ru-turbo/1940b046c6b5e125df11722b899130329d0a46da/RuSciBenchGRNTIClusteringP2P.json @@ -10,21 +10,21 @@ "languages": [ "rus-Cyrl" ], - "main_score": 0.5064389070110268, - "v_measure": 0.5064389070110268, - "v_measure_std": 0.01176930168172253, + "main_score": 0.5168634781503565, + "v_measure": 0.5168634781503565, + "v_measure_std": 0.010704087555300128, "v_measures": { "Level 0": [ - 0.49281135173400864, - 0.5087624779003245, - 0.5136909867366994, - 0.5009100506283635, - 0.5048009718242867, - 0.504690458821357, - 0.49977006835570603, - 0.4951496167190289, - 0.5369854226640096, - 0.5068176647264838 + 0.5377155919564779, + 0.5235460213785045, + 0.5190725969790164, + 0.5189098049129975, + 0.5042966756882047, + 0.521119070559439, + 0.5264872206575257, + 0.5020966622732258, + 0.5065934760541703, + 0.5087976610440036 ] } } diff --git a/results/sergeyzh__LaBSE-ru-turbo/1940b046c6b5e125df11722b899130329d0a46da/RuSciBenchOECDClusteringP2P.json b/results/sergeyzh__LaBSE-ru-turbo/1940b046c6b5e125df11722b899130329d0a46da/RuSciBenchOECDClusteringP2P.json index 2718a26ce..85935879c 100644 --- a/results/sergeyzh__LaBSE-ru-turbo/1940b046c6b5e125df11722b899130329d0a46da/RuSciBenchOECDClusteringP2P.json +++ b/results/sergeyzh__LaBSE-ru-turbo/1940b046c6b5e125df11722b899130329d0a46da/RuSciBenchOECDClusteringP2P.json @@ -10,21 +10,21 @@ "languages": [ "rus-Cyrl" ], - "main_score": 0.444828415881685, - "v_measure": 0.444828415881685, - "v_measure_std": 0.011894719691827666, + "main_score": 0.45564989763126507, + "v_measure": 0.45564989763126507, + "v_measure_std": 0.010652624187268568, "v_measures": { "Level 0": [ - 0.4620218825456935, - 0.437734533114208, - 0.43300097593964226, - 0.46843165352909294, - 0.4351918470474627, - 0.44788691457226104, - 0.45046687355966764, - 0.4313733681737988, - 0.4448877229368492, - 0.43728838739817383 + 0.45144157418817166, + 0.4468539562853618, + 0.45277266358543944, + 0.4724133354930541, + 0.43366482409715756, + 0.45951745537303307, + 0.45363315668347143, + 0.4551453277160861, + 0.4603988421875345, + 0.4706578407033405 ] } } diff --git a/results/sergeyzh__LaBSE-ru-turbo/1940b046c6b5e125df11722b899130329d0a46da/STS22.json b/results/sergeyzh__LaBSE-ru-turbo/1940b046c6b5e125df11722b899130329d0a46da/STS22.json index 65852bfbb..e5da73365 100644 --- a/results/sergeyzh__LaBSE-ru-turbo/1940b046c6b5e125df11722b899130329d0a46da/STS22.json +++ b/results/sergeyzh__LaBSE-ru-turbo/1940b046c6b5e125df11722b899130329d0a46da/STS22.json @@ -6,25 +6,19 @@ "scores": { "test": [ { - "cosine_pearson": 0.4066148085995761, - "cosine_spearman": 0.6289443980931155, - "euclidean_pearson": 0.4884936945039135, - "euclidean_spearman": 0.628888476631823, + "cosine_pearson": 0.6262649204002562, + "cosine_spearman": 0.6757304660384279, + "euclidean_pearson": 0.6382641225317527, + "euclidean_spearman": 0.6757304660384279, "hf_subset": "ru", "languages": [ "rus-Cyrl" ], - "main_score": 0.6289443980931155, - "manhattan_pearson": 0.4997705873213072, - "manhattan_spearman": 0.6287171648236973, - "pearson": [ - 0.4066148023073311, - 3.2589658265005604e-12 - ], - "spearman": [ - 0.628888476631823, - 3.043718148331608e-31 - ] + "main_score": 0.6757304660384279, + "manhattan_pearson": 0.638883931341251, + "manhattan_spearman": 0.6757065988640629, + "pearson": 0.6262649204002562, + "spearman": 0.6757304660384279 } ] }, diff --git a/results/sergeyzh__LaBSE-ru-turbo/1940b046c6b5e125df11722b899130329d0a46da/SensitiveTopicsClassification.json b/results/sergeyzh__LaBSE-ru-turbo/1940b046c6b5e125df11722b899130329d0a46da/SensitiveTopicsClassification.json index 8c0fad882..de2a01eec 100644 --- a/results/sergeyzh__LaBSE-ru-turbo/1940b046c6b5e125df11722b899130329d0a46da/SensitiveTopicsClassification.json +++ b/results/sergeyzh__LaBSE-ru-turbo/1940b046c6b5e125df11722b899130329d0a46da/SensitiveTopicsClassification.json @@ -6,64 +6,64 @@ "scores": { "test": [ { - "accuracy": 0.275244140625, - "f1": 0.29150075941334463, + "accuracy": 0.2875, + "f1": 0.3063527073655231, "hf_subset": "default", "languages": [ "rus-Cyrl" ], - "lrap": 0.4139275444878396, - "main_score": 0.275244140625, + "lrap": 0.43006388346353397, + "main_score": 0.2875, "scores_per_experiment": [ { - "accuracy": 0.29052734375, - "f1": 0.26712103086826444, - "lrap": 0.4333631727430479 + "accuracy": 0.25439453125, + "f1": 0.25868037757112394, + "lrap": 0.38678656684026996 }, { - "accuracy": 0.263671875, - "f1": 0.2975992837613153, - "lrap": 0.39885118272568665 + "accuracy": 0.2900390625, + "f1": 0.2997602900955247, + "lrap": 0.4292941623263811 }, { - "accuracy": 0.244140625, - "f1": 0.2615599599659626, - "lrap": 0.3748914930555482 + "accuracy": 0.28125, + "f1": 0.2713823982752672, + "lrap": 0.4122856987847143 }, { - "accuracy": 0.27685546875, - "f1": 0.3068034525405748, - "lrap": 0.4089287651909646 + "accuracy": 0.2744140625, + "f1": 0.25250045479350963, + "lrap": 0.4059109157986031 }, { - "accuracy": 0.26416015625, - "f1": 0.26058632149841443, - "lrap": 0.3940429687499923 + "accuracy": 0.27734375, + "f1": 0.33132418275621917, + "lrap": 0.4170803493923534 }, { - "accuracy": 0.2939453125, - "f1": 0.32408498616323694, - "lrap": 0.444132486979159 + "accuracy": 0.28369140625, + "f1": 0.295910788839725, + "lrap": 0.4380560980902704 }, { - "accuracy": 0.28125, - "f1": 0.3043677911005396, - "lrap": 0.440470377604159 + "accuracy": 0.31591796875, + "f1": 0.33387362847357155, + "lrap": 0.47064887152777035 }, { - "accuracy": 0.2763671875, - "f1": 0.2935574746912606, - "lrap": 0.4099460177951309 + "accuracy": 0.2861328125, + "f1": 0.32932281595213403, + "lrap": 0.4462212456597148 }, { - "accuracy": 0.27099609375, - "f1": 0.3072664138257324, - "lrap": 0.41335720486110344 + "accuracy": 0.3291015625, + "f1": 0.35992034799559436, + "lrap": 0.4588758680555478 }, { - "accuracy": 0.29052734375, - "f1": 0.2920608797181457, - "lrap": 0.42129177517360356 + "accuracy": 0.28271484375, + "f1": 0.3308517889025613, + "lrap": 0.4354790581597146 } ] } diff --git a/results/sergeyzh__LaBSE-ru-turbo/1940b046c6b5e125df11722b899130329d0a46da/model_meta.json b/results/sergeyzh__LaBSE-ru-turbo/1940b046c6b5e125df11722b899130329d0a46da/model_meta.json index 286ed600a..e651f12d0 100644 --- a/results/sergeyzh__LaBSE-ru-turbo/1940b046c6b5e125df11722b899130329d0a46da/model_meta.json +++ b/results/sergeyzh__LaBSE-ru-turbo/1940b046c6b5e125df11722b899130329d0a46da/model_meta.json @@ -1 +1,17 @@ -{"name": "sergeyzh/LaBSE-ru-turbo", "revision": "1940b046c6b5e125df11722b899130329d0a46da", "release_date": "2024-06-27", "languages": ["rus_Cyrl"], "n_parameters": null, "memory_usage": null, "max_tokens": null, "embed_dim": null, "license": null, "open_source": true, "similarity_fn_name": null, "framework": [], "loader": null} \ No newline at end of file +{ + "name": "sergeyzh/LaBSE-ru-turbo", + "revision": "1940b046c6b5e125df11722b899130329d0a46da", + "release_date": "2024-06-27", + "languages": [ + "rus_Cyrl" + ], + "n_parameters": null, + "memory_usage": null, + "max_tokens": null, + "embed_dim": null, + "license": null, + "open_source": true, + "similarity_fn_name": null, + "framework": [], + "loader": null +} \ No newline at end of file diff --git a/results/sergeyzh__rubert-tiny-turbo/8ce0cf757446ce9bb2d5f5a4ac8103c7a1049054/RuSciBenchGRNTIClusteringP2P.json b/results/sergeyzh__rubert-tiny-turbo/8ce0cf757446ce9bb2d5f5a4ac8103c7a1049054/RuSciBenchGRNTIClusteringP2P.json index b5ba1e843..e7171708b 100644 --- a/results/sergeyzh__rubert-tiny-turbo/8ce0cf757446ce9bb2d5f5a4ac8103c7a1049054/RuSciBenchGRNTIClusteringP2P.json +++ b/results/sergeyzh__rubert-tiny-turbo/8ce0cf757446ce9bb2d5f5a4ac8103c7a1049054/RuSciBenchGRNTIClusteringP2P.json @@ -10,21 +10,21 @@ "languages": [ "rus-Cyrl" ], - "main_score": 0.47548401486969843, - "v_measure": 0.47548401486969843, - "v_measure_std": 0.009652047055316595, + "main_score": 0.4810125905909669, + "v_measure": 0.4810125905909669, + "v_measure_std": 0.006922114327138789, "v_measures": { "Level 0": [ - 0.47474887236531765, - 0.4785849993294828, - 0.46367006851327036, - 0.4678173970077978, - 0.49276882256094645, - 0.467559239485672, - 0.4847182087337688, - 0.4647154510805738, - 0.47247429874787766, - 0.48778279087227633 + 0.48199325656558073, + 0.46891480854279516, + 0.4771265878718938, + 0.4809887460058498, + 0.4902230065187034, + 0.49087941403306307, + 0.4845275084599396, + 0.4866019111703341, + 0.4759245113316555, + 0.47294615540985435 ] } } diff --git a/results/sergeyzh__rubert-tiny-turbo/8ce0cf757446ce9bb2d5f5a4ac8103c7a1049054/RuSciBenchOECDClusteringP2P.json b/results/sergeyzh__rubert-tiny-turbo/8ce0cf757446ce9bb2d5f5a4ac8103c7a1049054/RuSciBenchOECDClusteringP2P.json index 0092492b4..764a839d2 100644 --- a/results/sergeyzh__rubert-tiny-turbo/8ce0cf757446ce9bb2d5f5a4ac8103c7a1049054/RuSciBenchOECDClusteringP2P.json +++ b/results/sergeyzh__rubert-tiny-turbo/8ce0cf757446ce9bb2d5f5a4ac8103c7a1049054/RuSciBenchOECDClusteringP2P.json @@ -10,21 +10,21 @@ "languages": [ "rus-Cyrl" ], - "main_score": 0.41440393356807953, - "v_measure": 0.41440393356807953, - "v_measure_std": 0.012447867997057735, + "main_score": 0.41058482743995717, + "v_measure": 0.41058482743995717, + "v_measure_std": 0.007434288295313363, "v_measures": { "Level 0": [ - 0.4277119271570462, - 0.41700888465701275, - 0.39089087539589434, - 0.40802787605475926, - 0.4068679181537213, - 0.4014026732700305, - 0.41785191568763097, - 0.43300862394884554, - 0.4280708813600974, - 0.4131977599957571 + 0.41635822586071797, + 0.4123597217338241, + 0.4015615402551029, + 0.4102551920874499, + 0.41028953505496524, + 0.42487561885179775, + 0.40629746462951877, + 0.41252779326389494, + 0.3967249069902914, + 0.41459827567200835 ] } } diff --git a/results/sergeyzh__rubert-tiny-turbo/8ce0cf757446ce9bb2d5f5a4ac8103c7a1049054/STS22.json b/results/sergeyzh__rubert-tiny-turbo/8ce0cf757446ce9bb2d5f5a4ac8103c7a1049054/STS22.json index 338988507..4f39974f2 100644 --- a/results/sergeyzh__rubert-tiny-turbo/8ce0cf757446ce9bb2d5f5a4ac8103c7a1049054/STS22.json +++ b/results/sergeyzh__rubert-tiny-turbo/8ce0cf757446ce9bb2d5f5a4ac8103c7a1049054/STS22.json @@ -71,25 +71,19 @@ ] }, { - "cosine_pearson": 0.514108131114559, - "cosine_spearman": 0.6005716921675363, - "euclidean_pearson": 0.5259520883430124, - "euclidean_spearman": 0.6005157835366834, + "cosine_pearson": 0.5931585595424974, + "cosine_spearman": 0.6455586599098249, + "euclidean_pearson": 0.6056847261991968, + "euclidean_spearman": 0.6455586599098249, "hf_subset": "ru", "languages": [ "rus-Cyrl" ], - "main_score": 0.6005716921675363, - "manhattan_pearson": 0.5249640999228367, - "manhattan_spearman": 0.5989412865698913, - "pearson": [ - 0.5141081326814505, - 1.1013760284137954e-19 - ], - "spearman": [ - 0.6005157835366834, - 6.02564522814787e-28 - ] + "main_score": 0.6455586599098249, + "manhattan_pearson": 0.6025431055720412, + "manhattan_spearman": 0.6435941561494153, + "pearson": 0.5931585595424974, + "spearman": 0.6455586599098249 }, { "cosine_pearson": 0.26610436064600534, diff --git a/results/sergeyzh__rubert-tiny-turbo/8ce0cf757446ce9bb2d5f5a4ac8103c7a1049054/SensitiveTopicsClassification.json b/results/sergeyzh__rubert-tiny-turbo/8ce0cf757446ce9bb2d5f5a4ac8103c7a1049054/SensitiveTopicsClassification.json index ebba51630..8a53d5498 100644 --- a/results/sergeyzh__rubert-tiny-turbo/8ce0cf757446ce9bb2d5f5a4ac8103c7a1049054/SensitiveTopicsClassification.json +++ b/results/sergeyzh__rubert-tiny-turbo/8ce0cf757446ce9bb2d5f5a4ac8103c7a1049054/SensitiveTopicsClassification.json @@ -6,64 +6,64 @@ "scores": { "test": [ { - "accuracy": 0.24443359375, - "f1": 0.21903258801323083, + "accuracy": 0.2515625, + "f1": 0.2417038584381097, "hf_subset": "default", "languages": [ "rus-Cyrl" ], - "lrap": 0.3634758843315896, - "main_score": 0.24443359375, + "lrap": 0.37687988281249235, + "main_score": 0.2515625, "scores_per_experiment": [ { - "accuracy": 0.26171875, - "f1": 0.21465062294682294, - "lrap": 0.3687201605902702 + "accuracy": 0.22900390625, + "f1": 0.18883668250910735, + "lrap": 0.3449435763888813 }, { - "accuracy": 0.2294921875, - "f1": 0.21798274554091368, - "lrap": 0.3449571397569371 + "accuracy": 0.22412109375, + "f1": 0.23112926644524956, + "lrap": 0.35724555121527035 }, { - "accuracy": 0.228515625, - "f1": 0.19234710952433062, - "lrap": 0.3424750434027702 + "accuracy": 0.25830078125, + "f1": 0.2358744743265705, + "lrap": 0.3744167751736033 }, { - "accuracy": 0.24365234375, - "f1": 0.2336792242766514, - "lrap": 0.36223687065971455 + "accuracy": 0.23583984375, + "f1": 0.17871755735752198, + "lrap": 0.34026421440971466 }, { - "accuracy": 0.24365234375, - "f1": 0.23315501157437682, - "lrap": 0.3790961371527703 + "accuracy": 0.24462890625, + "f1": 0.25438610804772255, + "lrap": 0.3749728732638814 }, { - "accuracy": 0.263671875, - "f1": 0.24366186735266945, - "lrap": 0.38992648654513135 + "accuracy": 0.25341796875, + "f1": 0.260179482846574, + "lrap": 0.38810221354165914 }, { - "accuracy": 0.23095703125, - "f1": 0.19499994950290167, - "lrap": 0.35484483506943687 + "accuracy": 0.26708984375, + "f1": 0.2689472404418445, + "lrap": 0.4032660590277701 }, { - "accuracy": 0.26220703125, - "f1": 0.2289944635729627, - "lrap": 0.37406412760415875 + "accuracy": 0.2607421875, + "f1": 0.25973839244631675, + "lrap": 0.3994004991319367 }, { - "accuracy": 0.22119140625, - "f1": 0.22753398492854832, - "lrap": 0.35607910156249234 + "accuracy": 0.283203125, + "f1": 0.2821463152172157, + "lrap": 0.39440917968749206 }, { "accuracy": 0.25927734375, - "f1": 0.20332090091213065, - "lrap": 0.36235894097221466 + "f1": 0.25708306474297404, + "lrap": 0.39177788628471466 } ] } diff --git a/results/sergeyzh__rubert-tiny-turbo/8ce0cf757446ce9bb2d5f5a4ac8103c7a1049054/model_meta.json b/results/sergeyzh__rubert-tiny-turbo/8ce0cf757446ce9bb2d5f5a4ac8103c7a1049054/model_meta.json index 739d340f9..1a66e74b8 100644 --- a/results/sergeyzh__rubert-tiny-turbo/8ce0cf757446ce9bb2d5f5a4ac8103c7a1049054/model_meta.json +++ b/results/sergeyzh__rubert-tiny-turbo/8ce0cf757446ce9bb2d5f5a4ac8103c7a1049054/model_meta.json @@ -1 +1,17 @@ -{"name": "sergeyzh/rubert-tiny-turbo", "revision": "8ce0cf757446ce9bb2d5f5a4ac8103c7a1049054", "release_date": "2024-06-21", "languages": ["rus_Cyrl"], "n_parameters": null, "memory_usage": null, "max_tokens": null, "embed_dim": null, "license": null, "open_source": true, "similarity_fn_name": null, "framework": [], "loader": null} \ No newline at end of file +{ + "name": "sergeyzh/rubert-tiny-turbo", + "revision": "8ce0cf757446ce9bb2d5f5a4ac8103c7a1049054", + "release_date": "2024-06-21", + "languages": [ + "rus_Cyrl" + ], + "n_parameters": null, + "memory_usage": null, + "max_tokens": null, + "embed_dim": null, + "license": null, + "open_source": true, + "similarity_fn_name": null, + "framework": [], + "loader": null +} \ No newline at end of file