Skip to content

Commit

Permalink
Add RuMTEB evaluation scores for most english top MTEB models (#76)
Browse files Browse the repository at this point in the history
  • Loading branch information
@AlexeyVatolin
AlexeyVatolin authored Dec 20, 2024
1 parent 71d8e5f commit 6fbbd2e
Show file tree
Hide file tree
Showing 417 changed files with 29,291 additions and 2,221 deletions.
73 changes: 73 additions & 0 deletions ..._gte-Qwen1.5-7B-instruct/07d27e5226328010336563bc1b564a5e3436a298/CEDRClassification.json
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,73 @@
{
"dataset_revision": "c0ba03d058e3e1b2f3fd20518875a4563dd12db4",
"evaluation_time": 49.17470383644104,
"kg_co2_emissions": null,
"mteb_version": "1.19.2",
"scores": {
"test": [
{
"accuracy": 0.5477151965993624,
"f1": 0.6036500719599494,
"hf_subset": "default",
"languages": [
"rus-Cyrl"
],
"lrap": 0.8649468650371996,
"main_score": 0.5477151965993624,
"scores_per_experiment": [
{
"accuracy": 0.5451647183846972,
"f1": 0.5850362532892082,
"lrap": 0.8438363443145652
},
{
"accuracy": 0.628586609989373,
"f1": 0.6422497078024308,
"lrap": 0.8565887353878912
},
{
"accuracy": 0.5371944739638682,
"f1": 0.602381004950392,
"lrap": 0.8797555791710995
},
{
"accuracy": 0.5143464399574921,
"f1": 0.5742093596923625,
"lrap": 0.8505844845908667
},
{
"accuracy": 0.5446333687566419,
"f1": 0.6001832009513983,
"lrap": 0.8803400637619601
},
{
"accuracy": 0.5712008501594049,
"f1": 0.6201225698955309,
"lrap": 0.8817215727949038
},
{
"accuracy": 0.502125398512221,
"f1": 0.5781829471349942,
"lrap": 0.8520191285866153
},
{
"accuracy": 0.5371944739638682,
"f1": 0.6032409719825884,
"lrap": 0.8667375132837453
},
{
"accuracy": 0.5749202975557917,
"f1": 0.622622514090637,
"lrap": 0.8739107332624917
},
{
"accuracy": 0.5217853347502657,
"f1": 0.6082721898099509,
"lrap": 0.8639744952178583
}
]
}
]
},
"task_name": "CEDRClassification"
}
73 changes: 73 additions & 0 deletions ...Qwen1.5-7B-instruct/07d27e5226328010336563bc1b564a5e3436a298/GeoreviewClassification.json
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,73 @@
{
"dataset_revision": "3765c0d1de6b7d264bc459433c45e5a75513839c",
"evaluation_time": 70.92497396469116,
"kg_co2_emissions": null,
"mteb_version": "1.19.2",
"scores": {
"test": [
{
"accuracy": 0.593115234375,
"f1": 0.5654370923929647,
"f1_weighted": 0.565366043274544,
"hf_subset": "default",
"languages": [
"rus-Cyrl"
],
"main_score": 0.593115234375,
"scores_per_experiment": [
{
"accuracy": 0.57275390625,
"f1": 0.5311404233477083,
"f1_weighted": 0.5310401242904998
},
{
"accuracy": 0.5693359375,
"f1": 0.5233095126267937,
"f1_weighted": 0.5231911107096815
},
{
"accuracy": 0.59619140625,
"f1": 0.5690305785888471,
"f1_weighted": 0.5689581598276597
},
{
"accuracy": 0.60595703125,
"f1": 0.5901413093480931,
"f1_weighted": 0.590105109669658
},
{
"accuracy": 0.6025390625,
"f1": 0.5729259523761623,
"f1_weighted": 0.5728395007156636
},
{
"accuracy": 0.5947265625,
"f1": 0.5692650108462612,
"f1_weighted": 0.569206137591517
},
{
"accuracy": 0.60693359375,
"f1": 0.5888821876084315,
"f1_weighted": 0.5888208744862033
},
{
"accuracy": 0.58935546875,
"f1": 0.55793654500316,
"f1_weighted": 0.5578614223389805
},
{
"accuracy": 0.60302734375,
"f1": 0.5833953830631733,
"f1_weighted": 0.5833356693161511
},
{
"accuracy": 0.59033203125,
"f1": 0.5683440211210167,
"f1_weighted": 0.5683023237994256
}
]
}
]
},
"task_name": "GeoreviewClassification"
}
34 changes: 34 additions & 0 deletions ...-Qwen1.5-7B-instruct/07d27e5226328010336563bc1b564a5e3436a298/GeoreviewClusteringP2P.json
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,34 @@
{
"dataset_revision": "97a313c8fc85b47f13f33e7e9a95c1ad888c7fec",
"evaluation_time": 106.16331720352173,
"kg_co2_emissions": null,
"mteb_version": "1.19.2",
"scores": {
"test": [
{
"hf_subset": "default",
"languages": [
"rus-Cyrl"
],
"main_score": 0.7490333015485914,
"v_measure": 0.7490333015485914,
"v_measure_std": 0.0034579399932174047,
"v_measures": {
"Level 0": [
0.752744253546754,
0.7440034439559667,
0.7466405375270816,
0.7504486865332559,
0.74656634640751,
0.750591472356923,
0.7485759760938262,
0.7532063350425257,
0.7439558077953369,
0.7536001562267337
]
}
}
]
},
"task_name": "GeoreviewClusteringP2P"
}
73 changes: 73 additions & 0 deletions ...-Qwen1.5-7B-instruct/07d27e5226328010336563bc1b564a5e3436a298/HeadlineClassification.json
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,73 @@
{
"dataset_revision": "2fe05ee6b5832cda29f2ef7aaad7b7fe6a3609eb",
"evaluation_time": 41.733545541763306,
"kg_co2_emissions": null,
"mteb_version": "1.19.2",
"scores": {
"test": [
{
"accuracy": 0.823193359375,
"f1": 0.8239465084764503,
"f1_weighted": 0.8239283463460406,
"hf_subset": "default",
"languages": [
"rus-Cyrl"
],
"main_score": 0.823193359375,
"scores_per_experiment": [
{
"accuracy": 0.80712890625,
"f1": 0.8089034546914284,
"f1_weighted": 0.8088769259452948
},
{
"accuracy": 0.8349609375,
"f1": 0.8357485899262026,
"f1_weighted": 0.8357459144685893
},
{
"accuracy": 0.81591796875,
"f1": 0.8175058255173461,
"f1_weighted": 0.8174854105287481
},
{
"accuracy": 0.81640625,
"f1": 0.8184209189049058,
"f1_weighted": 0.8184018752116653
},
{
"accuracy": 0.81494140625,
"f1": 0.816025755172741,
"f1_weighted": 0.8160000608774359
},
{
"accuracy": 0.8369140625,
"f1": 0.8376501552762083,
"f1_weighted": 0.8376405057751175
},
{
"accuracy": 0.82080078125,
"f1": 0.8200892484658041,
"f1_weighted": 0.8200637301606719
},
{
"accuracy": 0.814453125,
"f1": 0.8152234211475688,
"f1_weighted": 0.8151917129263095
},
{
"accuracy": 0.82958984375,
"f1": 0.8291598309049161,
"f1_weighted": 0.8291522960195152
},
{
"accuracy": 0.8408203125,
"f1": 0.840737884757382,
"f1_weighted": 0.84072503154706
}
]
}
]
},
"task_name": "HeadlineClassification"
}
95 changes: 95 additions & 0 deletions ...7B-instruct/07d27e5226328010336563bc1b564a5e3436a298/InappropriatenessClassification.json
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,95 @@
{
"dataset_revision": "601651fdc45ef243751676e62dd7a19f491c0285",
"evaluation_time": 37.14429306983948,
"kg_co2_emissions": null,
"mteb_version": "1.19.2",
"scores": {
"test": [
{
"accuracy": 0.732568359375,
"ap": 0.6710765411408649,
"ap_weighted": 0.6710765411408649,
"f1": 0.730367241384523,
"f1_weighted": 0.730367241384523,
"hf_subset": "default",
"languages": [
"rus-Cyrl"
],
"main_score": 0.732568359375,
"scores_per_experiment": [
{
"accuracy": 0.73291015625,
"ap": 0.678244924916574,
"ap_weighted": 0.678244924916574,
"f1": 0.7319114485702299,
"f1_weighted": 0.7319114485702299
},
{
"accuracy": 0.7568359375,
"ap": 0.696101186748497,
"ap_weighted": 0.696101186748497,
"f1": 0.7567967401972707,
"f1_weighted": 0.7567967401972707
},
{
"accuracy": 0.75341796875,
"ap": 0.6911183505754163,
"ap_weighted": 0.6911183505754163,
"f1": 0.753417439641227,
"f1_weighted": 0.753417439641227
},
{
"accuracy": 0.705078125,
"ap": 0.6430150082236842,
"ap_weighted": 0.6430150082236842,
"f1": 0.7049655782998275,
"f1_weighted": 0.7049655782998275
},
{
"accuracy": 0.7255859375,
"ap": 0.672415796624714,
"ap_weighted": 0.672415796624714,
"f1": 0.7241059263570389,
"f1_weighted": 0.7241059263570389
},
{
"accuracy": 0.6845703125,
"ap": 0.6162109375,
"ap_weighted": 0.6162109375,
"f1": 0.6697390979613315,
"f1_weighted": 0.6697390979613315
},
{
"accuracy": 0.77001953125,
"ap": 0.7068677063282002,
"ap_weighted": 0.7068677063282002,
"f1": 0.7700071934744195,
"f1_weighted": 0.7700071934744195
},
{
"accuracy": 0.75,
"ap": 0.675314465408805,
"ap_weighted": 0.675314465408805,
"f1": 0.7462795199380565,
"f1_weighted": 0.7462795199380565
},
{
"accuracy": 0.728515625,
"ap": 0.6725068933823529,
"ap_weighted": 0.6725068933823529,
"f1": 0.7277863998385874,
"f1_weighted": 0.7277863998385874
},
{
"accuracy": 0.71875,
"ap": 0.6589701417004049,
"ap_weighted": 0.6589701417004049,
"f1": 0.7186630695672415,
"f1_weighted": 0.7186630695672415
}
]
}
]
},
"task_name": "InappropriatenessClassification"
}
Loading

0 comments on commit 6fbbd2e

Please sign in to comment.