fix: update task metadata to allow for null

embeddings-benchmark · Nov 14, 2024 · 573f111 · 573f111
1 parent 3a1a470
commit 573f111
Show file tree

Hide file tree

Showing 3 changed files with 12 additions and 7 deletions.
diff --git a/mteb/leaderboard/figures.py b/mteb/leaderboard/figures.py
@@ -1,3 +1,5 @@
+from __future__ import annotations
+
 import numpy as np
 import pandas as pd
 import plotly.express as px
@@ -68,7 +70,7 @@ def performance_size_plot(df: pd.DataFrame) -> go.Figure:
         hover_name="Model",
     )
     fig.update_layout(
-        coloraxis_colorbar=dict(
+        coloraxis_colorbar=dict(  # noqa
             title="Max Tokens",
             tickvals=[2, 3, 4, 5],
             ticktext=[
@@ -78,7 +80,7 @@ def performance_size_plot(df: pd.DataFrame) -> go.Figure:
                 "100K",
             ],
         ),
-        hoverlabel=dict(
+        hoverlabel=dict(  # noqa
             bgcolor="white",
             font_size=16,
         ),
@@ -87,7 +89,7 @@ def performance_size_plot(df: pd.DataFrame) -> go.Figure:
         textposition="top center",
     )
     fig.update_layout(
-        font=dict(size=16, color="black"),
-        margin=dict(b=20, t=10, l=20, r=10),
+        font=dict(size=16, color="black"),  # noqa
+        margin=dict(b=20, t=10, l=20, r=10),  # noqa
     )
     return fig
diff --git a/mteb/leaderboard/table.py b/mteb/leaderboard/table.py
@@ -88,7 +88,7 @@ def get_means_per_types(df: pd.DataFrame) -> pd.DataFrame:
                 [name_to_score.get(task_name, np.nan) for task_name in task_names]
             )
             records.append(
-                dict(
+                dict(  # noqa
                     model_name=model_name,
                     model_revision=model_revision,
                     task_type=task_type,

diff --git a/mteb/load_results/task_results.py b/mteb/load_results/task_results.py
@@ -156,9 +156,9 @@ class TaskResult(BaseModel):
 
     dataset_revision: str
     task_name: str
-    mteb_version: str
+    mteb_version: str | None
     scores: dict[Split, list[ScoresDict]]
-    evaluation_time: float
+    evaluation_time: float | None
     kg_co2_emissions: float | None = None
 
     @classmethod
@@ -290,6 +290,9 @@ def from_disk(cls, path: Path, load_historic_data: bool = True) -> TaskResult:
                     f"Error loading TaskResult from disk. You can try to load historic data by setting `load_historic_data=True`. Error: {e}"
                 )
 
+        if data["mteb_version"] is None:
+            data.pop("mteb_version")
+
         pre_1_11_load = (
             (
                 "mteb_version" in data