Store everest results in ERT storage

src/ert/run_models/everest_run_model.py

@@ -12,7 +12,6 @@
 import shutil
 import threading
 import time
-from dataclasses import dataclass
 from pathlib import Path
 from types import TracebackType
 from typing import (
@@ -28,7 +27,6 @@
     TypedDict,
 )
 
-import seba_sqlite.sqlite_storage
 from ropt.enums import EventType, OptimizerExitCode
 from ropt.plan import BasicOptimizer, Event
 from seba_sqlite import SqliteStorage
@@ -37,6 +35,7 @@
 from ert.ensemble_evaluator import EvaluatorServerConfig
 from ert.storage import open_storage
 from everest.config import EverestConfig
+from everest.everest_storage import EverestStorage, OptimalResult
 from everest.optimizer.everest2ropt import everest2ropt
 from everest.simulator import Simulator
 from everest.simulator.everest_to_ert import everest_to_ert_config
@@ -291,24 +290,6 @@ class OptimizerCallback(Protocol):
     def __call__(self) -> str | None: ...
 
 
-@dataclass
-class OptimalResult:
-    batch: int
-    controls: List[Any]
-    total_objective: float
-
-    @staticmethod
-    def from_seba_optimal_result(
-        o: Optional[seba_sqlite.sqlite_storage.OptimalResult] = None,
-    ) -> "OptimalResult" | None:
-        if o is None:
-            return None
-
-        return OptimalResult(
-            batch=o.batch, controls=o.controls, total_objective=o.total_objective
-        )
-
-
 class EverestRunModel(BaseRunModel):
     def __init__(
         self,
@@ -420,6 +401,16 @@ def run_experiment(
         # Initialize the ropt optimizer:
         optimizer = self._create_optimizer(simulator)
 
+        self.ever_storage = EverestStorage(
+            output_dir=Path(self.everest_config.optimization_output_dir),
+        )
+        self.ever_storage.observe_optimizer(
+            optimizer,
+            Path(self.everest_config.optimization_output_dir)
+            / "dakota"
+            / "OPT_DEFAULT.out",
+        )
+
         # The SqliteStorage object is used to store optimization results from
         # Seba in an sqlite database. It reacts directly to events emitted by
         # Seba and is not called by Everest directly. The stored results are
@@ -437,6 +428,8 @@ def run_experiment(
         self._result = OptimalResult.from_seba_optimal_result(
             seba_storage.get_optimal_result()  # type: ignore
         )
+        optimal_result_from_everstorage = self.ever_storage.get_optimal_result()
+        assert self._result == optimal_result_from_everstorage
 
         if self._monitor_thread is not None:
             self._monitor_thread.stop()

src/everest/api/everest_data_api.py

@@ -1,46 +1,92 @@
 from collections import OrderedDict
+from pathlib import Path
 
 import pandas as pd
+import polars
 from seba_sqlite.snapshot import SebaSnapshot
 
 from ert.storage import open_storage
 from everest.config import EverestConfig, ServerConfig
 from everest.detached import ServerStatus, everserver_status
+from everest.everest_storage import EverestStorage
 
 
 class EverestDataAPI:
     def __init__(self, config: EverestConfig, filter_out_gradient=True):
         self._config = config
         output_folder = config.optimization_output_dir
         self._snapshot = SebaSnapshot(output_folder).get_snapshot(filter_out_gradient)
+        self._ever_storage = EverestStorage(Path(output_folder))
+        self._ever_storage.read_from_output_dir()
 
     @property
     def batches(self):
         batch_ids = list({opt.batch_id for opt in self._snapshot.optimization_data})
+        batch_ids2 = sorted(
+            b.batch_id
+            for b in self._ever_storage.data.batches
+            if b.batch_objectives is not None
+        )
+        assert batch_ids == batch_ids2
         return sorted(batch_ids)
 
     @property
     def accepted_batches(self):
         batch_ids = list(
             {opt.batch_id for opt in self._snapshot.optimization_data if opt.merit_flag}
         )
+        batch_ids2 = sorted(
+            b.batch_id for b in self._ever_storage.data.batches if b.is_improvement
+        )
+        assert batch_ids == batch_ids2
+
         return sorted(batch_ids)
 
     @property
     def objective_function_names(self):
-        return [fnc.name for fnc in self._snapshot.metadata.objectives.values()]
+        original = [fnc.name for fnc in self._snapshot.metadata.objectives.values()]
+        new = sorted(
+            self._ever_storage.data.objective_functions["objective_name"]
+            .unique()
+            .to_list()
+        )
+        assert original == new
+        return original
 
     @property
     def output_constraint_names(self):
-        return [fnc.name for fnc in self._snapshot.metadata.constraints.values()]
+        original = [fnc.name for fnc in self._snapshot.metadata.constraints.values()]
+        new = (
+            sorted(
+                self._ever_storage.data.nonlinear_constraints["constraint_name"]
+                .unique()
+                .to_list()
+            )
+            if self._ever_storage.data.nonlinear_constraints is not None
+            else []
+        )
+        assert original == new
+        return original
 
     def input_constraint(self, control):
         controls = [
             con
             for con in self._snapshot.metadata.controls.values()
             if con.name == control
         ]
-        return {"min": controls[0].min_value, "max": controls[0].max_value}
+
+        original = {"min": controls[0].min_value, "max": controls[0].max_value}
+
+        initial_values = self._ever_storage.data.initial_values
+        control_spec = initial_values.filter(
+            polars.col("control_name") == control
+        ).to_dicts()[0]
+        new = {
+            "min": control_spec.get("lower_bounds"),
+            "max": control_spec.get("upper_bounds"),
+        }
+        assert new == original
+        return original
 
     def output_constraint(self, constraint):
         """
@@ -55,30 +101,62 @@ def output_constraint(self, constraint):
             for con in self._snapshot.metadata.constraints.values()
             if con.name == constraint
         ]
-        return {
+
+        old = {
             "type": constraints[0].constraint_type,
             "right_hand_side": constraints[0].rhs_value,
         }
 
+        constraint_dict = self._ever_storage.data.nonlinear_constraints.to_dicts()[0]
+        new = {
+            "type": constraint_dict["constraint_type"],
+            "right_hand_side": constraint_dict["rhs_value"],
+        }
+
+        assert old == new
+        return new
+
     @property
     def realizations(self):
-        return list(
+        old = list(
             OrderedDict.fromkeys(
                 int(sim.realization) for sim in self._snapshot.simulation_data
             )
         )
+        new = sorted(
+            self._ever_storage.data.batches[0]
+            .realization_objectives["realization"]
+            .unique()
+            .to_list()
+        )
+        assert old == new
+        return new
 
     @property
     def simulations(self):
-        return list(
+        old = list(
             OrderedDict.fromkeys(
                 [int(sim.simulation) for sim in self._snapshot.simulation_data]
             )
         )
 
+        new = sorted(
+            self._ever_storage.data.batches[0]
+            .realization_objectives["result_id"]
+            .unique()
+            .to_list()
+        )
+        assert old == new
+        return new
+
     @property
     def control_names(self):
-        return [con.name for con in self._snapshot.metadata.controls.values()]
+        old = [con.name for con in self._snapshot.metadata.controls.values()]
+        new = sorted(
+            self._ever_storage.data.initial_values["control_name"].unique().to_list()
+        )
+        assert old == new
+        return new
 
     @property
     def control_values(self):
@@ -92,7 +170,7 @@ def control_values(self):
 
     @property
     def objective_values(self):
-        return [
+        old = [
             {
                 "function": objective.name,
                 "batch": sim.batch,
@@ -107,6 +185,14 @@ def objective_values(self):
             if objective.name in sim.objectives
         ]
 
+        new = [
+            b for b in self._ever_storage.data.batches if b.batch_objectives is not None
+        ]
+
+        assert old == new
+
+        return old
+
     @property
     def single_objective_values(self):
         single_obj = [