From b8092b7a95c7da92076ca6a87f2d6ae628807a0f Mon Sep 17 00:00:00 2001 From: Jon Holba Date: Thu, 14 Nov 2024 10:23:33 +0100 Subject: [PATCH 1/9] Fix typing for batching interval --- src/ert/ensemble_evaluator/evaluator.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ert/ensemble_evaluator/evaluator.py b/src/ert/ensemble_evaluator/evaluator.py index 3855ec85cac..cf86a1f6f60 100644 --- a/src/ert/ensemble_evaluator/evaluator.py +++ b/src/ert/ensemble_evaluator/evaluator.py @@ -86,7 +86,7 @@ def __init__(self, ensemble: Ensemble, config: EvaluatorServerConfig): List[Tuple[EVENT_HANDLER, Event]] ] = asyncio.Queue() self._max_batch_size: int = 500 - self._batching_interval: int = 2 + self._batching_interval: float = 2.0 self._complete_batch: asyncio.Event = asyncio.Event() async def _publisher(self) -> None: From 3796417c8495fdbbd724547f8dc1da0dd01b52a5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?H=C3=A5vard=20Berland?= Date: Fri, 15 Nov 2024 15:21:40 +0100 Subject: [PATCH 2/9] Remove outdated git lfs instruction "rpm -qa | grep git-lfs" reveals this is in place --- README.md | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 0e88a55fe52..a1470dbcc7e 100644 --- a/README.md +++ b/README.md @@ -80,10 +80,9 @@ code you can run a fast subset of the tests with pytest -n logical tests/ert/unit_tests -m "not integration_tests" ``` -[Git LFS](https://git-lfs.com/) must be installed to get all the files. This is packaged as `git-lfs` on Ubuntu, Fedora or macOS Homebrew. For Equinor RGS node users, it is possible to use `git` from Red Hat Software Collections: -```sh -source /opt/rh/rh-git227/enable -``` +[Git LFS](https://git-lfs.com/) must be installed to get all the files. This is +packaged as `git-lfs` on Ubuntu, Fedora or macOS Homebrew. For Equinor TGX +users, it is preinstalled. If you have not used git-lfs before, you might have to make changes to your global Git config for git-lfs to work properly. ```sh From 229671e79a1358a88a07c257d3765ae0b7fa019d Mon Sep 17 00:00:00 2001 From: Andreas Eknes Lie Date: Thu, 14 Nov 2024 12:48:50 +0100 Subject: [PATCH 3/9] Fix flaky rightclick plot-button test --- tests/ert/ui_tests/gui/test_main_window.py | 42 +++++++++++++++------- 1 file changed, 29 insertions(+), 13 deletions(-) diff --git a/tests/ert/ui_tests/gui/test_main_window.py b/tests/ert/ui_tests/gui/test_main_window.py index 709505ae50c..a50a160bf5f 100644 --- a/tests/ert/ui_tests/gui/test_main_window.py +++ b/tests/ert/ui_tests/gui/test_main_window.py @@ -4,11 +4,13 @@ import stat from pathlib import Path from textwrap import dedent +from typing import List from unittest.mock import MagicMock, Mock, patch import numpy as np import pytest from qtpy.QtCore import Qt, QTimer +from qtpy.QtGui import QWindow from qtpy.QtWidgets import ( QAction, QApplication, @@ -612,25 +614,39 @@ def test_right_click_plot_button_opens_external_plotter(qtbot, storage, monkeypa button_plot_tool = gui.findChild(SidebarToolButton, "button_Create_plot") assert button_plot_tool - qtbot.mouseClick(button_plot_tool, Qt.LeftButton) - plot_window = wait_for_child(gui, qtbot, PlotWindow) - assert plot_window + def top_level_plotter_windows() -> List[QWindow]: + top_level_plot_windows = [] + top_level_windows = QApplication.topLevelWindows() + for win in top_level_windows: + if "Plotting" in win.title() and win.isVisible(): + top_level_plot_windows.append(win) + return top_level_plot_windows - prev_open_windows = len(QApplication.topLevelWindows()) - - def detect_external_plot_widget_open_on_right_click(plot_count: int): - previous_count = plot_count - 1 - assert len(QApplication.topLevelWindows()) == previous_count + def right_click_plotter_button() -> None: + top_level_windows = len(top_level_plotter_windows()) qtbot.mouseClick(button_plot_tool, Qt.RightButton) qtbot.wait_until( - lambda: len(QApplication.topLevelWindows()) != previous_count, + lambda: len(top_level_plotter_windows()) > top_level_windows, timeout=5000, ) - assert len(QApplication.topLevelWindows()) == plot_count - detect_external_plot_widget_open_on_right_click(prev_open_windows + 1) - detect_external_plot_widget_open_on_right_click(prev_open_windows + 2) - detect_external_plot_widget_open_on_right_click(prev_open_windows + 3) + right_click_plotter_button() + right_click_plotter_button() + right_click_plotter_button() + + window_list = top_level_plotter_windows() + assert len(window_list) == 3 + + for window in window_list: + window.close() + + qtbot.wait_until(lambda: not top_level_plotter_windows(), timeout=5000) + + qtbot.mouseClick(button_plot_tool, Qt.LeftButton) + plot_window = wait_for_child(gui, qtbot, PlotWindow) + assert plot_window + assert "Plotting" in plot_window.windowTitle() + gui.close() From 67550e8ae19a36453e5b28a3b13002f563d778cc Mon Sep 17 00:00:00 2001 From: larsevj Date: Fri, 15 Nov 2024 09:52:31 +0100 Subject: [PATCH 4/9] Fetch all tags in readthedocs workflow --- .readthedocs.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.readthedocs.yaml b/.readthedocs.yaml index e97179bd62f..c44763bc858 100644 --- a/.readthedocs.yaml +++ b/.readthedocs.yaml @@ -15,3 +15,6 @@ build: os: ubuntu-22.04 tools: python: "3.12" + jobs: + post_checkout: + - git fetch --unshallow --tags || true From cba8c87635c6cb30d003a559fe129f13fb335bbe Mon Sep 17 00:00:00 2001 From: DanSava Date: Thu, 14 Nov 2024 13:32:10 +0200 Subject: [PATCH 5/9] Simplify test logic --- .../ert/unit_tests/config/test_ert_config.py | 34 +++++++------------ 1 file changed, 12 insertions(+), 22 deletions(-) diff --git a/tests/ert/unit_tests/config/test_ert_config.py b/tests/ert/unit_tests/config/test_ert_config.py index 93e5af10a2a..5aafff6b9d2 100644 --- a/tests/ert/unit_tests/config/test_ert_config.py +++ b/tests/ert/unit_tests/config/test_ert_config.py @@ -4,6 +4,7 @@ import os import os.path import stat +import warnings from datetime import date from pathlib import Path from textwrap import dedent @@ -1537,9 +1538,7 @@ def test_general_option_in_local_config_has_priority_over_site_config(): @pytest.mark.usefixtures("use_tmpdir") -def test_warning_raised_when_summary_key_and_no_simulation_job_present(caplog, recwarn): - caplog.set_level(logging.WARNING) - +def test_warning_raised_when_summary_key_and_no_simulation_job_present(): with open("job_file", "w", encoding="utf-8") as fout: fout.write("EXECUTABLE echo\nARGLIST \n") @@ -1548,22 +1547,18 @@ def test_warning_raised_when_summary_key_and_no_simulation_job_present(caplog, r fout.write("NUM_REALIZATIONS 1\n") fout.write("SUMMARY *\n") fout.write("ECLBASE RESULT_SUMMARY\n") - fout.write("INSTALL_JOB job_name job_file\n") fout.write( "FORWARD_MODEL job_name(=A/, =/x)\n" ) + with warnings.catch_warnings(record=True) as all_warnings: + ErtConfig.from_file("config_file.ert") - ErtConfig.from_file("config_file.ert") - - # Check no warning is logged when config contains - # forward model step with and as arguments - assert not caplog.text - assert len(recwarn) == 1 - assert issubclass(recwarn[0].category, ConfigWarning) - assert ( - recwarn[0].message.info.message + assert any( + str(w.message) == "Config contains a SUMMARY key but no forward model steps known to generate a summary file" + for w in all_warnings + if isinstance(w.message, ConfigWarning) ) @@ -1571,11 +1566,7 @@ def test_warning_raised_when_summary_key_and_no_simulation_job_present(caplog, r "job_name", ["eclipse", "eclipse100", "flow", "FLOW", "ECLIPSE100"] ) @pytest.mark.usefixtures("use_tmpdir") -def test_no_warning_when_summary_key_and_simulation_job_present( - caplog, recwarn, job_name -): - caplog.set_level(logging.WARNING) - +def test_no_warning_when_summary_key_and_simulation_job_present(job_name): with open("job_file", "w", encoding="utf-8") as fout: fout.write("EXECUTABLE echo\nARGLIST \n") @@ -1589,9 +1580,8 @@ def test_no_warning_when_summary_key_and_simulation_job_present( fout.write( f"FORWARD_MODEL {job_name}(=A/, =/x)\n" ) - - ErtConfig.from_file("config_file.ert") - # Check no warning is logged when config contains # forward model step with and as arguments - assert not any(w.message for w in recwarn if issubclass(w.category, ConfigWarning)) + with warnings.catch_warnings(): + warnings.simplefilter("error", category=ConfigWarning) + ErtConfig.from_file("config_file.ert") From 3e84fabb459221f8a36abbd3b6259a492658ef0a Mon Sep 17 00:00:00 2001 From: Frode Aarstad Date: Fri, 15 Nov 2024 13:59:59 +0100 Subject: [PATCH 6/9] Remove comment --- src/ert/simulator/batch_simulator.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/ert/simulator/batch_simulator.py b/src/ert/simulator/batch_simulator.py index 517e7232267..00b24710c4a 100644 --- a/src/ert/simulator/batch_simulator.py +++ b/src/ert/simulator/batch_simulator.py @@ -171,9 +171,6 @@ def _check_suffix( for control_name, control in controls.items(): ext_config = self.parameter_configurations[control_name] - - # fix this - if isinstance(ext_config, ExtParamConfig): if len(ext_config) != len(control.keys()): raise KeyError( From 5bd0744a48c2b986bd58247dde12ca707504232e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?H=C3=A5vard=20Berland?= Date: Fri, 15 Nov 2024 15:15:58 +0100 Subject: [PATCH 7/9] Leave test machine usable when running rapid-tests Running with a lower CPU priority helps this --- justfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/justfile b/justfile index 652f8e6f9d7..bef28cdd31a 100644 --- a/justfile +++ b/justfile @@ -10,4 +10,4 @@ snake_oil: # execute rapid unittests rapid-tests: - pytest -n logical tests/ert/unit_tests -m "not integration_tests" + nice pytest -n logical tests/ert/unit_tests -m "not integration_tests" From d27eb29fcf431911fa97d13b13f5a0303beb206d Mon Sep 17 00:00:00 2001 From: Eivind Jahren Date: Mon, 18 Nov 2024 07:24:19 +0100 Subject: [PATCH 8/9] Add test for mean calculation in dark_storage common --- .../unit_tests/dark_storage/test_common.py | 72 +++++++++++++++++++ 1 file changed, 72 insertions(+) create mode 100644 tests/ert/unit_tests/dark_storage/test_common.py diff --git a/tests/ert/unit_tests/dark_storage/test_common.py b/tests/ert/unit_tests/dark_storage/test_common.py new file mode 100644 index 00000000000..b05ed558c79 --- /dev/null +++ b/tests/ert/unit_tests/dark_storage/test_common.py @@ -0,0 +1,72 @@ +import pandas as pd +import pytest + +from ert.config import SummaryConfig +from ert.dark_storage.common import data_for_key +from ert.storage import open_storage +from tests.ert.unit_tests.config.summary_generator import ( + Date, + Simulator, + Smspec, + SmspecIntehead, + SummaryMiniStep, + SummaryStep, + UnitSystem, + Unsmry, +) + + +def test_data_for_key_gives_mean_for_duplicate_values(tmp_path): + value1 = 1.1 + value2 = 1.0e19 + with open_storage(tmp_path / "storage", mode="w") as storage: + summary_config = SummaryConfig(name="summary", input_files=["CASE"], keys=["*"]) + experiment = storage.create_experiment( + observations={}, + parameters=[], + responses=[summary_config], + ) + ensemble = experiment.create_ensemble(name="ensemble", ensemble_size=1) + unsmry = Unsmry( + steps=[ + SummaryStep( + seqnum=0, + ministeps=[ + SummaryMiniStep(mini_step=0, params=[0.0, 5.629901e16]), + SummaryMiniStep(mini_step=1, params=[365.0, value1]), + ], + ), + SummaryStep( + seqnum=1, + ministeps=[SummaryMiniStep(mini_step=2, params=[365.0, value2])], + ), + ] + ) + smspec = Smspec( + nx=4, + ny=4, + nz=10, + restarted_from_step=0, + num_keywords=2, + restart=" ", + keywords=["TIME ", "NRPPR"], + well_names=[":+:+:+:+", "WELLNAME"], + region_numbers=[-32676, 0], + units=["HOURS ", "SM3"], + start_date=Date( + day=1, month=1, year=2014, hour=0, minutes=0, micro_seconds=0 + ), + intehead=SmspecIntehead( + unit=UnitSystem.METRIC, + simulator=Simulator.ECLIPSE_100, + ), + ) + smspec.to_file(tmp_path / "CASE.SMSPEC") + unsmry.to_file(tmp_path / "CASE.UNSMRY") + ds = summary_config.read_from_file(tmp_path, 0) + ensemble.save_response(summary_config.response_type, ds, 0) + df = data_for_key(ensemble, "NRPPR:WELLNAME") + assert list(df.columns) == [pd.Timestamp("2014-01-16 05:00:00")] + assert df[pd.Timestamp("2014-01-16 05:00:00")][0] == pytest.approx( + (value1 + value2) / 2 + ) From d3c10f6341863bd06f61832e16302b7a1fe5b0e8 Mon Sep 17 00:00:00 2001 From: Eivind Jahren Date: Tue, 19 Nov 2024 15:06:17 +0100 Subject: [PATCH 9/9] Fix issue where dark_storage did not handle empty responses --- src/ert/dark_storage/common.py | 3 +- .../unit_tests/dark_storage/test_common.py | 32 ++++++++++++++++++- 2 files changed, 33 insertions(+), 2 deletions(-) diff --git a/src/ert/dark_storage/common.py b/src/ert/dark_storage/common.py index c6871c3e452..3ef7ac3f1ea 100644 --- a/src/ert/dark_storage/common.py +++ b/src/ert/dark_storage/common.py @@ -6,6 +6,7 @@ import pandas as pd import polars import xarray as xr +from polars.exceptions import ColumnNotFoundError from ert.config import GenDataConfig, GenKwConfig from ert.config.field import Field @@ -184,7 +185,7 @@ def data_for_key( return data.astype(float) except ValueError: return data - except (ValueError, KeyError): + except (ValueError, KeyError, ColumnNotFoundError): return pd.DataFrame() return pd.DataFrame() diff --git a/tests/ert/unit_tests/dark_storage/test_common.py b/tests/ert/unit_tests/dark_storage/test_common.py index b05ed558c79..1473e3e80e1 100644 --- a/tests/ert/unit_tests/dark_storage/test_common.py +++ b/tests/ert/unit_tests/dark_storage/test_common.py @@ -1,7 +1,8 @@ import pandas as pd +import polars import pytest -from ert.config import SummaryConfig +from ert.config import GenDataConfig, SummaryConfig from ert.dark_storage.common import data_for_key from ert.storage import open_storage from tests.ert.unit_tests.config.summary_generator import ( @@ -70,3 +71,32 @@ def test_data_for_key_gives_mean_for_duplicate_values(tmp_path): assert df[pd.Timestamp("2014-01-16 05:00:00")][0] == pytest.approx( (value1 + value2) / 2 ) + + +def test_data_for_key_returns_empty_gen_data_config(tmp_path): + with open_storage(tmp_path / "storage", mode="w") as storage: + gen_data_config = GenDataConfig(keys=["response"]) + experiment = storage.create_experiment( + observations={}, + parameters=[], + responses=[gen_data_config], + ) + ensemble = experiment.create_ensemble(name="ensemble", ensemble_size=1) + + data = data_for_key(ensemble, "response@0") + assert data.empty + + ensemble.save_response( + "gen_data", + polars.DataFrame( + { + "response_key": "response", + "report_step": polars.Series([0], dtype=polars.UInt16), + "index": polars.Series([0], dtype=polars.UInt16), + "values": polars.Series([0.0], dtype=polars.Float32), + } + ), + 0, + ) + data = data_for_key(ensemble, "response@0") + assert not data.empty