From 37256479c5b397e5d73ef600d71635f85cb8e6d6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Therese=20Natter=C3=B8y?= <61694854+tnatt@users.noreply.github.com> Date: Tue, 22 Oct 2024 10:22:27 +0200 Subject: [PATCH] DEP: Writing metadata on json format --- docs/src/dataio_3_migration.rst | 4 ++++ src/fmu/dataio/_utils.py | 31 +++++++----------------- src/fmu/dataio/aggregation.py | 10 ++++++-- src/fmu/dataio/case.py | 9 ++----- src/fmu/dataio/dataio.py | 10 ++++++-- src/fmu/dataio/preprocessed.py | 4 +--- tests/test_units/test_dataio.py | 34 ++++++++++++++++----------- tests/test_units/test_preprocessed.py | 4 ++-- tests/test_units/test_utils.py | 7 +----- 9 files changed, 54 insertions(+), 59 deletions(-) diff --git a/docs/src/dataio_3_migration.rst b/docs/src/dataio_3_migration.rst index 71bf19a57..d6e634e82 100644 --- a/docs/src/dataio_3_migration.rst +++ b/docs/src/dataio_3_migration.rst @@ -91,6 +91,7 @@ safely be removed if present in the code. * ``ExportData.legacy_time_format`` * ``ExportData.table_include_index`` * ``ExportData.verifyfolder`` + * ``ExportData.meta_format`` .. code-block:: python @@ -309,3 +310,6 @@ Changes to method arguments Deprecated methods - The ``generate_aggregation_metadata()`` method is deprecated. Replace it with the identical ``generate_metadata()`` method instead. + +Deprecated class variables + * ``AggregatedData.meta_format`` - metadata will always be exported in yaml format diff --git a/src/fmu/dataio/_utils.py b/src/fmu/dataio/_utils.py index bff9131dc..cbb0bf06d 100644 --- a/src/fmu/dataio/_utils.py +++ b/src/fmu/dataio/_utils.py @@ -11,7 +11,7 @@ from io import BufferedIOBase, BytesIO from pathlib import Path from tempfile import NamedTemporaryFile -from typing import Any, Final, Literal +from typing import Any, Final import numpy as np import pandas as pd @@ -52,35 +52,20 @@ def dataio_examples() -> bool: return "RUN_DATAIO_EXAMPLES" in os.environ -def export_metadata_file( - file: Path, - metadata: dict, - savefmt: Literal["yaml", "json"] = "yaml", -) -> None: +def export_metadata_file(file: Path, metadata: dict) -> None: """Export genericly and ordered to the complementary metadata file.""" if not metadata: raise RuntimeError( "Export of metadata was requested, but no metadata are present." ) - if savefmt == "yaml": - with open(file, "w", encoding="utf8") as stream: - stream.write( - yaml.safe_dump( - metadata, - allow_unicode=True, - ) - ) - else: - with open(file.replace(file.with_suffix(".json")), "w") as stream: - stream.write( - json.dumps( - metadata, - default=str, - indent=2, - ensure_ascii=False, - ) + with open(file, "w", encoding="utf8") as stream: + stream.write( + yaml.safe_dump( + metadata, + allow_unicode=True, ) + ) logger.info("Yaml file on: %s", file) diff --git a/src/fmu/dataio/aggregation.py b/src/fmu/dataio/aggregation.py index afc757456..ad64b8d94 100644 --- a/src/fmu/dataio/aggregation.py +++ b/src/fmu/dataio/aggregation.py @@ -54,7 +54,7 @@ class AggregatedData: """ # class variable(s) - meta_format: ClassVar[Literal["yaml", "json"]] = "yaml" + meta_format: ClassVar[Optional[Literal["yaml", "json"]]] = None # deprecated # instance aggregation_id: Optional[str] = None @@ -77,6 +77,12 @@ def __post_init__(self) -> None: "manner instead.", UserWarning, ) + if self.meta_format: + warnings.warn( + "The 'meta_format' option is deprecated and should be removed. " + "Metadata will only be exported in yaml format.", + UserWarning, + ) @staticmethod def _generate_aggr_uuid(uuids: list[str]) -> str: @@ -397,7 +403,7 @@ def export(self, obj: types.Inferrable, **kwargs: object) -> str: logger.info("Export to file and export metadata file.") _utils.export_file(obj, outfile) - _utils.export_metadata_file(metafile, metadata, savefmt=self.meta_format) + _utils.export_metadata_file(metafile, metadata) logger.info("Actual file is: %s", outfile) logger.info("Metadata file is: %s", metafile) diff --git a/src/fmu/dataio/case.py b/src/fmu/dataio/case.py index 5251be398..039343945 100644 --- a/src/fmu/dataio/case.py +++ b/src/fmu/dataio/case.py @@ -5,7 +5,7 @@ import warnings from dataclasses import dataclass, field from pathlib import Path -from typing import ClassVar, Final, Literal, Optional, Union +from typing import Final, Optional, Union from pydantic import ValidationError @@ -45,9 +45,6 @@ class CreateCaseMetadata: # pylint: disable=too-few-public-methods description (Optional): Description text as string or list of strings. """ - # class variables - meta_format: ClassVar[Literal["yaml", "json"]] = "yaml" - config: dict rootfolder: str | Path casename: str @@ -149,8 +146,6 @@ def export(self) -> str: Full path of metadata file. """ if self.generate_metadata(): - _utils.export_metadata_file( - self._metafile, self._metadata, savefmt=self.meta_format - ) + _utils.export_metadata_file(self._metafile, self._metadata) logger.info("METAFILE %s", self._metafile) return str(self._metafile) diff --git a/src/fmu/dataio/dataio.py b/src/fmu/dataio/dataio.py index cb9aca795..7168cf8a9 100644 --- a/src/fmu/dataio/dataio.py +++ b/src/fmu/dataio/dataio.py @@ -363,7 +363,7 @@ class ExportData: grid_fformat: ClassVar[str] = "roff" include_ertjobs: ClassVar[bool] = False # deprecated legacy_time_format: ClassVar[bool] = False # deprecated - meta_format: ClassVar[Literal["yaml", "json"]] = "yaml" + meta_format: ClassVar[Optional[Literal["yaml", "json"]]] = None # deprecated polygons_fformat: ClassVar[str] = "csv" # or use "csv|xtgeo" points_fformat: ClassVar[str] = "csv" # or use "csv|xtgeo" surface_fformat: ClassVar[str] = "irap_binary" @@ -651,6 +651,12 @@ def _show_deprecations_or_notimplemented(self) -> None: "key instead to set the reference for the given 'vertical_domain'.", UserWarning, ) + if self.meta_format: + warn( + "The 'meta_format' option is deprecated and should be removed. " + "Metadata will only be exported in yaml format.", + UserWarning, + ) def _validate_and_establish_fmucontext(self) -> None: """ @@ -941,6 +947,6 @@ def export( export_file(obj, outfile, fmt=metadata["data"].get("format", "")) logger.info("Actual file is: %s", outfile) - export_metadata_file(metafile, metadata, savefmt=self.meta_format) + export_metadata_file(metafile, metadata) logger.info("Metadata file is: %s", metafile) return str(outfile) diff --git a/src/fmu/dataio/preprocessed.py b/src/fmu/dataio/preprocessed.py index b1151d085..d54e509d8 100644 --- a/src/fmu/dataio/preprocessed.py +++ b/src/fmu/dataio/preprocessed.py @@ -249,9 +249,7 @@ def export(self, obj: str | Path) -> str: else: # store metafile to updated path metafile = outfile.parent / f".{outfile.name}.yml" - export_metadata_file( - file=metafile, metadata=meta_updated, savefmt="yaml" - ) + export_metadata_file(file=metafile, metadata=meta_updated) logger.info("Updated metadata file is: %s", metafile) else: warnings.warn( diff --git a/tests/test_units/test_dataio.py b/tests/test_units/test_dataio.py index 503255485..6d9ec9681 100644 --- a/tests/test_units/test_dataio.py +++ b/tests/test_units/test_dataio.py @@ -33,7 +33,7 @@ def test_generate_metadata_simple(globalconfig1): assert edata.config.model.name == "Test" - assert edata.meta_format == "yaml" + assert edata.meta_format is None assert edata.grid_fformat == "grdecl" assert edata.name == "" @@ -973,24 +973,30 @@ def test_norwegian_letters_globalconfig( assert meta2["masterdata"]["smda"]["field"][0]["identifier"] == "DRÅGØN" -def test_norwegian_letters_globalconfig_as_json( - globalvars_norwegian_letters, - regsurf, -): - """Testing using norwegian letters in global config, with json output.""" - - path, cfg, _ = globalvars_norwegian_letters - os.chdir(path) +def test_metadata_format_deprecated(globalconfig1, regsurf, tmp_path, monkeypatch): + """ + Test that setting the class variable "metadata_format" gives a warning, + and that writing metadata on json format does not work + """ + monkeypatch.chdir(tmp_path) ExportData.meta_format = "json" - edata = ExportData(config=cfg, name="TopBlåbær", content="depth") + with pytest.warns(UserWarning, match="meta_format"): + result = ExportData( + config=globalconfig1, name="TopBlåbær", content="depth" + ).export(regsurf) - result = pathlib.Path(edata.export(regsurf)) + result = pathlib.Path(result) metafile = result.parent / ("." + str(result.stem) + ".gri.json") - with open(metafile, encoding="utf-8") as stream: - assert "DRÅGØN" in stream.read() + assert not metafile.exists() + assert not metafile.with_suffix(".yaml").exists() + + # test that also value "yaml" will cause warning + ExportData.meta_format = "yaml" + with pytest.warns(UserWarning, match="meta_format"): + ExportData(config=globalconfig1, name="TopBlåbær", content="depth") - ExportData.meta_format = "yaml" # reset + ExportData.meta_format = None # reset def test_establish_runpath(tmp_path, globalconfig2): diff --git a/tests/test_units/test_preprocessed.py b/tests/test_units/test_preprocessed.py index 30d490972..555f5867c 100644 --- a/tests/test_units/test_preprocessed.py +++ b/tests/test_units/test_preprocessed.py @@ -130,7 +130,7 @@ def test_outdated_metadata(fmurun_prehook, rmsglobalconfig, regsurf, monkeypatch # modify existing metadata file to make it 'outdated' metadata = read_metadata(metafile) del metadata["data"] # pretend data was not required before - utils.export_metadata_file(file=metafile, metadata=metadata, savefmt="yaml") + utils.export_metadata_file(file=metafile, metadata=metadata) # run the re-export of the preprocessed data inside an mocked FMU run set_ert_env_prehook(monkeypatch) @@ -189,7 +189,7 @@ def test_preprocessed_surface_modified_post_export( # modify existing metadata file to make the md5sum inconsistent metadata = read_metadata(metafile) metadata["file"]["checksum_md5"] = "dummy_modified" - utils.export_metadata_file(file=metafile, metadata=metadata, savefmt="yaml") + utils.export_metadata_file(file=metafile, metadata=metadata) # run the re-export of the preprocessed data inside an mocked FMU run set_ert_env_prehook(monkeypatch) diff --git a/tests/test_units/test_utils.py b/tests/test_units/test_utils.py index 9e9c70b06..3fd3a2852 100644 --- a/tests/test_units/test_utils.py +++ b/tests/test_units/test_utils.py @@ -97,12 +97,7 @@ def test_non_metadata_export_metadata_file(): with NamedTemporaryFile(buffering=0, suffix=".yaml") as tf, pytest.raises( RuntimeError ): - utils.export_metadata_file(Path(tf.name), {}, savefmt="json") - - with NamedTemporaryFile(buffering=0, suffix=".yaml") as tf, pytest.raises( - RuntimeError - ): - utils.export_metadata_file(Path(tf.name), {}, savefmt="yaml") + utils.export_metadata_file(Path(tf.name), {}) def test_export_file_raises():