Merge pull request #1568 from ESMValGroup/version

Increase version to v2.0.0b3

CITATION.cff

@@ -308,5 +308,5 @@ license: "Apache-2.0"
 message: "If you use this software, please cite it using these metadata."
 repository-code: "https://github.com/ESMValGroup/ESMValTool/"
 title: ESMValTool
-version: "v2.0.0b2"
+version: "v2.0.0b3"
 ...

environment.yml

@@ -7,7 +7,7 @@ channels:
 dependencies:
   # Python packages that cannot be installed from PyPI:
   - gdal
-  - esmvalcore>=2.0.0b6,<2.1
+  - esmvalcore>=2.0.0b7,<2.1
   # Non-Python dependencies
   - graphviz
   - cdo>=1.9.7

esmvaltool/__init__.py

@@ -1,2 +1,2 @@
 """ESMValTool diagnostics package."""
-__version__ = '2.0.0b2'
+__version__ = '2.0.0b3'

meta.yaml

@@ -5,7 +5,7 @@
 # conda build . -c conda-forge -c esmvalgroup
 
 # Package version number
-{% set version = "2.0.0b2" %}
+{% set version = "2.0.0b3" %}
 
 package:
   name: esmvaltool
@@ -51,7 +51,7 @@ requirements:
     - ecmwf-api-client  # in esmvalgroup channel
     - eofs
     - esmpy
-    - esmvalcore>=2.0.0b6,<2.1  # in esmvalgroup channel
+    - esmvalcore>=2.0.0b7,<2.1  # in esmvalgroup channel
     - jinja2
     - matplotlib
     - nc-time-axis

setup.py

@@ -30,7 +30,7 @@
         'cython',
         'ecmwf-api-client',
         'eofs',
-        'esmvalcore>=2.0.0b6,<2.1',
+        'esmvalcore>=2.0.0b7,<2.1',
         'fiona',
         'jinja2',
         'matplotlib<3',

tests/integration/test_recipes_loading.py

@@ -1,157 +1,77 @@
 """Test recipes are well formed."""
-
-import os
 from pathlib import Path
+from unittest.mock import create_autospec
 
-import iris
-import numpy as np
 import pytest
+import yaml
 
 import esmvalcore
 import esmvaltool
-from esmvalcore import _data_finder, _recipe_checks
-from esmvalcore._config import read_config_user_file
-from esmvalcore._recipe import read_recipe_file
 
 from .test_diagnostic_run import write_config_user_file
 
 
-def _get_recipes():
-    recipes_path = Path(esmvaltool.__file__).absolute().parent / 'recipes'
-    recipes = recipes_path.glob("**/recipe*.yml")
-    return recipes
-
-
-def _tracking_ids(i=0):
-    while True:
-        yield i
-        i += 1
-
-
-def _create_test_file(filename, tracking_id=None):
-    """Generate dummy data file for recipe checker."""
-    dirname = os.path.dirname(filename)
-    if not os.path.exists(dirname):
-        os.makedirs(dirname)
-
-    attributes = {}
-    if tracking_id is not None:
-        attributes['tracking_id'] = tracking_id
-
-    xcoord = iris.coords.DimCoord(np.linspace(0, 5, 5),
-                                  standard_name="longitude")
-    ycoord = iris.coords.DimCoord(np.linspace(0, 5, 12),
-                                  standard_name="latitude")
-    zcoord = iris.coords.DimCoord(np.linspace(0, 5, 17),
-                                  standard_name="height",
-                                  attributes={'positive': 'up'})
-    cube = iris.cube.Cube(np.zeros((5, 12, 17), np.float32),
-                          dim_coords_and_dims=[(xcoord, 0), (ycoord, 1),
-                                               (zcoord, 2)],
-                          attributes=attributes)
-    iris.save(cube, filename)
-
-
-def _get_dummy_filenames(drs):
-    """Generate list of realistic dummy filename(s) according to drs.
-
-    drs is the directory structure used to find input files in ESMValTool
-    """
-    dummy_filenames = []
-
-    # Time-invariant (fx) variables don't have years in their filename
-    if 'fx' in drs:
-        if drs.endswith('[_.]*nc'):
-            dummy_filename = drs.replace('[_.]*', '.')
-        elif drs.endswith('*.nc'):
-            dummy_filename = drs.replace('*', '')
-        dummy_filenames.append(dummy_filename)
-    # For other variables, add custom (large) intervals in dummy filename
-    elif '*' in drs:
-        if drs.endswith('[_.]*nc'):
-            dummy_filename = drs[:-len('[_.]*nc')]
-        elif drs.endswith('*.nc'):
-            dummy_filename = drs[:-len('*.nc')]
-        # Spread dummy data over multiple files for realistic test
-        # Note: adding too many intervals here makes the tests really slow!
-        for interval in ['0000_1849', '1850_9999']:
-            dummy_filenames.append(dummy_filename + '_' + interval + '.nc')
-    # Provide for the possibility of filename drss without *.
-    else:
-        dummy_filename = drs
-        dummy_filenames.append(dummy_filename)
-    return dummy_filenames
-
-
 @pytest.fixture
 def config_user(tmp_path):
     """Generate dummy config-user file for testing purposes."""
     filename = write_config_user_file(tmp_path)
-    cfg = read_config_user_file(filename, 'recipe_test')
+    cfg = esmvalcore._config.read_config_user_file(filename, 'recipe_test')
     cfg['synda_download'] = False
+    cfg['auxiliary_data_dir'] = str(tmp_path / 'auxiliary_data_dir')
     return cfg
 
 
-@pytest.fixture
-def patched_datafinder(tmp_path, monkeypatch):
-    """Replace `_datafinder.find_files()`.
-
-    Creates and points to dummy data input files instead of searching for
-    existing data.
-    """
-    def find_files(_, filenames):
-        drs = filenames[0]
-        dummyfiles = str(tmp_path / 'input' / drs)
-        filenames = _get_dummy_filenames(dummyfiles)
-
-        for file in filenames:
-            _create_test_file(file, next(tracking_id))
-
-        return filenames
-
-    tracking_id = _tracking_ids()
-    monkeypatch.setattr(_data_finder, 'find_files', find_files)
-
-
-@pytest.fixture
-def patched_extract_shape(monkeypatch):
-    """Replace `_recipe_checks.extract_shape`.
-
-    Skips check that shapefile exists.
-    """
-    def extract_shape(settings):
-        valid = {
-            'method': {'contains', 'representative'},
-            'crop': {True, False},
-        }
-
-        for key in valid:
-            value = settings.get(key)
-            if not (value is None or value in valid[key]):
-                raise _recipe_checks.RecipeError(
-                    "In preprocessor function `extract_shape`: Invalid value"
-                    f"'{value}' for argument '{key}', choose from "
-                    "{}".format(', '.join(f"'{k}'".lower()
-                                          for k in valid[key])))
-
-    monkeypatch.setattr(_recipe_checks, 'extract_shape', extract_shape)
-
-
-@pytest.fixture
-def patched_get_reference_levels(monkeypatch):
-    """Replace `_regrid.get_reference_levels`.
+def _get_recipes():
+    recipes_path = Path(esmvaltool.__file__).absolute().parent / 'recipes'
+    recipes = tuple(recipes_path.glob("**/recipe*.yml"))
+    ids = tuple(str(p.relative_to(recipes_path)) for p in recipes)
+    return recipes, ids
 
-    Return a random set of reference levels
-    """
-    def get_reference_levels(*_, **_a):
-        return [1, 2]
 
-    monkeypatch.setattr(esmvalcore._recipe, 'get_reference_levels',
-                        get_reference_levels)
+RECIPES, IDS = _get_recipes()
 
 
-@pytest.mark.parametrize('recipe_file', _get_recipes())
-def test_diagnostic_run(recipe_file, config_user, patched_datafinder,
-                        patched_extract_shape, patched_get_reference_levels):
+@pytest.mark.parametrize('recipe_file', RECIPES, ids=IDS)
+def test_recipe_valid(recipe_file, config_user, monkeypatch):
     """Check that recipe files are valid ESMValTool recipes."""
-    read_recipe_file(recipe_file, config_user)
+    # Mock input files
+    find_files = create_autospec(esmvalcore._data_finder.find_files,
+                                 spec_set=True)
+    find_files.side_effect = lambda *_, **__: [
+        'test_0000-1849.nc',
+        'test_1850-9999.nc',
+    ]
+    monkeypatch.setattr(esmvalcore._data_finder, 'find_files', find_files)
+
+    # Mock vertical levels
+    levels = create_autospec(esmvalcore._recipe.get_reference_levels,
+                             spec_set=True)
+    levels.side_effect = lambda *_, **__: [1, 2]
+    monkeypatch.setattr(esmvalcore._recipe, 'get_reference_levels', levels)
+
+    # Mock valid NCL version
+    ncl_version = create_autospec(esmvalcore._recipe_checks.ncl_version,
+                                  spec_set=True)
+    monkeypatch.setattr(esmvalcore._recipe_checks, 'ncl_version', ncl_version)
+
+    # Mock interpreters installed
+    def which(executable):
+        if executable in ('julia', 'ncl', 'python', 'Rscript'):
+            path = '/path/to/' + executable
+        else:
+            path = None
+        return path
+
+    monkeypatch.setattr(esmvalcore._task, 'which', which)
+
+    # Create a shapefile for extract_shape preprocessor if needed
+    recipe = yaml.safe_load(recipe_file.read_text())
+    for preproc in recipe.get('preprocessors', {}).values():
+        extract_shape = preproc.get('extract_shape')
+        if extract_shape and 'shapefile' in extract_shape:
+            filename = Path(
+                config_user['auxiliary_data_dir']) / extract_shape['shapefile']
+            filename.parent.mkdir(parents=True, exist_ok=True)
+            filename.touch()
+
+    esmvalcore._recipe.read_recipe_file(recipe_file, config_user)