From b0d30f65ccc34b9a85b3edcc722f77560fba2adf Mon Sep 17 00:00:00 2001 From: Manuel Schlund <32543114+schlunma@users.noreply.github.com> Date: Thu, 1 Feb 2024 16:13:33 +0100 Subject: [PATCH] Refactor regridding (#2231) Co-authored-by: Valeriu Predoi --- doc/api/esmvalcore.regridding_schemes.rst | 23 + doc/api/esmvalcore.rst | 1 + doc/quickstart/find_data.rst | 12 +- doc/recipe/preprocessor.rst | 74 ++-- esmvalcore/_recipe/check.py | 46 ++ esmvalcore/_recipe/recipe.py | 1 + esmvalcore/cmor/_fixes/fix.py | 6 +- esmvalcore/cmor/_utils.py | 14 - esmvalcore/cmor/check.py | 4 +- esmvalcore/iris_helpers.py | 52 ++- esmvalcore/preprocessor/_io.py | 2 +- esmvalcore/preprocessor/_regrid.py | 419 +++++++++++------- esmvalcore/preprocessor/_regrid_esmpy.py | 180 ++++++-- .../preprocessor/_regrid_unstructured.py | 49 ++ esmvalcore/preprocessor/regrid_schemes.py | 121 +++++ .../preprocessor/_derive/test_sispeed.py | 11 +- .../preprocessor/_regrid/test_regrid.py | 136 ++++-- .../_regrid/test_regrid_schemes.py | 55 +++ .../_regrid/test_regrid_unstructured.py | 97 ++++ tests/integration/recipe/test_recipe.py | 92 ++++ .../preprocessor/_regrid/test__stock_cube.py | 16 +- .../unit/preprocessor/_regrid/test_regrid.py | 32 +- .../_regrid_esmpy/test_regrid_esmpy.py | 61 ++- tests/unit/test_iris_helpers.py | 155 +++++++ 24 files changed, 1340 insertions(+), 319 deletions(-) create mode 100644 doc/api/esmvalcore.regridding_schemes.rst create mode 100644 esmvalcore/preprocessor/_regrid_unstructured.py create mode 100644 esmvalcore/preprocessor/regrid_schemes.py create mode 100644 tests/integration/preprocessor/_regrid/test_regrid_schemes.py create mode 100644 tests/integration/preprocessor/_regrid/test_regrid_unstructured.py diff --git a/doc/api/esmvalcore.regridding_schemes.rst b/doc/api/esmvalcore.regridding_schemes.rst new file mode 100644 index 0000000000..960b8c4b7d --- /dev/null +++ b/doc/api/esmvalcore.regridding_schemes.rst @@ -0,0 +1,23 @@ +.. _regridding_schemes: + +Regridding schemes +================== + +Iris natively supports data regridding with its :meth:`iris.cube.Cube.regrid` +method and a set of predefined regridding schemes provided in the +:mod:`~iris.analysis` module (further details are given on `this +`__ +page). +Here, further regridding schemes are provided that are compatible with +:meth:`iris.cube.Cube.regrid`. + +Example: + +.. code:: python + + from esmvalcore.preprocessor.regrid_schemes import ESMPyAreaWeighted + + regridded_cube = cube.regrid(target_grid, ESMPyAreaWeighted()) + +.. automodule:: esmvalcore.preprocessor.regrid_schemes + :no-show-inheritance: diff --git a/doc/api/esmvalcore.rst b/doc/api/esmvalcore.rst index e1490a08d8..d160246243 100644 --- a/doc/api/esmvalcore.rst +++ b/doc/api/esmvalcore.rst @@ -17,5 +17,6 @@ library. This section documents the public API of ESMValCore. esmvalcore.iris_helpers esmvalcore.local esmvalcore.preprocessor + esmvalcore.regridding_schemes esmvalcore.typing esmvalcore.experimental diff --git a/doc/quickstart/find_data.rst b/doc/quickstart/find_data.rst index 5062d9fe15..31ee262ef5 100644 --- a/doc/quickstart/find_data.rst +++ b/doc/quickstart/find_data.rst @@ -121,13 +121,13 @@ ERA5 - Supported variables: ``cl``, ``clt``, ``evspsbl``, ``evspsblpot``, ``mrro``, ``pr``, ``prsn``, ``ps``, ``psl``, ``ptype``, ``rls``, ``rlds``, ``rsds``, ``rsdt``, ``rss``, ``uas``, ``vas``, ``tas``, ``tasmax``, ``tasmin``, ``tdps``, ``ts``, ``tsn`` (``E1hr``/``Amon``), ``orog`` (``fx``) - Tier: 3 -.. note:: According to the description of Evapotranspiration and potential Evapotranspiration on the Copernicus page +.. note:: According to the description of Evapotranspiration and potential Evapotranspiration on the Copernicus page (https://cds.climate.copernicus.eu/cdsapp#!/dataset/reanalysis-era5-single-levels-monthly-means?tab=overview): - "The ECMWF Integrated Forecasting System (IFS) convention is that downward fluxes are positive. + "The ECMWF Integrated Forecasting System (IFS) convention is that downward fluxes are positive. Therefore, negative values indicate evaporation and positive values indicate condensation." - + In the CMOR table, these fluxes are defined as positive, if they go from the surface into the atmosphere: - "Evaporation at surface (also known as evapotranspiration): flux of water into the atmosphere due to conversion + "Evaporation at surface (also known as evapotranspiration): flux of water into the atmosphere due to conversion of both liquid and solid phases to vapor (from underlying surface and vegetation)." Therefore, the ERA5 (and ERA5-Land) CMORizer switches the signs of ``evspsbl`` and ``evspsblpot`` to be compatible with the CMOR standard used e.g. by the CMIP models. @@ -398,7 +398,7 @@ The UGRID conventions provide a standardized format to store data on unstructured grids, which is required by many software packages or tools to work correctly. An example is the horizontal regridding of native ICON data to a regular grid. -While the built-in :ref:`unstructured_nearest scheme ` can handle unstructured grids not in UGRID format, using more complex regridding algorithms (for example provided by the :doc:`iris-esmf-regrid:index` package through :ref:`generic regridding @@ -420,7 +420,7 @@ This automatic UGRIDization is enabled by default, but can be switched off with the facet ``ugrid: false`` in the recipe or the extra facets (see below). This is useful for diagnostics that do not support input data in UGRID format (yet) like the :ref:`Psyplot diagnostic ` or -if you want to use the built-in :ref:`unstructured_nearest scheme ` regridding scheme. For 3D ICON variables, ESMValCore tries to add the pressure level information diff --git a/doc/recipe/preprocessor.rst b/doc/recipe/preprocessor.rst index 271641c857..5c994ff001 100644 --- a/doc/recipe/preprocessor.rst +++ b/doc/recipe/preprocessor.rst @@ -898,43 +898,27 @@ third party regridding schemes designed for use with :doc:`Iris Built-in regridding schemes ~~~~~~~~~~~~~~~~~~~~~~~~~~~ -The schemes used for the interpolation and extrapolation operations needed by -the horizontal regridding functionality directly map to their corresponding -implementations in :mod:`iris`: - -* ``linear``: Linear interpolation without extrapolation, i.e., extrapolation - points will be masked even if the source data is not a masked array (uses - ``Linear(extrapolation_mode='mask')``, see :obj:`iris.analysis.Linear`). -* ``linear_extrapolate``: Linear interpolation with extrapolation, i.e., - extrapolation points will be calculated by extending the gradient of the - closest two points (uses ``Linear(extrapolation_mode='extrapolate')``, see - :obj:`iris.analysis.Linear`). -* ``nearest``: Nearest-neighbour interpolation without extrapolation, i.e., - extrapolation points will be masked even if the source data is not a masked - array (uses ``Nearest(extrapolation_mode='mask')``, see - :obj:`iris.analysis.Nearest`). -* ``area_weighted``: Area-weighted regridding (uses ``AreaWeighted()``, see - :obj:`iris.analysis.AreaWeighted`). -* ``unstructured_nearest``: Nearest-neighbour interpolation for unstructured - grids (uses ``UnstructuredNearest()``, see - :obj:`iris.analysis.UnstructuredNearest`). +* ``linear``: Bilinear regridding. + For source data on a regular grid, uses :obj:`~iris.analysis.Linear` with + `extrapolation_mode='mask'`. + For source data on an irregular grid, uses + :class:`~esmvalcore.preprocessor.regrid_schemes.ESMPyLinear`. + Source data on an unstructured grid is not supported, yet. +* ``nearest``: Nearest-neighbor regridding. + For source data on a regular grid, uses :obj:`~iris.analysis.Nearest` with + `extrapolation_mode='mask'`. + For source data on an irregular grid, uses + :class:`~esmvalcore.preprocessor.regrid_schemes.ESMPyNearest`. + For source data on an unstructured grid, uses + :class:`~esmvalcore.preprocessor.regrid_schemes.UnstructuredNearest`. +* ``area_weighted``: First-order conservative (area-weighted) regridding. + For source data on a regular grid, uses :obj:`~iris.analysis.AreaWeighted`. + For source data on an irregular grid, uses + :class:`~esmvalcore.preprocessor.regrid_schemes.ESMPyAreaWeighted`. + Source data on an unstructured grid is not supported, yet. See also :func:`esmvalcore.preprocessor.regrid` -.. note:: - - Controlling the extrapolation mode allows us to avoid situations where - extrapolating values makes little physical sense (e.g. extrapolating beyond - the last data point). - -.. note:: - - The regridding mechanism is (at the moment) done with fully realized data in - memory, so depending on how fine the target grid is, it may use a rather - large amount of memory. Empirically target grids of up to ``0.5x0.5`` - degrees should not produce any memory-related issues, but be advised that - for resolutions of ``< 0.5`` degrees the regridding becomes very slow and - will use a lot of memory. .. _generic regridding schemes: @@ -971,6 +955,28 @@ tolerance. reference: iris.analysis:AreaWeighted mdtol: 0.7 +Another example is bilinear regridding with extrapolation. +This can be achieved with the :class:`iris.analysis.Linear` scheme and the +``extrapolation_mode`` keyword. +Extrapolation points will be calculated by extending the gradient of the +closest two points. + +.. code-block:: yaml + + preprocessors: + regrid_preprocessor: + regrid: + target_grid: 2.5x2.5 + scheme: + reference: iris.analysis:Linear + extrapolation_mode: extrapolate + +.. note:: + + Controlling the extrapolation mode allows us to avoid situations where + extrapolating values makes little physical sense (e.g. extrapolating beyond + the last data point). + The value of the ``reference`` key has two parts that are separated by a ``:`` with no surrounding spaces. The first part is an importable Python module, the second refers to the scheme, i.e. some callable that will be called diff --git a/esmvalcore/_recipe/check.py b/esmvalcore/_recipe/check.py index f80487ecee..f3de14e2ad 100644 --- a/esmvalcore/_recipe/check.py +++ b/esmvalcore/_recipe/check.py @@ -17,6 +17,12 @@ from esmvalcore.local import _get_start_end_year, _parse_period from esmvalcore.preprocessor import TIME_PREPROCESSORS, PreprocessingTask from esmvalcore.preprocessor._multimodel import _get_operator_and_kwargs +from esmvalcore.preprocessor._regrid import ( + HORIZONTAL_SCHEMES_IRREGULAR, + HORIZONTAL_SCHEMES_REGULAR, + HORIZONTAL_SCHEMES_UNSTRUCTURED, + _load_generic_scheme, +) from esmvalcore.preprocessor._shared import get_iris_aggregator from esmvalcore.preprocessor._supplementary_vars import ( PREPROCESSOR_SUPPLEMENTARIES, @@ -486,3 +492,43 @@ def _check_mm_stat(step, step_settings): raise RecipeError( f"Invalid options for {step}: {exc}" ) + + +def regridding_schemes(settings: dict): + """Check :obj:`str` regridding schemes.""" + if 'regrid' not in settings: + return + + # Note: If 'scheme' is missing, this will be detected in + # PreprocessorFile.check() later + scheme = settings['regrid'].get('scheme') + + # Check built-in regridding schemes (given as str) + if isinstance(scheme, str): + scheme = settings['regrid']['scheme'] + allowed_regridding_schemes = list( + set( + list(HORIZONTAL_SCHEMES_IRREGULAR) + + list(HORIZONTAL_SCHEMES_REGULAR) + + list(HORIZONTAL_SCHEMES_UNSTRUCTURED) + ) + ) + if scheme not in allowed_regridding_schemes: + raise RecipeError( + f"Got invalid built-in regridding scheme '{scheme}', expected " + f"one of {allowed_regridding_schemes} or a generic scheme " + f"(see https://docs.esmvaltool.org/projects/ESMValCore/en/" + f"latest/recipe/preprocessor.html#generic-regridding-schemes)." + ) + + # Check generic regridding schemes (given as dict) + if isinstance(scheme, dict): + try: + _load_generic_scheme(scheme) + except ValueError as exc: + raise RecipeError( + f"Failed to load generic regridding scheme: {str(exc)} See " + f"https://docs.esmvaltool.org/projects/ESMValCore/en/latest" + f"/recipe/preprocessor.html#generic-regridding-schemes for " + f"details." + ) diff --git a/esmvalcore/_recipe/recipe.py b/esmvalcore/_recipe/recipe.py index 5ea3199812..4369a7d4ac 100644 --- a/esmvalcore/_recipe/recipe.py +++ b/esmvalcore/_recipe/recipe.py @@ -656,6 +656,7 @@ def _update_preproc_functions(settings, dataset, datasets, missing_vars): if dataset.facets.get('frequency') == 'fx': check.check_for_temporal_preprocs(settings) check.statistics_preprocessors(settings) + check.regridding_schemes(settings) check.bias_type(settings) diff --git a/esmvalcore/cmor/_fixes/fix.py b/esmvalcore/cmor/_fixes/fix.py index f0e794f24d..def33fccc5 100644 --- a/esmvalcore/cmor/_fixes/fix.py +++ b/esmvalcore/cmor/_fixes/fix.py @@ -23,10 +23,10 @@ _get_new_generic_level_coord, _get_simplified_calendar, _get_single_cube, - _is_unstructured_grid, ) from esmvalcore.cmor.fixes import get_time_bounds from esmvalcore.cmor.table import get_var_info +from esmvalcore.iris_helpers import has_unstructured_grid if TYPE_CHECKING: from esmvalcore.cmor.table import CoordinateInfo, VariableInfo @@ -727,7 +727,7 @@ def _fix_coord_bounds( return # Skip guessing bounds for unstructured grids - if _is_unstructured_grid(cube) and cube_coord.standard_name in ( + if has_unstructured_grid(cube) and cube_coord.standard_name in ( 'latitude', 'longitude'): self._debug_msg( cube, @@ -763,7 +763,7 @@ def _fix_coord_direction( return (cube, cube_coord) if cube_coord.dtype.kind == 'U': return (cube, cube_coord) - if _is_unstructured_grid(cube) and cube_coord.standard_name in ( + if has_unstructured_grid(cube) and cube_coord.standard_name in ( 'latitude', 'longitude' ): return (cube, cube_coord) diff --git a/esmvalcore/cmor/_utils.py b/esmvalcore/cmor/_utils.py index 4b083f19de..be837d9c10 100644 --- a/esmvalcore/cmor/_utils.py +++ b/esmvalcore/cmor/_utils.py @@ -7,7 +7,6 @@ from iris.coords import Coord from iris.cube import Cube -from iris.exceptions import CoordinateNotFoundError from esmvalcore.cmor.table import CMOR_TABLES, CoordinateInfo, VariableInfo @@ -173,19 +172,6 @@ def _get_simplified_calendar(calendar: str) -> str: return calendar_aliases.get(calendar, calendar) -def _is_unstructured_grid(cube: Cube) -> bool: - """Check if cube uses unstructured grid.""" - try: - lat = cube.coord('latitude') - lon = cube.coord('longitude') - except CoordinateNotFoundError: - pass - else: - if lat.ndim == 1 and (cube.coord_dims(lat) == cube.coord_dims(lon)): - return True - return False - - def _get_single_cube( cube_list: Sequence[Cube], short_name: str, diff --git a/esmvalcore/cmor/check.py b/esmvalcore/cmor/check.py index cba7e347ae..09f7a6331d 100644 --- a/esmvalcore/cmor/check.py +++ b/esmvalcore/cmor/check.py @@ -25,10 +25,10 @@ _get_generic_lev_coord_names, _get_new_generic_level_coord, _get_simplified_calendar, - _is_unstructured_grid, ) from esmvalcore.cmor.table import CoordinateInfo, get_var_info from esmvalcore.exceptions import ESMValCoreDeprecationWarning +from esmvalcore.iris_helpers import has_unstructured_grid class CheckLevels(IntEnum): @@ -140,7 +140,7 @@ def __init__(self, @cached_property def _unstructured_grid(self) -> bool: """Cube uses unstructured grid.""" - return _is_unstructured_grid(self._cube) + return has_unstructured_grid(self._cube) def check_metadata(self, logger: Optional[logging.Logger] = None) -> Cube: """Check the cube metadata. diff --git a/esmvalcore/iris_helpers.py b/esmvalcore/iris_helpers.py index 8fbd794b3e..9b5fddbfe1 100644 --- a/esmvalcore/iris_helpers.py +++ b/esmvalcore/iris_helpers.py @@ -10,7 +10,7 @@ import numpy as np from iris.coords import Coord from iris.cube import Cube -from iris.exceptions import CoordinateMultiDimError +from iris.exceptions import CoordinateMultiDimError, CoordinateNotFoundError from esmvalcore.typing import NetCDFAttr @@ -268,3 +268,53 @@ def rechunk_cube( _rechunk_dim_metadata(cube, complete_dims, remaining_dims=remaining_dims) return cube + + +def has_irregular_grid(cube: Cube) -> bool: + """Check if a cube has an irregular grid. + + Parameters + ---------- + cube: + Cube to be checked. + + Returns + ------- + bool + ``True`` if input cube has an irregular grid, else ``False``. + + """ + try: + lat = cube.coord('latitude') + lon = cube.coord('longitude') + except CoordinateNotFoundError: + return False + if lat.ndim == 2 and lon.ndim == 2: + return True + return False + + +def has_unstructured_grid(cube: Cube) -> bool: + """Check if a cube has an unstructured grid. + + Parameters + ---------- + cube: + Cube to be checked. + + Returns + ------- + bool + ``True`` if input cube has an unstructured grid, else ``False``. + + """ + try: + lat = cube.coord('latitude') + lon = cube.coord('longitude') + except CoordinateNotFoundError: + return False + if lat.ndim != 1 or lon.ndim != 1: + return False + if cube.coord_dims(lat) != cube.coord_dims(lon): + return False + return True diff --git a/esmvalcore/preprocessor/_io.py b/esmvalcore/preprocessor/_io.py index f1586a9f17..d53955b132 100644 --- a/esmvalcore/preprocessor/_io.py +++ b/esmvalcore/preprocessor/_io.py @@ -6,7 +6,7 @@ import os from itertools import groupby from pathlib import Path -from typing import Optional, NamedTuple +from typing import NamedTuple, Optional from warnings import catch_warnings, filterwarnings import cftime diff --git a/esmvalcore/preprocessor/_regrid.py b/esmvalcore/preprocessor/_regrid.py index 1008cf67ad..8b3c483154 100644 --- a/esmvalcore/preprocessor/_regrid.py +++ b/esmvalcore/preprocessor/_regrid.py @@ -1,30 +1,50 @@ """Horizontal and vertical regridding module.""" +from __future__ import annotations +import functools import importlib import inspect import logging import os import re import ssl +import warnings from copy import deepcopy from decimal import Decimal from pathlib import Path -from typing import Dict +from typing import TYPE_CHECKING, Any import dask.array as da import iris import numpy as np import stratify from geopy.geocoders import Nominatim -from iris.analysis import AreaWeighted, Linear, Nearest, UnstructuredNearest +from iris.analysis import AreaWeighted, Linear, Nearest +from iris.cube import Cube from iris.util import broadcast_to_shape -from ..cmor._fixes.shared import add_altitude_from_plev, add_plev_from_altitude -from ..cmor.table import CMOR_TABLES -from ._other import get_array_module -from ._regrid_esmpy import ESMF_REGRID_METHODS -from ._regrid_esmpy import regrid as esmpy_regrid -from ._supplementary_vars import add_ancillary_variable, add_cell_measure +from esmvalcore.cmor._fixes.shared import ( + add_altitude_from_plev, + add_plev_from_altitude, +) +from esmvalcore.cmor.table import CMOR_TABLES +from esmvalcore.exceptions import ESMValCoreDeprecationWarning +from esmvalcore.iris_helpers import has_irregular_grid, has_unstructured_grid +from esmvalcore.preprocessor._other import get_array_module +from esmvalcore.preprocessor._supplementary_vars import ( + add_ancillary_variable, + add_cell_measure, +) +from esmvalcore.preprocessor.regrid_schemes import ( + ESMPyAreaWeighted, + ESMPyLinear, + ESMPyNearest, + GenericFuncScheme, + UnstructuredNearest, +) + +if TYPE_CHECKING: + from esmvalcore.dataset import Dataset logger = logging.getLogger(__name__) @@ -48,22 +68,29 @@ _LON_MAX = 360.0 _LON_RANGE = _LON_MAX - _LON_MIN -# A cached stock of standard horizontal target grids. -_CACHE: Dict[str, iris.cube.Cube] = {} - # Supported point interpolation schemes. POINT_INTERPOLATION_SCHEMES = { 'linear': Linear(extrapolation_mode='mask'), 'nearest': Nearest(extrapolation_mode='mask'), } -# Supported horizontal regridding schemes. -HORIZONTAL_SCHEMES = { +# Supported horizontal regridding schemes for regular grids +HORIZONTAL_SCHEMES_REGULAR = { + 'area_weighted': AreaWeighted(), 'linear': Linear(extrapolation_mode='mask'), - 'linear_extrapolate': Linear(extrapolation_mode='extrapolate'), 'nearest': Nearest(extrapolation_mode='mask'), - 'area_weighted': AreaWeighted(), - 'unstructured_nearest': UnstructuredNearest(), +} + +# Supported horizontal regridding schemes for irregular grids +HORIZONTAL_SCHEMES_IRREGULAR = { + 'area_weighted': ESMPyAreaWeighted(), + 'linear': ESMPyLinear(), + 'nearest': ESMPyNearest(), +} + +# Supported horizontal regridding schemes for unstructured grids +HORIZONTAL_SCHEMES_UNSTRUCTURED = { + 'nearest': UnstructuredNearest(), } # Supported vertical interpolation schemes. @@ -132,7 +159,7 @@ def _generate_cube_from_dimcoords(latdata, londata, circular: bool = False): Returns ------- - :class:`~iris.cube.Cube` + iris.cube.Cube """ lats = iris.coords.DimCoord(latdata, standard_name='latitude', @@ -155,11 +182,12 @@ def _generate_cube_from_dimcoords(latdata, londata, circular: bool = False): shape = (latdata.size, londata.size) dummy = np.empty(shape, dtype=np.dtype('int8')) coords_spec = [(lats, 0), (lons, 1)] - cube = iris.cube.Cube(dummy, dim_coords_and_dims=coords_spec) + cube = Cube(dummy, dim_coords_and_dims=coords_spec) return cube +@functools.lru_cache def _global_stock_cube(spec, lat_offset=True, lon_offset=True): """Create a stock cube. @@ -185,7 +213,7 @@ def _global_stock_cube(spec, lat_offset=True, lon_offset=True): Returns ------- - :class:`~iris.cube.Cube` + iris.cube.Cube """ dlon, dlat = parse_cell_spec(spec) mid_dlon, mid_dlat = dlon / 2, dlat / 2 @@ -280,7 +308,7 @@ def _regional_stock_cube(spec: dict): Returns ------- - :class:`~iris.cube.Cube`. + iris.cube.Cube """ latdata, londata = _spec_to_latlonvals(**spec) @@ -300,19 +328,6 @@ def add_bounds_from_step(coord, step): return cube -def _attempt_irregular_regridding(cube, scheme): - """Check if irregular regridding with ESMF should be used.""" - if isinstance(scheme, str) and scheme in ESMF_REGRID_METHODS: - try: - lat_dim = cube.coord('latitude').ndim - lon_dim = cube.coord('longitude').ndim - if lat_dim == lon_dim == 2: - return True - except iris.exceptions.CoordinateNotFoundError: - pass - return False - - def extract_location(cube, location, scheme): """Extract a point using a location name, with interpolation. @@ -415,7 +430,7 @@ def extract_point(cube, latitude, longitude, scheme): Returns ------- - :py:class:`~iris.cube.Cube` + iris.cube.Cube Returns a cube with the extracted point(s), and with adjusted latitude and longitude coordinates (see above). If desired point outside values for at least one coordinate, this cube will have fully @@ -470,19 +485,180 @@ def is_dataset(dataset): return hasattr(dataset, 'facets') -def regrid(cube, target_grid, scheme, lat_offset=True, lon_offset=True): +def _get_target_grid_cube( + cube: Cube, + target_grid: Cube | Dataset | Path | str | dict, + lat_offset: bool = True, + lon_offset: bool = True, +) -> Cube: + """Get target grid cube.""" + if is_dataset(target_grid): + target_grid = target_grid.copy() # type: ignore + target_grid.supplementaries.clear() # type: ignore + target_grid.files = [target_grid.files[0]] # type: ignore + target_grid_cube = target_grid.load() # type: ignore + elif isinstance(target_grid, (str, Path)) and os.path.isfile(target_grid): + target_grid_cube = iris.load_cube(target_grid) + elif isinstance(target_grid, str): + # Generate a target grid from the provided cell-specification, + # and cache the resulting stock cube for later use. + target_grid_cube = _global_stock_cube( + target_grid, lat_offset, lon_offset + ) + # Align the target grid coordinate system to the source + # coordinate system. + src_cs = cube.coord_system() + xcoord = target_grid_cube.coord(axis='x', dim_coords=True) + ycoord = target_grid_cube.coord(axis='y', dim_coords=True) + xcoord.coord_system = src_cs + ycoord.coord_system = src_cs + elif isinstance(target_grid, dict): + # Generate a target grid from the provided specification, + target_grid_cube = _regional_stock_cube(target_grid) + else: + target_grid_cube = target_grid + + if not isinstance(target_grid_cube, Cube): + raise ValueError(f'Expecting a cube, got {target_grid}.') + + return target_grid_cube + + +def _attempt_irregular_regridding(cube: Cube, scheme: str) -> bool: + """Check if irregular regridding with ESMF should be used.""" + if not has_irregular_grid(cube): + return False + if scheme not in HORIZONTAL_SCHEMES_IRREGULAR: + raise ValueError( + f"Regridding scheme '{scheme}' does not support irregular data, " + f"expected one of {list(HORIZONTAL_SCHEMES_IRREGULAR)}" + ) + return True + + +def _attempt_unstructured_regridding(cube: Cube, scheme: str) -> bool: + """Check if unstructured regridding should be used.""" + if not has_unstructured_grid(cube): + return False + if scheme not in HORIZONTAL_SCHEMES_UNSTRUCTURED: + raise ValueError( + f"Regridding scheme '{scheme}' does not support unstructured " + f"data, expected one of {list(HORIZONTAL_SCHEMES_UNSTRUCTURED)}" + ) + return True + + +def _load_scheme(src_cube: Cube, scheme: str | dict): + """Return scheme that can be used in :meth:`iris.cube.Cube.regrid`.""" + loaded_scheme: Any = None + + # Deprecations + if scheme == 'unstructured_nearest': + msg = ( + "The regridding scheme `unstructured_nearest` has been deprecated " + "in ESMValCore version 2.11.0 and is scheduled for removal in " + "version 2.13.0. Please use the scheme `nearest` instead. This is " + "an exact replacement for data on unstructured grids. Since " + "version 2.11.0, ESMValCore is able to determine the most " + "suitable regridding scheme based on the input data." + ) + warnings.warn(msg, ESMValCoreDeprecationWarning) + scheme = 'nearest' + + if scheme == 'linear_extrapolate': + msg = ( + "The regridding scheme `linear_extrapolate` has been deprecated " + "in ESMValCore version 2.11.0 and is scheduled for removal in " + "version 2.13.0. Please use a generic scheme with `reference: " + "iris.analysis:Linear` and `extrapolation_mode: extrapolate` " + "instead (see https://docs.esmvaltool.org/projects/ESMValCore/en/" + "latest/recipe/preprocessor.html#generic-regridding-schemes)." + "This is an exact replacement." + ) + warnings.warn(msg, ESMValCoreDeprecationWarning) + scheme = 'linear' + loaded_scheme = Linear(extrapolation_mode='extrapolate') + logger.debug("Loaded regridding scheme %s", loaded_scheme) + return loaded_scheme + + # Scheme is a dict -> assume this describes a generic regridding scheme + if isinstance(scheme, dict): + loaded_scheme = _load_generic_scheme(scheme) + + # Scheme is a str -> load appropriate regridding scheme depending on the + # type of input data + elif _attempt_irregular_regridding(src_cube, scheme): + loaded_scheme = HORIZONTAL_SCHEMES_IRREGULAR[scheme] + elif _attempt_unstructured_regridding(src_cube, scheme): + loaded_scheme = HORIZONTAL_SCHEMES_UNSTRUCTURED[scheme] + else: + loaded_scheme = HORIZONTAL_SCHEMES_REGULAR.get(scheme) + + if loaded_scheme is None: + raise ValueError( + f"Got invalid regridding scheme string '{scheme}', expected one " + f"of {list(HORIZONTAL_SCHEMES_REGULAR)}" + ) + + logger.debug("Loaded regridding scheme %s", loaded_scheme) + + return loaded_scheme + + +def _load_generic_scheme(scheme: dict): + """Load generic regridding scheme.""" + scheme = dict(scheme) # do not overwrite original scheme + + try: + object_ref = scheme.pop("reference") + except KeyError as key_err: + raise ValueError( + "No reference specified for generic regridding." + ) from key_err + module_name, separator, scheme_name = object_ref.partition(":") + try: + obj: Any = importlib.import_module(module_name) + except ImportError as import_err: + raise ValueError( + f"Could not import specified generic regridding module " + f"'{module_name}'. Please double check spelling and that the " + f"required module is installed." + ) from import_err + if separator: + for attr in scheme_name.split('.'): + obj = getattr(obj, attr) + + # If `obj` is a function that requires `src_cube` and `grid_cube`, use + # GenericFuncScheme + scheme_args = inspect.getfullargspec(obj).args + if 'src_cube' in scheme_args and 'grid_cube' in scheme_args: + loaded_scheme = GenericFuncScheme(obj, **scheme) + else: + loaded_scheme = obj(**scheme) + + return loaded_scheme + + +def regrid( + cube: Cube, + target_grid: Cube | Dataset | Path | str | dict, + scheme: str | dict, + lat_offset: bool = True, + lon_offset: bool = True, +) -> Cube: """Perform horizontal regridding. - Note that the target grid can be a cube (:py:class:`~iris.cube.Cube`), - path to a cube (``str``), a grid spec (``str``) in the form - of `MxN`, or a ``dict`` specifying the target grid. + Note that the target grid can be a :class:`~iris.cube.Cube`, a + :class:`~esmvalcore.dataset.Dataset`, a path to a cube + (:class:`~pathlib.Path` or :obj:`str`), a grid spec (:obj:`str`) in the + form of `MxN`, or a :obj:`dict` specifying the target grid. - For the latter, the ``target_grid`` should be a ``dict`` with the + For the latter, the `target_grid` should be a :obj:`dict` with the following keys: - ``start_longitude``: longitude at the center of the first grid cell. - ``end_longitude``: longitude at the center of the last grid cell. - - ``step_longitude``: constant longitude distance between grid cell \ + - ``step_longitude``: constant longitude distance between grid cell centers. - ``start_latitude``: latitude at the center of the first grid cell. - ``end_latitude``: longitude at the center of the last grid cell. @@ -490,39 +666,40 @@ def regrid(cube, target_grid, scheme, lat_offset=True, lon_offset=True): Parameters ---------- - cube : :py:class:`~iris.cube.Cube` + cube: The source cube to be regridded. - target_grid : Cube or str or dict + target_grid: The (location of a) cube that specifies the target or reference grid for the regridding operation. + Alternatively, a :class:`~esmvalcore.dataset.Dataset` can be provided. Alternatively, a string cell specification may be provided, of the form ``MxN``, which specifies the extent of the cell, longitude by latitude (degrees) for a global, regular target grid. Alternatively, a dictionary with a regional target grid may be specified (see above). - scheme : str or dict - The regridding scheme to perform. If both source and target grid are - structured (regular or irregular), can be one of the built-in schemes - ``linear``, ``linear_extrapolate``, ``nearest``, ``area_weighted``, - ``unstructured_nearest``. - Alternatively, a `dict` that specifies generic regridding (see below). - lat_offset : bool - Offset the grid centers of the latitude coordinate w.r.t. the - pole by half a grid step. This argument is ignored if ``target_grid`` - is a cube or file. - lon_offset : bool + scheme: + The regridding scheme to perform. If the source grid is structured + (regular or irregular), can be one of the built-in schemes ``linear``, + ``nearest``, ``area_weighted``. If the source grid is unstructured, can + be one of the built-in schemes ``nearest``. Alternatively, a `dict` + that specifies generic regridding can be given (see below). + lat_offset: + Offset the grid centers of the latitude coordinate w.r.t. the pole by + half a grid step. This argument is ignored if `target_grid` is a cube + or file. + lon_offset: Offset the grid centers of the longitude coordinate w.r.t. Greenwich - meridian by half a grid step. - This argument is ignored if ``target_grid`` is a cube or file. + meridian by half a grid step. This argument is ignored if + `target_grid` is a cube or file. Returns ------- - :py:class:`~iris.cube.Cube` + iris.cube.Cube Regridded cube. See Also -------- - extract_levels : Perform vertical regridding. + extract_levels: Perform vertical regridding. Notes ----- @@ -563,105 +740,34 @@ def regrid(cube, target_grid, scheme, lat_offset=True, lon_offset=True): reference: esmf_regrid.schemes:ESMFAreaWeighted """ - if is_dataset(target_grid): - target_grid = target_grid.copy() - target_grid.supplementaries.clear() - target_grid.files = [target_grid.files[0]] - target_grid = target_grid.load() - elif isinstance(target_grid, (str, Path)) and os.path.isfile(target_grid): - target_grid = iris.load_cube(target_grid) - elif isinstance(target_grid, str): - # Generate a target grid from the provided cell-specification, - # and cache the resulting stock cube for later use. - target_grid = _CACHE.setdefault( - target_grid, - _global_stock_cube(target_grid, lat_offset, lon_offset), - ) - # Align the target grid coordinate system to the source - # coordinate system. - src_cs = cube.coord_system() - xcoord = target_grid.coord(axis='x', dim_coords=True) - ycoord = target_grid.coord(axis='y', dim_coords=True) - xcoord.coord_system = src_cs - ycoord.coord_system = src_cs - elif isinstance(target_grid, dict): - # Generate a target grid from the provided specification, - target_grid = _regional_stock_cube(target_grid) - - if not isinstance(target_grid, iris.cube.Cube): - raise ValueError(f'Expecting a cube, got {target_grid}.') - - if isinstance(scheme, dict): - scheme = dict(scheme) # do not overwrite original scheme - try: - object_ref = scheme.pop("reference") - except KeyError as key_err: - raise ValueError( - "No reference specified for generic regridding.") from key_err - module_name, separator, scheme_name = object_ref.partition(":") - try: - obj = importlib.import_module(module_name) - except ImportError as import_err: - raise ValueError( - "Could not import specified generic regridding module. " - "Please double check spelling and that the required module is " - "installed.") from import_err - if separator: - for attr in scheme_name.split('.'): - obj = getattr(obj, attr) - - scheme_args = inspect.getfullargspec(obj).args - # Add source and target cubes as arguments if required - if 'src_cube' in scheme_args: - scheme['src_cube'] = cube - if 'grid_cube' in scheme_args: - scheme['grid_cube'] = target_grid - - loaded_scheme = obj(**scheme) - else: - loaded_scheme = HORIZONTAL_SCHEMES.get(scheme.lower()) - - if loaded_scheme is None: - emsg = 'Unknown regridding scheme, got {!r}.' - raise ValueError(emsg.format(scheme)) - - # Unstructured regridding requires x2 2d spatial coordinates, - # so ensure to purge any 1d native spatial dimension coordinates - # for the regridder. - if scheme == 'unstructured_nearest': - for axis in ['x', 'y']: - coords = cube.coords(axis=axis, dim_coords=True) - if coords: - [coord] = coords - cube.remove_coord(coord) + # Load target grid and select appropriate scheme + target_grid_cube = _get_target_grid_cube( + cube, target_grid, lat_offset=lat_offset, lon_offset=lon_offset, + ) # Horizontal grids from source and target (almost) match # -> Return source cube with target coordinates - if _horizontal_grid_is_close(cube, target_grid): + if _horizontal_grid_is_close(cube, target_grid_cube): for coord in ['latitude', 'longitude']: - cube.coord(coord).points = target_grid.coord(coord).points - cube.coord(coord).bounds = target_grid.coord(coord).bounds + cube.coord(coord).points = ( + target_grid_cube.coord(coord).core_points() + ) + cube.coord(coord).bounds = ( + target_grid_cube.coord(coord).core_bounds() + ) return cube - # Horizontal grids from source and target do not match - # -> Regrid - if _attempt_irregular_regridding(cube, scheme): - cube = esmpy_regrid(cube, target_grid, scheme) - elif isinstance(loaded_scheme, iris.cube.Cube): - # Return regridded cube in cases in which the - # scheme is a function f(src_cube, grid_cube) -> Cube - cube = loaded_scheme - else: - cube = _rechunk(cube, target_grid) - cube = cube.regrid(target_grid, loaded_scheme) + # Load scheme, rechunk and regrid + if isinstance(scheme, str): + scheme = scheme.lower() + loaded_scheme = _load_scheme(cube, scheme) + cube = _rechunk(cube, target_grid_cube) + cube = cube.regrid(target_grid_cube, loaded_scheme) return cube -def _rechunk( - cube: iris.cube.Cube, - target_grid: iris.cube.Cube, -) -> iris.cube.Cube: +def _rechunk(cube: Cube, target_grid: Cube) -> Cube: """Re-chunk cube with optimal chunk sizes for target grid.""" if not cube.has_lazy_data() or cube.ndim < 3: # Only rechunk lazy multidimensional data @@ -698,29 +804,30 @@ def _rechunk( return cube -def _horizontal_grid_is_close(cube1, cube2): +def _horizontal_grid_is_close(cube1: Cube, cube2: Cube) -> bool: """Check if two cubes have the same horizontal grid definition. The result of the function is a boolean answer, if both cubes have the same horizontal grid definition. The function checks both longitude and latitude, based on extent and resolution. + Note + ---- + The current implementation checks if the bounds and the grid shapes are the + same. Exits on first difference. + Parameters ---------- - cube1 : cube + cube1: The first of the cubes to be checked. - cube2 : cube + cube2: The second of the cubes to be checked. Returns ------- bool + ``True`` if grids are close; ``False`` if not. - .. note:: - - The current implementation checks if the bounds and the - grid shapes are the same. - Exits on first difference. """ # Go through the 2 expected horizontal coordinates longitude and latitude. for coord in ['latitude', 'longitude']: @@ -779,7 +886,7 @@ def _create_cube(src_cube, data, src_levels, levels): # Construct the resultant cube with the interpolated data # and the source cube metadata. kwargs = deepcopy(src_cube.metadata)._asdict() - result = iris.cube.Cube(data, **kwargs) + result = Cube(data, **kwargs) # Add the appropriate coordinates to the cube, excluding # any coordinates that span the z-dimension of interpolation. @@ -1117,7 +1224,7 @@ def extract_coordinate_points(cube, definition, scheme): Returns ------- - :py:class:`~iris.cube.Cube` + iris.cube.Cube Returns a cube with the extracted point(s), and with adjusted latitude and longitude coordinates (see above). If desired point outside values for at least one coordinate, this cube will have fully diff --git a/esmvalcore/preprocessor/_regrid_esmpy.py b/esmvalcore/preprocessor/_regrid_esmpy.py index c7df8bbb36..7da6dd63c1 100755 --- a/esmvalcore/preprocessor/_regrid_esmpy.py +++ b/esmvalcore/preprocessor/_regrid_esmpy.py @@ -11,6 +11,7 @@ raise exc import iris import numpy as np +from iris.cube import Cube from ._mapping import get_empty_data, map_slices, ref_to_dims_index @@ -39,6 +40,149 @@ # } +class ESMPyRegridder: + """General ESMPy regridder. + + Parameters + ---------- + src_cube: + Cube defining the source grid. + tgt_cube: + Cube defining the target grid. + method: + Regridding algorithm. Must be one of `linear`, `area_weighted`, + `nearest`. + mask_threshold: + Threshold used to regrid mask of input cube. + + """ + + def __init__( + self, + src_cube: Cube, + tgt_cube: Cube, + method: str = 'linear', + mask_threshold: float = 0.99, + ): + """Initialize class instance.""" + self.src_cube = src_cube + self.tgt_cube = tgt_cube + self.method = method + self.mask_threshold = mask_threshold + + def __call__(self, cube: Cube) -> Cube: + """Perform regridding. + + Parameters + ---------- + cube: + Cube to be regridded. + + Returns + ------- + Cube + Regridded cube. + + """ + src_rep, dst_rep = get_grid_representants(cube, self.tgt_cube) + regridder = build_regridder( + src_rep, dst_rep, self.method, mask_threshold=self.mask_threshold + ) + result = map_slices(cube, regridder, src_rep, dst_rep) + return result + + +class _ESMPyScheme: + """General irregular regridding scheme. + + This class can be used in :meth:`iris.cube.Cube.regrid`. + + Note + ---- + See `ESMPy `__ for more details on + this. + + Parameters + ---------- + mask_threshold: + Threshold used to regrid mask of source cube. + + """ + + _METHOD = '' + + def __init__(self, mask_threshold: float = 0.99): + """Initialize class instance.""" + self.mask_threshold = mask_threshold + + def __repr__(self) -> str: + """Return string representation of class.""" + return ( + f'{self.__class__.__name__}(mask_threshold={self.mask_threshold})' + ) + + def regridder(self, src_cube: Cube, tgt_cube: Cube) -> ESMPyRegridder: + """Get regridder. + + Parameters + ---------- + src_cube: + Cube defining the source grid. + tgt_cube: + Cube defining the target grid. + + Returns + ------- + ESMPyRegridder + Regridder instance. + + """ + return ESMPyRegridder( + src_cube, + tgt_cube, + method=self._METHOD, + mask_threshold=self.mask_threshold, + ) + + +class ESMPyAreaWeighted(_ESMPyScheme): + """ESMPy area-weighted regridding scheme. + + This class can be used in :meth:`iris.cube.Cube.regrid`. + + Does not support lazy regridding. + + """ + + _METHOD = 'area_weighted' + + +class ESMPyLinear(_ESMPyScheme): + """ESMPy bilinear regridding scheme. + + This class can be used in :meth:`iris.cube.Cube.regrid`. + + Does not support lazy regridding. + + """ + + _METHOD = 'linear' + + +class ESMPyNearest(_ESMPyScheme): + """ESMPy nearest-neighbor regridding scheme. + + This class can be used in :meth:`iris.cube.Cube.regrid`. + + Does not support lazy regridding. + + """ + + _METHOD = 'nearest' + + def cf_2d_bounds_to_esmpy_corners(bounds, circular): """Convert cf style 2d bounds to normal (esmpy style) corners.""" no_lat_points, no_lon_points = bounds.shape[:2] @@ -121,7 +265,7 @@ def is_lon_circular(lon): else: raise NotImplementedError('AuxCoord longitude is higher ' 'dimensional than 2d. Giving up.') - circular = np.alltrue(abs(seam) % 360. < 1.e-3) + circular = np.all(abs(seam) % 360. < 1.e-3) else: raise ValueError('longitude is neither DimCoord nor AuxCoord. ' 'Giving up.') @@ -318,37 +462,3 @@ def get_grid_representants(src, dst): aux_coords_and_dims=aux_coords_and_dims, ) return src_rep, dst_rep - - -def regrid(src, dst, method='linear'): - """ - Regrid src_cube to the grid defined by dst_cube. - - Regrid the data in src_cube onto the grid defined by dst_cube. - - Parameters - ---------- - src: :class:`iris.cube.Cube` - Source data. Must have latitude and longitude coords. - These can be 1d or 2d and should have bounds. - dst: :class:`iris.cube.Cube` - Defines the target grid. - method: - Selects the regridding method. - Can be 'linear', 'area_weighted', - or 'nearest'. See ESMPy_. - - Returns - ------- - :class:`iris.cube.Cube`: - The regridded cube. - - - .. _ESMPy: http://www.earthsystemmodeling.org/ - esmf_releases/non_public/ESMF_7_0_0/esmpy_doc/html/ - RegridMethod.html#ESMF.api.constants.RegridMethod - """ - src_rep, dst_rep = get_grid_representants(src, dst) - regridder = build_regridder(src_rep, dst_rep, method) - res = map_slices(src, regridder, src_rep, dst_rep) - return res diff --git a/esmvalcore/preprocessor/_regrid_unstructured.py b/esmvalcore/preprocessor/_regrid_unstructured.py new file mode 100644 index 0000000000..d23464cf42 --- /dev/null +++ b/esmvalcore/preprocessor/_regrid_unstructured.py @@ -0,0 +1,49 @@ +"""Unstructured grid regridding.""" +from __future__ import annotations + +import logging + +from iris.analysis import UnstructuredNearest as IrisUnstructuredNearest +from iris.analysis.trajectory import UnstructuredNearestNeigbourRegridder +from iris.cube import Cube + +logger = logging.getLogger(__name__) + + +class UnstructuredNearest(IrisUnstructuredNearest): + """Unstructured nearest-neighbor regridding scheme. + + This class is a wrapper around :class:`iris.analysis.UnstructuredNearest` + that removes any additional X or Y coordinates prior to regridding if + necessary. It can be used in :meth:`iris.cube.Cube.regrid`. + + """ + + def regridder( + self, + src_cube: Cube, + tgt_cube: Cube, + ) -> UnstructuredNearestNeigbourRegridder: + """Get regridder. + + Parameters + ---------- + src_cube: + Cube defining the source grid. + tgt_cube: + Cube defining the target grid. + + Returns + ------- + UnstructuredNearestNeigbourRegridder + Regridder instance. + + """ + # Unstructured nearest-neighbor regridding requires exactly one X and + # one Y coordinate (latitude and longitude). Remove any X or Y + # dimensional coordinates if necessary. + for axis in ['x', 'y']: + if src_cube.coords(axis=axis, dim_coords=True): + coord = src_cube.coord(axis=axis, dim_coords=True) + src_cube.remove_coord(coord) + return super().regridder(src_cube, tgt_cube) diff --git a/esmvalcore/preprocessor/regrid_schemes.py b/esmvalcore/preprocessor/regrid_schemes.py new file mode 100644 index 0000000000..13dee49a0d --- /dev/null +++ b/esmvalcore/preprocessor/regrid_schemes.py @@ -0,0 +1,121 @@ +"""Regridding schemes.""" +from __future__ import annotations + +import logging +from collections.abc import Callable + +from iris.cube import Cube + +from esmvalcore.preprocessor._regrid_esmpy import ( + ESMPyAreaWeighted, + ESMPyLinear, + ESMPyNearest, + ESMPyRegridder, +) +from esmvalcore.preprocessor._regrid_unstructured import UnstructuredNearest + +logger = logging.getLogger(__name__) + + +__all__ = [ + 'ESMPyAreaWeighted', + 'ESMPyLinear', + 'ESMPyNearest', + 'ESMPyRegridder', + 'GenericFuncScheme', + 'GenericRegridder', + 'UnstructuredNearest', +] + + +class GenericRegridder: + r"""Generic function regridder. + + Parameters + ---------- + src_cube: + Cube defining the source grid. + tgt_cube: + Cube defining the target grid. + func: + Generic regridding function with signature f(src_cube: Cube, grid_cube: + Cube, \*\*kwargs) -> Cube. + **kwargs: + Keyword arguments for the generic regridding function. + + """ + + def __init__( + self, + src_cube: Cube, + tgt_cube: Cube, + func: Callable, + **kwargs, + ): + """Initialize class instance.""" + self.src_cube = src_cube + self.tgt_cube = tgt_cube + self.func = func + self.kwargs = kwargs + + def __call__(self, cube: Cube) -> Cube: + """Perform regridding. + + Parameters + ---------- + cube: + Cube to be regridded. + + Returns + ------- + Cube + Regridded cube. + + """ + return self.func(cube, self.tgt_cube, **self.kwargs) + + +class GenericFuncScheme: + r"""Regridding with a generic function. + + This class can be used in :meth:`iris.cube.Cube.regrid`. + + Does support lazy regridding if `func` does. + + Parameters + ---------- + func: + Generic regridding function with signature f(src_cube: Cube, grid_cube: + Cube, \*\*kwargs) -> Cube. + **kwargs: + Keyword arguments for the generic regridding function. + + """ + + def __init__(self, func: Callable, **kwargs): + """Initialize class instance.""" + self.func = func + self.kwargs = kwargs + + def __repr__(self) -> str: + """Return string representation of class.""" + kwargs = ', '.join(f"{k}={v}" for (k, v) in self.kwargs.items()) + return f'GenericFuncScheme({self.func.__name__}, {kwargs})' + + def regridder(self, src_cube: Cube, tgt_cube: Cube) -> GenericRegridder: + """Get regridder. + + Parameters + ---------- + src_cube: + Cube defining the source grid. + tgt_cube: + Cube defining the target grid. + + Returns + ------- + GenericRegridder + Regridder instance. + + """ + return GenericRegridder(src_cube, tgt_cube, self.func, **self.kwargs) diff --git a/tests/integration/preprocessor/_derive/test_sispeed.py b/tests/integration/preprocessor/_derive/test_sispeed.py index 3bb8188ac1..9daee38954 100644 --- a/tests/integration/preprocessor/_derive/test_sispeed.py +++ b/tests/integration/preprocessor/_derive/test_sispeed.py @@ -4,9 +4,8 @@ from unittest import mock import numpy as np - -from iris.cube import Cube, CubeList from iris.coords import AuxCoord +from iris.cube import Cube, CubeList from esmvalcore.preprocessor._derive.sispeed import DerivedVariable @@ -23,7 +22,9 @@ def get_cube(name, lat=((0.5, 1.5), (2.5, 3.5)), lon=((0.5, 1.5), (2.5, 3.5))): @mock.patch( - 'esmvalcore.preprocessor._regrid.esmpy_regrid', autospec=True) + 'esmvalcore.preprocessor._regrid_esmpy.ESMPyRegridder.__call__', + autospec=True, +) def test_sispeed_calculation(mock_regrid): """Test calculation of `sispeed.""" siu = get_cube('sea_ice_x_velocity') @@ -37,7 +38,9 @@ def test_sispeed_calculation(mock_regrid): @mock.patch( - 'esmvalcore.preprocessor._regrid.esmpy_regrid', autospec=True) + 'esmvalcore.preprocessor._regrid_esmpy.ESMPyRegridder.__call__', + autospec=True, +) def test_sispeed_calculation_coord_differ(mock_regrid): """Test calculation of `sispeed.""" siu = get_cube('sea_ice_x_velocity') diff --git a/tests/integration/preprocessor/_regrid/test_regrid.py b/tests/integration/preprocessor/_regrid/test_regrid.py index de7a20749d..c774433a42 100644 --- a/tests/integration/preprocessor/_regrid/test_regrid.py +++ b/tests/integration/preprocessor/_regrid/test_regrid.py @@ -10,6 +10,7 @@ from numpy import ma from esmvalcore.dataset import Dataset +from esmvalcore.exceptions import ESMValCoreDeprecationWarning from esmvalcore.preprocessor import regrid from tests import assert_array_equal from tests.unit.preprocessor._regrid import _make_cube @@ -54,42 +55,56 @@ def setUp(self): units='degrees_north', coord_system=self.cs) coords_spec = [(lats, 0), (lons, 1)] - self.grid_for_unstructured_nearest = iris.cube.Cube( + self.tgt_grid_for_unstructured = iris.cube.Cube( data, dim_coords_and_dims=coords_spec) - # Replace 1d spatial coords with 2d spatial coords. lons = self.cube.coord('longitude') lats = self.cube.coord('latitude') x, y = np.meshgrid(lons.points, lats.points) lats = iris.coords.AuxCoord( - y, + y.ravel(), standard_name=lats.metadata.standard_name, long_name=lats.metadata.long_name, var_name=lats.metadata.var_name, units=lats.metadata.units, attributes=lats.metadata.attributes, coord_system=lats.metadata.coord_system, - climatological=lats.metadata.climatological) + climatological=lats.metadata.climatological, + ) lons = iris.coords.AuxCoord( - x, + x.ravel(), standard_name=lons.metadata.standard_name, long_name=lons.metadata.long_name, var_name=lons.metadata.var_name, units=lons.metadata.units, attributes=lons.metadata.attributes, coord_system=lons.metadata.coord_system, - climatological=lons.metadata.climatological) - - self.unstructured_grid_cube = self.cube.copy() - self.unstructured_grid_cube.remove_coord('longitude') - self.unstructured_grid_cube.remove_coord('latitude') - self.unstructured_grid_cube.remove_coord('Pressure Slice') - self.unstructured_grid_cube.add_aux_coord(lons, (1, 2)) - self.unstructured_grid_cube.add_aux_coord(lats, (1, 2)) - self.unstructured_grid_cube.data = np.ma.masked_less( - self.cube.data.astype(np.float32), 3.5 + climatological=lons.metadata.climatological, + ) + + unstructured_data = np.ma.masked_less( + self.cube.data.reshape(3, 4).astype(np.float32), 3.5 + ) + + self.unstructured_grid_cube = iris.cube.Cube( + unstructured_data, + dim_coords_and_dims=[(self.cube.coord('air_pressure'), 0)], + aux_coords_and_dims=[(lats, 1), (lons, 1)], + ) + self.unstructured_grid_cube.metadata = self.cube.metadata + + # Setup irregular cube and grid + lons_2d = iris.coords.AuxCoord( + [[0, 1]], standard_name='longitude', units='degrees_east' + ) + lats_2d = iris.coords.AuxCoord( + [[0, 1]], standard_name='latitude', units='degrees_north' + ) + self.irregular_grid = iris.cube.Cube( + [[1, 1]], + aux_coords_and_dims=[(lats_2d, (0, 1)), (lons_2d, (0, 1))], ) def test_regrid__linear(self): @@ -155,7 +170,7 @@ def test_regrid__linear_do_not_preserve_dtype(self): assert np.issubdtype(self.cube.dtype, np.integer) assert np.issubdtype(result.dtype, np.floating) - def test_regrid__linear_extrapolate(self): + def test_regrid__linear_with_extrapolation(self): data = np.empty((3, 3)) lons = iris.coords.DimCoord([0, 1.5, 3], standard_name='longitude', @@ -169,13 +184,17 @@ def test_regrid__linear_extrapolate(self): coord_system=self.cs) coords_spec = [(lats, 0), (lons, 1)] grid = iris.cube.Cube(data, dim_coords_and_dims=coords_spec) - result = regrid(self.cube, grid, 'linear_extrapolate') + scheme = { + 'reference': 'iris.analysis:Linear', + 'extrapolation_mode': 'extrapolate', + } + result = regrid(self.cube, grid, scheme) expected = [[[-3., -1.5, 0.], [0., 1.5, 3.], [3., 4.5, 6.]], [[1., 2.5, 4.], [4., 5.5, 7.], [7., 8.5, 10.]], [[5., 6.5, 8.], [8., 9.5, 11.], [11., 12.5, 14.]]] assert_array_equal(result.data, expected) - def test_regrid__linear_extrapolate_with_mask(self): + def test_regrid__linear_with_mask(self): data = np.empty((3, 3)) grid = iris.cube.Cube(data) lons = iris.coords.DimCoord([0, 1.5, 3], @@ -273,11 +292,11 @@ def test_regrid__esmf_area_weighted(self): expected = np.array([[[1.499886]], [[5.499886]], [[9.499886]]]) np.testing.assert_array_almost_equal(result.data, expected, decimal=6) - def test_regrid__unstructured_nearest_float(self): - """Test unstructured_nearest regridding with cube of floats.""" + def test_regrid_nearest_unstructured_grid_float(self): + """Test `nearest` regridding with unstructured cube of floats.""" result = regrid(self.unstructured_grid_cube, - self.grid_for_unstructured_nearest, - 'unstructured_nearest') + self.tgt_grid_for_unstructured, + 'nearest') expected = np.ma.array([[[3.0]], [[7.0]], [[11.0]]], mask=[[[True]], [[False]], [[False]]]) np.testing.assert_array_equal(result.data.mask, expected.mask) @@ -289,11 +308,74 @@ def test_regrid__unstructured_nearest_float(self): assert self.unstructured_grid_cube.dtype == np.float32 assert result.dtype == np.float32 - def test_regrid__unstructured_nearest_int(self): - """Test unstructured_nearest regridding with cube of ints.""" - self.unstructured_grid_cube.data = np.ones((3, 2, 2), dtype=int) + def test_regrid_nearest_unstructured_grid_int(self): + """Test `nearest` regridding with unstructured cube of ints.""" + self.unstructured_grid_cube.data = np.ones((3, 4), dtype=int) result = regrid(self.unstructured_grid_cube, - self.grid_for_unstructured_nearest, - 'unstructured_nearest') + self.tgt_grid_for_unstructured, + 'nearest') expected = np.array([[[1]], [[1]], [[1]]]) np.testing.assert_array_equal(result.data, expected) + + def test_invalid_scheme_for_unstructured_grid(self): + """Test invalid scheme for unstructured cube.""" + msg = ( + "Regridding scheme 'invalid' does not support unstructured data, " + ) + with pytest.raises(ValueError, match=msg): + regrid( + self.unstructured_grid_cube, + self.tgt_grid_for_unstructured, + 'invalid', + ) + + def test_invalid_scheme_for_irregular_grid(self): + """Test invalid scheme for irregular cube.""" + msg = ( + "Regridding scheme 'invalid' does not support irregular data, " + ) + with pytest.raises(ValueError, match=msg): + regrid( + self.irregular_grid, + self.tgt_grid_for_unstructured, + 'invalid', + ) + + def test_deprecate_unstrucured_nearest(self): + """Test deprecation of `unstructured_nearest` regridding scheme.""" + with pytest.warns(ESMValCoreDeprecationWarning): + result = regrid( + self.unstructured_grid_cube, + self.tgt_grid_for_unstructured, + 'unstructured_nearest', + ) + expected = np.ma.array( + [[[3.0]], [[7.0]], [[11.0]]], + mask=[[[True]], [[False]], [[False]]], + ) + np.testing.assert_array_equal(result.data.mask, expected.mask) + np.testing.assert_array_almost_equal(result.data, expected, decimal=6) + + def test_deprecate_linear_extrapolate(self): + """Test deprecation of `linear_extrapolate` regridding scheme.""" + data = np.empty((3, 3)) + lons = iris.coords.DimCoord([0, 1.5, 3], + standard_name='longitude', + bounds=[[0, 1], [1, 2], [2, 3]], + units='degrees_east', + coord_system=self.cs) + lats = iris.coords.DimCoord([0, 1.5, 3], + standard_name='latitude', + bounds=[[0, 1], [1, 2], [2, 3]], + units='degrees_north', + coord_system=self.cs) + coords_spec = [(lats, 0), (lons, 1)] + grid = iris.cube.Cube(data, dim_coords_and_dims=coords_spec) + + with pytest.warns(ESMValCoreDeprecationWarning): + result = regrid(self.cube, grid, 'linear_extrapolate') + + expected = [[[-3., -1.5, 0.], [0., 1.5, 3.], [3., 4.5, 6.]], + [[1., 2.5, 4.], [4., 5.5, 7.], [7., 8.5, 10.]], + [[5., 6.5, 8.], [8., 9.5, 11.], [11., 12.5, 14.]]] + assert_array_equal(result.data, expected) diff --git a/tests/integration/preprocessor/_regrid/test_regrid_schemes.py b/tests/integration/preprocessor/_regrid/test_regrid_schemes.py new file mode 100644 index 0000000000..b4dd039f44 --- /dev/null +++ b/tests/integration/preprocessor/_regrid/test_regrid_schemes.py @@ -0,0 +1,55 @@ +"""Integration tests for regrid schemes.""" +import numpy as np +import pytest +from iris.cube import Cube + +from esmvalcore.preprocessor.regrid_schemes import ( + GenericFuncScheme, + GenericRegridder, +) + + +def set_data_to_const(cube, _, const=1.0): + """Dummy function to test ``GenericFuncScheme``.""" + cube = cube.copy(np.full(cube.shape, const)) + return cube + + +@pytest.fixture +def generic_func_scheme(): + """Generic function scheme.""" + return GenericFuncScheme(set_data_to_const, const=2) + + +def test_generic_func_scheme_init(generic_func_scheme): + """Test ``GenericFuncScheme``.""" + assert generic_func_scheme.func == set_data_to_const + assert generic_func_scheme.kwargs == {'const': 2} + + +def test_generic_func_scheme_repr(generic_func_scheme): + """Test ``GenericFuncScheme``.""" + repr = generic_func_scheme.__repr__() + assert repr == 'GenericFuncScheme(set_data_to_const, const=2)' + + +def test_generic_func_scheme_regridder(generic_func_scheme, mocker): + """Test ``GenericFuncScheme``.""" + regridder = generic_func_scheme.regridder( + mocker.sentinel.src_cube, + mocker.sentinel.tgt_cube, + ) + assert isinstance(regridder, GenericRegridder) + assert regridder.src_cube == mocker.sentinel.src_cube + assert regridder.tgt_cube == mocker.sentinel.tgt_cube + assert regridder.func == set_data_to_const + assert regridder.kwargs == {'const': 2} + + +def test_generic_func_scheme_regrid(generic_func_scheme, mocker): + """Test ``GenericFuncScheme``.""" + cube = Cube([0.0, 0.0], var_name='x') + + result = cube.regrid(mocker.sentinel.tgt_grid, generic_func_scheme) + + assert result == Cube([2, 2], var_name='x') diff --git a/tests/integration/preprocessor/_regrid/test_regrid_unstructured.py b/tests/integration/preprocessor/_regrid/test_regrid_unstructured.py new file mode 100644 index 0000000000..2dec8a946c --- /dev/null +++ b/tests/integration/preprocessor/_regrid/test_regrid_unstructured.py @@ -0,0 +1,97 @@ +""" Integration tests for unstructured regridding.""" + +import numpy as np +import pytest +from iris.coords import AuxCoord, DimCoord +from iris.cube import Cube + +from esmvalcore.preprocessor._regrid import _global_stock_cube +from esmvalcore.preprocessor._regrid_unstructured import UnstructuredNearest + + +@pytest.fixture +def unstructured_grid_cube(): + """Sample cube with unstructured grid.""" + time = DimCoord( + [0.0, 1.0], standard_name='time', units='days since 1950-01-01' + ) + lat = AuxCoord( + [-1.0, -1.0, 1.0, 1.0], standard_name='latitude', units='degrees_north' + ) + lon = AuxCoord( + [179.0, 180.0, 180.0, 179.0], + standard_name='longitude', + units='degrees_east', + ) + cube = Cube( + np.array([[0.0, 1.0, 2.0, 3.0], [0.0, 0.0, 0.0, 0.0]]), + standard_name='air_temperature', + units='K', + dim_coords_and_dims=[(time, 0)], + aux_coords_and_dims=[(lat, 1), (lon, 1)], + ) + return cube + + +@pytest.fixture +def target_grid(): + """Sample cube with regular grid.""" + return _global_stock_cube('120x60') + + +class TestUnstructuredNearest: + """Test ``UnstructuredNearest``.""" + + def test_regridding(self, unstructured_grid_cube, target_grid): + """Test regridding.""" + src_cube = unstructured_grid_cube.copy() + + result = src_cube.regrid(target_grid, UnstructuredNearest()) + + assert src_cube == unstructured_grid_cube + assert result.shape == (2, 3, 3) + assert result.coord('time') == src_cube.coord('time') + assert result.coord('latitude') == target_grid.coord('latitude') + assert result.coord('longitude') == target_grid.coord('longitude') + np.testing.assert_allclose( + result.data, + [[[0.0, 1.0, 1.0], + [0.0, 2.0, 1.0], + [3.0, 2.0, 2.0]], + [[0.0, 0.0, 0.0], + [0.0, 0.0, 0.0], + [0.0, 0.0, 0.0]]], + ) + + def test_regridding_with_dim_coord( + self, + unstructured_grid_cube, + target_grid, + ): + """Test regridding.""" + src_cube = unstructured_grid_cube.copy() + dim_coord = DimCoord( + [0, 1, 2, 3], + var_name='x', + standard_name='grid_latitude', + ) + src_cube.add_dim_coord(dim_coord, 1) + assert src_cube != unstructured_grid_cube + + result = src_cube.regrid(target_grid, UnstructuredNearest()) + + assert src_cube == unstructured_grid_cube + assert not src_cube.coords('grid_latitude') + assert result.shape == (2, 3, 3) + assert result.coord('time') == src_cube.coord('time') + assert result.coord('latitude') == target_grid.coord('latitude') + assert result.coord('longitude') == target_grid.coord('longitude') + np.testing.assert_allclose( + result.data, + [[[0.0, 1.0, 1.0], + [0.0, 2.0, 1.0], + [3.0, 2.0, 2.0]], + [[0.0, 0.0, 0.0], + [0.0, 0.0, 0.0], + [0.0, 0.0, 0.0]]], + ) diff --git a/tests/integration/recipe/test_recipe.py b/tests/integration/recipe/test_recipe.py index eb1c260411..f93557f93e 100644 --- a/tests/integration/recipe/test_recipe.py +++ b/tests/integration/recipe/test_recipe.py @@ -2874,3 +2874,95 @@ def test_inlvaid_bias_type(tmp_path, patched_datafinder, session): get_recipe(tmp_path, content, session) assert str(exc.value) == INITIALIZATION_ERROR_MSG assert exc.value.failed_tasks[0].message == msg + + +def test_invalid_builtin_regridding_scheme( + tmp_path, patched_datafinder, session +): + content = dedent(""" + preprocessors: + test: + regrid: + scheme: INVALID + diagnostics: + diagnostic_name: + variables: + tas: + mip: Amon + preprocessor: test + timerange: '2000/2010' + additional_datasets: + - {project: CMIP5, dataset: CanESM2, exp: amip, + ensemble: r1i1p1} + scripts: null + """) + msg = ( + "Got invalid built-in regridding scheme 'INVALID', expected one of " + ) + with pytest.raises(RecipeError) as rec_err_exp: + get_recipe(tmp_path, content, session) + assert str(rec_err_exp.value) == INITIALIZATION_ERROR_MSG + assert msg in str(rec_err_exp.value.failed_tasks[0].message) + + +def test_generic_regridding_scheme_no_ref( + tmp_path, patched_datafinder, session +): + content = dedent(""" + preprocessors: + test: + regrid: + scheme: + no_reference: given + diagnostics: + diagnostic_name: + variables: + tas: + mip: Amon + preprocessor: test + timerange: '2000/2010' + additional_datasets: + - {project: CMIP5, dataset: CanESM2, exp: amip, + ensemble: r1i1p1} + scripts: null + """) + msg = ( + "Failed to load generic regridding scheme: No reference specified for " + "generic regridding. See " + ) + with pytest.raises(RecipeError) as rec_err_exp: + get_recipe(tmp_path, content, session) + assert str(rec_err_exp.value) == INITIALIZATION_ERROR_MSG + assert msg in str(rec_err_exp.value.failed_tasks[0].message) + + +def test_invalid_generic_regridding_scheme( + tmp_path, patched_datafinder, session +): + content = dedent(""" + preprocessors: + test: + regrid: + scheme: + reference: invalid.module:and.function + diagnostics: + diagnostic_name: + variables: + tas: + mip: Amon + preprocessor: test + timerange: '2000/2010' + additional_datasets: + - {project: CMIP5, dataset: CanESM2, exp: amip, + ensemble: r1i1p1} + scripts: null + """) + msg = ( + "Failed to load generic regridding scheme: Could not import specified " + "generic regridding module 'invalid.module'. Please double check " + "spelling and that the required module is installed. " + ) + with pytest.raises(RecipeError) as rec_err_exp: + get_recipe(tmp_path, content, session) + assert str(rec_err_exp.value) == INITIALIZATION_ERROR_MSG + assert msg in str(rec_err_exp.value.failed_tasks[0].message) diff --git a/tests/unit/preprocessor/_regrid/test__stock_cube.py b/tests/unit/preprocessor/_regrid/test__stock_cube.py index 7946d18dc4..782fa2f6e6 100644 --- a/tests/unit/preprocessor/_regrid/test__stock_cube.py +++ b/tests/unit/preprocessor/_regrid/test__stock_cube.py @@ -11,9 +11,15 @@ import numpy as np import tests -from esmvalcore.preprocessor._regrid import (_LAT_MAX, _LAT_MIN, _LAT_RANGE, - _LON_MAX, _LON_MIN, _LON_RANGE) -from esmvalcore.preprocessor._regrid import _global_stock_cube +from esmvalcore.preprocessor._regrid import ( + _LAT_MAX, + _LAT_MIN, + _LAT_RANGE, + _LON_MAX, + _LON_MIN, + _LON_RANGE, + _global_stock_cube, +) class Test(tests.Test): @@ -71,7 +77,9 @@ def _check(self, dx, dy, lat_off=True, lon_off=True): def setUp(self): self.Cube = mock.sentinel.Cube - self.mock_Cube = self.patch('iris.cube.Cube', return_value=self.Cube) + self.mock_Cube = self.patch( + 'esmvalcore.preprocessor._regrid.Cube', return_value=self.Cube + ) self.mock_coord = mock.Mock(spec=iris.coords.DimCoord) self.mock_DimCoord = self.patch( 'iris.coords.DimCoord', return_value=self.mock_coord) diff --git a/tests/unit/preprocessor/_regrid/test_regrid.py b/tests/unit/preprocessor/_regrid/test_regrid.py index b6334d0340..046da17c85 100644 --- a/tests/unit/preprocessor/_regrid/test_regrid.py +++ b/tests/unit/preprocessor/_regrid/test_regrid.py @@ -13,8 +13,7 @@ import tests from esmvalcore.preprocessor import regrid from esmvalcore.preprocessor._regrid import ( - _CACHE, - HORIZONTAL_SCHEMES, + HORIZONTAL_SCHEMES_REGULAR, _horizontal_grid_is_close, _rechunk, ) @@ -23,12 +22,10 @@ class Test(tests.Test): def _check(self, tgt_grid, scheme, spec=False): - expected_scheme = HORIZONTAL_SCHEMES[scheme] + expected_scheme = HORIZONTAL_SCHEMES_REGULAR[scheme] if spec: spec = tgt_grid - self.assertIn(spec, _CACHE) - self.assertEqual(_CACHE[spec], self.tgt_grid) self.coord_system.asset_called_once() expected_calls = [ mock.call(axis='x', dim_coords=True), @@ -37,14 +34,6 @@ def _check(self, tgt_grid, scheme, spec=False): self.assertEqual(self.tgt_grid_coord.mock_calls, expected_calls) self.regrid.assert_called_once_with(self.tgt_grid, expected_scheme) else: - if scheme == 'unstructured_nearest': - expected_calls = [ - mock.call(axis='x', dim_coords=True), - mock.call(axis='y', dim_coords=True) - ] - self.assertEqual(self.coords.mock_calls, expected_calls) - expected_calls = [mock.call(self.coord), mock.call(self.coord)] - self.assertEqual(self.remove_coord.mock_calls, expected_calls) self.regrid.assert_called_once_with(tgt_grid, expected_scheme) # Reset the mocks to enable multiple calls per test-case. @@ -73,10 +62,7 @@ def setUp(self): self.tgt_grid_coord = mock.Mock() self.tgt_grid = mock.Mock(spec=iris.cube.Cube, coord=self.tgt_grid_coord) - self.regrid_schemes = [ - 'linear', 'linear_extrapolate', 'nearest', 'area_weighted', - 'unstructured_nearest' - ] + self.regrid_schemes = ['linear', 'nearest', 'area_weighted'] def _mock_horizontal_grid_is_close(src, tgt): return False @@ -107,12 +93,12 @@ def test_invalid_tgt_grid__unknown(self): regrid(self.src_cube, dummy, scheme) def test_invalid_scheme__unknown(self): - emsg = 'Unknown regridding scheme' + emsg = "Got invalid regridding scheme string 'wibble'" with self.assertRaisesRegex(ValueError, emsg): regrid(self.src_cube, self.src_cube, 'wibble') def test_horizontal_schemes(self): - self.assertEqual(set(HORIZONTAL_SCHEMES.keys()), + self.assertEqual(set(HORIZONTAL_SCHEMES_REGULAR.keys()), set(self.regrid_schemes)) def test_regrid__horizontal_schemes(self): @@ -123,11 +109,6 @@ def test_regrid__horizontal_schemes(self): self._check(self.tgt_grid, scheme) def test_regrid__cell_specification(self): - # Clear cache before and after the test to avoid poisoning - # the cache with Mocked cubes - # https://github.com/ESMValGroup/ESMValCore/issues/953 - _CACHE.clear() - specs = ['1x1', '2x2', '3x3', '4x4', '5x5'] scheme = 'linear' for spec in specs: @@ -135,9 +116,6 @@ def test_regrid__cell_specification(self): self.assertEqual(result, self.regridded_cube) self.assertEqual(result.data, mock.sentinel.data) self._check(spec, scheme, spec=True) - self.assertEqual(set(_CACHE.keys()), set(specs)) - - _CACHE.clear() def test_regrid_generic_missing_reference(self): emsg = "No reference specified for generic regridding." diff --git a/tests/unit/preprocessor/_regrid_esmpy/test_regrid_esmpy.py b/tests/unit/preprocessor/_regrid_esmpy/test_regrid_esmpy.py index 107ab2a5c5..611966ab75 100644 --- a/tests/unit/preprocessor/_regrid_esmpy/test_regrid_esmpy.py +++ b/tests/unit/preprocessor/_regrid_esmpy/test_regrid_esmpy.py @@ -10,6 +10,9 @@ import tests from esmvalcore.preprocessor._regrid_esmpy import ( + ESMPyAreaWeighted, + ESMPyLinear, + ESMPyNearest, build_regridder, build_regridder_2d, coords_iris_to_esmpy, @@ -19,7 +22,6 @@ get_grid_representants, get_representant, is_lon_circular, - regrid, ) @@ -664,13 +666,62 @@ def test_get_grid_representants_2d_src(self, mock_cube, @mock.patch('esmvalcore.preprocessor._regrid_esmpy.build_regridder') @mock.patch('esmvalcore.preprocessor._regrid_esmpy.get_grid_representants', mock.Mock(side_effect=identity)) - def test_regrid(self, mock_build_regridder, mock_map_slices): + def test_regrid_nearest(self, mock_build_regridder, mock_map_slices): """Test full regrid method.""" mock_build_regridder.return_value = mock.sentinel.regridder mock_map_slices.return_value = mock.sentinel.regridded - regrid(self.cube_3d, self.cube) - mock_build_regridder.assert_called_once_with(self.cube_3d, self.cube, - 'linear') + regridder = ESMPyNearest().regridder(self.cube_3d, self.cube) + regridder(self.cube_3d) + mock_build_regridder.assert_called_once_with( + self.cube_3d, self.cube, 'nearest', mask_threshold=0.99 + ) + mock_map_slices.assert_called_once_with(self.cube_3d, + mock.sentinel.regridder, + self.cube_3d, self.cube) + + @mock.patch('esmvalcore.preprocessor._regrid_esmpy.map_slices') + @mock.patch('esmvalcore.preprocessor._regrid_esmpy.build_regridder') + @mock.patch('esmvalcore.preprocessor._regrid_esmpy.get_grid_representants', + mock.Mock(side_effect=identity)) + def test_regrid_linear(self, mock_build_regridder, mock_map_slices): + """Test full regrid method.""" + mock_build_regridder.return_value = mock.sentinel.regridder + mock_map_slices.return_value = mock.sentinel.regridded + regridder = ESMPyLinear().regridder(self.cube_3d, self.cube) + regridder(self.cube_3d) + mock_build_regridder.assert_called_once_with( + self.cube_3d, self.cube, 'linear', mask_threshold=0.99 + ) + mock_map_slices.assert_called_once_with(self.cube_3d, + mock.sentinel.regridder, + self.cube_3d, self.cube) + + @mock.patch('esmvalcore.preprocessor._regrid_esmpy.map_slices') + @mock.patch('esmvalcore.preprocessor._regrid_esmpy.build_regridder') + @mock.patch('esmvalcore.preprocessor._regrid_esmpy.get_grid_representants', + mock.Mock(side_effect=identity)) + def test_regrid_area_weighted(self, mock_build_regridder, mock_map_slices): + """Test full regrid method.""" + mock_build_regridder.return_value = mock.sentinel.regridder + mock_map_slices.return_value = mock.sentinel.regridded + regridder = ESMPyAreaWeighted().regridder(self.cube_3d, self.cube) + regridder(self.cube_3d) + mock_build_regridder.assert_called_once_with( + self.cube_3d, self.cube, 'area_weighted', mask_threshold=0.99 + ) mock_map_slices.assert_called_once_with(self.cube_3d, mock.sentinel.regridder, self.cube_3d, self.cube) + + +@pytest.mark.parametrize( + 'scheme,output', + [ + (ESMPyAreaWeighted(), 'ESMPyAreaWeighted(mask_threshold=0.99)'), + (ESMPyLinear(), 'ESMPyLinear(mask_threshold=0.99)'), + (ESMPyNearest(), 'ESMPyNearest(mask_threshold=0.99)'), + ] +) +def test_scheme_repr(scheme, output): + """Test ``_ESMPyScheme.__repr__``.""" + assert scheme.__repr__() == output diff --git a/tests/unit/test_iris_helpers.py b/tests/unit/test_iris_helpers.py index e91f2f70a1..0b742ffe30 100644 --- a/tests/unit/test_iris_helpers.py +++ b/tests/unit/test_iris_helpers.py @@ -21,6 +21,8 @@ from esmvalcore.iris_helpers import ( add_leading_dim_to_cube, date2num, + has_irregular_grid, + has_unstructured_grid, merge_cube_attributes, rechunk_cube, ) @@ -338,3 +340,156 @@ def test_rechunk_cube_invalid_coord_fail(cube_3d): ) with pytest.raises(CoordinateMultiDimError, match=msg): rechunk_cube(cube_3d, ['xy']) + + +@pytest.fixture +def lat_coord_1d(): + """1D latitude coordinate.""" + return DimCoord([0, 1], standard_name='latitude') + + +@pytest.fixture +def lon_coord_1d(): + """1D longitude coordinate.""" + return DimCoord([0, 1], standard_name='longitude') + + +@pytest.fixture +def lat_coord_2d(): + """2D latitude coordinate.""" + return AuxCoord([[0, 1]], standard_name='latitude') + + +@pytest.fixture +def lon_coord_2d(): + """2D longitude coordinate.""" + return AuxCoord([[0, 1]], standard_name='longitude') + + +def test_has_irregular_grid_no_lat_lon(): + """Test `has_irregular_grid`.""" + cube = Cube(0) + assert has_irregular_grid(cube) is False + + +def test_has_irregular_grid_no_lat(lon_coord_2d): + """Test `has_irregular_grid`.""" + cube = Cube([[0, 1]], aux_coords_and_dims=[(lon_coord_2d, (0, 1))]) + assert has_irregular_grid(cube) is False + + +def test_has_irregular_grid_no_lon(lat_coord_2d): + """Test `has_irregular_grid`.""" + cube = Cube([[0, 1]], aux_coords_and_dims=[(lat_coord_2d, (0, 1))]) + assert has_irregular_grid(cube) is False + + +def test_has_irregular_grid_1d_lon(lat_coord_2d, lon_coord_1d): + """Test `has_irregular_grid`.""" + cube = Cube( + [[0, 1]], + dim_coords_and_dims=[(lon_coord_1d, 1)], + aux_coords_and_dims=[(lat_coord_2d, (0, 1))], + ) + assert has_irregular_grid(cube) is False + + +def test_has_irregular_grid_1d_lat(lat_coord_1d, lon_coord_2d): + """Test `has_irregular_grid`.""" + cube = Cube( + [[0, 1]], + dim_coords_and_dims=[(lat_coord_1d, 1)], + aux_coords_and_dims=[(lon_coord_2d, (0, 1))], + ) + assert has_irregular_grid(cube) is False + + +def test_has_irregular_grid_1d_lat_lon(lat_coord_1d, lon_coord_1d): + """Test `has_irregular_grid`.""" + cube = Cube( + [0, 1], aux_coords_and_dims=[(lat_coord_1d, 0), (lon_coord_1d, 0)] + ) + assert has_irregular_grid(cube) is False + + +def test_has_irregular_grid_regular_grid(lat_coord_1d, lon_coord_1d): + """Test `has_irregular_grid`.""" + cube = Cube( + [[0, 1], [2, 3]], + dim_coords_and_dims=[(lat_coord_1d, 0), (lon_coord_1d, 1)], + ) + assert has_irregular_grid(cube) is False + + +def test_has_irregular_grid_true(lat_coord_2d, lon_coord_2d): + """Test `has_irregular_grid`.""" + cube = Cube( + [[0, 1]], + aux_coords_and_dims=[(lat_coord_2d, (0, 1)), (lon_coord_2d, (0, 1))], + ) + assert has_irregular_grid(cube) is True + + +def test_has_unstructured_grid_no_lat_lon(): + """Test `has_unstructured_grid`.""" + cube = Cube(0) + assert has_unstructured_grid(cube) is False + + +def test_has_unstructured_grid_no_lat(lon_coord_1d): + """Test `has_unstructured_grid`.""" + cube = Cube([0, 1], dim_coords_and_dims=[(lon_coord_1d, 0)]) + assert has_unstructured_grid(cube) is False + + +def test_has_unstructured_grid_no_lon(lat_coord_1d): + """Test `has_unstructured_grid`.""" + cube = Cube([0, 1], dim_coords_and_dims=[(lat_coord_1d, 0)]) + assert has_unstructured_grid(cube) is False + + +def test_has_unstructured_grid_2d_lat(lat_coord_2d, lon_coord_1d): + """Test `has_unstructured_grid`.""" + cube = Cube( + [[0, 1]], + dim_coords_and_dims=[(lon_coord_1d, 1)], + aux_coords_and_dims=[(lat_coord_2d, (0, 1))], + ) + assert has_unstructured_grid(cube) is False + + +def test_has_unstructured_grid_2d_lon(lat_coord_1d, lon_coord_2d): + """Test `has_unstructured_grid`.""" + cube = Cube( + [[0, 1]], + dim_coords_and_dims=[(lat_coord_1d, 1)], + aux_coords_and_dims=[(lon_coord_2d, (0, 1))], + ) + assert has_unstructured_grid(cube) is False + + +def test_has_unstructured_grid_2d_lat_lon(lat_coord_2d, lon_coord_2d): + """Test `has_unstructured_grid`.""" + cube = Cube( + [[0, 1]], + aux_coords_and_dims=[(lat_coord_2d, (0, 1)), (lon_coord_2d, (0, 1))], + ) + assert has_unstructured_grid(cube) is False + + +def test_has_unstructured_grid_regular_grid(lat_coord_1d, lon_coord_1d): + """Test `has_unstructured_grid`.""" + cube = Cube( + [[0, 1], [2, 3]], + dim_coords_and_dims=[(lat_coord_1d, 0), (lon_coord_1d, 1)], + ) + assert has_unstructured_grid(cube) is False + + +def test_has_unstructured_grid_true(lat_coord_1d, lon_coord_1d): + """Test `has_unstructured_grid`.""" + cube = Cube( + [[0, 1], [2, 3]], + aux_coords_and_dims=[(lat_coord_1d, 0), (lon_coord_1d, 0)], + ) + assert has_unstructured_grid(cube) is True