diff --git a/esmvalcore/cmor/check.py b/esmvalcore/cmor/check.py index 342d0fff66..a75dcdaab4 100644 --- a/esmvalcore/cmor/check.py +++ b/esmvalcore/cmor/check.py @@ -3,7 +3,6 @@ from __future__ import annotations import logging -import warnings from collections import namedtuple from collections.abc import Callable from enum import IntEnum @@ -20,7 +19,6 @@ from iris.coords import Coord from iris.cube import Cube -from esmvalcore.cmor._fixes.fix import GenericFix from esmvalcore.cmor._utils import ( _get_alternative_generic_lev_coord, _get_generic_lev_coord_names, @@ -28,7 +26,6 @@ _get_simplified_calendar, ) from esmvalcore.cmor.table import CoordinateInfo, get_var_info -from esmvalcore.exceptions import ESMValCoreDeprecationWarning from esmvalcore.iris_helpers import has_unstructured_grid @@ -70,18 +67,6 @@ class CMORCheck: fail_on_error: bool If true, CMORCheck stops on the first error. If false, it collects all possible errors before stopping. - automatic_fixes: bool - If True, CMORCheck will try to apply automatic fixes for any - detected error, if possible. - - .. deprecated:: 2.10.0 - This option has been deprecated in ESMValCore version 2.10.0 and is - scheduled for removal in version 2.12.0. Please use the functions - :func:`~esmvalcore.preprocessor.fix_metadata`, - :func:`~esmvalcore.preprocessor.fix_data`, or - :meth:`esmvalcore.dataset.Dataset.load` (which automatically - includes the first two functions) instead. Fixes and CMOR checks - have been clearly separated in ESMValCore version 2.10.0. check_level: CheckLevels Level of strictness of the checks. @@ -104,7 +89,6 @@ def __init__( frequency=None, fail_on_error=False, check_level=CheckLevels.DEFAULT, - automatic_fixes=False, ): self._cube = cube self._failerr = fail_on_error @@ -118,26 +102,6 @@ def __init__( if not frequency: frequency = self._cmor_var.frequency self.frequency = frequency - self.automatic_fixes = automatic_fixes - - # Deprecate automatic_fixes (remove in v2.12) - if automatic_fixes: - msg = ( - "The option `automatic_fixes` has been deprecated in " - "ESMValCore version 2.10.0 and is scheduled for removal in " - "version 2.12.0. Please use the functions " - "esmvalcore.preprocessor.fix_metadata(), " - "esmvalcore.preprocessor.fix_data(), or " - "esmvalcore.dataset.Dataset.load() (which automatically " - "includes the first two functions) instead. Fixes and CMOR " - "checks have been clearly separated in ESMValCore version " - "2.10.0." - ) - warnings.warn(msg, ESMValCoreDeprecationWarning) - - # TODO: remove in v2.12 - - self._generic_fix = GenericFix(var_info, frequency=frequency) @cached_property def _unstructured_grid(self) -> bool: @@ -171,10 +135,6 @@ def check_metadata(self, logger: Optional[logging.Logger] = None) -> Cube: if logger is not None: self._logger = logger - # TODO: remove in v2.12 - if self.automatic_fixes: - [self._cube] = self._generic_fix.fix_metadata([self._cube]) - self._check_var_metadata() self._check_fill_value() self._check_multiple_coords_same_stdname() @@ -220,10 +180,6 @@ def check_data(self, logger: Optional[logging.Logger] = None) -> Cube: if logger is not None: self._logger = logger - # TODO: remove in v2.12 - if self.automatic_fixes: - self._cube = self._generic_fix.fix_data(self._cube) - self._check_coords_data() self.report_debug_messages() @@ -345,7 +301,6 @@ def _check_var_metadata(self): def _get_effective_units(self): """Get effective units.""" - # TODO: remove entire function in v2.12 if self._cmor_var.units.lower() == "psu": units = "1.0" else: @@ -606,12 +561,6 @@ def _check_coords_data(self): except iris.exceptions.CoordinateNotFoundError: continue - # TODO: remove in v2.12 - if self.automatic_fixes: - (self._cube, coord) = self._generic_fix._fix_coord_direction( - self._cube, coordinate, coord - ) - self._check_coord_monotonicity_and_direction( coordinate, coord, var_name ) @@ -935,7 +884,6 @@ def _get_cmor_checker( frequency: None | str = None, fail_on_error: bool = False, check_level: CheckLevels = CheckLevels.DEFAULT, - automatic_fixes: bool = False, # TODO: remove in v2.12 ) -> Callable[[Cube], CMORCheck]: """Get a CMOR checker.""" var_info = get_var_info(project, mip, short_name) @@ -947,7 +895,6 @@ def _checker(cube: Cube) -> CMORCheck: frequency=frequency, fail_on_error=fail_on_error, check_level=check_level, - automatic_fixes=automatic_fixes, ) return _checker diff --git a/esmvalcore/cmor/fix.py b/esmvalcore/cmor/fix.py index 2e3209897d..ab81353cfb 100644 --- a/esmvalcore/cmor/fix.py +++ b/esmvalcore/cmor/fix.py @@ -8,7 +8,6 @@ from __future__ import annotations import logging -import warnings from collections import defaultdict from collections.abc import Sequence from pathlib import Path @@ -17,8 +16,6 @@ from iris.cube import Cube, CubeList from esmvalcore.cmor._fixes.fix import Fix -from esmvalcore.cmor.check import CheckLevels, _get_cmor_checker -from esmvalcore.exceptions import ESMValCoreDeprecationWarning if TYPE_CHECKING: from ..config import Session @@ -109,7 +106,6 @@ def fix_metadata( dataset: str, mip: str, frequency: Optional[str] = None, - check_level: CheckLevels = CheckLevels.DEFAULT, session: Optional[Session] = None, **extra_facets, ) -> CubeList: @@ -132,17 +128,6 @@ def fix_metadata( Variable's MIP. frequency: Variable's data frequency, if available. - check_level: - Level of strictness of the checks. - - .. deprecated:: 2.10.0 - This option has been deprecated in ESMValCore version 2.10.0 and is - scheduled for removal in version 2.12.0. Please use the functions - :func:`~esmvalcore.preprocessor.cmor_check_metadata`, - :func:`~esmvalcore.preprocessor.cmor_check_data`, or - :meth:`~esmvalcore.cmor.check.cmor_check` instead. This function - will no longer perform CMOR checks. Fixes and CMOR checks have been - clearly separated in ESMValCore version 2.10.0. session: Current session which includes configuration and directory information. **extra_facets: @@ -155,20 +140,6 @@ def fix_metadata( Fixed cubes. """ - # Deprecate CMOR checks (remove in v2.12) - if check_level != CheckLevels.DEFAULT: - msg = ( - "The option `check_level` has been deprecated in ESMValCore " - "version 2.10.0 and is scheduled for removal in version 2.12.0. " - "Please use the functions " - "esmvalcore.preprocessor.cmor_check_metadata, " - "esmvalcore.preprocessor.cmor_check_data, or " - "esmvalcore.cmor.check.cmor_check instead. This function will no " - "longer perform CMOR checks. Fixes and CMOR checks have been " - "clearly separated in ESMValCore version 2.10.0." - ) - warnings.warn(msg, ESMValCoreDeprecationWarning) - # Update extra_facets with variable information given as regular arguments # to this function extra_facets.update( @@ -207,18 +178,6 @@ def fix_metadata( # returns a single cube cube = cube_list[0] - # Perform CMOR checks - # TODO: remove in v2.12 - checker = _get_cmor_checker( - project, - mip, - short_name, - frequency, - fail_on_error=False, - check_level=check_level, - ) - cube = checker(cube).check_metadata() - cube.attributes.pop("source_file", None) fixed_cubes.append(cube) @@ -232,7 +191,6 @@ def fix_data( dataset: str, mip: str, frequency: Optional[str] = None, - check_level: CheckLevels = CheckLevels.DEFAULT, session: Optional[Session] = None, **extra_facets, ) -> Cube: @@ -257,17 +215,6 @@ def fix_data( Variable's MIP. frequency: Variable's data frequency, if available. - check_level: - Level of strictness of the checks. - - .. deprecated:: 2.10.0 - This option has been deprecated in ESMValCore version 2.10.0 and is - scheduled for removal in version 2.12.0. Please use the functions - :func:`~esmvalcore.preprocessor.cmor_check_metadata`, - :func:`~esmvalcore.preprocessor.cmor_check_data`, or - :meth:`~esmvalcore.cmor.check.cmor_check` instead. This function - will no longer perform CMOR checks. Fixes and CMOR checks have been - clearly separated in ESMValCore version 2.10.0. session: Current session which includes configuration and directory information. **extra_facets: @@ -280,20 +227,6 @@ def fix_data( Fixed cube. """ - # Deprecate CMOR checks (remove in v2.12) - if check_level != CheckLevels.DEFAULT: - msg = ( - "The option `check_level` has been deprecated in ESMValCore " - "version 2.10.0 and is scheduled for removal in version 2.12.0. " - "Please use the functions " - "esmvalcore.preprocessor.cmor_check_metadata, " - "esmvalcore.preprocessor.cmor_check_data, or " - "esmvalcore.cmor.check.cmor_check instead. This function will no " - "longer perform CMOR checks. Fixes and CMOR checks have been " - "clearly separated in ESMValCore version 2.10.0." - ) - warnings.warn(msg, ESMValCoreDeprecationWarning) - # Update extra_facets with variable information given as regular arguments # to this function extra_facets.update( @@ -317,16 +250,4 @@ def fix_data( ): cube = fix.fix_data(cube) - # Perform CMOR checks - # TODO: remove in v2.12 - checker = _get_cmor_checker( - project, - mip, - short_name, - frequency, - fail_on_error=False, - check_level=check_level, - ) - cube = checker(cube).check_data() - return cube diff --git a/esmvalcore/dataset.py b/esmvalcore/dataset.py index c436edfe8b..6717485e38 100644 --- a/esmvalcore/dataset.py +++ b/esmvalcore/dataset.py @@ -761,7 +761,6 @@ def _load(self) -> Cube: ), } settings["fix_metadata"] = { - "check_level": self.session["check_level"], "session": self.session, **self.facets, } @@ -778,7 +777,6 @@ def _load(self) -> Cube: "timerange": self.facets["timerange"], } settings["fix_data"] = { - "check_level": self.session["check_level"], "session": self.session, **self.facets, } diff --git a/tests/integration/cmor/_fixes/obs4mips/test_airs_2_0.py b/tests/integration/cmor/_fixes/obs4mips/test_airs_2_0.py index d82aba1640..8370f15e97 100644 --- a/tests/integration/cmor/_fixes/obs4mips/test_airs_2_0.py +++ b/tests/integration/cmor/_fixes/obs4mips/test_airs_2_0.py @@ -20,14 +20,7 @@ def test_fix_metadata_hur(): ] ) - fixed_cubes = fix_metadata( - cubes, - "hur", - "obs4MIPs", - "AIRS-2-0", - "Amon", - check_level=5, - ) + fixed_cubes = fix_metadata(cubes, "hur", "obs4MIPs", "AIRS-2-0", "Amon") assert len(fixed_cubes) == 1 fixed_cube = fixed_cubes[0] diff --git a/tests/integration/cmor/test_fix.py b/tests/integration/cmor/test_fix.py index 63b4767841..74a0a08841 100644 --- a/tests/integration/cmor/test_fix.py +++ b/tests/integration/cmor/test_fix.py @@ -8,8 +8,7 @@ from iris.coords import AuxCoord, DimCoord from iris.cube import Cube, CubeList -from esmvalcore.cmor.check import CheckLevels, CMORCheckError -from esmvalcore.exceptions import ESMValCoreDeprecationWarning +from esmvalcore.cmor.check import CMORCheckError from esmvalcore.preprocessor import ( cmor_check_data, cmor_check_metadata, @@ -18,25 +17,6 @@ ) -# TODO: remove in v2.12 -@pytest.fixture(autouse=True) -def disable_fix_cmor_checker(mocker): - """Disable the CMOR checker in fixes (will be default in v2.12).""" - - class MockChecker: - def __init__(self, cube): - self._cube = cube - - def check_metadata(self): - return self._cube - - def check_data(self): - return self._cube - - mock = mocker.patch("esmvalcore.cmor.fix._get_cmor_checker") - mock.return_value = MockChecker - - class TestGenericFix: """Tests for ``GenericFix``.""" @@ -887,27 +867,3 @@ def test_fix_data_amon_tas(self): assert self.mock_debug.call_count == 0 assert self.mock_warning.call_count == 0 - - def test_deprecate_check_level_fix_metadata(self): - """Test deprecation of check level in ``fix_metadata``.""" - with pytest.warns(ESMValCoreDeprecationWarning): - fix_metadata( - self.cubes_4d, - "ta", - "CMIP6", - "MODEL", - "Amon", - check_level=CheckLevels.RELAXED, - ) - - def test_deprecate_check_level_fix_data(self): - """Test deprecation of check level in ``fix_data``.""" - with pytest.warns(ESMValCoreDeprecationWarning): - fix_metadata( - self.cubes_4d, - "ta", - "CMIP6", - "MODEL", - "Amon", - check_level=CheckLevels.RELAXED, - ) diff --git a/tests/unit/cmor/test_cmor_check.py b/tests/unit/cmor/test_cmor_check.py index 331f3b6273..a7822ec03b 100644 --- a/tests/unit/cmor/test_cmor_check.py +++ b/tests/unit/cmor/test_cmor_check.py @@ -20,7 +20,6 @@ CMORCheckError, _get_cmor_checker, ) -from esmvalcore.exceptions import ESMValCoreDeprecationWarning logger = logging.getLogger(__name__) @@ -299,14 +298,6 @@ def test_valid_generic_level(self): checker.check_metadata() checker.check_data() - # TODO: remove in v2.12 - def test_valid_generic_level_automatic_fixes(self): - """Test valid generic level coordinate with automatic fixes.""" - self._setup_generic_level_var() - checker = CMORCheck(self.cube, self.var_info, automatic_fixes=True) - checker.check_metadata() - checker.check_data() - def test_invalid_generic_level(self): """Test invalid generic level coordinate.""" self._setup_generic_level_var() @@ -680,20 +671,6 @@ def test_non_decreasing(self): self.var_info.coordinates["lat"].stored_direction = "decreasing" self._check_fails_in_metadata() - # TODO: remove in v2.12 - def test_non_decreasing_automatic_fix_metadata(self): - """Automatic fix for decreasing coordinate.""" - self.var_info.coordinates["lat"].stored_direction = "decreasing" - checker = CMORCheck(self.cube, self.var_info, automatic_fixes=True) - checker.check_metadata() - - # TODO: remove in v2.12 - def test_non_decreasing_automatic_fix_data(self): - """Automatic fix for decreasing coordinate.""" - self.var_info.coordinates["lat"].stored_direction = "decreasing" - checker = CMORCheck(self.cube, self.var_info, automatic_fixes=True) - checker.check_data() - def test_lat_non_monotonic(self): """Test fail for non monotonic latitude.""" lat = self.cube.coord("latitude") @@ -1258,11 +1235,5 @@ def test_get_cmor_checker_invalid_project_fail(): _get_cmor_checker("INVALID_PROJECT", "mip", "short_name", "frequency") -def test_deprecate_automatic_fixes(): - """Test deprecation of automatic_fixes.""" - with pytest.warns(ESMValCoreDeprecationWarning): - CMORCheck("cube", "var_info", "frequency", automatic_fixes=True) - - if __name__ == "__main__": unittest.main() diff --git a/tests/unit/cmor/test_fix.py b/tests/unit/cmor/test_fix.py index 01038d2786..8c005f1400 100644 --- a/tests/unit/cmor/test_fix.py +++ b/tests/unit/cmor/test_fix.py @@ -117,12 +117,9 @@ class TestFixMetadata: def setUp(self): """Prepare for testing.""" self.cube = self._create_mock_cube() - self.intermediate_cube = self._create_mock_cube() self.fixed_cube = self._create_mock_cube() self.mock_fix = Mock() - self.mock_fix.fix_metadata.return_value = [self.intermediate_cube] - self.checker = Mock() - self.check_metadata = self.checker.return_value.check_metadata + self.mock_fix.fix_metadata.return_value = [self.fixed_cube] self.expected_get_fixes_call = { "project": "project", "dataset": "model", @@ -148,81 +145,58 @@ def _create_mock_cube(var_name="short_name"): def test_fix(self): """Check that the returned fix is applied.""" - self.check_metadata.side_effect = lambda: self.fixed_cube with patch( "esmvalcore.cmor._fixes.fix.Fix.get_fixes", return_value=[self.mock_fix], ) as mock_get_fixes: - with patch( - "esmvalcore.cmor.fix._get_cmor_checker", - return_value=self.checker, - ): - cube_returned = fix_metadata( - cubes=[self.cube], - short_name="short_name", - project="project", - dataset="model", - mip="mip", - frequency="frequency", - session=sentinel.session, - )[0] - self.checker.assert_called_once_with(self.intermediate_cube) - self.check_metadata.assert_called_once_with() - assert cube_returned is not self.cube - assert cube_returned is not self.intermediate_cube - assert cube_returned is self.fixed_cube - mock_get_fixes.assert_called_once_with( - **self.expected_get_fixes_call - ) + cube_returned = fix_metadata( + cubes=[self.cube], + short_name="short_name", + project="project", + dataset="model", + mip="mip", + frequency="frequency", + session=sentinel.session, + )[0] + assert cube_returned is not self.cube + assert cube_returned is self.fixed_cube + mock_get_fixes.assert_called_once_with( + **self.expected_get_fixes_call + ) def test_nofix(self): """Check that the same cube is returned if no fix is available.""" - self.check_metadata.side_effect = lambda: self.cube with patch( "esmvalcore.cmor._fixes.fix.Fix.get_fixes", return_value=[] ) as mock_get_fixes: - with patch( - "esmvalcore.cmor.fix._get_cmor_checker", - return_value=self.checker, - ): - cube_returned = fix_metadata( - cubes=[self.cube], - short_name="short_name", - project="project", - dataset="model", - mip="mip", - frequency="frequency", - session=sentinel.session, - )[0] - self.checker.assert_called_once_with(self.cube) - self.check_metadata.assert_called_once_with() - assert cube_returned is self.cube - assert cube_returned is not self.intermediate_cube - assert cube_returned is not self.fixed_cube - mock_get_fixes.assert_called_once_with( - **self.expected_get_fixes_call - ) + cube_returned = fix_metadata( + cubes=[self.cube], + short_name="short_name", + project="project", + dataset="model", + mip="mip", + frequency="frequency", + session=sentinel.session, + )[0] + assert cube_returned is self.cube + assert cube_returned is not self.fixed_cube + mock_get_fixes.assert_called_once_with( + **self.expected_get_fixes_call + ) def test_select_var(self): """Check that the same cube is returned if no fix is available.""" - self.check_metadata.side_effect = lambda: self.cube with patch( "esmvalcore.cmor._fixes.fix.Fix.get_fixes", return_value=[] ): - with patch( - "esmvalcore.cmor.fix._get_cmor_checker", - return_value=self.checker, - ): - cube_returned = fix_metadata( - cubes=[self.cube, self._create_mock_cube("extra")], - short_name="short_name", - project="CMIP6", - dataset="model", - mip="mip", - )[0] - self.checker.assert_called_once_with(self.cube) - self.check_metadata.assert_called_once_with() - assert cube_returned is self.cube + cube_returned = fix_metadata( + cubes=[self.cube, self._create_mock_cube("extra")], + short_name="short_name", + project="CMIP6", + dataset="model", + mip="mip", + )[0] + assert cube_returned is self.cube def test_select_var_failed_if_bad_var_name(self): """Check that an error is raised if short_names do not match.""" @@ -247,12 +221,9 @@ class TestFixData: def setUp(self): """Prepare for testing.""" self.cube = Mock() - self.intermediate_cube = Mock() self.fixed_cube = Mock() self.mock_fix = Mock() - self.mock_fix.fix_data.return_value = self.intermediate_cube - self.checker = Mock() - self.check_data = self.checker.return_value.check_data + self.mock_fix.fix_data.return_value = self.fixed_cube self.expected_get_fixes_call = { "project": "project", "dataset": "model", @@ -271,57 +242,41 @@ def setUp(self): def test_fix(self): """Check that the returned fix is applied.""" - self.check_data.side_effect = lambda: self.fixed_cube with patch( "esmvalcore.cmor._fixes.fix.Fix.get_fixes", return_value=[self.mock_fix], ) as mock_get_fixes: - with patch( - "esmvalcore.cmor.fix._get_cmor_checker", - return_value=self.checker, - ): - cube_returned = fix_data( - self.cube, - short_name="short_name", - project="project", - dataset="model", - mip="mip", - frequency="frequency", - session=sentinel.session, - ) - self.checker.assert_called_once_with(self.intermediate_cube) - self.check_data.assert_called_once_with() - assert cube_returned is not self.cube - assert cube_returned is not self.intermediate_cube - assert cube_returned is self.fixed_cube - mock_get_fixes.assert_called_once_with( - **self.expected_get_fixes_call - ) + cube_returned = fix_data( + self.cube, + short_name="short_name", + project="project", + dataset="model", + mip="mip", + frequency="frequency", + session=sentinel.session, + ) + assert cube_returned is not self.cube + assert cube_returned is self.fixed_cube + mock_get_fixes.assert_called_once_with( + **self.expected_get_fixes_call + ) def test_nofix(self): """Check that the same cube is returned if no fix is available.""" - self.check_data.side_effect = lambda: self.cube with patch( "esmvalcore.cmor._fixes.fix.Fix.get_fixes", return_value=[] ) as mock_get_fixes: - with patch( - "esmvalcore.cmor.fix._get_cmor_checker", - return_value=self.checker, - ): - cube_returned = fix_data( - self.cube, - short_name="short_name", - project="project", - dataset="model", - mip="mip", - frequency="frequency", - session=sentinel.session, - ) - self.checker.assert_called_once_with(self.cube) - self.check_data.assert_called_once_with() - assert cube_returned is self.cube - assert cube_returned is not self.intermediate_cube - assert cube_returned is not self.fixed_cube - mock_get_fixes.assert_called_once_with( - **self.expected_get_fixes_call - ) + cube_returned = fix_data( + self.cube, + short_name="short_name", + project="project", + dataset="model", + mip="mip", + frequency="frequency", + session=sentinel.session, + ) + assert cube_returned is self.cube + assert cube_returned is not self.fixed_cube + mock_get_fixes.assert_called_once_with( + **self.expected_get_fixes_call + ) diff --git a/tests/unit/test_dataset.py b/tests/unit/test_dataset.py index 8408c622b9..1348dc0ebf 100644 --- a/tests/unit/test_dataset.py +++ b/tests/unit/test_dataset.py @@ -1735,7 +1735,6 @@ def mock_preprocess( "timerange": "2000/2005", }, "fix_metadata": { - "check_level": CheckLevels.DEFAULT, "session": session, "dataset": "CanESM2", "ensemble": "r1i1p1", @@ -1757,7 +1756,6 @@ def mock_preprocess( "timerange": "2000/2005", }, "fix_data": { - "check_level": CheckLevels.DEFAULT, "session": session, "dataset": "CanESM2", "ensemble": "r1i1p1",