diff --git a/esmvalcore/_recipe/check.py b/esmvalcore/_recipe/check.py index bb935cbc0a..a2f8e68978 100644 --- a/esmvalcore/_recipe/check.py +++ b/esmvalcore/_recipe/check.py @@ -36,6 +36,7 @@ from pathlib import Path from esmvalcore._task import TaskSet + from esmvalcore.config import Session from esmvalcore.dataset import Dataset from esmvalcore.typing import Facets @@ -43,7 +44,7 @@ logger = logging.getLogger(__name__) -def align_metadata(step_settings: dict[str, Any]) -> None: +def align_metadata(step_settings: dict[str, Any], session: Session) -> None: """Check settings of preprocessor ``align_metadata``.""" project = step_settings.get("target_project") mip = step_settings.get("target_mip") @@ -61,6 +62,7 @@ def align_metadata(step_settings: dict[str, Any]) -> None: mip, short_name, branding_suffix=branding_suffix, + session=session, ) except ValueError as exc: if strict: diff --git a/esmvalcore/_recipe/recipe.py b/esmvalcore/_recipe/recipe.py index 7d8e15b1c4..b0c4e026ea 100644 --- a/esmvalcore/_recipe/recipe.py +++ b/esmvalcore/_recipe/recipe.py @@ -146,6 +146,7 @@ def _update_target_levels( settings["extract_levels"]["levels"] = get_cmor_levels( levels["cmor_table"], levels["coordinate"], + session=dataset.session, ) elif "dataset" in levels: dataset_name = levels["dataset"] @@ -600,7 +601,7 @@ def _update_align_metadata( "target_short_name", dataset.facets["short_name"], ) - check.align_metadata(settings["align_metadata"]) + check.align_metadata(settings["align_metadata"], dataset.session) def _update_extract_shape( diff --git a/esmvalcore/cmor/_fixes/fix.py b/esmvalcore/cmor/_fixes/fix.py index 3bfd524e69..dde5f4352f 100644 --- a/esmvalcore/cmor/_fixes/fix.py +++ b/esmvalcore/cmor/_fixes/fix.py @@ -7,6 +7,7 @@ import inspect import logging import tempfile +import warnings from pathlib import Path from typing import TYPE_CHECKING, Any @@ -26,7 +27,7 @@ _get_single_cube, ) from esmvalcore.cmor.fixes import get_time_bounds -from esmvalcore.cmor.table import get_var_info +from esmvalcore.cmor.table import get_tables from esmvalcore.iris_helpers import ( has_unstructured_grid, safe_convert_units, @@ -51,8 +52,9 @@ class Fix: def __init__( self, vardef: VariableInfo, + *, + session: Session, extra_facets: dict | None = None, - session: Session | None = None, frequency: str | None = None, ) -> None: """Initialize fix object. @@ -61,11 +63,11 @@ def __init__( ---------- vardef: CMOR table entry of the variable. - extra_facets: - Extra facets. For details, see :ref:`config-extra-facets`. session: Current session which includes configuration and directory information. + extra_facets: + Extra facets. For details, see :ref:`config-extra-facets`. frequency: Expected frequency of the variable. If not given, use the one from the CMOR table entry of the variable. @@ -210,7 +212,7 @@ def get_fixes( extra_facets: dict | None = None, session: Session | None = None, frequency: str | None = None, - ) -> list: + ) -> list[Fix]: """Get the fixes that must be applied for a given dataset. It will look for them at the module @@ -247,15 +249,25 @@ def get_fixes( Returns ------- - list[Fix] + : Fixes to apply for the given data. """ + if session is None: + warnings.warn( + "Not providing a `session` argument or using `session=None` " + "is deprecated and will no longer be supported in v2.17.0.", + DeprecationWarning, + stacklevel=2, + ) + from esmvalcore.config import CFG # noqa: PLC0415 + + session = CFG.start_session("fix") + if extra_facets is None: extra_facets = {} - vardef = get_var_info( - project, + vardef = get_tables(session, project).get_variable( mip, short_name, branding_suffix=extra_facets.get("branding_suffix"), @@ -372,7 +384,10 @@ def fix_metadata(self, cubes: Sequence[Cube]) -> CubeList: Fixed cubes. """ - # Make sure the this fix also works when no extra_facets are given + # Make sure the this fix also works when no extra_facets are given. + # Note that this never happens in practice because `"project"` and + # `"dataset"` are inserted into `extra_facets` in + # `esmvalcore.cmor.fix.fix_*`. if "project" in self.extra_facets and "dataset" in self.extra_facets: dataset_str = ( f"{self.extra_facets['project']}:" @@ -598,7 +613,8 @@ def _fix_alternative_generic_level_coords(self, cube: Cube) -> Cube: _get_alternative_generic_lev_coord( cube, coord_name, - self.vardef.table_type, + project=self.extra_facets["project"], + session=self.session, ) ) except ValueError: # no alternatives found diff --git a/esmvalcore/cmor/_utils.py b/esmvalcore/cmor/_utils.py index e2d33b762d..1d96927eba 100644 --- a/esmvalcore/cmor/_utils.py +++ b/esmvalcore/cmor/_utils.py @@ -5,7 +5,7 @@ import logging from typing import TYPE_CHECKING -import esmvalcore.cmor.table +from esmvalcore.cmor.table import get_tables if TYPE_CHECKING: from collections.abc import Sequence @@ -14,17 +14,18 @@ from iris.cube import Cube from esmvalcore.cmor.table import CoordinateInfo, VariableInfo + from esmvalcore.config import Session logger = logging.getLogger(__name__) _ALTERNATIVE_GENERIC_LEV_COORDS = { "alevel": { - "CMIP5": ["alt40", "plevs"], - "CMIP6": ["alt16", "plev3"], - "obs4MIPs": ["alt16", "plev3"], + "CMIP5Info": ["alt40", "plevs"], + "CMIP6Info": ["alt16", "plev3"], + "Obs4MIPsInfo": ["alt16", "plev3"], }, "zlevel": { - "CMIP3": ["pressure"], + "CMIP3Info": ["pressure"], }, } @@ -32,7 +33,8 @@ def _get_alternative_generic_lev_coord( cube: Cube, coord_name: str, - cmor_table_type: str, + project: str, + session: Session, ) -> tuple[CoordinateInfo, Coord]: """Find alternative generic level coordinate in cube. @@ -42,10 +44,10 @@ def _get_alternative_generic_lev_coord( Cube to be checked. coord_name: Name of the generic level coordinate. - cmor_table_type: - CMOR table type, e.g., CMIP3, CMIP5, CMIP6. Note: This is NOT the - project of the dataset, but rather the entry `cmor_type` in - `config-developer.yml`. + project: + Project that the dataset belongs to. + session: + The session to use. Returns ------- @@ -63,14 +65,16 @@ def _get_alternative_generic_lev_coord( coord_name, {}, ) - allowed_alternatives = alternatives_for_coord.get(cmor_table_type, []) + tables = get_tables(session, project) + allowed_alternatives = alternatives_for_coord.get( + tables.__class__.__name__, + [], + ) # Check if any of the allowed alternative coordinates is present in the # cube for allowed_alternative in allowed_alternatives: - cmor_coord = esmvalcore.cmor.table.CMOR_TABLES[cmor_table_type].coords[ - allowed_alternative - ] + cmor_coord = tables.coords[allowed_alternative] if cube.coords(var_name=cmor_coord.out_name): cube_coord = cube.coord(var_name=cmor_coord.out_name) return (cmor_coord, cube_coord) diff --git a/esmvalcore/cmor/check.py b/esmvalcore/cmor/check.py index 046a4bb03d..c2613a91db 100644 --- a/esmvalcore/cmor/check.py +++ b/esmvalcore/cmor/check.py @@ -3,6 +3,7 @@ from __future__ import annotations import logging +import warnings from enum import IntEnum from functools import cached_property from typing import TYPE_CHECKING, NamedTuple @@ -22,7 +23,7 @@ _get_new_generic_level_coord, _get_simplified_calendar, ) -from esmvalcore.cmor.table import get_var_info +from esmvalcore.cmor.table import get_tables from esmvalcore.iris_helpers import has_unstructured_grid if TYPE_CHECKING: @@ -32,6 +33,7 @@ from iris.cube import Cube from esmvalcore.cmor.table import CoordinateInfo + from esmvalcore.config import Session class CheckLevels(IntEnum): @@ -947,14 +949,24 @@ def _get_cmor_checker( mip: str, short_name: str, *, + session: Session | None, branding_suffix: str | None = None, frequency: None | str = None, - fail_on_error: bool = False, check_level: CheckLevels = CheckLevels.DEFAULT, ) -> Callable[[Cube], CMORCheck]: """Get a CMOR checker.""" - var_info = get_var_info( - project, + if session is None: + warnings.warn( + "Not providing a `session` argument or using `session=None` " + "is deprecated and will no longer be supported in v2.17.0.", + DeprecationWarning, + stacklevel=2, + ) + from esmvalcore.config import CFG # noqa: PLC0415 + + session = CFG.start_session("cmor_check") + + var_info = get_tables(session, project).get_variable( mip, short_name, branding_suffix=branding_suffix, @@ -965,19 +977,19 @@ def _checker(cube: Cube) -> CMORCheck: cube, var_info, frequency=frequency, - fail_on_error=fail_on_error, check_level=check_level, ) return _checker -def cmor_check_metadata( +def cmor_check_metadata( # noqa: PLR0913 cube: Cube, cmor_table: str, mip: str, short_name: str, *, + session: Session | None = None, branding_suffix: str | None = None, frequency: str | None = None, check_level: CheckLevels = CheckLevels.DEFAULT, @@ -996,6 +1008,8 @@ def cmor_check_metadata( Variable's MIP. short_name: Variable's short name. + session: + The session to use. branding_suffix: A suffix that will be appended to ``short_name`` when looking up the variable in the CMOR table. Used by the CMIP7 project. @@ -1005,6 +1019,11 @@ def cmor_check_metadata( check_level: Level of strictness of the checks. + .. deprecated: 2.15.0 + The ``check_level`` parameter is deprecated and will be removed in + version 2.17.0. Please set the desired strictness level using + ``session["check_level"]`` instead. + Returns ------- iris.cube.Cube @@ -1017,17 +1036,19 @@ def cmor_check_metadata( short_name, branding_suffix=branding_suffix, frequency=frequency, + session=session, check_level=check_level, ) return checker(cube).check_metadata() -def cmor_check_data( +def cmor_check_data( # noqa: PLR0913 cube: Cube, cmor_table: str, mip: str, short_name: str, *, + session: Session | None = None, branding_suffix: str | None = None, frequency: str | None = None, check_level: CheckLevels = CheckLevels.DEFAULT, @@ -1044,6 +1065,8 @@ def cmor_check_data( Variable's MIP. short_name: Variable's short name + session: + The session to use. branding_suffix: A suffix that will be appended to ``short_name`` when looking up the variable in the CMOR table. Used by the CMIP7 project. @@ -1053,6 +1076,11 @@ def cmor_check_data( check_level: Level of strictness of the checks. + .. deprecated: 2.15.0 + The ``check_level`` parameter is deprecated and will be removed in + version 2.17.0. Please set the desired strictness level using + ``session["check_level"]`` instead. + Returns ------- iris.cube.Cube @@ -1065,17 +1093,19 @@ def cmor_check_data( short_name, branding_suffix=branding_suffix, frequency=frequency, + session=session, check_level=check_level, ) return checker(cube).check_data() -def cmor_check( +def cmor_check( # noqa: PLR0913 cube: Cube, cmor_table: str, mip: str, short_name: str, *, + session: Session | None = None, branding_suffix: str | None = None, frequency: str | None = None, check_level: CheckLevels = CheckLevels.DEFAULT, @@ -1095,6 +1125,8 @@ def cmor_check( Variable's MIP. short_name: Variable's short name. + session: + The session to use. branding_suffix: A suffix that will be appended to ``short_name`` when looking up the variable in the CMOR table. Used by the CMIP7 project. @@ -1104,6 +1136,11 @@ def cmor_check( check_level: Level of strictness of the checks. + .. deprecated: 2.15.0 + The ``check_level`` parameter is deprecated and will be removed in + version 2.17.0. Please set the desired strictness level using + ``session["check_level"]`` instead. + Returns ------- iris.cube.Cube @@ -1117,6 +1154,7 @@ def cmor_check( short_name, branding_suffix=branding_suffix, frequency=frequency, + session=session, check_level=check_level, ) return cmor_check_data( @@ -1126,5 +1164,6 @@ def cmor_check( short_name, branding_suffix=branding_suffix, frequency=frequency, + session=session, check_level=check_level, ) diff --git a/esmvalcore/cmor/fix.py b/esmvalcore/cmor/fix.py index ef23022846..aab3502dda 100644 --- a/esmvalcore/cmor/fix.py +++ b/esmvalcore/cmor/fix.py @@ -99,11 +99,12 @@ def fix_file( # noqa: PLR0913 session=session, frequency=frequency, ): - result = fix.fix_file( - result, - output_dir, - add_unique_suffix=add_unique_suffix, - ) + if isinstance(result, Path): + result = fix.fix_file( + result, + output_dir, + add_unique_suffix=add_unique_suffix, + ) if isinstance(file, LocalFile): # This happens when this function is called from diff --git a/esmvalcore/cmor/table.py b/esmvalcore/cmor/table.py index 6db325fe80..337aca7c3c 100644 --- a/esmvalcore/cmor/table.py +++ b/esmvalcore/cmor/table.py @@ -156,6 +156,12 @@ def get_var_info( ) -> VariableInfo | None: """Get variable information. + .. deprecated:: 2.15.0 + The ``get_var_info`` function is deprecated and will be removed in + ESMValCore v2.17.0. Please use :func:`~esmvalcore.cmor.table.get_tables` + to retrieve the tables for a project and :meth:`InfoBase.get_variable` + to retrieve a variable from a table. + Note ---- If `project=CORDEX` and the `mip` ends with 'hr', it is cropped to 'h' @@ -211,7 +217,7 @@ def read_cmor_tables(cfg_developer: Path | None = None) -> None: The config-developer.yml file based configuration is deprecated and will no longer be supported in ESMValCore v2.16.0. Please use - :func:`~esmvalcore.cmor.table.load_cmor_tables` instead of this function. + :func:`~esmvalcore.cmor.table.get_tables` instead of this function. Parameters ---------- diff --git a/esmvalcore/dataset.py b/esmvalcore/dataset.py index 597a081c62..4557fa9636 100644 --- a/esmvalcore/dataset.py +++ b/esmvalcore/dataset.py @@ -829,10 +829,8 @@ def _load(self) -> Cube: **self.facets, } settings["concatenate"] = {"check_level": self.session["check_level"]} - if cmor_tables_available: settings["cmor_check_metadata"] = { - "check_level": self.session["check_level"], "cmor_table": self.facets["project"], "mip": self.facets["mip"], "frequency": self.facets["frequency"], @@ -849,7 +847,6 @@ def _load(self) -> Cube: } if cmor_tables_available: settings["cmor_check_data"] = { - "check_level": self.session["check_level"], "cmor_table": self.facets["project"], "mip": self.facets["mip"], "frequency": self.facets["frequency"], diff --git a/esmvalcore/preprocessor/_other.py b/esmvalcore/preprocessor/_other.py index 7dac3d14c7..3de99e2d43 100644 --- a/esmvalcore/preprocessor/_other.py +++ b/esmvalcore/preprocessor/_other.py @@ -14,7 +14,7 @@ from iris.cube import Cube from iris.exceptions import CoordinateMultiDimError -from esmvalcore.cmor.table import get_var_info +from esmvalcore.cmor.table import get_tables from esmvalcore.iris_helpers import ( ignore_iris_vague_metadata_warnings, rechunk_cube, @@ -34,6 +34,7 @@ from iris.coords import Coord from esmvalcore.cmor.table import VariableInfo + from esmvalcore.config import Session logger = logging.getLogger(__name__) @@ -43,6 +44,8 @@ def align_metadata( target_project: str, target_mip: str, target_short_name: str, + *, + session: Session, # TODO: Add something to pass the session in when using this from the recipe. target_branding_suffix: str | None = None, strict: bool = True, ) -> Cube: @@ -65,6 +68,8 @@ def align_metadata( MIP table from which target metadata is read. target_short_name: Variable short name from which target metadata is read. + session: + The session to use. target_branding_suffix: Branding suffix from which target metadata is read. strict: @@ -90,10 +95,11 @@ def align_metadata( try: var_info = _get_var_info( - target_project, - target_mip, - target_short_name, + project=target_project, + mip=target_mip, + short_name=target_short_name, branding_suffix=target_branding_suffix, + session=session, ) except ValueError as exc: if strict: @@ -114,12 +120,12 @@ def _get_var_info( mip: str, short_name: str, branding_suffix: str | None, + session: Session, ) -> VariableInfo: """Get variable information.""" - var_info = get_var_info( - project, - mip, - short_name, + var_info = get_tables(session, project).get_variable( + table_name=mip, + short_name=short_name, branding_suffix=branding_suffix, ) if var_info is None: diff --git a/esmvalcore/preprocessor/_regrid.py b/esmvalcore/preprocessor/_regrid.py index 49c8343ee6..50d6faa9b2 100644 --- a/esmvalcore/preprocessor/_regrid.py +++ b/esmvalcore/preprocessor/_regrid.py @@ -29,11 +29,11 @@ from iris.cube import Cube from iris.util import broadcast_to_shape -import esmvalcore.cmor.table from esmvalcore.cmor._fixes.shared import ( add_altitude_from_plev, add_plev_from_altitude, ) +from esmvalcore.cmor.table import get_tables from esmvalcore.iris_helpers import has_irregular_grid, has_unstructured_grid from esmvalcore.preprocessor._shared import ( _rechunk_aux_factory_dependencies, @@ -58,6 +58,7 @@ from iris.analysis import Regridder, RegriddingScheme from numpy.typing import ArrayLike + from esmvalcore.config import Session from esmvalcore.dataset import Dataset logger = logging.getLogger(__name__) @@ -1455,7 +1456,11 @@ def extract_levels( return result -def get_cmor_levels(cmor_table: str, coordinate: str) -> list[float]: +def get_cmor_levels( + cmor_table: str, + coordinate: str, + session: Session, +) -> list[float]: """Get level definition from a CMOR coordinate. Parameters @@ -1464,10 +1469,13 @@ def get_cmor_levels(cmor_table: str, coordinate: str) -> list[float]: CMOR table name coordinate: CMOR coordinate name + session: + The session to use. Returns ------- - list[float] + : + A list of levels. Raises ------ @@ -1475,15 +1483,13 @@ def get_cmor_levels(cmor_table: str, coordinate: str) -> list[float]: If the CMOR table is not defined, the coordinate does not specify any levels or the string is badly formatted. """ - if cmor_table not in esmvalcore.cmor.table.CMOR_TABLES: - msg = f"Level definition cmor_table '{cmor_table}' not available" - raise ValueError(msg) + cmor_tables = get_tables(session, project=cmor_table) - if coordinate not in esmvalcore.cmor.table.CMOR_TABLES[cmor_table].coords: + if coordinate not in cmor_tables.coords: msg = f"Coordinate {coordinate} not available for {cmor_table}" raise ValueError(msg) - cmor = esmvalcore.cmor.table.CMOR_TABLES[cmor_table].coords[coordinate] + cmor = cmor_tables.coords[coordinate] if cmor.requested: return [float(level) for level in cmor.requested] diff --git a/tests/integration/cmor/_fixes/cordex/test_cordex_fixes.py b/tests/integration/cmor/_fixes/cordex/test_cordex_fixes.py index 590b61e7ca..764f776c46 100644 --- a/tests/integration/cmor/_fixes/cordex/test_cordex_fixes.py +++ b/tests/integration/cmor/_fixes/cordex/test_cordex_fixes.py @@ -22,6 +22,8 @@ if TYPE_CHECKING: import pytest_mock + from esmvalcore.config import Session + @pytest.fixture def cubes(): @@ -145,10 +147,11 @@ def test_mohchadrem3ga705_fix_metadata(cubes, coord, var_name, long_name): def test_mohchadrem3ga705_fix_metadata_no_time_coord( cubes: iris.cube.CubeList, + session: Session, ) -> None: for cube in cubes: cube.remove_coord("time") - fix = MOHCHadREM3GA705(None) # type: ignore[arg-type] + fix = MOHCHadREM3GA705(None, session=session) # type: ignore[arg-type] out_cubes = fix.fix_metadata(cubes) assert cubes is out_cubes for cube in out_cubes: @@ -167,6 +170,7 @@ def test_timelongname_fix_metadata(cubes): @pytest.mark.parametrize("has_time_coord", [True, False]) def test_clmcomcclm4817_fix_metadata_time( cubes: iris.cube.CubeList, + session: Session, has_time_coord: bool, ) -> None: if has_time_coord: @@ -189,7 +193,7 @@ def test_clmcomcclm4817_fix_metadata_time( lat.guess_bounds() lat.bounds = lat.core_bounds().astype(">f4", casting="same_kind") - fix = CLMcomCCLM4817(None) # type: ignore[arg-type] + fix = CLMcomCCLM4817(None, session=session) # type: ignore[arg-type] out_cubes = fix.fix_metadata(cubes) assert cubes is out_cubes for cube in out_cubes: