From b5c72035ec8f6f4d8b7dc5494fdc20910259b560 Mon Sep 17 00:00:00 2001 From: Bouwe Andela Date: Thu, 2 Jul 2026 15:37:27 +0200 Subject: [PATCH 01/10] Add a converter for writing recipes as YAML --- esmvalcore/_recipe/writer.py | 305 ++++++++++++++++++ esmvalcore/experimental/recipe.py | 24 +- .../experimental/test_run_recipe.py | 17 + 3 files changed, 342 insertions(+), 4 deletions(-) create mode 100644 esmvalcore/_recipe/writer.py diff --git a/esmvalcore/_recipe/writer.py b/esmvalcore/_recipe/writer.py new file mode 100644 index 0000000000..50add0ddea --- /dev/null +++ b/esmvalcore/_recipe/writer.py @@ -0,0 +1,305 @@ +"""Write ESMValTool recipes to YAML files.""" + +from __future__ import annotations + +from functools import total_ordering +from pathlib import Path +from typing import TYPE_CHECKING + +import yaml + +if TYPE_CHECKING: + import os + + +class RecipeDumper(yaml.SafeDumper): + """Custom YAML dumper for recipes.""" + + def increase_indent( + self, + flow=False, + indentless=False, # noqa: ARG002 + ): + """Increase indentation level.""" + # Increase the indentation level for lists that are values in a mapping. + return super().increase_indent(flow, False) + + +class Recipe(dict): + """Recipe.""" + + +RecipeDumper.add_representer( + Recipe, + lambda dumper, data: dumper.represent_mapping( + "tag:yaml.org,2002:map", + { + k: data[k] + for k in [ + "documentation", + "preprocessors", + "datasets", + "diagnostics", + ] + if k in data + }.items(), + flow_style=False, + ), +) + + +class RecipeDocumentation(dict): + """Recipe documentation section.""" + + +def strip(txt: str) -> str: + """Strip leading and trailing whitespace from each line in a multi-line string.""" + return "\n".join(line.strip() for line in txt.splitlines()) + + +RecipeDumper.add_representer( + RecipeDocumentation, + lambda dumper, data: dumper.represent_mapping( + "tag:yaml.org,2002:map", + { + k: strip(data[k]) if k == "description" else data[k] + for k in [ + "title", + "description", + "authors", + "maintainer", + ] + if k in data + }.items(), + flow_style=False, + ), +) + + +class RecipePreprocessor(dict): + """Preprocessor steps.""" + + +RecipeDumper.add_representer( + RecipePreprocessor, + lambda dumper, data: dumper.represent_mapping( + "tag:yaml.org,2002:map", + data.items(), + flow_style=False, + ), +) + + +class RecipeDatasetList(list): + """List of datasets.""" + + +RecipeDumper.add_representer( + RecipeDatasetList, + lambda dumper, data: dumper.represent_sequence( + "tag:yaml.org,2002:seq", + sorted( + data, + key=lambda x: (x.get("project", ""), x.items()), + ), + flow_style=False, + ), +) + + +class RecipeDataset(dict): + """Dataset entry.""" + + +RecipeDumper.add_representer( + RecipeDataset, + lambda dumper, data: dumper.represent_mapping( + "tag:yaml.org,2002:map", + { + k: data[k] + for k in sorted( + data, + key=lambda x: ( + { + "start_year": 1, + "end_year": 2, + "supplementary_variables": 3, + }.get(x, 0), + x, + ), + ) + }.items(), + flow_style=True, + ), +) + + +class RecipeVariable(dict): + """Variable entry.""" + + +RecipeDumper.add_representer( + RecipeVariable, + lambda dumper, data: dumper.represent_mapping( + "tag:yaml.org,2002:map", + { + k: data[k] + for k in sorted( + data, + key=lambda x: ( + { + "start_year": 1, + "end_year": 2, + "additional_datasets": 3, + }.get(x, 0), + x, + ), + ) + }.items(), + flow_style=False, + ), +) + + +class RecipeDiagnostic(dict): + """Diagnostic entry.""" + + +RecipeDumper.add_representer( + RecipeDiagnostic, + lambda dumper, data: dumper.represent_mapping( + "tag:yaml.org,2002:map", + { + k: data[k] + for k in sorted( + data, + key=lambda x: ( + { + "description": 0, + "realms": 1, + "themes": 2, + "variables": 3, + "additional_datasets": 4, + "scripts": 5, + }.get(x, 6), + x, + ), + ) + }.items(), + flow_style=False, + ), +) + + +class RecipeScript(dict): + """Diagnostic script entry.""" + + +RecipeDumper.add_representer( + RecipeScript, + lambda dumper, data: dumper.represent_mapping( + "tag:yaml.org,2002:map", + { + k: data[k] for k in sorted(data, key=lambda x: (x != "script", x)) + }.items(), + flow_style=False, + ), +) + + +def convert_to_recipe_objects(recipe: dict) -> Recipe: + """Convert the recipe dictionary to Recipe objects for YAML representation.""" + recipe = Recipe(recipe) + if "documentation" in recipe: + recipe["documentation"] = RecipeDocumentation(recipe["documentation"]) + if "datasets" in recipe: + recipe["datasets"] = RecipeDatasetList( + RecipeDataset(d) for d in recipe["datasets"] + ) + if "preprocessors" in recipe: + recipe["preprocessors"] = { + k: RecipePreprocessor(v) + for k, v in recipe["preprocessors"].items() + } + for diagnostic_name, diagnostic in recipe.get("diagnostics", {}).items(): + if "additional_datasets" in diagnostic: + diagnostic["additional_datasets"] = RecipeDatasetList( + RecipeDataset(d) for d in diagnostic["additional_datasets"] + ) + for variable_group, variable in diagnostic.get( + "variables", + {}, + ).items(): + if variable is None: + continue + if "additional_datasets" in variable: + variable["additional_datasets"] = RecipeDatasetList( + RecipeDataset(d) for d in variable["additional_datasets"] + ) + recipe["diagnostics"][diagnostic_name]["variables"][ + variable_group + ] = RecipeVariable(variable) + if scripts := diagnostic.get("scripts"): + for script_name, script in scripts.items(): + scripts[script_name] = RecipeScript(script) + recipe["diagnostics"][diagnostic_name] = RecipeDiagnostic(diagnostic) + return recipe + + +def add_blank_lines_between_top_level_items(yaml_text: str) -> str: + """Insert blank lines between top-level YAML sections.""" + lines = yaml_text.splitlines() + if not lines: + return yaml_text + + top_level_keys = { + "documentation", + "preprocessors", + "datasets", + "diagnostics", + } + formatted_lines = [lines[0]] + + for line in lines[1:]: + for key in top_level_keys: + if line and line.startswith(key): + formatted_lines.append("") + break + formatted_lines.append(line) + + return "\n".join(formatted_lines) + + +def to_yaml( + recipe: dict, + file: os.PathLike | None = None, +) -> str: + """Convert a recipe to YAML format. + + Parameters + ---------- + recipe: + The recipe to write. + file: + If provided, the recipe will be written to this file. + + Returns + ------- + : + The YAML representation of the recipe. + """ + recipe = convert_to_recipe_objects(recipe) + txt = yaml.dump( + recipe, + Dumper=RecipeDumper, + allow_unicode=True, + width=200, + ) + txt = add_blank_lines_between_top_level_items(txt) + + if file: + recipe_file = Path(file) + recipe_file.parent.mkdir(parents=True, exist_ok=True) + recipe_file.write_text(txt, encoding="utf-8") + + return txt diff --git a/esmvalcore/experimental/recipe.py b/esmvalcore/experimental/recipe.py index 1da67e9ce1..93cb0e8fba 100644 --- a/esmvalcore/experimental/recipe.py +++ b/esmvalcore/experimental/recipe.py @@ -11,11 +11,11 @@ import yaml from esmvalcore._recipe.recipe import Recipe as RecipeEngine +from esmvalcore._recipe.writer import to_yaml from esmvalcore.config import CFG - -from ._logging import log_to_dir -from .recipe_info import RecipeInfo -from .recipe_output import RecipeOutput +from esmvalcore.experimental._logging import log_to_dir +from esmvalcore.experimental.recipe_info import RecipeInfo +from esmvalcore.experimental.recipe_output import RecipeOutput if TYPE_CHECKING: import os @@ -59,6 +59,22 @@ def _repr_html_(self) -> str: """Return html representation.""" return self.render() + def to_yaml(self, file: os.PathLike | None = None) -> str | None: + """Write recipe to a YAML file. + + Parameters + ---------- + file : + A path-like object to write the YAML data to. + + Returns + ------- + : + The YAML representation of the recipe as a string. + + """ + return to_yaml(self.data, file=file) + def render(self, template=None): """Render output as html. diff --git a/tests/sample_data/experimental/test_run_recipe.py b/tests/sample_data/experimental/test_run_recipe.py index 03e290ea21..caed22240b 100644 --- a/tests/sample_data/experimental/test_run_recipe.py +++ b/tests/sample_data/experimental/test_run_recipe.py @@ -10,6 +10,7 @@ import iris import pytest +import yaml import esmvalcore._task from esmvalcore.config._diagnostics import TAGS @@ -138,3 +139,19 @@ def test_run_recipe_diagnostic_failing(monkeypatch, recipe, tmp_path): task = "example/non-existent" with pytest.raises(RecipeError): _ = recipe.run(task, session) + + +def test_recipe_to_yaml(recipe: Recipe, tmp_path: Path) -> None: + """Test writing a recipe to YAML.""" + recipe_file = tmp_path / "recipe.yml" + recipe_yaml = recipe.to_yaml(file=recipe_file) + print() + print(recipe_yaml) + assert isinstance(recipe_yaml, str) + assert recipe_file.exists() + recipe_dict = yaml.safe_load(recipe_yaml) + assert isinstance(recipe_dict, dict) + assert recipe_dict == recipe.data + assert ( + yaml.safe_load(recipe_file.read_text(encoding="utf-8")) == recipe.data + ) From 90c090c3b56248477d13bdcdb4dfbfbcd528212c Mon Sep 17 00:00:00 2001 From: Bouwe Andela Date: Thu, 2 Jul 2026 16:26:41 +0200 Subject: [PATCH 02/10] Avoid losing items --- esmvalcore/_recipe/writer.py | 62 +++++++++++++++++++++++++----------- 1 file changed, 43 insertions(+), 19 deletions(-) diff --git a/esmvalcore/_recipe/writer.py b/esmvalcore/_recipe/writer.py index 50add0ddea..49578f22a6 100644 --- a/esmvalcore/_recipe/writer.py +++ b/esmvalcore/_recipe/writer.py @@ -2,7 +2,6 @@ from __future__ import annotations -from functools import total_ordering from pathlib import Path from typing import TYPE_CHECKING @@ -35,13 +34,18 @@ class Recipe(dict): "tag:yaml.org,2002:map", { k: data[k] - for k in [ - "documentation", - "preprocessors", - "datasets", - "diagnostics", - ] - if k in data + for k in sorted( + data, + key=lambda x: ( + { + "documentation": 0, + "preprocessors": 2, + "datasets": 3, + "diagnostics": 4, + }.get(x, 1), + x, + ), + ) }.items(), flow_style=False, ), @@ -63,13 +67,18 @@ def strip(txt: str) -> str: "tag:yaml.org,2002:map", { k: strip(data[k]) if k == "description" else data[k] - for k in [ - "title", - "description", - "authors", - "maintainer", - ] - if k in data + for k in sorted( + data, + key=lambda x: ( + { + "title": 0, + "description": 1, + "authors": 2, + "maintainer": 3, + }.get(x, 4), + x, + ), + ) }.items(), flow_style=False, ), @@ -84,7 +93,7 @@ class RecipePreprocessor(dict): RecipePreprocessor, lambda dumper, data: dumper.represent_mapping( "tag:yaml.org,2002:map", - data.items(), + data.items(), # Prevents sorting. flow_style=False, ), ) @@ -100,7 +109,10 @@ class RecipeDatasetList(list): "tag:yaml.org,2002:seq", sorted( data, - key=lambda x: (x.get("project", ""), x.items()), + key=lambda x: ( + x.get("project", ""), + tuple((k, str(v)) for k, v in x.items()), + ), ), flow_style=False, ), @@ -207,8 +219,20 @@ class RecipeScript(dict): ) -def convert_to_recipe_objects(recipe: dict) -> Recipe: - """Convert the recipe dictionary to Recipe objects for YAML representation.""" +def convert_to_recipe_objects(recipe: dict) -> Recipe: # noqa: C901 + """Convert the recipe dictionary to Recipe objects for YAML representation. + + Parameters + ---------- + recipe: + The recipe to convert. + + Returns + ------- + : + The recipe with relevant elements replaced by Recipe* classes used + to tell the YAML dumper how to write the recipe. + """ recipe = Recipe(recipe) if "documentation" in recipe: recipe["documentation"] = RecipeDocumentation(recipe["documentation"]) From 47a3bf583b7bb915c306a5311fc2b5b11b4516e0 Mon Sep 17 00:00:00 2001 From: Bouwe Andela Date: Thu, 2 Jul 2026 21:18:45 +0200 Subject: [PATCH 03/10] Improve dataset sorting --- esmvalcore/_recipe/writer.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/esmvalcore/_recipe/writer.py b/esmvalcore/_recipe/writer.py index 49578f22a6..9b961cd444 100644 --- a/esmvalcore/_recipe/writer.py +++ b/esmvalcore/_recipe/writer.py @@ -2,6 +2,7 @@ from __future__ import annotations +import re from pathlib import Path from typing import TYPE_CHECKING @@ -111,7 +112,15 @@ class RecipeDatasetList(list): data, key=lambda x: ( x.get("project", ""), - tuple((k, str(v)) for k, v in x.items()), + tuple( + ( + k, + [int(i) if i else 0 for i in re.split(r"\D", x[k])] + if k == "ensemble" and isinstance(x[k], str) + else str(x[k]), + ) + for k in sorted(x) + ), ), ), flow_style=False, From 7c92ed49c2e44bb1e062f44b5d74ba7488f81217 Mon Sep 17 00:00:00 2001 From: Bouwe Andela Date: Thu, 9 Jul 2026 17:34:37 +0200 Subject: [PATCH 04/10] Integrate with writing filled recipe --- esmvalcore/_recipe/recipe.py | 19 +++++++++---------- esmvalcore/_recipe/writer.py | 2 +- 2 files changed, 10 insertions(+), 11 deletions(-) diff --git a/esmvalcore/_recipe/recipe.py b/esmvalcore/_recipe/recipe.py index f15e61e92a..d56833c5c7 100644 --- a/esmvalcore/_recipe/recipe.py +++ b/esmvalcore/_recipe/recipe.py @@ -17,6 +17,14 @@ import esmvalcore.io.esgf from esmvalcore import __version__ from esmvalcore._provenance import get_recipe_provenance +from esmvalcore._recipe import check +from esmvalcore._recipe.from_datasets import datasets_to_recipe +from esmvalcore._recipe.to_datasets import ( + _derive_needed, + _get_input_datasets, + _representative_datasets, +) +from esmvalcore._recipe.writer import to_yaml from esmvalcore._task import DiagnosticTask, ResumeTask, TaskSet from esmvalcore.config._config import TASKSEP from esmvalcore.config._dask import validate_dask_config @@ -50,14 +58,6 @@ ) from esmvalcore.preprocessor._shared import _group_products -from . import check -from .from_datasets import datasets_to_recipe -from .to_datasets import ( - _derive_needed, - _get_input_datasets, - _representative_datasets, -) - if TYPE_CHECKING: from collections.abc import Iterable, Sequence @@ -1377,8 +1377,7 @@ def write_filled_recipe(self) -> Path: """Write copy of recipe with filled wildcards.""" recipe = datasets_to_recipe(USED_DATASETS, self._raw_recipe) filename = self.session.run_dir / f"{self._filename.stem}_filled.yml" - with filename.open("w", encoding="utf-8") as file: - yaml.safe_dump(recipe, file, sort_keys=False) + to_yaml(recipe, filename) logger.info( "Wrote recipe with version numbers and wildcards to:\nfile://%s", filename, diff --git a/esmvalcore/_recipe/writer.py b/esmvalcore/_recipe/writer.py index 9b961cd444..25be963a99 100644 --- a/esmvalcore/_recipe/writer.py +++ b/esmvalcore/_recipe/writer.py @@ -333,6 +333,6 @@ def to_yaml( if file: recipe_file = Path(file) recipe_file.parent.mkdir(parents=True, exist_ok=True) - recipe_file.write_text(txt, encoding="utf-8") + recipe_file.write_text(f"{txt}\n", encoding="utf-8") return txt From 37cda20ca92f63a0ab3664dad6f36c6190f740cb Mon Sep 17 00:00:00 2001 From: Bouwe Andela Date: Fri, 10 Jul 2026 16:38:48 +0200 Subject: [PATCH 05/10] Add tests --- esmvalcore/_recipe/writer.py | 16 ++- pixi.lock | 122 ++++++++++++------ pyproject.toml | 1 + tests/unit/recipe/writer/__init__.py | 0 .../recipe/writer/recipes/recipe_test1.yml | 108 ++++++++++++++++ .../recipe/writer/recipes/recipe_test2.yml | 38 ++++++ .../recipe/writer/recipes/recipe_test3.yml | 27 ++++ .../writer/reference_recipes/recipe_test1.yml | 85 ++++++++++++ .../writer/reference_recipes/recipe_test2.yml | 30 +++++ .../writer/reference_recipes/recipe_test3.yml | 26 ++++ tests/unit/recipe/writer/test_writer.py | 48 +++++++ 11 files changed, 459 insertions(+), 42 deletions(-) create mode 100644 tests/unit/recipe/writer/__init__.py create mode 100644 tests/unit/recipe/writer/recipes/recipe_test1.yml create mode 100644 tests/unit/recipe/writer/recipes/recipe_test2.yml create mode 100644 tests/unit/recipe/writer/recipes/recipe_test3.yml create mode 100644 tests/unit/recipe/writer/reference_recipes/recipe_test1.yml create mode 100644 tests/unit/recipe/writer/reference_recipes/recipe_test2.yml create mode 100644 tests/unit/recipe/writer/reference_recipes/recipe_test3.yml create mode 100644 tests/unit/recipe/writer/test_writer.py diff --git a/esmvalcore/_recipe/writer.py b/esmvalcore/_recipe/writer.py index 25be963a99..2bb0970474 100644 --- a/esmvalcore/_recipe/writer.py +++ b/esmvalcore/_recipe/writer.py @@ -199,8 +199,8 @@ class RecipeDiagnostic(dict): "description": 0, "realms": 1, "themes": 2, - "variables": 3, - "additional_datasets": 4, + "additional_datasets": 3, + "variables": 4, "scripts": 5, }.get(x, 6), x, @@ -221,7 +221,17 @@ class RecipeScript(dict): lambda dumper, data: dumper.represent_mapping( "tag:yaml.org,2002:map", { - k: data[k] for k in sorted(data, key=lambda x: (x != "script", x)) + k: data[k] + for k in sorted( + data, + key=lambda x: ( + { + "ancestors": 0, + "script": 1, + }.get(x, 2), + x, + ), + ) }.items(), flow_style=False, ), diff --git a/pixi.lock b/pixi.lock index 24329fb9b5..d01aad9b21 100644 --- a/pixi.lock +++ b/pixi.lock @@ -456,10 +456,12 @@ environments: - conda: https://conda.anaconda.org/conda-forge/noarch/pystac-client-0.9.0-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/pytest-9.1.1-pyhc364b38_2.conda - conda: https://conda.anaconda.org/conda-forge/noarch/pytest-cov-7.1.0-pyhcf101f3_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/pytest-datadir-1.8.0-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/pytest-env-1.6.0-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/pytest-html-4.2.0-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/pytest-metadata-3.1.1-pyhd8ed1ab_1.conda - conda: https://conda.anaconda.org/conda-forge/noarch/pytest-mock-3.15.1-pyhd8ed1ab_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/pytest-regressions-2.11.0-pyhc455866_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/pytest-xdist-3.8.0-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/python-dateutil-2.9.0.post0-pyhe01879c_2.conda - conda: https://conda.anaconda.org/conda-forge/noarch/python-discovery-1.4.2-pyhcf101f3_0.conda @@ -717,10 +719,12 @@ environments: - conda: https://conda.anaconda.org/conda-forge/noarch/pystac-client-0.9.0-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/pytest-9.1.1-pyhc364b38_2.conda - conda: https://conda.anaconda.org/conda-forge/noarch/pytest-cov-7.1.0-pyhcf101f3_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/pytest-datadir-1.8.0-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/pytest-env-1.6.0-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/pytest-html-4.2.0-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/pytest-metadata-3.1.1-pyhd8ed1ab_1.conda - conda: https://conda.anaconda.org/conda-forge/noarch/pytest-mock-3.15.1-pyhd8ed1ab_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/pytest-regressions-2.11.0-pyhc455866_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/pytest-xdist-3.8.0-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/python-dateutil-2.9.0.post0-pyhe01879c_2.conda - conda: https://conda.anaconda.org/conda-forge/noarch/python-discovery-1.4.2-pyhcf101f3_0.conda @@ -2145,10 +2149,12 @@ environments: - conda: https://conda.anaconda.org/conda-forge/noarch/pystac-client-0.9.0-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/pytest-9.1.1-pyhc364b38_2.conda - conda: https://conda.anaconda.org/conda-forge/noarch/pytest-cov-7.1.0-pyhcf101f3_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/pytest-datadir-1.8.0-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/pytest-env-1.6.0-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/pytest-html-4.2.0-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/pytest-metadata-3.1.1-pyhd8ed1ab_1.conda - conda: https://conda.anaconda.org/conda-forge/noarch/pytest-mock-3.15.1-pyhd8ed1ab_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/pytest-regressions-2.11.0-pyhc455866_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/pytest-xdist-3.8.0-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/python-dateutil-2.9.0.post0-pyhe01879c_2.conda - conda: https://conda.anaconda.org/conda-forge/noarch/python-dotenv-1.2.2-pyhcf101f3_0.conda @@ -2286,10 +2292,12 @@ environments: - conda: https://conda.anaconda.org/conda-forge/noarch/pystac-client-0.9.0-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/pytest-9.1.1-pyhc364b38_2.conda - conda: https://conda.anaconda.org/conda-forge/noarch/pytest-cov-7.1.0-pyhcf101f3_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/pytest-datadir-1.8.0-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/pytest-env-1.6.0-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/pytest-html-4.2.0-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/pytest-metadata-3.1.1-pyhd8ed1ab_1.conda - conda: https://conda.anaconda.org/conda-forge/noarch/pytest-mock-3.15.1-pyhd8ed1ab_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/pytest-regressions-2.11.0-pyhc455866_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/pytest-xdist-3.8.0-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/python-dateutil-2.9.0.post0-pyhe01879c_2.conda - conda: https://conda.anaconda.org/conda-forge/noarch/python-dotenv-1.2.2-pyhcf101f3_0.conda @@ -2885,10 +2893,12 @@ environments: - conda: https://conda.anaconda.org/conda-forge/noarch/pystac-client-0.9.0-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/pytest-9.1.1-pyhc364b38_2.conda - conda: https://conda.anaconda.org/conda-forge/noarch/pytest-cov-7.1.0-pyhcf101f3_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/pytest-datadir-1.8.0-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/pytest-env-1.6.0-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/pytest-html-4.2.0-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/pytest-metadata-3.1.1-pyhd8ed1ab_1.conda - conda: https://conda.anaconda.org/conda-forge/noarch/pytest-mock-3.15.1-pyhd8ed1ab_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/pytest-regressions-2.11.0-pyhc455866_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/pytest-xdist-3.8.0-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/python-dateutil-2.9.0.post0-pyhe01879c_2.conda - conda: https://conda.anaconda.org/conda-forge/noarch/python-dotenv-1.2.2-pyhcf101f3_0.conda @@ -3025,10 +3035,12 @@ environments: - conda: https://conda.anaconda.org/conda-forge/noarch/pystac-client-0.9.0-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/pytest-9.1.1-pyhc364b38_2.conda - conda: https://conda.anaconda.org/conda-forge/noarch/pytest-cov-7.1.0-pyhcf101f3_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/pytest-datadir-1.8.0-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/pytest-env-1.6.0-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/pytest-html-4.2.0-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/pytest-metadata-3.1.1-pyhd8ed1ab_1.conda - conda: https://conda.anaconda.org/conda-forge/noarch/pytest-mock-3.15.1-pyhd8ed1ab_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/pytest-regressions-2.11.0-pyhc455866_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/pytest-xdist-3.8.0-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/python-dateutil-2.9.0.post0-pyhe01879c_2.conda - conda: https://conda.anaconda.org/conda-forge/noarch/python-dotenv-1.2.2-pyhcf101f3_0.conda @@ -3646,10 +3658,12 @@ environments: - conda: https://conda.anaconda.org/conda-forge/noarch/pystac-client-0.9.0-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/pytest-9.1.1-pyhc364b38_2.conda - conda: https://conda.anaconda.org/conda-forge/noarch/pytest-cov-7.1.0-pyhcf101f3_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/pytest-datadir-1.8.0-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/pytest-env-1.6.0-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/pytest-html-4.2.0-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/pytest-metadata-3.1.1-pyhd8ed1ab_1.conda - conda: https://conda.anaconda.org/conda-forge/noarch/pytest-mock-3.15.1-pyhd8ed1ab_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/pytest-regressions-2.11.0-pyhc455866_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/pytest-xdist-3.8.0-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/python-dateutil-2.9.0.post0-pyhe01879c_2.conda - conda: https://conda.anaconda.org/conda-forge/noarch/python-dotenv-1.2.2-pyhcf101f3_0.conda @@ -3823,10 +3837,12 @@ environments: - conda: https://conda.anaconda.org/conda-forge/noarch/pystac-client-0.9.0-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/pytest-9.1.1-pyhc364b38_2.conda - conda: https://conda.anaconda.org/conda-forge/noarch/pytest-cov-7.1.0-pyhcf101f3_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/pytest-datadir-1.8.0-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/pytest-env-1.6.0-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/pytest-html-4.2.0-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/pytest-metadata-3.1.1-pyhd8ed1ab_1.conda - conda: https://conda.anaconda.org/conda-forge/noarch/pytest-mock-3.15.1-pyhd8ed1ab_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/pytest-regressions-2.11.0-pyhc455866_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/pytest-xdist-3.8.0-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/python-dateutil-2.9.0.post0-pyhe01879c_2.conda - conda: https://conda.anaconda.org/conda-forge/noarch/python-dotenv-1.2.2-pyhcf101f3_0.conda @@ -4921,7 +4937,7 @@ packages: license: MIT AND Apache-2.0 license_family: Apache purls: - - pkg:pypi/aiohttp?source=compressed-mapping + - pkg:pypi/aiohttp?source=hash-mapping run_exports: {} size: 1078273 timestamp: 1780913823270 @@ -4944,7 +4960,7 @@ packages: license: MIT AND Apache-2.0 license_family: Apache purls: - - pkg:pypi/aiohttp?source=hash-mapping + - pkg:pypi/aiohttp?source=compressed-mapping run_exports: {} size: 1083744 timestamp: 1780914191006 @@ -5375,7 +5391,7 @@ packages: - zstd >=1.5.7,<1.6.0a0 license: BSD-3-Clause AND MIT AND EPL-2.0 purls: - - pkg:pypi/backports-zstd?source=compressed-mapping + - pkg:pypi/backports-zstd?source=hash-mapping run_exports: {} size: 242845 timestamp: 1781450812380 @@ -6329,7 +6345,7 @@ packages: license: MIT license_family: MIT purls: - - pkg:pypi/fonttools?source=compressed-mapping + - pkg:pypi/fonttools?source=hash-mapping run_exports: {} size: 3007892 timestamp: 1778770568019 @@ -6430,7 +6446,7 @@ packages: license: Apache-2.0 license_family: APACHE purls: - - pkg:pypi/frozenlist?source=hash-mapping + - pkg:pypi/frozenlist?source=compressed-mapping run_exports: {} size: 55016 timestamp: 1779999817627 @@ -9006,7 +9022,7 @@ packages: license: Apache-2.0 license_family: APACHE purls: - - pkg:pypi/msgpack?source=compressed-mapping + - pkg:pypi/msgpack?source=hash-mapping run_exports: {} size: 112800 timestamp: 1782460774570 @@ -9022,7 +9038,7 @@ packages: license: Apache-2.0 license_family: APACHE purls: - - pkg:pypi/msgpack?source=compressed-mapping + - pkg:pypi/msgpack?source=hash-mapping run_exports: {} size: 112798 timestamp: 1782460772873 @@ -9143,7 +9159,7 @@ packages: license: MIT license_family: MIT purls: - - pkg:pypi/netcdf4?source=compressed-mapping + - pkg:pypi/netcdf4?source=hash-mapping run_exports: {} size: 1094073 timestamp: 1782151628345 @@ -9261,7 +9277,7 @@ packages: license: BSD-3-Clause license_family: BSD purls: - - pkg:pypi/numpy?source=compressed-mapping + - pkg:pypi/numpy?source=hash-mapping run_exports: weak: - numpy >=1.25,<3 @@ -9400,7 +9416,7 @@ packages: license: BSD-3-Clause license_family: BSD purls: - - pkg:pypi/pandas?source=hash-mapping + - pkg:pypi/pandas?source=compressed-mapping run_exports: {} size: 14872605 timestamp: 1778602625175 @@ -9458,7 +9474,7 @@ packages: license: BSD-3-Clause license_family: BSD purls: - - pkg:pypi/pandas?source=compressed-mapping + - pkg:pypi/pandas?source=hash-mapping run_exports: {} size: 15001668 timestamp: 1778602610159 @@ -9744,7 +9760,7 @@ packages: license: Apache-2.0 license_family: APACHE purls: - - pkg:pypi/propcache?source=compressed-mapping + - pkg:pypi/propcache?source=hash-mapping run_exports: {} size: 51586 timestamp: 1780037816755 @@ -10778,7 +10794,7 @@ packages: license: Apache-2.0 license_family: Apache purls: - - pkg:pypi/tornado?source=compressed-mapping + - pkg:pypi/tornado?source=hash-mapping run_exports: {} size: 864705 timestamp: 1781006801632 @@ -11746,7 +11762,7 @@ packages: license: MIT license_family: MIT purls: - - pkg:pypi/beautifulsoup4?source=hash-mapping + - pkg:pypi/beautifulsoup4?source=compressed-mapping run_exports: {} size: 92704 timestamp: 1780853175566 @@ -11761,7 +11777,7 @@ packages: - tinycss2 >=1.1.0,<1.5 license: Apache-2.0 AND MIT purls: - - pkg:pypi/bleach?source=compressed-mapping + - pkg:pypi/bleach?source=hash-mapping run_exports: {} size: 142246 timestamp: 1780675823953 @@ -11807,7 +11823,7 @@ packages: license: BSD-3-Clause license_family: BSD purls: - - pkg:pypi/bokeh?source=compressed-mapping + - pkg:pypi/bokeh?source=hash-mapping run_exports: {} size: 4526315 timestamp: 1781002115296 @@ -12195,7 +12211,7 @@ packages: license: BSD-3-Clause license_family: BSD purls: - - pkg:pypi/distributed?source=compressed-mapping + - pkg:pypi/distributed?source=hash-mapping run_exports: {} size: 838296 timestamp: 1781563082788 @@ -12327,7 +12343,7 @@ packages: license: Apache-2.0 license_family: APACHE purls: - - pkg:pypi/fastcore?source=compressed-mapping + - pkg:pypi/fastcore?source=hash-mapping run_exports: {} size: 104111 timestamp: 1782729278094 @@ -12510,7 +12526,7 @@ packages: license: Apache-2.0 license_family: APACHE purls: - - pkg:pypi/frozenlist?source=compressed-mapping + - pkg:pypi/frozenlist?source=hash-mapping run_exports: {} size: 19878 timestamp: 1779999782801 @@ -13215,7 +13231,7 @@ packages: license: BSD-3-Clause license_family: BSD purls: - - pkg:pypi/jupyter-client?source=compressed-mapping + - pkg:pypi/jupyter-client?source=hash-mapping run_exports: {} size: 117954 timestamp: 1781019994076 @@ -13286,7 +13302,7 @@ packages: license: BSD-3-Clause license_family: BSD purls: - - pkg:pypi/jupyter-server?source=compressed-mapping + - pkg:pypi/jupyter-server?source=hash-mapping run_exports: {} size: 363068 timestamp: 1781713810089 @@ -13540,7 +13556,7 @@ packages: license: MIT license_family: MIT purls: - - pkg:pypi/mdit-py-plugins?source=compressed-mapping + - pkg:pypi/mdit-py-plugins?source=hash-mapping run_exports: {} size: 50460 timestamp: 1778692223625 @@ -13676,7 +13692,7 @@ packages: license: MIT license_family: MIT purls: - - pkg:pypi/narwhals?source=compressed-mapping + - pkg:pypi/narwhals?source=hash-mapping run_exports: {} size: 285532 timestamp: 1780672242196 @@ -13957,7 +13973,7 @@ packages: license: MIT license_family: MIT purls: - - pkg:pypi/pip?source=compressed-mapping + - pkg:pypi/pip?source=hash-mapping run_exports: {} size: 1203173 timestamp: 1780262795392 @@ -14387,6 +14403,19 @@ packages: run_exports: {} size: 29559 timestamp: 1774139250481 +- conda: https://conda.anaconda.org/conda-forge/noarch/pytest-datadir-1.8.0-pyhd8ed1ab_0.conda + sha256: 9bb02764223157d4b58f7b193f8cdaf82f36a1ade1cd7df611e09e824ece43d8 + md5: e46da0323c9a50ef1b53c20d33e243a0 + depends: + - pytest >=7.0 + - python >=3.9 + license: MIT + license_family: MIT + purls: + - pkg:pypi/pytest-datadir?source=hash-mapping + run_exports: {} + size: 12519 + timestamp: 1753915123443 - conda: https://conda.anaconda.org/conda-forge/noarch/pytest-env-1.6.0-pyhd8ed1ab_0.conda sha256: d288b55b224a1d6a7eca5ca0ac0e026169ffa6458e64dcb990f62f99e37b8010 md5: 27f9775b96f64454633682cc6a0b0768 @@ -14443,6 +14472,21 @@ packages: run_exports: {} size: 22968 timestamp: 1758101248317 +- conda: https://conda.anaconda.org/conda-forge/noarch/pytest-regressions-2.11.0-pyhc455866_0.conda + sha256: ba4413898fc01903ef1291a18842689345364b2d79eaa315851b829754d38818 + md5: b73569e55629580f6d277fe57cc3c9e3 + depends: + - pytest >=6.2.0 + - pytest-datadir >=1.7.0 + - python >=3.10 + - pyyaml + license: MIT + license_family: MIT + purls: + - pkg:pypi/pytest-regressions?source=hash-mapping + run_exports: {} + size: 78330 + timestamp: 1779792061225 - conda: https://conda.anaconda.org/conda-forge/noarch/pytest-xdist-3.8.0-pyhd8ed1ab_0.conda sha256: b7b58a5be090883198411337b99afb6404127809c3d1c9f96e99b59f36177a96 md5: 8375cfbda7c57fbceeda18229be10417 @@ -14484,7 +14528,7 @@ packages: license: MIT license_family: MIT purls: - - pkg:pypi/python-discovery?source=compressed-mapping + - pkg:pypi/python-discovery?source=hash-mapping run_exports: {} size: 35514 timestamp: 1781257630962 @@ -15005,7 +15049,7 @@ packages: license: BSD-2-Clause license_family: BSD purls: - - pkg:pypi/sphinxcontrib-serializinghtml?source=compressed-mapping + - pkg:pypi/sphinxcontrib-serializinghtml?source=hash-mapping run_exports: {} size: 30640 timestamp: 1781260357443 @@ -15035,7 +15079,7 @@ packages: license: BSD-3-Clause license_family: BSD purls: - - pkg:pypi/starlette?source=compressed-mapping + - pkg:pypi/starlette?source=hash-mapping run_exports: {} size: 64264 timestamp: 1781447071524 @@ -15285,7 +15329,7 @@ packages: license: BSD-3-Clause license_family: BSD purls: - - pkg:pypi/uvicorn?source=hash-mapping + - pkg:pypi/uvicorn?source=compressed-mapping run_exports: {} size: 56228 timestamp: 1780574319325 @@ -15316,7 +15360,7 @@ packages: license: MIT license_family: MIT purls: - - pkg:pypi/virtualenv?source=compressed-mapping + - pkg:pypi/virtualenv?source=hash-mapping run_exports: {} size: 3111990 timestamp: 1781651033074 @@ -15555,7 +15599,7 @@ packages: license: MIT AND Apache-2.0 license_family: Apache purls: - - pkg:pypi/aiohttp?source=compressed-mapping + - pkg:pypi/aiohttp?source=hash-mapping run_exports: {} size: 1049144 timestamp: 1780913992605 @@ -15954,7 +15998,7 @@ packages: - python_abi 3.12.* *_cp312 license: BSD-3-Clause AND MIT AND EPL-2.0 purls: - - pkg:pypi/backports-zstd?source=compressed-mapping + - pkg:pypi/backports-zstd?source=hash-mapping run_exports: {} size: 240925 timestamp: 1781450816363 @@ -16785,7 +16829,7 @@ packages: license: Apache-2.0 license_family: APACHE purls: - - pkg:pypi/cython?source=compressed-mapping + - pkg:pypi/cython?source=hash-mapping run_exports: {} size: 3453779 timestamp: 1782821623606 @@ -19664,7 +19708,7 @@ packages: license: BSD-3-Clause license_family: BSD purls: - - pkg:pypi/numpy?source=compressed-mapping + - pkg:pypi/numpy?source=hash-mapping run_exports: weak: - numpy >=1.25,<3 @@ -19708,7 +19752,7 @@ packages: license: BSD-3-Clause license_family: BSD purls: - - pkg:pypi/numpy?source=compressed-mapping + - pkg:pypi/numpy?source=hash-mapping run_exports: weak: - numpy >=1.25,<3 @@ -20937,7 +20981,7 @@ packages: license: MIT license_family: MIT purls: - - pkg:pypi/rpds-py?source=compressed-mapping + - pkg:pypi/rpds-py?source=hash-mapping run_exports: {} size: 285627 timestamp: 1782831308617 @@ -20960,7 +21004,7 @@ packages: license: BSD-3-Clause license_family: BSD purls: - - pkg:pypi/scipy?source=compressed-mapping + - pkg:pypi/scipy?source=hash-mapping run_exports: {} size: 14165071 timestamp: 1781912652942 @@ -21217,7 +21261,7 @@ packages: license: Apache-2.0 license_family: Apache purls: - - pkg:pypi/tornado?source=compressed-mapping + - pkg:pypi/tornado?source=hash-mapping run_exports: {} size: 915857 timestamp: 1781007345425 @@ -21354,7 +21398,7 @@ packages: license: BSD-2-Clause license_family: BSD purls: - - pkg:pypi/wrapt?source=compressed-mapping + - pkg:pypi/wrapt?source=hash-mapping run_exports: {} size: 112459 timestamp: 1782134073014 @@ -21369,7 +21413,7 @@ packages: license: BSD-2-Clause license_family: BSD purls: - - pkg:pypi/wrapt?source=compressed-mapping + - pkg:pypi/wrapt?source=hash-mapping run_exports: {} size: 113551 timestamp: 1782133966372 diff --git a/pyproject.toml b/pyproject.toml index d8bc48dc15..747b4eb998 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -278,6 +278,7 @@ test-r = { features = ["test-r"], solve-group = "test-r" } "pytest-html" = "!=2.1.0" "pytest-metadata" = ">=1.5.1" "pytest-mock" = "*" +"pytest-regressions" = "*" "pytest-xdist" = "*" # Test R support diff --git a/tests/unit/recipe/writer/__init__.py b/tests/unit/recipe/writer/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/unit/recipe/writer/recipes/recipe_test1.yml b/tests/unit/recipe/writer/recipes/recipe_test1.yml new file mode 100644 index 0000000000..a933fca548 --- /dev/null +++ b/tests/unit/recipe/writer/recipes/recipe_test1.yml @@ -0,0 +1,108 @@ +# ESMValTool +# recipe_python.yml +# +# See https://docs.esmvaltool.org/en/latest/recipes/recipe_examples.html +# for a description of this recipe. +# +# See https://docs.esmvaltool.org/projects/esmvalcore/en/latest/recipe/overview.html +# for a description of the recipe format. +--- +documentation: + description: | + Example recipe that plots a map and timeseries of temperature. + + title: Recipe that runs an example diagnostic written in Python. + + authors: + - andela_bouwe + - righi_mattia + + maintainer: + - schlund_manuel + + references: + - acknow_project + + projects: + - esmval + - c3s-magic + +datasets: + - {dataset: BCC-ESM1, project: CMIP6, exp: historical, ensemble: r1i1p1f1, grid: gn} + - {dataset: bcc-csm1-1, version: v1, project: CMIP5, exp: historical, ensemble: r1i1p1} + +preprocessors: + # See https://docs.esmvaltool.org/projects/esmvalcore/en/latest/recipe/preprocessor.html + # for a description of the preprocessor functions. + + to_degrees_c: + convert_units: + units: degrees_C + + annual_mean_amsterdam: + extract_location: + location: Amsterdam + scheme: linear + annual_statistics: + operator: mean + multi_model_statistics: + statistics: + - mean + span: overlap + convert_units: + units: degrees_C + + annual_mean_global: + area_statistics: + operator: mean + annual_statistics: + operator: mean + convert_units: + units: degrees_C + +diagnostics: + + map: + description: Global map of temperature in January 2000. + themes: + - phys + realms: + - atmos + variables: + tas: + mip: Amon + preprocessor: to_degrees_c + timerange: 2000/P1M + caption: >- + Global map of {long_name} in January 2000 according to {dataset}. + scripts: + script1: + script: examples/diagnostic.py + quickplot: + plot_type: pcolormesh + cmap: Reds + + timeseries: + description: Annual mean temperature in Amsterdam and global mean since 1850. + themes: + - phys + realms: + - atmos + variables: + tas_amsterdam: + short_name: tas + mip: Amon + preprocessor: annual_mean_amsterdam + timerange: 1850/2000 + caption: Annual mean {long_name} in Amsterdam according to {dataset}. + tas_global: + short_name: tas + mip: Amon + preprocessor: annual_mean_global + timerange: 1850/2000 + caption: Annual global mean {long_name} according to {dataset}. + scripts: + script1: + script: examples/diagnostic.py + quickplot: + plot_type: plot diff --git a/tests/unit/recipe/writer/recipes/recipe_test2.yml b/tests/unit/recipe/writer/recipes/recipe_test2.yml new file mode 100644 index 0000000000..3402718318 --- /dev/null +++ b/tests/unit/recipe/writer/recipes/recipe_test2.yml @@ -0,0 +1,38 @@ +documentation: + title: Test recipe with additional datasets. + description: Test recipe + authors: [andela_bouwe] + maintainer: [andela_bouwe] + references: [acknow_project] + +diagnostics: + + diagnostic: + variables: + tas: + mip: Amon + timerange: "2000/2010" + project: CMIP6 + additional_datasets: + - dataset: bcc-csm1-1 + project: CMIP5 + exp: historical + ensemble: r1i1p1 + supplementary_variables: [{short_name: sflft, mip: fx}] + tos: + mip: Omon + timerange: "2000/2010" + project: CMIP6 + + scripts: + script2: + script: diagnostic2.py + script1: + script: diagnostic1.py + + additional_datasets: + - dataset: BCC-ESM1 + project: CMIP6 + exp: historical + ensemble: r1i1p1f1 + grid: gn diff --git a/tests/unit/recipe/writer/recipes/recipe_test3.yml b/tests/unit/recipe/writer/recipes/recipe_test3.yml new file mode 100644 index 0000000000..a49e55fb8c --- /dev/null +++ b/tests/unit/recipe/writer/recipes/recipe_test3.yml @@ -0,0 +1,27 @@ +documentation: + title: Test recipe with without scripts. + description: Test recipe + authors: [andela_bouwe] + maintainer: [andela_bouwe] + references: [acknow_project] + +diagnostics: + + diagnostic1: + variables: + tas: + mip: Amon + timerange: "2000/2010" + project: CMIP6 + additional_datasets: + - dataset: bcc-csm1-1 + project: CMIP5 + exp: historical + ensemble: r1i1p1 + scripts: null + + diagnostic2: + scripts: + script2: + script: diagnostic.py + ancestors: [diagnostic1/tas] diff --git a/tests/unit/recipe/writer/reference_recipes/recipe_test1.yml b/tests/unit/recipe/writer/reference_recipes/recipe_test1.yml new file mode 100644 index 0000000000..a1df164c10 --- /dev/null +++ b/tests/unit/recipe/writer/reference_recipes/recipe_test1.yml @@ -0,0 +1,85 @@ +documentation: + title: Recipe that runs an example diagnostic written in Python. + description: Example recipe that plots a map and timeseries of temperature. + authors: + - andela_bouwe + - righi_mattia + maintainer: + - schlund_manuel + projects: + - esmval + - c3s-magic + references: + - acknow_project + +preprocessors: + annual_mean_amsterdam: + extract_location: + location: Amsterdam + scheme: linear + annual_statistics: + operator: mean + multi_model_statistics: + span: overlap + statistics: + - mean + convert_units: + units: degrees_C + annual_mean_global: + area_statistics: + operator: mean + annual_statistics: + operator: mean + convert_units: + units: degrees_C + to_degrees_c: + convert_units: + units: degrees_C + +datasets: + - {dataset: bcc-csm1-1, ensemble: r1i1p1, exp: historical, project: CMIP5, version: v1} + - {dataset: BCC-ESM1, ensemble: r1i1p1f1, exp: historical, grid: gn, project: CMIP6} + +diagnostics: + map: + description: Global map of temperature in January 2000. + realms: + - atmos + themes: + - phys + variables: + tas: + caption: Global map of {long_name} in January 2000 according to {dataset}. + mip: Amon + preprocessor: to_degrees_c + timerange: 2000/P1M + scripts: + script1: + script: examples/diagnostic.py + quickplot: + cmap: Reds + plot_type: pcolormesh + timeseries: + description: Annual mean temperature in Amsterdam and global mean since 1850. + realms: + - atmos + themes: + - phys + variables: + tas_amsterdam: + caption: Annual mean {long_name} in Amsterdam according to {dataset}. + mip: Amon + preprocessor: annual_mean_amsterdam + short_name: tas + timerange: 1850/2000 + tas_global: + caption: Annual global mean {long_name} according to {dataset}. + mip: Amon + preprocessor: annual_mean_global + short_name: tas + timerange: 1850/2000 + scripts: + script1: + script: examples/diagnostic.py + quickplot: + plot_type: plot diff --git a/tests/unit/recipe/writer/reference_recipes/recipe_test2.yml b/tests/unit/recipe/writer/reference_recipes/recipe_test2.yml new file mode 100644 index 0000000000..bc23a0f485 --- /dev/null +++ b/tests/unit/recipe/writer/reference_recipes/recipe_test2.yml @@ -0,0 +1,30 @@ +documentation: + title: Test recipe with additional datasets. + description: Test recipe + authors: + - andela_bouwe + maintainer: + - andela_bouwe + references: + - acknow_project + +diagnostics: + diagnostic: + additional_datasets: + - {dataset: BCC-ESM1, ensemble: r1i1p1f1, exp: historical, grid: gn, project: CMIP6} + variables: + tas: + mip: Amon + project: CMIP6 + timerange: 2000/2010 + additional_datasets: + - {dataset: bcc-csm1-1, ensemble: r1i1p1, exp: historical, project: CMIP5, supplementary_variables: [{mip: fx, short_name: sflft}]} + tos: + mip: Omon + project: CMIP6 + timerange: 2000/2010 + scripts: + script1: + script: diagnostic1.py + script2: + script: diagnostic2.py diff --git a/tests/unit/recipe/writer/reference_recipes/recipe_test3.yml b/tests/unit/recipe/writer/reference_recipes/recipe_test3.yml new file mode 100644 index 0000000000..a901e3da82 --- /dev/null +++ b/tests/unit/recipe/writer/reference_recipes/recipe_test3.yml @@ -0,0 +1,26 @@ +documentation: + title: Test recipe with without scripts. + description: Test recipe + authors: + - andela_bouwe + maintainer: + - andela_bouwe + references: + - acknow_project + +diagnostics: + diagnostic1: + variables: + tas: + mip: Amon + project: CMIP6 + timerange: 2000/2010 + additional_datasets: + - {dataset: bcc-csm1-1, ensemble: r1i1p1, exp: historical, project: CMIP5} + scripts: null + diagnostic2: + scripts: + script2: + ancestors: + - diagnostic1/tas + script: diagnostic.py diff --git a/tests/unit/recipe/writer/test_writer.py b/tests/unit/recipe/writer/test_writer.py new file mode 100644 index 0000000000..6d6911f59e --- /dev/null +++ b/tests/unit/recipe/writer/test_writer.py @@ -0,0 +1,48 @@ +"""Unit tests for the recipe writer.""" + +from __future__ import annotations + +from pathlib import Path +from typing import TYPE_CHECKING + +import pytest +import yaml + +from esmvalcore._recipe.writer import to_yaml + +if TYPE_CHECKING: + from pytest_regressions import FileRegressionFixture + + +@pytest.mark.parametrize( + "recipe", + list(Path(__file__).parent.glob("recipes/*.yml")), +) +def test_writer( + tmp_path: Path, + file_regression: FileRegressionFixture, + recipe: Path, +) -> None: + """Test that the recipe writer produces the expected output.""" + # Run pytest tests/unit/recipe/writer/test_writer.py --force-regen to + # regenerate the expected output files. + load_recipe = yaml.safe_load(recipe.read_text(encoding="utf-8")) + reference_data_path = Path(__file__).parent / "reference_recipes" + file_regression.check( + contents=f"{to_yaml(load_recipe)}\n", + encoding="utf-8", + fullpath=reference_data_path / recipe.name, + ) + + +def test_empty_recipe() -> None: + """Test that the code does not crash on an empty recipe.""" + recipe_text = to_yaml({}) + assert recipe_text == "{}" + + +def test_recipe_file_endswith_newline(tmp_path: Path) -> None: + """Test that the recipe ends with a newline.""" + filename = tmp_path / "recipe.yml" + to_yaml({}, file=filename) + assert filename.read_text(encoding="utf-8").endswith("\n") From 5a8a5a0c304baf1d5ead7538ac621297cac57b28 Mon Sep 17 00:00:00 2001 From: Bouwe Andela Date: Fri, 10 Jul 2026 17:22:48 +0200 Subject: [PATCH 06/10] Test empty variables too --- .../recipe/writer/recipes/recipe_test3.yml | 22 ++++++++++--------- .../writer/reference_recipes/recipe_test3.yml | 13 +++++------ 2 files changed, 18 insertions(+), 17 deletions(-) diff --git a/tests/unit/recipe/writer/recipes/recipe_test3.yml b/tests/unit/recipe/writer/recipes/recipe_test3.yml index a49e55fb8c..8f8e5cb48b 100644 --- a/tests/unit/recipe/writer/recipes/recipe_test3.yml +++ b/tests/unit/recipe/writer/recipes/recipe_test3.yml @@ -1,24 +1,26 @@ documentation: - title: Test recipe with without scripts. + title: Test recipe with without scripts and ancestors. description: Test recipe authors: [andela_bouwe] maintainer: [andela_bouwe] references: [acknow_project] +datasets: + - dataset: BCC-ESM1 + mip: Amon + timerange: "2000/2010" + project: CMIP6 + exp: historical + ensemble: r1i1p1f1 + grid: gn + diagnostics: diagnostic1: variables: tas: - mip: Amon - timerange: "2000/2010" - project: CMIP6 - additional_datasets: - - dataset: bcc-csm1-1 - project: CMIP5 - exp: historical - ensemble: r1i1p1 - scripts: null + pr: + scripts: diagnostic2: scripts: diff --git a/tests/unit/recipe/writer/reference_recipes/recipe_test3.yml b/tests/unit/recipe/writer/reference_recipes/recipe_test3.yml index a901e3da82..b24c8d6f6f 100644 --- a/tests/unit/recipe/writer/reference_recipes/recipe_test3.yml +++ b/tests/unit/recipe/writer/reference_recipes/recipe_test3.yml @@ -1,5 +1,5 @@ documentation: - title: Test recipe with without scripts. + title: Test recipe with without scripts and ancestors. description: Test recipe authors: - andela_bouwe @@ -8,15 +8,14 @@ documentation: references: - acknow_project +datasets: + - {dataset: BCC-ESM1, ensemble: r1i1p1f1, exp: historical, grid: gn, mip: Amon, project: CMIP6, timerange: 2000/2010} + diagnostics: diagnostic1: variables: - tas: - mip: Amon - project: CMIP6 - timerange: 2000/2010 - additional_datasets: - - {dataset: bcc-csm1-1, ensemble: r1i1p1, exp: historical, project: CMIP5} + pr: null + tas: null scripts: null diagnostic2: scripts: From 962377bf61ea713b8065a4ca69e1bd5ee2c47952 Mon Sep 17 00:00:00 2001 From: Bouwe Andela Date: Fri, 10 Jul 2026 17:27:16 +0200 Subject: [PATCH 07/10] Remove unnessary code --- esmvalcore/_recipe/writer.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/esmvalcore/_recipe/writer.py b/esmvalcore/_recipe/writer.py index 2bb0970474..96d0670669 100644 --- a/esmvalcore/_recipe/writer.py +++ b/esmvalcore/_recipe/writer.py @@ -292,8 +292,6 @@ def convert_to_recipe_objects(recipe: dict) -> Recipe: # noqa: C901 def add_blank_lines_between_top_level_items(yaml_text: str) -> str: """Insert blank lines between top-level YAML sections.""" lines = yaml_text.splitlines() - if not lines: - return yaml_text top_level_keys = { "documentation", From 85af1e6e8f1e646bdc9073fd8450c63b4ca58c9e Mon Sep 17 00:00:00 2001 From: Bouwe Andela Date: Fri, 10 Jul 2026 17:32:45 +0200 Subject: [PATCH 08/10] Add comment and a few more type hints --- esmvalcore/_recipe/writer.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/esmvalcore/_recipe/writer.py b/esmvalcore/_recipe/writer.py index 96d0670669..73826b447c 100644 --- a/esmvalcore/_recipe/writer.py +++ b/esmvalcore/_recipe/writer.py @@ -17,14 +17,18 @@ class RecipeDumper(yaml.SafeDumper): def increase_indent( self, - flow=False, - indentless=False, # noqa: ARG002 - ): + flow: bool = False, + indentless: bool = False, # noqa: ARG002 + ) -> None: """Increase indentation level.""" # Increase the indentation level for lists that are values in a mapping. return super().increase_indent(flow, False) +# The following classes are used to mark certain elements in the recipe so +# the YAML dumper recognizes them and knows how to format them. + + class Recipe(dict): """Recipe.""" From 4ef3e68c02e1e7103bb8a7beb26976b1d78b2cd3 Mon Sep 17 00:00:00 2001 From: Bouwe Andela Date: Fri, 10 Jul 2026 17:57:09 +0200 Subject: [PATCH 09/10] Update Jupyter notebook --- notebooks/composing-recipes.ipynb | 583 +++++++++--------------------- 1 file changed, 167 insertions(+), 416 deletions(-) diff --git a/notebooks/composing-recipes.ipynb b/notebooks/composing-recipes.ipynb index 5156618d88..8f9b945554 100644 --- a/notebooks/composing-recipes.ipynb +++ b/notebooks/composing-recipes.ipynb @@ -17,10 +17,13 @@ "metadata": {}, "outputs": [], "source": [ + "from pathlib import Path\n", + "\n", "import yaml\n", "\n", "from esmvalcore.config import CFG\n", - "from esmvalcore.dataset import Dataset, datasets_to_recipe" + "from esmvalcore.dataset import Dataset, datasets_to_recipe\n", + "from esmvalcore.experimental.recipe import Recipe" ] }, { @@ -39,7 +42,7 @@ "metadata": {}, "outputs": [], "source": [ - "CFG[\"search_esgf\"] = \"always\"" + "CFG[\"search_data\"] = \"complete\"" ] }, { @@ -118,7 +121,7 @@ { "data": { "text/plain": [ - "778" + "1011" ] }, "execution_count": 4, @@ -160,110 +163,6 @@ "output_type": "stream", "text": [ "datasets:\n", - "- dataset: TaiESM1\n", - " ensemble: r(1:2)i1p1f1\n", - " grid: gn\n", - " institute: AS-RCEC\n", - "- dataset: AWI-CM-1-1-MR\n", - " ensemble: r(1:5)i1p1f1\n", - " grid: gn\n", - " institute: AWI\n", - "- dataset: AWI-ESM-1-1-LR\n", - " ensemble: r1i1p1f1\n", - " grid: gn\n", - " institute: AWI\n", - "- dataset: BCC-CSM2-MR\n", - " ensemble: r(1:3)i1p1f1\n", - " grid: gn\n", - " institute: BCC\n", - "- dataset: BCC-ESM1\n", - " ensemble: r(1:3)i1p1f1\n", - " grid: gn\n", - " institute: BCC\n", - "- dataset: CAMS-CSM1-0\n", - " ensemble: r1i1p1f2\n", - " grid: gn\n", - " institute: CAMS\n", - "- dataset: CAMS-CSM1-0\n", - " ensemble: r(1:2)i1p1f1\n", - " grid: gn\n", - " institute: CAMS\n", - "- dataset: CAS-ESM2-0\n", - " ensemble: r(1:4)i1p1f1\n", - " grid: gn\n", - " institute: CAS\n", - "- dataset: FGOALS-f3-L\n", - " ensemble: r(1:3)i1p1f1\n", - " grid: gr\n", - " institute: CAS\n", - "- dataset: FGOALS-g3\n", - " ensemble: r(1:6)i1p1f1\n", - " grid: gn\n", - " institute: CAS\n", - "- dataset: IITM-ESM\n", - " ensemble: r1i1p1f1\n", - " grid: gn\n", - " institute: CCCR-IITM\n", - "- dataset: CanESM5-1\n", - " ensemble: r(1:20)i1p1f1\n", - " grid: gn\n", - " institute: CCCma\n", - "- dataset: CanESM5-1\n", - " ensemble: r(1:25)i1p2f1\n", - " grid: gn\n", - " institute: CCCma\n", - "- dataset: CanESM5-1\n", - " ensemble: r22i1p1f1\n", - " grid: gn\n", - " institute: CCCma\n", - "- dataset: CanESM5-1\n", - " ensemble: r(24:39)i1p1f1\n", - " grid: gn\n", - " institute: CCCma\n", - "- dataset: CanESM5-1\n", - " ensemble: r(41:50)i1p1f1\n", - " grid: gn\n", - " institute: CCCma\n", - "- dataset: CanESM5-CanOE\n", - " ensemble: r(1:3)i1p2f1\n", - " grid: gn\n", - " institute: CCCma\n", - "- dataset: CanESM5\n", - " ensemble: r(1:25)i1p1f1\n", - " grid: gn\n", - " institute: CCCma\n", - "- dataset: CanESM5\n", - " ensemble: r(1:40)i1p2f1\n", - " grid: gn\n", - " institute: CCCma\n", - "- dataset: CMCC-CM2-HR4\n", - " ensemble: r1i1p1f1\n", - " grid: gn\n", - " institute: CMCC\n", - "- dataset: CMCC-CM2-SR5\n", - " ensemble: r1i1p1f1\n", - " grid: gn\n", - " institute: CMCC\n", - "- dataset: CMCC-CM2-SR5\n", - " ensemble: r(2:11)i1p2f1\n", - " grid: gn\n", - " institute: CMCC\n", - "- dataset: CMCC-ESM2\n", - " ensemble: r1i1p1f1\n", - " grid: gn\n", - " institute: CMCC\n", - "- dataset: CNRM-CM6-1-HR\n", - " ensemble: r1i1p1f2\n", - " grid: gr\n", - " institute: CNRM-CERFACS\n", - "- dataset: CNRM-CM6-1\n", - " ensemble: r(1:30)i1p1f2\n", - " grid: gr\n", - " institute: CNRM-CERFACS\n", - "- dataset: CNRM-ESM2-1\n", - " ensemble: r(1:11)i1p1f2\n", - " grid: gr\n", - " institute: CNRM-CERFACS\n", "- dataset: ACCESS-CM2\n", " ensemble: r(1:10)i1p1f1\n", " grid: gn\n", @@ -272,327 +171,184 @@ " ensemble: r(1:40)i1p1f1\n", " grid: gn\n", " institute: CSIRO\n", - "- dataset: E3SM-1-0\n", - " ensemble: r(1:5)i1p1f1\n", - " grid: gr\n", - " institute: E3SM-Project\n", - "- dataset: E3SM-1-1-ECA\n", - " ensemble: r1i1p1f1\n", - " grid: gr\n", - " institute: E3SM-Project\n", - "- dataset: E3SM-1-1\n", - " ensemble: r1i1p1f1\n", - " grid: gr\n", - " institute: E3SM-Project\n", - "- dataset: E3SM-2-0\n", - " ensemble: r(1:5)i1p1f1\n", - " grid: gr\n", - " institute: E3SM-Project\n", - "- dataset: EC-Earth3-AerChem\n", - " ensemble: r1i1p1f1\n", - " grid: gr\n", - " institute: EC-Earth-Consortium\n", - "- dataset: EC-Earth3-AerChem\n", - " ensemble: r(3:4)i1p1f1\n", - " grid: gr\n", - " institute: EC-Earth-Consortium\n", - "- dataset: EC-Earth3-CC\n", - " ensemble: r1i1p1f1\n", - " grid: gr\n", - " institute: EC-Earth-Consortium\n", - "- dataset: EC-Earth3-CC\n", - " ensemble: r4i1p1f1\n", - " grid: gr\n", - " institute: EC-Earth-Consortium\n", - "- dataset: EC-Earth3-CC\n", - " ensemble: r(6:13)i1p1f1\n", - " grid: gr\n", - " institute: EC-Earth-Consortium\n", - "- dataset: EC-Earth3-Veg-LR\n", - " ensemble: r(1:3)i1p1f1\n", - " grid: gr\n", - " institute: EC-Earth-Consortium\n", - "- dataset: EC-Earth3-Veg\n", - " ensemble: r(1:6)i1p1f1\n", - " grid: gr\n", - " institute: EC-Earth-Consortium\n", - "- dataset: EC-Earth3-Veg\n", - " ensemble: r10i1p1f1\n", - " grid: gr\n", - " institute: EC-Earth-Consortium\n", - "- dataset: EC-Earth3-Veg\n", - " ensemble: r12i1p1f1\n", - " grid: gr\n", - " institute: EC-Earth-Consortium\n", - "- dataset: EC-Earth3-Veg\n", - " ensemble: r14i1p1f1\n", - " grid: gr\n", - " institute: EC-Earth-Consortium\n", - "- dataset: EC-Earth3\n", - " ensemble: r(1:7)i1p1f1\n", - " grid: gr\n", - " institute: EC-Earth-Consortium\n", - "- dataset: EC-Earth3\n", - " ensemble: r(9:25)i1p1f1\n", - " grid: gr\n", - " institute: EC-Earth-Consortium\n", - "- dataset: EC-Earth3\n", - " ensemble: r(101:150)i1p1f1\n", - " grid: gr\n", - " institute: EC-Earth-Consortium\n", - "- dataset: FIO-ESM-2-0\n", - " ensemble: r(1:3)i1p1f1\n", - " grid: gn\n", - " institute: FIO-QLNM\n", - "- dataset: MPI-ESM-1-2-HAM\n", - " ensemble: r(1:3)i1p1f1\n", - " grid: gn\n", - " institute: HAMMOZ-Consortium\n", - "- dataset: INM-CM4-8\n", - " ensemble: r1i1p1f1\n", - " grid: gr1\n", - " institute: INM\n", - "- dataset: INM-CM5-0\n", - " ensemble: r(1:10)i1p1f1\n", - " grid: gr1\n", - " institute: INM\n", - "- dataset: IPSL-CM5A2-INCA\n", - " ensemble: r1i1p1f1\n", - " grid: gr\n", - " institute: IPSL\n", - "- dataset: IPSL-CM6A-LR-INCA\n", - " ensemble: r1i1p1f1\n", - " grid: gr\n", - " institute: IPSL\n", - "- dataset: IPSL-CM6A-LR\n", - " ensemble: r(1:33)i1p1f1\n", - " grid: gr\n", - " institute: IPSL\n", - "- dataset: KIOST-ESM\n", - " ensemble: r1i1p1f1\n", - " grid: gr1\n", - " institute: KIOST\n", - "- dataset: MIROC-ES2H\n", - " ensemble: r1i1p(1:3)f2\n", - " grid: gn\n", - " institute: MIROC\n", - "- dataset: MIROC-ES2H\n", - " ensemble: r(1:3)i1p4f2\n", - " grid: gn\n", - " institute: MIROC\n", - "- dataset: MIROC-ES2L\n", - " ensemble: r1i1000p1f2\n", - " grid: gn\n", - " institute: MIROC\n", - "- dataset: MIROC-ES2L\n", - " ensemble: r(1:30)i1p1f2\n", - " grid: gn\n", - " institute: MIROC\n", - "- dataset: MIROC6\n", - " ensemble: r(1:50)i1p1f1\n", - " grid: gn\n", - " institute: MIROC\n", - "- dataset: HadGEM3-GC31-LL\n", - " ensemble: r(1:5)i1p1f3\n", - " grid: gn\n", - " institute: MOHC\n", - "- dataset: HadGEM3-GC31-MM\n", - " ensemble: r(1:4)i1p1f3\n", - " grid: gn\n", - " institute: MOHC\n", - "- dataset: UKESM1-0-LL\n", - " ensemble: r(1:4)i1p1f2\n", - " grid: gn\n", - " institute: MOHC\n", - "- dataset: UKESM1-0-LL\n", - " ensemble: r(5:7)i1p1f3\n", - " grid: gn\n", - " institute: MOHC\n", - "- dataset: UKESM1-0-LL\n", - " ensemble: r(8:12)i1p1f2\n", - " grid: gn\n", - " institute: MOHC\n", - "- dataset: UKESM1-0-LL\n", - " ensemble: r(16:19)i1p1f2\n", - " grid: gn\n", - " institute: MOHC\n", - "- dataset: UKESM1-1-LL\n", - " ensemble: r1i1p1f2\n", - " grid: gn\n", - " institute: MOHC\n", - "- dataset: ICON-ESM-LR\n", - " ensemble: r(1:5)i1p1f1\n", - " grid: gn\n", - " institute: MPI-M\n", - "- dataset: MPI-ESM1-2-HR\n", - " ensemble: r(1:10)i1p1f1\n", - " grid: gn\n", - " institute: MPI-M\n", - "- dataset: MPI-ESM1-2-LR\n", - " ensemble: r1i2000p1f1\n", - " grid: gn\n", - " institute: MPI-M\n", - "- dataset: MPI-ESM1-2-LR\n", - " ensemble: r(1:30)i1p1f1\n", - " grid: gn\n", - " institute: MPI-M\n", - "- dataset: MRI-ESM2-0\n", - " ensemble: r1i2p1f1\n", - " grid: gn\n", - " institute: MRI\n", - "- dataset: MRI-ESM2-0\n", - " ensemble: r1i1000p1f1\n", - " grid: gn\n", - " institute: MRI\n", - "- dataset: MRI-ESM2-0\n", - " ensemble: r(1:10)i1p1f1\n", - " grid: gn\n", - " institute: MRI\n", - "- dataset: GISS-E2-1-G-CC\n", - " ensemble: r1i1p1f1\n", - " grid: gn\n", - " institute: NASA-GISS\n", - "- dataset: GISS-E2-1-G\n", - " ensemble: r(1:4)i1p5f1\n", - " grid: gn\n", - " institute: NASA-GISS\n", - "- dataset: GISS-E2-1-G\n", - " ensemble: r(1:5)i1p1f3\n", - " grid: gn\n", - " institute: NASA-GISS\n", - "- dataset: GISS-E2-1-G\n", - " ensemble: r(1:10)i1p1f1\n", - " grid: gn\n", - " institute: NASA-GISS\n", - "- dataset: GISS-E2-1-G\n", - " ensemble: r(1:10)i1p3f1\n", - " grid: gn\n", - " institute: NASA-GISS\n", - "- dataset: GISS-E2-1-G\n", - " ensemble: r(1:11)i1p1f2\n", - " grid: gn\n", - " institute: NASA-GISS\n", - "- dataset: GISS-E2-1-G\n", - " ensemble: r(6:10)i1p5f1\n", - " grid: gn\n", - " institute: NASA-GISS\n", - "- dataset: GISS-E2-1-G\n", - " ensemble: r(101:102)i1p1f1\n", - " grid: gn\n", - " institute: NASA-GISS\n", - "- dataset: GISS-E2-1-H\n", - " ensemble: r(1:5)i1p1f2\n", - " grid: gn\n", - " institute: NASA-GISS\n", - "- dataset: GISS-E2-1-H\n", - " ensemble: r(1:5)i1p3f1\n", - " grid: gn\n", - " institute: NASA-GISS\n", - "- dataset: GISS-E2-1-H\n", - " ensemble: r(1:5)i1p5f1\n", - " grid: gn\n", - " institute: NASA-GISS\n", - "- dataset: GISS-E2-1-H\n", - " ensemble: r(1:10)i1p1f1\n", - " grid: gn\n", - " institute: NASA-GISS\n", - "- dataset: GISS-E2-2-G\n", - " ensemble: r(1:5)i1p3f1\n", - " grid: gn\n", - " institute: NASA-GISS\n", - "- dataset: GISS-E2-2-G\n", - " ensemble: r(1:6)i1p1f1\n", - " grid: gn\n", - " institute: NASA-GISS\n", - "- dataset: GISS-E2-2-H\n", - " ensemble: r(1:5)i1p1f1\n", - " grid: gn\n", - " institute: NASA-GISS\n", - "- dataset: CESM2-FV2\n", - " ensemble: r1i2p2f1\n", - " grid: gn\n", - " institute: NCAR\n", - "- dataset: CESM2-FV2\n", - " ensemble: r(1:3)i1p1f1\n", - " grid: gn\n", - " institute: NCAR\n", - "- dataset: CESM2-WACCM-FV2\n", - " ensemble: r(1:3)i1p1f1\n", - " grid: gn\n", - " institute: NCAR\n", - "- dataset: CESM2-WACCM\n", - " ensemble: r(1:3)i1p1f1\n", - " grid: gn\n", - " institute: NCAR\n", - "- dataset: CESM2\n", - " ensemble: r(1:11)i1p1f1\n", - " grid: gn\n", - " institute: NCAR\n", - "- dataset: NorCPM1\n", - " ensemble: r(1:30)i1p1f1\n", - " grid: gn\n", - " institute: NCC\n", - "- dataset: NorESM2-LM\n", - " ensemble: r(1:3)i1p1f1\n", - " grid: gn\n", - " institute: NCC\n", - "- dataset: NorESM2-MM\n", - " ensemble: r(1:3)i1p1f1\n", - " grid: gn\n", - " institute: NCC\n", - "- dataset: KACE-1-0-G\n", - " ensemble: r(1:3)i1p1f1\n", - " grid: gr\n", - " institute: NIMS-KMA\n", - "- dataset: UKESM1-0-LL\n", - " ensemble: r(13:15)i1p1f2\n", - " grid: gn\n", - " institute: NIMS-KMA\n", - "- dataset: GFDL-CM4\n", - " ensemble: r1i1p1f1\n", - " grid: gr1\n", - " institute: NOAA-GFDL\n", - "- dataset: GFDL-ESM4\n", - " ensemble: r(1:3)i1p1f1\n", - " grid: gr1\n", - " institute: NOAA-GFDL\n", - "- dataset: NESM3\n", + "- dataset: AWI-CM-1-1-MR\n", " ensemble: r(1:5)i1p1f1\n", " grid: gn\n", - " institute: NUIST\n", - "- dataset: SAM0-UNICON\n", + " institute: AWI\n", + "- dataset: AWI-ESM-1-1-LR\n", " ensemble: r1i1p1f1\n", - " grid: gn\n", - " institute: SNU\n", - "- dataset: CIESM\n", - " ensemble: r(1:3)i1p1f1\n", - " grid: gr\n", - " institute: THU\n", - "- dataset: MCM-UA-1-0\n", - " ensemble: r1i1p1f(1:2)\n", - " grid: gn\n", - " institute: UA\n", + "...\n" + ] + } + ], + "source": [ + "for dataset in datasets:\n", + " dataset.facets[\"diagnostic\"] = \"diagnostic_name\"\n", + "recipe = Path(\"recipe.yml\")\n", + "with recipe.open(\"w\") as file:\n", + " yaml.safe_dump(datasets_to_recipe(datasets), file, encoding=\"utf-8\")\n", + "\n", + "print(recipe.read_text(encoding=\"utf-8\")[:300])\n", + "print(\"...\")" + ] + }, + { + "cell_type": "markdown", + "id": "56728d24", + "metadata": {}, + "source": [ + "The above is a rather long list of datasets, therefore we provide a function\n", + "to format recipes in a more compact way:" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "f034972c", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "datasets:\n", + " - {dataset: ACCESS-CM2, ensemble: 'r(1:10)i1p1f1', grid: gn, institute: CSIRO-ARCCSS}\n", + " - {dataset: ACCESS-ESM1-5, ensemble: 'r(1:40)i1p1f1', grid: gn, institute: CSIRO}\n", + " - {dataset: AWI-CM-1-1-MR, ensemble: 'r(1:5)i1p1f1', grid: gn, institute: AWI}\n", + " - {dataset: AWI-ESM-1-1-LR, ensemble: r1i1p1f1, grid: gn, institute: AWI}\n", + " - {dataset: AWI-ESM-1-REcoM, ensemble: r1i1p1f1, grid: gn, institute: AWI}\n", + " - {dataset: BCC-CSM2-MR, ensemble: 'r(1:3)i1p1f1', grid: gn, institute: BCC}\n", + " - {dataset: BCC-ESM1, ensemble: 'r(1:3)i1p1f1', grid: gn, institute: BCC}\n", + " - {dataset: CAMS-CSM1-0, ensemble: 'r(1:2)i1p1f1', grid: gn, institute: CAMS}\n", + " - {dataset: CAMS-CSM1-0, ensemble: r1i1p1f2, grid: gn, institute: CAMS}\n", + " - {dataset: CAS-ESM2-0, ensemble: 'r(1:4)i1p1f1', grid: gn, institute: CAS}\n", + " - {dataset: CESM2, ensemble: 'r(1:11)i1p1f1', grid: gn, institute: NCAR}\n", + " - {dataset: CESM2-FV2, ensemble: 'r(1:3)i1p1f1', grid: gn, institute: NCAR}\n", + " - {dataset: CESM2-FV2, ensemble: r1i2p2f1, grid: gn, institute: NCAR}\n", + " - {dataset: CESM2-WACCM, ensemble: 'r(1:3)i1p1f1', grid: gn, institute: NCAR}\n", + " - {dataset: CESM2-WACCM-FV2, ensemble: 'r(1:3)i1p1f1', grid: gn, institute: NCAR}\n", + " - {dataset: CIESM, ensemble: 'r(1:3)i1p1f1', grid: gr, institute: THU}\n", + " - {dataset: CMCC-CM2-HR4, ensemble: r1i1p1f1, grid: gn, institute: CMCC}\n", + " - {dataset: CMCC-CM2-SR5, ensemble: 'r(2:11)i1p2f1', grid: gn, institute: CMCC}\n", + " - {dataset: CMCC-CM2-SR5, ensemble: r1i1p1f1, grid: gn, institute: CMCC}\n", + " - {dataset: CMCC-ESM2, ensemble: r1i1p1f1, grid: gn, institute: CMCC}\n", + " - {dataset: CNRM-CM6-1, ensemble: 'r(1:30)i1p1f2', grid: gr, institute: CNRM-CERFACS}\n", + " - {dataset: CNRM-CM6-1-HR, ensemble: r1i1p1f2, grid: gr, institute: CNRM-CERFACS}\n", + " - {dataset: CNRM-ESM2-1, ensemble: 'r(1:15)i1p1f2', grid: gr, institute: CNRM-CERFACS}\n", + " - {dataset: CanESM5, ensemble: 'r(1:25)i1p1f1', grid: gn, institute: CCCma}\n", + " - {dataset: CanESM5, ensemble: 'r(1:40)i1p2f1', grid: gn, institute: CCCma}\n", + " - {dataset: CanESM5-1, ensemble: 'r(1:20)i1p1f1', grid: gn, institute: CCCma}\n", + " - {dataset: CanESM5-1, ensemble: 'r(1:25)i1p2f1', grid: gn, institute: CCCma}\n", + " - {dataset: CanESM5-1, ensemble: 'r(24:39)i1p1f1', grid: gn, institute: CCCma}\n", + " - {dataset: CanESM5-1, ensemble: 'r(41:50)i1p1f1', grid: gn, institute: CCCma}\n", + " - {dataset: CanESM5-1, ensemble: r22i1p1f1, grid: gn, institute: CCCma}\n", + " - {dataset: CanESM5-CanOE, ensemble: 'r(1:3)i1p2f1', grid: gn, institute: CCCma}\n", + " - {dataset: E3SM-1-0, ensemble: 'r(1:5)i1p1f1', grid: gr, institute: E3SM-Project}\n", + " - {dataset: E3SM-1-0, ensemble: 'r(1:20)i2p2f1', grid: gr, institute: UCSB}\n", + " - {dataset: E3SM-1-1, ensemble: r1i1p1f1, grid: gr, institute: E3SM-Project}\n", + " - {dataset: E3SM-1-1-ECA, ensemble: r1i1p1f1, grid: gr, institute: E3SM-Project}\n", + " - {dataset: E3SM-2-0, ensemble: 'r(1:21)i1p1f1', grid: gr, institute: E3SM-Project}\n", + " - {dataset: E3SM-2-0-NARRM, ensemble: 'r(1:5)i1p1f1', grid: gr, institute: E3SM-Project}\n", + " - {dataset: E3SM-2-1, ensemble: 'r(1:5)i1p1f1', grid: gr, institute: E3SM-Project}\n", + " - {dataset: EC-Earth3, ensemble: 'r(1:7)i1p1f1', grid: gr, institute: EC-Earth-Consortium}\n", + " - {dataset: EC-Earth3, ensemble: 'r(9:25)i1p1f1', grid: gr, institute: EC-Earth-Consortium}\n", + " - {dataset: EC-Earth3, ensemble: 'r(101:150)i1p1f1', grid: gr, institute: EC-Earth-Consortium}\n", + " - {dataset: EC-Earth3-AerChem, ensemble: 'r(3:4)i1p1f1', grid: gr, institute: EC-Earth-Consortium}\n", + " - {dataset: EC-Earth3-AerChem, ensemble: r1i1p1f1, grid: gr, institute: EC-Earth-Consortium}\n", + " - {dataset: EC-Earth3-AerChem, ensemble: r1i1p4f1, grid: gr, institute: EC-Earth-Consortium}\n", + " - {dataset: EC-Earth3-CC, ensemble: 'r(6:13)i1p1f1', grid: gr, institute: EC-Earth-Consortium}\n", + " - {dataset: EC-Earth3-CC, ensemble: r1i1p1f1, grid: gr, institute: EC-Earth-Consortium}\n", + " - {dataset: EC-Earth3-CC, ensemble: r4i1p1f1, grid: gr, institute: EC-Earth-Consortium}\n", + " - {dataset: EC-Earth3-ESM-1, ensemble: r5i1p1f1, grid: gr, institute: EC-Earth-Consortium}\n", + " - {dataset: EC-Earth3-HR, ensemble: r1i1p1f1, grid: gr, institute: EC-Earth-Consortium}\n", + " - {dataset: EC-Earth3-Veg, ensemble: 'r(1:6)i1p1f1', grid: gr, institute: EC-Earth-Consortium}\n", + " - {dataset: EC-Earth3-Veg, ensemble: 'r(10:14)i1p1f1', grid: gr, institute: EC-Earth-Consortium}\n", + " - {dataset: EC-Earth3-Veg-LR, ensemble: 'r(1:3)i1p1f1', grid: gr, institute: EC-Earth-Consortium}\n", + " - {dataset: FGOALS-f3-L, ensemble: 'r(1:3)i1p1f1', grid: gr, institute: CAS}\n", + " - {dataset: FGOALS-g3, ensemble: 'r(1:6)i1p1f1', grid: gn, institute: CAS}\n", + " - {dataset: FIO-ESM-2-0, ensemble: 'r(1:3)i1p1f1', grid: gn, institute: FIO-QLNM}\n", + " - {dataset: GFDL-CM4, ensemble: r1i1p1f1, grid: gr1, institute: NOAA-GFDL}\n", + " - {dataset: GFDL-ESM4, ensemble: 'r(1:3)i1p1f1', grid: gr1, institute: NOAA-GFDL}\n", + " - {dataset: GISS-E2-1-G, ensemble: 'r(1:4)i1p5f1', grid: gn, institute: NASA-GISS}\n", + " - {dataset: GISS-E2-1-G, ensemble: 'r(1:5)i1p1f3', grid: gn, institute: NASA-GISS}\n", + " - {dataset: GISS-E2-1-G, ensemble: 'r(1:10)i1p1f1', grid: gn, institute: NASA-GISS}\n", + " - {dataset: GISS-E2-1-G, ensemble: 'r(1:20)i1p1f2', grid: gn, institute: NASA-GISS}\n", + " - {dataset: GISS-E2-1-G, ensemble: 'r(1:20)i1p3f1', grid: gn, institute: NASA-GISS}\n", + " - {dataset: GISS-E2-1-G, ensemble: 'r(6:10)i1p5f1', grid: gn, institute: NASA-GISS}\n", + " - {dataset: GISS-E2-1-G, ensemble: 'r(101:102)i1p1f1', grid: gn, institute: NASA-GISS}\n", + " - {dataset: GISS-E2-1-G, ensemble: 'r(201:202)i1p1f(5:6)', grid: gn, institute: NASA-GISS}\n", + " - {dataset: GISS-E2-1-G, ensemble: 'r(201:210)i1p1f2', grid: gn, institute: NASA-GISS}\n", + " - {dataset: GISS-E2-1-G, ensemble: 'r(201:210)i1p1f4', grid: gn, institute: NASA-GISS}\n", + " - {dataset: GISS-E2-1-G, ensemble: 'r(301:310)i1p1f2', grid: gn, institute: NASA-GISS}\n", + " - {dataset: GISS-E2-1-G, ensemble: 'r(301:310)i1p1f4', grid: gn, institute: NASA-GISS}\n", + " - {dataset: GISS-E2-1-G, ensemble: r1i1p1f5, grid: gn, institute: NASA-GISS}\n", + " - {dataset: GISS-E2-1-G-CC, ensemble: r1i1p1f1, grid: gn, institute: NASA-GISS}\n", + " - {dataset: GISS-E2-1-H, ensemble: 'r(1:5)i1p1f2', grid: gn, institute: NASA-GISS}\n", + " - {dataset: GISS-E2-1-H, ensemble: 'r(1:5)i1p3f1', grid: gn, institute: NASA-GISS}\n", + " - {dataset: GISS-E2-1-H, ensemble: 'r(1:5)i1p5f1', grid: gn, institute: NASA-GISS}\n", + " - {dataset: GISS-E2-1-H, ensemble: 'r(1:10)i1p1f1', grid: gn, institute: NASA-GISS}\n", + " - {dataset: GISS-E2-2-G, ensemble: 'r(1:5)i1p3f1', grid: gn, institute: NASA-GISS}\n", + " - {dataset: GISS-E2-2-G, ensemble: 'r(1:6)i1p1f1', grid: gn, institute: NASA-GISS}\n", + " - {dataset: GISS-E2-2-H, ensemble: 'r(1:5)i1p1f1', grid: gn, institute: NASA-GISS}\n", + " - {dataset: GISS-E3-G, ensemble: r1i1p101f1, grid: gr, institute: NASA-GISS}\n", + " - {dataset: GISS-E3-G, ensemble: r1i1p103f1, grid: gr, institute: NASA-GISS}\n", + " - {dataset: HadGEM3-GC31-LL, ensemble: 'r(1:5)i1p1f3', grid: gn, institute: MOHC}\n", + " - {dataset: HadGEM3-GC31-LL, ensemble: 'r(11:60)i1p1f3', grid: gn, institute: MOHC}\n", + " - {dataset: HadGEM3-GC31-MM, ensemble: 'r(1:4)i1p1f3', grid: gn, institute: MOHC}\n", + " - {dataset: ICON-ESM-LR, ensemble: 'r(1:5)i1p1f1', grid: gn, institute: MPI-M}\n", + " - {dataset: IITM-ESM, ensemble: r1i1p1f1, grid: gn, institute: CCCR-IITM}\n", + " - {dataset: INM-CM4-8, ensemble: r1i1p1f1, grid: gr1, institute: INM}\n", + " - {dataset: INM-CM5-0, ensemble: 'r(1:10)i1p1f1', grid: gr1, institute: INM}\n", + " - {dataset: IPSL-CM5A2-INCA, ensemble: r1i1p1f1, grid: gr, institute: IPSL}\n", + " - {dataset: IPSL-CM6A-LR, ensemble: 'r(1:33)i1p1f1', grid: gr, institute: IPSL}\n", + " - {dataset: IPSL-CM6A-LR-INCA, ensemble: r1i1p1f1, grid: gr, institute: IPSL}\n", + " - {dataset: KACE-1-0-G, ensemble: 'r(1:3)i1p1f1', grid: gr, institute: NIMS-KMA}\n", + " - {dataset: KIOST-ESM, ensemble: r1i1p1f1, grid: gr1, institute: KIOST}\n", + " - {dataset: MCM-UA-1-0, ensemble: 'r1i1p1f(1:2)', grid: gn, institute: UA}\n", + " - {dataset: MIROC-ES2H, ensemble: 'r(1:3)i1p4f2', grid: gn, institute: MIROC}\n", + " - {dataset: MIROC-ES2H, ensemble: 'r1i1p(1:3)f2', grid: gn, institute: MIROC}\n", + " - {dataset: MIROC-ES2L, ensemble: 'r(1:30)i1p1f2', grid: gn, institute: MIROC}\n", + " - {dataset: MIROC-ES2L, ensemble: r1i1000p1f2, grid: gn, institute: MIROC}\n", + " - {dataset: MIROC6, ensemble: 'r(1:50)i1p1f1', grid: gn, institute: MIROC}\n", + " - {dataset: MPI-ESM-1-2-HAM, ensemble: 'r(1:3)i1p1f1', grid: gn, institute: HAMMOZ-Consortium}\n", + " - {dataset: MPI-ESM1-2-HR, ensemble: 'r(1:10)i1p1f1', grid: gn, institute: MPI-M}\n", + " - {dataset: MPI-ESM1-2-LR, ensemble: 'r(1:50)i1p1f1', grid: gn, institute: MPI-M}\n", + " - {dataset: MPI-ESM1-2-LR, ensemble: r1i2000p1f1, grid: gn, institute: MPI-M}\n", + " - {dataset: MRI-ESM2-0, ensemble: 'r(1:10)i1p1f1', grid: gn, institute: MRI}\n", + " - {dataset: MRI-ESM2-0, ensemble: r1i2p1f1, grid: gn, institute: MRI}\n", + " - {dataset: MRI-ESM2-0, ensemble: r1i1000p1f1, grid: gn, institute: MRI}\n", + " - {dataset: NESM3, ensemble: 'r(1:5)i1p1f1', grid: gn, institute: NUIST}\n", + " - {dataset: NorCPM1, ensemble: 'r(1:30)i1p1f1', grid: gn, institute: NCC}\n", + " - {dataset: NorESM2-LM, ensemble: 'r(1:43)i1p1f1', grid: gn, institute: NCC}\n", + " - {dataset: NorESM2-LM, ensemble: r1i1p4f1, grid: gn, institute: NCC}\n", + " - {dataset: NorESM2-MM, ensemble: 'r(1:3)i1p1f1', grid: gn, institute: NCC}\n", + " - {dataset: SAM0-UNICON, ensemble: r1i1p1f1, grid: gn, institute: SNU}\n", + " - {dataset: TaiESM1, ensemble: 'r(1:2)i1p1f1', grid: gn, institute: AS-RCEC}\n", + " - {dataset: UKESM1-0-LL, ensemble: 'r(1:4)i1p1f2', grid: gn, institute: MOHC}\n", + " - {dataset: UKESM1-0-LL, ensemble: 'r(5:7)i1p1f3', grid: gn, institute: MOHC}\n", + " - {dataset: UKESM1-0-LL, ensemble: 'r(8:12)i1p1f2', grid: gn, institute: MOHC}\n", + " - {dataset: UKESM1-0-LL, ensemble: 'r(13:15)i1p1f2', grid: gn, institute: NIMS-KMA}\n", + " - {dataset: UKESM1-0-LL, ensemble: 'r(16:19)i1p1f2', grid: gn, institute: MOHC}\n", + " - {dataset: UKESM1-1-LL, ensemble: r1i1p1f2, grid: gn, institute: MOHC}\n", + "\n", "diagnostics:\n", " diagnostic_name:\n", " variables:\n", " tas:\n", " exp: historical\n", " mip: Amon\n", - " project: CMIP6\n", - "\n" + " project: CMIP6\n" ] } ], "source": [ - "for dataset in datasets:\n", - " dataset.facets[\"diagnostic\"] = \"diagnostic_name\"\n", - "print(yaml.safe_dump(datasets_to_recipe(datasets)))" + "print(Recipe(recipe).to_yaml())" ] } ], "metadata": { "kernelspec": { - "display_name": "esm", + "display_name": "Python 3", "language": "python", "name": "python3" }, @@ -606,12 +362,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.10.9" - }, - "vscode": { - "interpreter": { - "hash": "17e81e49408864327be43d3caebcb8eca32ff92a01becb15aa27be73c37f0517" - } + "version": "3.14.6" } }, "nbformat": 4, From c1733641bd8b56d596416802d649714bf7d1aa7c Mon Sep 17 00:00:00 2001 From: Bouwe Andela Date: Sat, 11 Jul 2026 19:13:02 +0200 Subject: [PATCH 10/10] Fix type hint and docstring --- esmvalcore/experimental/recipe.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/esmvalcore/experimental/recipe.py b/esmvalcore/experimental/recipe.py index 93cb0e8fba..8093c729fd 100644 --- a/esmvalcore/experimental/recipe.py +++ b/esmvalcore/experimental/recipe.py @@ -59,13 +59,13 @@ def _repr_html_(self) -> str: """Return html representation.""" return self.render() - def to_yaml(self, file: os.PathLike | None = None) -> str | None: + def to_yaml(self, file: os.PathLike | None = None) -> str: """Write recipe to a YAML file. Parameters ---------- - file : - A path-like object to write the YAML data to. + file: + If provided, the recipe will be written to this file too. Returns -------