Add cloud scatterplots

changelog/261.feature.md

@@ -0,0 +1 @@
+Added a diagnostic to create scatterplots of two cloud-relevant variables.

packages/climate-ref-core/src/climate_ref_core/datasets.py

@@ -5,7 +5,7 @@
 import enum
 import functools
 import hashlib
-from collections.abc import Collection, Iterable
+from collections.abc import Collection, Iterable, Iterator
 from typing import Any, Self
 
 import pandas as pd
@@ -172,9 +172,24 @@ def __getitem__(self, key: SourceDatasetType | str) -> DatasetCollection:
     def __hash__(self) -> int:
         return hash(self.hash)
 
+    def __iter__(self) -> Iterator[SourceDatasetType]:
+        return iter(self._collection)
+
+    def keys(self) -> Iterable[SourceDatasetType]:
+        """
+        Iterate over the source types in the collection.
+        """
+        return self._collection.keys()
+
+    def values(self) -> Iterable[DatasetCollection]:
+        """
+        Iterate over the datasets in the collection.
+        """
+        return self._collection.values()
+
     def items(self) -> Iterable[tuple[SourceDatasetType, DatasetCollection]]:
         """
-        Iterate over the datasets in the collection
+        Iterate over the items in the collection.
         """
         return self._collection.items()
 

packages/climate-ref-core/tests/unit/test_datasets.py

@@ -35,6 +35,18 @@ def test_get_item_missing(self, metric_dataset):
         with pytest.raises(KeyError):
             metric_dataset["cmip7"]
 
+    def test_iter(self, metric_dataset):
+        assert tuple(iter(metric_dataset)) == tuple(iter(metric_dataset._collection))
+
+    def test_keys(self, metric_dataset):
+        assert metric_dataset.keys() == metric_dataset._collection.keys()
+
+    def test_values(self, metric_dataset):
+        assert tuple(metric_dataset.values()) == tuple(metric_dataset._collection.values())
+
+    def test_items(self, metric_dataset):
+        assert metric_dataset.items() == metric_dataset._collection.items()
+
     def test_python_hash(self, metric_dataset, cmip6_data_catalog, data_regression):
         dataset_hash = hash(metric_dataset)
 

packages/climate-ref-esmvaltool/pyproject.toml

@@ -23,9 +23,9 @@ classifiers = [
     "License :: OSI Approved :: Apache Software License",
 ]
 dependencies = [
-    "pooch >= 1.8",
     "climate-ref-core",
-    "ruamel.yaml >= 0.18",
+    "pooch >= 1.8",
+    "pyyaml",
     "xarray >= 2023.3.0",
 ]
 

packages/climate-ref-esmvaltool/src/climate_ref_esmvaltool/dataset_registry/data.txt

@@ -3,6 +3,9 @@ ESMValTool/OBS/Tier2/CERES-EBAF/OBS_CERES-EBAF_sat_Ed4.2_Amon_rlut_200003-202311
 ESMValTool/OBS/Tier2/CERES-EBAF/OBS_CERES-EBAF_sat_Ed4.2_Amon_rlutcs_200003-202311.nc e70e3273092edf01527970693271641fc6474d1974887d7d272e7d656bab83c2
 ESMValTool/OBS/Tier2/CERES-EBAF/OBS_CERES-EBAF_sat_Ed4.2_Amon_rsut_200003-202311.nc e31e648886c4fa9c09686672a06ab18fbba687ff0d6de2891616d4c8b74e215d
 ESMValTool/OBS/Tier2/CERES-EBAF/OBS_CERES-EBAF_sat_Ed4.2_Amon_rsutcs_200003-202311.nc eb96edd9274670aa705eab2a6d1ee0cca11e01ac17096706463e032b58e6be47
+ESMValTool/OBS/Tier2/ESACCI-CLOUD/OBS_ESACCI-CLOUD_sat_AVHRR-AMPM-fv3.0_Amon_clivi_198201-201612.nc 13bc6e3a46397386a14a36776fdd6bdbf5c45147c8dc695d4a7387883d449775
+ESMValTool/OBS/Tier2/ESACCI-CLOUD/OBS_ESACCI-CLOUD_sat_AVHRR-AMPM-fv3.0_Amon_clt_198201-201612.nc 4a430d77dbe9164dba2d1fdef4bb89ea9358b6d10023f3fbee50917422446ed0
+ESMValTool/OBS/Tier2/ESACCI-CLOUD/OBS_ESACCI-CLOUD_sat_AVHRR-AMPM-fv3.0_Amon_clwvi_198201-201612.nc e0ffa31369d9552be16b920110b24013d31e116b180ee3f70b3d0aaa5281eff0
 ESMValTool/OBS/Tier2/ESACCI-CLOUD/OBS_ESACCI-CLOUD_sat_AVHRR-AMPM-fv3.0_Amon_rlut_198201-201612.nc 075144d673a9f2ff49fbe59e701535bf80c04908797a9dca83781000a9b1b7f2
 ESMValTool/OBS/Tier2/ESACCI-CLOUD/OBS_ESACCI-CLOUD_sat_AVHRR-AMPM-fv3.0_Amon_rlutcs_198201-201612.nc 21f096ecafff659e5c7e3338060425f7194e5d1b39c9510865496e04ecac3d75
 ESMValTool/OBS/Tier2/ESACCI-CLOUD/OBS_ESACCI-CLOUD_sat_AVHRR-AMPM-fv3.0_Amon_rsut_198201-201612.nc f2c3f3afcdc2e730df7985c210a3de89b0d4f83b150e0c3846f7ac3c5fa9c54a
@@ -85,5 +88,6 @@ ESMValTool/OBS/Tier2/OSI-450-sh/OBS_OSI-450-sh_reanaly_v3_OImon_sic_201101-20111
 ESMValTool/OBS/Tier2/OSI-450-sh/OBS_OSI-450-sh_reanaly_v3_OImon_sic_201201-201212.nc 86187c3d1174053f2cba6dad010af49ceab77d368aa9314bf53c330b5f2217b9
 ESMValTool/OBS/Tier2/OSI-450-sh/OBS_OSI-450-sh_reanaly_v3_OImon_sic_201301-201312.nc 8820353570884b2ef182caaffb5986ed6268bbe199fd867f61b56e798ca01f1a
 ESMValTool/OBS/Tier2/OSI-450-sh/OBS_OSI-450-sh_reanaly_v3_OImon_sic_201401-201412.nc 7102d0db3dc02c5b0eb0cfe3535ee50171007ef5b43eb9aae1220ac21b0b98e9
+ESMValTool/OBS/Tier3/CALIPSO-ICECLOUD/OBS_CALIPSO-ICECLOUD_sat_1-00_Amon_cli_200701-201512.nc 977824810e8f9dbe7df278c59397ffc3f78491a9eb5a0b70e6b28ac66db8e12d
 ESMValTool/OBS/Tier2/TROPFLUX/OBS6_TROPFLUX_reanaly_v1_Amon_tauu_197901-201812.nc bf313e661b42341d5090038b501ed1ff09e58201009c3fccfe45b78e116fdd78
 ESMValTool/OBS/Tier2/TROPFLUX/OBS6_TROPFLUX_reanaly_v1_Omon_tos_197901-201812.nc 5f10a5a2aa47f5d21378ad3178bf8e4b577b0ed72ef8402dc04f5ff6fc99ec07

packages/climate-ref-esmvaltool/src/climate_ref_esmvaltool/diagnostics/__init__.py

@@ -2,6 +2,13 @@
 
 from climate_ref_esmvaltool.diagnostics.climate_at_global_warming_levels import ClimateAtGlobalWarmingLevels
 from climate_ref_esmvaltool.diagnostics.cloud_radiative_effects import CloudRadiativeEffects
+from climate_ref_esmvaltool.diagnostics.cloud_scatterplots import (
+    CloudScatterplotCliTa,
+    CloudScatterplotCliviLwcre,
+    CloudScatterplotCltSwcre,
+    CloudScatterplotClwviPr,
+    CloudScatterplotsReference,
+)
 from climate_ref_esmvaltool.diagnostics.ecs import EquilibriumClimateSensitivity
 from climate_ref_esmvaltool.diagnostics.enso import ENSOBasicClimatology, ENSOCharacteristics
 from climate_ref_esmvaltool.diagnostics.example import GlobalMeanTimeseries
@@ -13,6 +20,11 @@
 __all__ = [
     "ClimateAtGlobalWarmingLevels",
     "CloudRadiativeEffects",
+    "CloudScatterplotCliTa",
+    "CloudScatterplotCliviLwcre",
+    "CloudScatterplotCltSwcre",
+    "CloudScatterplotClwviPr",
+    "CloudScatterplotsReference",
     "ENSOBasicClimatology",
     "ENSOCharacteristics",
     "EquilibriumClimateSensitivity",

packages/climate-ref-esmvaltool/src/climate_ref_esmvaltool/diagnostics/base.py

@@ -4,8 +4,8 @@
 from typing import ClassVar
 
 import pandas
+import yaml
 from loguru import logger
-from ruamel.yaml import YAML
 
 from climate_ref_core.dataset_registry import dataset_registry_manager
 from climate_ref_core.datasets import ExecutionDatasetCollection, SourceDatasetType
@@ -19,8 +19,6 @@
 from climate_ref_esmvaltool.recipe import load_recipe, prepare_climate_data
 from climate_ref_esmvaltool.types import MetricBundleArgs, OutputBundleArgs, Recipe
 
-yaml = YAML()
-
 
 class ESMValToolDiagnostic(CommandLineDiagnostic):
     """ESMValTool Diagnostic base class."""
@@ -29,7 +27,10 @@ class ESMValToolDiagnostic(CommandLineDiagnostic):
 
     @staticmethod
     @abstractmethod
-    def update_recipe(recipe: Recipe, input_files: pandas.DataFrame) -> None:
+    def update_recipe(
+        recipe: Recipe,
+        input_files: dict[SourceDatasetType, pandas.DataFrame],
+    ) -> None:
         """
         Update the base recipe for the run.
 
@@ -83,20 +84,21 @@ def build_cmd(self, definition: ExecutionDefinition) -> Iterable[str]:
         :
             The result of running the diagnostic.
         """
-        input_files = definition.datasets[SourceDatasetType.CMIP6].datasets
+        input_files = {project: definition.datasets[project].datasets for project in definition.datasets}
         recipe = load_recipe(self.base_recipe)
         self.update_recipe(recipe, input_files)
 
         recipe_path = definition.to_output_path("recipe.yml")
         with recipe_path.open("w", encoding="utf-8") as file:
-            yaml.dump(recipe, file)
+            yaml.safe_dump(recipe, file)
 
         climate_data = definition.to_output_path("climate_data")
 
-        prepare_climate_data(
-            definition.datasets[SourceDatasetType.CMIP6].datasets,
-            climate_data_dir=climate_data,
-        )
+        for metric_dataset in definition.datasets.values():
+            prepare_climate_data(
+                metric_dataset.datasets,
+                climate_data_dir=climate_data,
+            )
 
         config = {
             "drs": {
@@ -143,7 +145,7 @@ def build_cmd(self, definition: ExecutionDefinition) -> Iterable[str]:
         config_dir = definition.to_output_path("config")
         config_dir.mkdir()
         with (config_dir / "config.yml").open("w", encoding="utf-8") as file:
-            yaml.dump(config, file)
+            yaml.safe_dump(config, file)
 
         return [
             "esmvaltool",
@@ -177,7 +179,7 @@ def build_execution_result(
         # Add the plots and data files
         plot_suffixes = {".png", ".jpg", ".pdf", ".ps"}
         for metadata_file in result_dir.glob("run/*/*/diagnostic_provenance.yml"):
-            metadata = yaml.load(metadata_file.read_text(encoding="utf-8"))
+            metadata = yaml.safe_load(metadata_file.read_text(encoding="utf-8"))
             for filename in metadata:
                 caption = metadata[filename].get("caption", "")
                 relative_path = definition.as_relative_path(filename)

packages/climate-ref-esmvaltool/src/climate_ref_esmvaltool/diagnostics/climate_at_global_warming_levels.py

@@ -65,13 +65,16 @@ class ClimateAtGlobalWarmingLevels(ESMValToolDiagnostic):
     facets = ()
 
     @staticmethod
-    def update_recipe(recipe: Recipe, input_files: pandas.DataFrame) -> None:
+    def update_recipe(
+        recipe: Recipe,
+        input_files: dict[SourceDatasetType, pandas.DataFrame],
+    ) -> None:
         """Update the recipe."""
         # Set up the datasets
         diagnostics = recipe["diagnostics"]
         for diagnostic in diagnostics.values():
             diagnostic.pop("additional_datasets")
-        recipe_variables = dataframe_to_recipe(input_files)
+        recipe_variables = dataframe_to_recipe(input_files[SourceDatasetType.CMIP6])
         datasets = recipe_variables["tas"]["additional_datasets"]
         datasets = [ds for ds in datasets if ds["exp"] != "historical"]
         for dataset in datasets:

packages/climate-ref-esmvaltool/src/climate_ref_esmvaltool/diagnostics/cloud_radiative_effects.py

@@ -53,9 +53,9 @@ class CloudRadiativeEffects(ESMValToolDiagnostic):
     )
 
     @staticmethod
-    def update_recipe(recipe: Recipe, input_files: pandas.DataFrame) -> None:
+    def update_recipe(recipe: Recipe, input_files: dict[SourceDatasetType, pandas.DataFrame]) -> None:
         """Update the recipe."""
-        recipe_variables = dataframe_to_recipe(input_files)
+        recipe_variables = dataframe_to_recipe(input_files[SourceDatasetType.CMIP6])
         recipe_variables = {k: v for k, v in recipe_variables.items() if k != "areacella"}
 
         # Select a timerange covered by all datasets.