diff --git a/doc/source/enhancements.rst b/doc/source/enhancements.rst index 87cdecb23f..2892b0ef73 100644 --- a/doc/source/enhancements.rst +++ b/doc/source/enhancements.rst @@ -20,8 +20,10 @@ as well. See :ref:`component_configuration` for more information. Enhancements can be defined in a ``generic.yaml`` file that is always loaded for all data or in an instrument-specific file (e.g. ``seviri.yaml``) -corresponding to the ``.attrs["sensor"]`` metadata of the ``DataArray`` being -processed. Generic enhancements are loaded first followed by sensor-specific +corresponding to the ``.attrs["instruments"]`` metadata of the ``DataArray`` +being processed. For the filename, instruments are normalized using +:meth:`satpy._instruments.normalize_instrument_name`. +Generic enhancements are loaded first followed by instrument-specific enhancement files. Enhancement YAML Format @@ -95,7 +97,7 @@ implementation depends on the following keys: 1. ``name`` 2. ``reader`` 3. ``platform_name`` -4. ``sensor`` +4. ``instruments`` (previously ``sensor`` in Satpy <1.0) 5. ``standard_name`` 6. ``units`` diff --git a/satpy/decision_tree.py b/satpy/decision_tree.py index c1f5e25162..2550171a03 100644 --- a/satpy/decision_tree.py +++ b/satpy/decision_tree.py @@ -163,8 +163,8 @@ def _build_tree(self, conf): for match_level, match_key in enumerate(self._match_keys): # or None is necessary if they have empty strings this_attr_val = sect_attrs.get(match_key, self.any_key) or None - if match_key in self._multival_keys and isinstance(this_attr_val, list): - this_attr_val = tuple(sorted(this_attr_val)) + if match_key in self._multival_keys: + this_attr_val = self._normalize_multival_attr(this_attr_val) is_last_key = match_key == self._match_keys[-1] level_needs_init = this_attr_val not in curr_level if is_last_key: @@ -176,10 +176,32 @@ def _build_tree(self, conf): curr_level[this_attr_val] = _DecisionDict(self._match_keys[match_level + 1], match_level + 1) curr_level = curr_level[this_attr_val] + def _normalize_multival_attr(self, attr): + """Convert multival attributes from list/str to sorted tuple.""" + if isinstance(attr, list): + return tuple(sorted(attr)) + elif isinstance(attr, str): + return (attr,) + return attr + @staticmethod def _convert_query_val_to_hashable(query_val): + """Prepare multival query for matching. + + First priority is exact matches with the sorted values. + If not found, search each of the values individually in + alphabetical order - both as tuple with a single element + and string. + """ _sorted_query_val = sorted(query_val) - query_vals = [tuple(_sorted_query_val)] + _sorted_query_val + # Exact match + query_vals = [tuple(_sorted_query_val)] + # Each of the values individually, in alphabetical order, + # as tuple with a single element. + query_vals += [tuple([v]) for v in _sorted_query_val] + # Each of the values individually, in alphabetical order, + # as string. + query_vals += _sorted_query_val query_vals += query_val return query_vals diff --git a/satpy/enhancements/enhancer.py b/satpy/enhancements/enhancer.py index fb2299b660..0b3c09515b 100644 --- a/satpy/enhancements/enhancer.py +++ b/satpy/enhancements/enhancer.py @@ -17,6 +17,7 @@ from __future__ import annotations import os +import warnings from pathlib import Path import yaml @@ -42,12 +43,12 @@ def __init__(self, *decision_dicts, **kwargs): ("name", "reader", "platform_name", - "sensor", + "instruments", "standard_name", "units", )) self.prefix = kwargs.pop("config_section", "enhancements") - multival_keys = kwargs.pop("multival_keys", ["sensor"]) + multival_keys = kwargs.pop("multival_keys", ["instruments"]) super(EnhancementDecisionTree, self).__init__( decision_dicts, match_keys, multival_keys) @@ -57,6 +58,9 @@ def add_config_to_tree(self, *decision_dict: str | Path | dict) -> None: for config_file in decision_dict: config_dict = self._get_config_dict_from_user(config_file) recursive_dict_update(conf, config_dict) + # 8< v1.0 + self._ensure_compat(conf) + # >8 v1.0 self._build_tree(conf) def _get_config_dict_from_user(self, config_file: str | Path | dict) -> dict: @@ -87,6 +91,21 @@ def _get_yaml_enhancement_dict(self, config_file: str | Path) -> dict: LOG.debug(f"Adding enhancement configuration from file: {config_file}") return enhancement_section + # 8< v1.0 + def _ensure_compat(self, config_dict: dict) -> None: + for enh_name, enh_config in config_dict.items(): + if "sensor" in enh_config: + warnings.warn( + "Renaming the 'sensor' enhancement attribute to 'instruments'. " + "This will raise an exception in Satpy v1.0 when the 'sensor' " + "attribute will be removed. To silence this warning, rename " + "'sensor' to 'instruments' in your enhancement YAML file.", + DeprecationWarning, + stacklevel=3 + ) + inst_utils.set_instruments_attr(config_dict[enh_name], enh_config["sensor"]) + # >8 v1.0 + def find_match(self, **query_dict): """Find a match.""" try: diff --git a/satpy/etc/enhancements/abi.yaml b/satpy/etc/enhancements/abi.yaml index 9920f842a3..e72e7a7f63 100644 --- a/satpy/etc/enhancements/abi.yaml +++ b/satpy/etc/enhancements/abi.yaml @@ -1,7 +1,7 @@ enhancements: cimss_true_color: standard_name: cimss_true_color - sensor: abi + instruments: [abi] operations: - name: linear_stretch method: !!python/name:satpy.enhancements.contrast.stretch @@ -35,7 +35,7 @@ enhancements: true_color_with_night_fires: standard_name: true_color_with_night_fires - sensor: abi + instruments: [abi] operations: - name: stretch method: !!python/name:satpy.enhancements.contrast.stretch @@ -77,7 +77,7 @@ enhancements: ash_abi: ## RGB Ash recipe source: http://rammb.cira.colostate.edu/training/visit/quick_guides/GOES_Ash_RGB.pdf standard_name: ash - sensor: abi + instruments: [abi] operations: - name: stretch method: !!python/name:satpy.enhancements.contrast.stretch @@ -89,7 +89,7 @@ enhancements: dust_abi: ## RGB Dust recipe source: http://rammb.cira.colostate.edu/training/visit/quick_guides/Dust_RGB_Quick_Guide.pdf standard_name: dust - sensor: abi + instruments: [abi] operations: - name: stretch method: !!python/name:satpy.enhancements.contrast.stretch @@ -105,7 +105,7 @@ enhancements: convection_abi: ## RGB Convection recipe source: http://rammb.cira.colostate.edu/training/visit/quick_guides/QuickGuide_GOESR_DayConvectionRGB_final.pdf standard_name: convection - sensor: abi + instruments: [abi] operations: - name: stretch method: !!python/name:satpy.enhancements.contrast.stretch @@ -142,7 +142,7 @@ enhancements: night_microphysics_abi: ## RGB Nighttime Microphysics recipe source: http://rammb.cira.colostate.edu/training/visit/quick_guides/QuickGuide_GOESR_NtMicroRGB_final.pdf standard_name: night_microphysics - sensor: abi + instruments: [abi] operations: - name: stretch method: !!python/name:satpy.enhancements.contrast.stretch @@ -154,7 +154,7 @@ enhancements: night_microphysics_tropical_abi: ## RGB Nighttime Microphysics recipe source: http://rammb.cira.colostate.edu/training/visit/quick_guides/QuickGuide_GOESR_NtMicroRGB_final.pdf standard_name: night_microphysics_tropical - sensor: abi + instruments: [abi] operations: - name: stretch method: !!python/name:satpy.enhancements.contrast.stretch @@ -170,7 +170,7 @@ enhancements: land_cloud_fire: ## RGB Day Land Cloud Fire recipe source: http://rammb.cira.colostate.edu/training/visit/quick_guides/QuickGuide_GOESR_DayLandCloudFireRGB_final.pdf standard_name: land_cloud_fire - sensor: abi + instruments: [abi] operations: - name: stretch method: !!python/name:satpy.enhancements.contrast.stretch @@ -179,7 +179,7 @@ enhancements: land_cloud: ## RGB Day Land Cloud Fire recipe source: http://rammb.cira.colostate.edu/training/visit/quick_guides/QuickGuide_GOESR_daylandcloudRGB_final.pdf standard_name: land_cloud - sensor: abi + instruments: [abi] operations: - name: stretch method: !!python/name:satpy.enhancements.contrast.stretch @@ -191,7 +191,7 @@ enhancements: # IR with white clouds highlighted_brightness_temperature: standard_name: highlighted_toa_brightness_temperature - sensor: abi + instruments: [abi] operations: - name: btemp_threshold method: !!python/name:satpy.enhancements.contrast.btemp_threshold @@ -308,7 +308,7 @@ enhancements: ## RGB Recipe: https://rammb2.cira.colostate.edu/wp-content/uploads/2024/11/GOES-BlowingSnowRGB1_QuickGuide_24April2024.pdf ## Modified to match recommendations from RGB Workshop 2025 standard_name: day_blowing_snow - sensor: abi + instruments: [abi] operations: - name: stretch method: !!python/name:satpy.enhancements.contrast.stretch @@ -323,7 +323,7 @@ enhancements: day_cloud_type: # Recipe PDF: http://cimss.ssec.wisc.edu/goes/OCLOFactSheetPDFs/ABIQuickGuide_Day_Cloud_Type_RGB.pdf standard_name: day_cloud_type - sensor: abi + instruments: [abi] operations: - name: stretch method: !!python/name:satpy.enhancements.contrast.stretch diff --git a/satpy/etc/enhancements/ahi.yaml b/satpy/etc/enhancements/ahi.yaml index 5496be3cd8..3f73b9c1a4 100644 --- a/satpy/etc/enhancements/ahi.yaml +++ b/satpy/etc/enhancements/ahi.yaml @@ -12,7 +12,7 @@ enhancements: day_severe_storms: standard_name: day_severe_storms - sensor: ahi + instruments: [ahi] operations: - name: stretch method: !!python/name:satpy.enhancements.contrast.stretch @@ -27,7 +27,7 @@ enhancements: night_microphysics_tropical: standard_name: night_microphysics_tropical - sensor: ahi + instruments: [ahi] operations: - name: stretch method: !!python/name:satpy.enhancements.contrast.stretch diff --git a/satpy/etc/enhancements/amsr2.yaml b/satpy/etc/enhancements/amsr2.yaml index eb56f944ec..788b52cc2c 100644 --- a/satpy/etc/enhancements/amsr2.yaml +++ b/satpy/etc/enhancements/amsr2.yaml @@ -3,28 +3,28 @@ enhancements: # https://www.ospo.noaa.gov/Products/atmosphere/gpds/maps.html?GPRR#gpdsMaps gaasp_clw: name: CLW - sensor: amsr2 + instruments: [amsr2] operations: - name: linear_stretch method: !!python/name:satpy.enhancements.contrast.stretch kwargs: {stretch: 'crude', min_stretch: 0.0, max_stretch: 0.5} gaasp_sst: name: SST - sensor: amsr2 + instruments: [amsr2] operations: - name: linear_stretch method: !!python/name:satpy.enhancements.contrast.stretch kwargs: {stretch: 'crude', min_stretch: -5.0, max_stretch: 35} gaasp_tpw: name: TPW - sensor: amsr2 + instruments: [amsr2] operations: - name: linear_stretch method: !!python/name:satpy.enhancements.contrast.stretch kwargs: {stretch: 'crude', min_stretch: 0.0, max_stretch: 75.0} gaasp_wspd: name: WSPD - sensor: amsr2 + instruments: [amsr2] operations: - name: linear_stretch method: !!python/name:satpy.enhancements.contrast.stretch @@ -32,56 +32,56 @@ enhancements: # Snow_Cover unscaled (category product) gaasp_snow_depth: name: Snow_Depth - sensor: amsr2 + instruments: [amsr2] operations: - name: linear_stretch method: !!python/name:satpy.enhancements.contrast.stretch kwargs: {stretch: 'crude', min_stretch: 0.0, max_stretch: 150.0} gaasp_swe: name: SWE - sensor: amsr2 + instruments: [amsr2] operations: - name: linear_stretch method: !!python/name:satpy.enhancements.contrast.stretch kwargs: {stretch: 'crude', min_stretch: 0.0, max_stretch: 16.0} gaasp_soil_moisture: name: Soil_Moisture - sensor: amsr2 + instruments: [amsr2] operations: - name: linear_stretch method: !!python/name:satpy.enhancements.contrast.stretch kwargs: {stretch: 'crude', min_stretch: 0.0, max_stretch: 100.0} gaasp_ice_concentration_nh: name: NASA_Team_2_Ice_Concentration_NH - sensor: amsr2 + instruments: [amsr2] operations: - name: linear_stretch method: !!python/name:satpy.enhancements.contrast.stretch kwargs: {stretch: 'crude', min_stretch: 0.0, max_stretch: 100.0} gaasp_ice_concentration_sh: name: NASA_Team_2_Ice_Concentration_SH - sensor: amsr2 + instruments: [amsr2] operations: - name: linear_stretch method: !!python/name:satpy.enhancements.contrast.stretch kwargs: {stretch: 'crude', min_stretch: 0.0, max_stretch: 100.0} # gaasp_latency_nh: # name: Latency_NH -# sensor: amsr2 +# instruments: [amsr2] # operations: # - name: linear_stretch # method: !!python/name:satpy.enhancements.contrast.stretch # kwargs: {stretch: 'crude', min_stretch: 0.0, max_stretch: 100.0} # gaasp_latency_sh: # name: Latency_SH -# sensor: amsr2 +# instruments: [amsr2] # operations: # - name: linear_stretch # method: !!python/name:satpy.enhancements.contrast.stretch # kwargs: {stretch: 'crude', min_stretch: 0.0, max_stretch: 100.0} gaasp_rain_rate: name: Rain_Rate - sensor: amsr2 + instruments: [amsr2] operations: - name: linear_stretch method: !!python/name:satpy.enhancements.contrast.stretch diff --git a/satpy/tests/enhancement_tests/test_enhancer.py b/satpy/tests/enhancement_tests/test_enhancer.py index c175f11a9b..06e03da02a 100644 --- a/satpy/tests/enhancement_tests/test_enhancer.py +++ b/satpy/tests/enhancement_tests/test_enhancer.py @@ -144,7 +144,7 @@ class TestComplexSensorEnhancerConfigs(_BaseCustomEnhancementConfigTests): enhancements: test1_sensor1_specific: name: test1 - sensor: test_sensor1 + instruments: [test_sensor1] operations: - name: stretch method: !!python/name:satpy.enhancements.contrast.stretch @@ -160,14 +160,14 @@ class TestComplexSensorEnhancerConfigs(_BaseCustomEnhancementConfigTests): kwargs: {stretch: crude, min_stretch: 0, max_stretch: 100} test1_sensor2_specific: name: test1 - sensor: test_sensor2 + instruments: [test_sensor2] operations: - name: stretch method: !!python/name:satpy.enhancements.contrast.stretch kwargs: {stretch: crude, min_stretch: 0, max_stretch: 50} exact_multisensor_comp: name: my_comp - sensor: [test_sensor1, test_sensor2] + instruments: [test_sensor1, test_sensor2] operations: - name: stretch method: !!python/name:satpy.enhancements.contrast.stretch @@ -183,7 +183,7 @@ def test_multisensor_choice(self, test_configs_path): ds = DataArray(np.arange(1, 11.).reshape((2, 5)), attrs={ "name": "test1", - "sensor": {"test_sensor2", "test_sensor1"}, + "instruments": {"test_sensor2", "test_sensor1"}, "mode": "L" }, dims=["y", "x"]) @@ -206,7 +206,7 @@ def test_multisensor_exact(self, test_configs_path): ds = DataArray(np.arange(1, 11.).reshape((2, 5)), attrs={ "name": "my_comp", - "sensor": {"test_sensor2", "test_sensor1"}, + "instruments": {"test_sensor2", "test_sensor1"}, "mode": "L" }, dims=["y", "x"]) @@ -227,7 +227,7 @@ def test_enhance_bad_query_value(self): from satpy.enhancements.enhancer import Enhancer, get_enhanced_image ds = DataArray(np.arange(1, 11.).reshape((2, 5)), - attrs=dict(name=["I", "am", "invalid"], sensor="test_sensor2", mode="L"), + attrs=dict(name=["I", "am", "invalid"], instruments={"test_sensor2"}, mode="L"), dims=["y", "x"]) e = Enhancer() assert e.enhancement_tree is not None @@ -282,7 +282,7 @@ def test_enhance_empty_config(self, test_configs_path): from satpy.enhancements.enhancer import Enhancer, get_enhanced_image ds = DataArray(np.arange(1, 11.).reshape((2, 5)), - attrs=dict(sensor="test_empty", mode="L"), + attrs=dict(instruments={"test_empty"}, mode="L"), dims=["y", "x"]) e = Enhancer() assert e.enhancement_tree is not None @@ -296,7 +296,7 @@ def test_enhance_with_sensor_no_entry(self, test_configs_path): from satpy.enhancements.enhancer import Enhancer, get_enhanced_image ds = DataArray(np.arange(1, 11.).reshape((2, 5)), - attrs=dict(sensor="test_sensor2", mode="L"), + attrs=dict(instruments={"test_sensor2"}, mode="L"), dims=["y", "x"]) e = Enhancer() assert e.enhancement_tree is not None @@ -311,7 +311,7 @@ def test_no_enhance(self): from satpy.enhancements.enhancer import get_enhanced_image ds = DataArray(np.arange(1, 11.).reshape((2, 5)), - attrs=dict(name="test1", sensor="test_sensor", mode="L"), + attrs=dict(name="test1", instruments={"test_sensor"}, mode="L"), dims=["y", "x"]) img = get_enhanced_image(ds, enhance=False) np.testing.assert_allclose(img.data.data.compute().squeeze(), ds.data) @@ -320,7 +320,7 @@ def test_writer_no_enhance(self): """Test turning off enhancements with writer.""" from xarray import DataArray ds = DataArray(np.arange(1, 11.).reshape((2, 5)), - attrs=dict(name="test1", sensor="test_sensor", mode="L"), + attrs=dict(name="test1", instruments={"test_sensor"}, mode="L"), dims=["y", "x"]) writer = _CustomImageWriter(enhance=False) writer.save_datasets((ds,), compute=False) @@ -333,7 +333,7 @@ def test_writer_custom_enhance(self): from satpy.enhancements.enhancer import Enhancer ds = DataArray(np.arange(1, 11.).reshape((2, 5)), - attrs=dict(name="test1", sensor="test_sensor", mode="L"), + attrs=dict(name="test1", instruments={"test_sensor"}, mode="L"), dims=["y", "x"]) enhance = Enhancer() writer = _CustomImageWriter(enhance=enhance) @@ -347,7 +347,7 @@ def test_enhance_with_sensor_entry(self, test_configs_path): from satpy.enhancements.enhancer import Enhancer, get_enhanced_image ds = DataArray(np.arange(1, 11.).reshape((2, 5)), - attrs=dict(name="test1", sensor="test_sensor", mode="L"), + attrs=dict(name="test1", instruments={"test_sensor"}, mode="L"), dims=["y", "x"]) e = Enhancer() assert e.enhancement_tree is not None @@ -359,7 +359,7 @@ def test_enhance_with_sensor_entry(self, test_configs_path): 1.) ds = DataArray(da.arange(1, 11., chunks=5).reshape((2, 5)), - attrs=dict(name="test1", sensor="test_sensor", mode="L"), + attrs=dict(name="test1", instruments={"test_sensor"}, mode="L"), dims=["y", "x"]) e = Enhancer() assert e.enhancement_tree is not None @@ -376,7 +376,7 @@ def test_enhance_with_sensor_entry2(self, test_configs_path): from satpy.enhancements.enhancer import Enhancer, get_enhanced_image ds = DataArray(np.arange(1, 11.).reshape((2, 5)), attrs=dict(name="test1", units="kelvin", - sensor="test_sensor", mode="L"), + instruments={"test_sensor"}, mode="L"), dims=["y", "x"]) e = Enhancer() assert e.enhancement_tree is not None @@ -432,7 +432,7 @@ def _get_test_data_array(self): ds = DataArray(np.arange(1, 11.).reshape((2, 5)), attrs={ "name": "test1", - "sensor": "test_sensor1", + "instruments": {"test_sensor1"}, "mode": "L", }, dims=["y", "x"])