diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 38aeed0b47..fbb852bb20 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -27,7 +27,7 @@ repos: - types-setuptools - types-PyYAML - types-requests - args: ["--python-version", "3.10", "--ignore-missing-imports"] + args: ["--python-version", "3.11", "--ignore-missing-imports"] - repo: https://github.com/pycqa/isort rev: 9.0.0a3 hooks: diff --git a/doc/source/dev_guide/custom_reader.rst b/doc/source/dev_guide/custom_reader.rst index 0475e07144..0098e14f13 100644 --- a/doc/source/dev_guide/custom_reader.rst +++ b/doc/source/dev_guide/custom_reader.rst @@ -494,7 +494,7 @@ needs to implement a few methods: successful, containing the data and :ref:`metadata ` of the loaded dataset, or return None if the loading was unsuccessful. If the reader is reading satellite data the returned xarray.DataArrays should also have the - attributes ``platform_name`` and ``sensor`` with names according to https://space.oscar.wmo.int/. + attributes ``platform_name`` and ``instruments`` with names according to https://space.oscar.wmo.int/. The DataArray should at least have a ``y`` dimension. For data covering a 2D region on the Earth, their should be at least a ``y`` and ``x`` diff --git a/doc/source/dev_guide/xarray_migration.rst b/doc/source/dev_guide/xarray_migration.rst index 1a3b02f61e..6bd2d1d248 100644 --- a/doc/source/dev_guide/xarray_migration.rst +++ b/doc/source/dev_guide/xarray_migration.rst @@ -35,7 +35,7 @@ To create such an array, you can do for example my_dataarray = xr.DataArray(my_data, dims=['y', 'x'], coords={'x': np.arange(...)}, - attrs={'sensor': 'olci'}) + attrs={'instruments': {'OLCI'}) where ``my_data`` can be a regular numpy array, a numpy memmap, or, if you want to keep things lazy, a dask array (more on dask later). Satpy uses dask @@ -105,7 +105,7 @@ Some metadata that should always be present in our dataarrays: - ``area`` the area of the dataset. This should be handled in the reader. - ``start_time``, ``end_time`` -- ``sensor`` +- ``instruments`` Operations on DataArrays ************************ diff --git a/doc/source/reading.rst b/doc/source/reading.rst index 288bb3d6d4..ac2e6718b2 100644 --- a/doc/source/reading.rst +++ b/doc/source/reading.rst @@ -225,6 +225,16 @@ time etc. The following attributes are standardized across all readers: * ``sensor``: The name of the sensor that recorded the data. For full support through Satpy this should be all lowercase. If the dataset is the result of observations from multiple sensors a ``set`` object can be used to specify more than one sensor name. + + .. deprecated:: 0.61.0 + + The ``sensor`` attribute will be replaced by ``instruments`` in Satpy v1.0. + +* ``instruments``: Names of instruments that recorded the data, stored in a ``set`` object. + Instrument names follow the WMO OSCAR naming conventions. + + .. versionadded:: 0.61.0 + * ``reader``: The name of the Satpy reader that produced the dataset. * ``orbital_parameters``: Dictionary of orbital parameters describing the satellite's position. See the :ref:`orbital_parameters` section below for more information. diff --git a/satpy/_config.py b/satpy/_config.py index 47012742d6..8ed4dbf93b 100644 --- a/satpy/_config.py +++ b/satpy/_config.py @@ -54,6 +54,7 @@ "readers": { "clip_negative_radiances": False, }, + "instruments_key": "sensor", } # Satpy main configuration object diff --git a/satpy/_instruments.py b/satpy/_instruments.py new file mode 100644 index 0000000000..2c9bf22b96 --- /dev/null +++ b/satpy/_instruments.py @@ -0,0 +1,177 @@ +# Copyright (c) 2026 Satpy developers +# +# This file is part of satpy. +# +# satpy is free software: you can redistribute it and/or modify it under the +# terms of the GNU General Public License as published by the Free Software +# Foundation, either version 3 of the License, or (at your option) any later +# version. +# +# satpy is distributed in the hope that it will be useful, but WITHOUT ANY +# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR +# A PARTICULAR PURPOSE. See the GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along with +# satpy. If not, see . +"""Helpers for accessing and modifying instrument attributes.""" + +import logging +import warnings +from enum import StrEnum +from typing import Any + +import satpy + +logger = logging.getLogger(__name__) + +def get_instruments_from_attrs(attrs: dict[str,Any], to_internal: bool=False) -> set[str]: + """Get instrument names from dataset attributes. + + String type attributes are converted to set. This can be + removed once all file handlers provide instruments as a + set. + """ + instruments = attrs.get("instruments", set()) + # 8< v1.0 + sensor = attrs.get("sensor", set()) + if sensor: + warnings.warn( + "Satpy will ignore the 'sensor' attribute as of v1.0. " + "Use the 'instruments' attribute instead.", + DeprecationWarning, + stacklevel=2 + ) + if not instruments: + instruments = sensor + if isinstance(instruments, str): + warnings.warn( + "Converting 'instruments' attribute from string to set. " + "This will result in an error in v1.0, when Satpy will require " + "set type instruments attributes.", + DeprecationWarning, + stacklevel=2 + ) + instruments = set([instruments]) + # >8 v1.0 + if to_internal: + return { + wmo_to_internal(inst) for inst in instruments + } + return instruments + + + +def wmo_to_internal(instrument: str) -> str: + """Convert WMO to internal instrument name.""" + sep_map = { + "-": "-", + "(": "", + ")": "", + " ": "_", + "/": "-" + } + sep_trans = str.maketrans(sep_map) + return instrument.translate(sep_trans).lower() + + +def get_one_instrument_from_attrs(attrs: dict[str,Any]) -> str: + """Get a single instrument name from dataset attributes.""" + instruments = get_instruments_from_attrs(attrs) + if not instruments: + raise KeyError("No 'instruments' in dataset attribute") + if len(instruments) > 1: + logger.warning(f"More than one instrument in dataset attributes, will use the first value: {instruments}") + return list(instruments)[0] + + +def get_pyspectral_instrument_name(instrument: str) -> str: + """Get instrument name expected by pyspectral.""" + return wmo_to_internal(instrument) + + +def join_instrument_names(instruments: set[str]) -> str: + """Join a set of instrument names.""" + return "-".join(sorted(instruments)) + + +def set_instruments_attr(attrs: dict[str,Any], instruments: set[str]|str) -> None: + """Set 'instruments' dataset atrribute.""" + key = get_instruments_key() + attrs[key] = instruments + + +def get_instruments_key(): + """Get key for instruments in dataset attributes.""" + return satpy.config.get("instruments_key") + + +class OSCAR(StrEnum): + """WMO OSCAR instrument names.""" + ABI = "ABI" + AHI = "AHI" + AMSR_2 = "AMSR2" + AMSU_A = "AMSU-A" + AMSU_B = "AMSU-B" + ATMS = "ATMS" + AVHRR = "AVHRR" + AVHRR_2 = "AVHRR/2" + AVHRR_3 = "AVHRR/3" + CRIS = "CrIS" + EPIC = "EPIC" + ETM_PLUS = "ETM+" + FCI = "FCI" + GLM = "GLM" + GMI = "GMI" + IASI = "IASI" + IASI_NG = "IASI-NG" + IMAGER_GOES_12_15 = "IMAGER (GOES 12-15)" + IMAGER_GOES_8_11 = "IMAGER (GOES 8-11)" + IMAGER_INSAT = "IMAGER (INSAT)" + IMAGER_MTSAT_2 = "IMAGER (MTSAT-2)" + JAMI = "JAMI" + LI = "LI" + MERIS = "MERIS" + MERSI_1 = "MERSI-1" + MERSI_2 = "MERSI-2" + MERSI_3 = "MERSI-3" + MERSI_LL = "MERSI-LL" + MERSI_RM = "MERSI-RM" + METIMAGE = "METimage" + MHS = "MHS" + MODIS = "MODIS" + MSS = "MSS" + MSU_GS = "MSU-GS" + MSU_GS_A = "MSU-GS/A" + MVIRI = "MVIRI" + # OSCAR lists "MWR (Sterna)", "MWR (AWS)" etc. + # But to avoid enhancement/composite duplication + # we just use "MWR". + MWR = "MWR" + OCI = "OCI" + OLCI = "OLCI" + OLI = "OLI" + SEAWIFS = "SeaWiFS" + SEVIRI = "SEVIRI" + SGLI = "SGLI" + SLSTR = "SLSTR" + SSMIS = "SSMIS" + TIRS = "TIRS" + TM = "TM" + VIIRS = "VIIRS" + VISSR = "VISSR" + VISSR_HIMAWARI_5 = "VISSR (Himawari-5)" + + +def enum_to_str(instruments: set[StrEnum]) -> set[str]: + """Convert OSCAR enums to string.""" + return {str(i) for i in instruments} + + +_INTERNAL_TO_WMO = { + wmo_to_internal(inst): str(inst) + for inst in OSCAR +} + +def internal_to_wmo(instrument: str) -> str: + """Convert internal to WMO instrument name.""" + return _INTERNAL_TO_WMO.get(instrument, instrument) diff --git a/satpy/composites/aux_data.py b/satpy/composites/aux_data.py index 3179f826a5..10c0eeab50 100644 --- a/satpy/composites/aux_data.py +++ b/satpy/composites/aux_data.py @@ -22,6 +22,7 @@ import os import satpy +import satpy._instruments as inst_utils from satpy.aux_download import DataDownloadMixin from .core import GenericCompositor @@ -156,7 +157,7 @@ def __call__(self, *args, **kwargs): if self.area is None: raise AttributeError("Area definition needs to be configured") img.attrs["area"] = self.area - img.attrs["sensor"] = None + inst_utils.set_instruments_attr(img.attrs, set()) img.attrs["mode"] = "".join(img.bands.data) img.attrs.pop("modifiers", None) img.attrs.pop("calibration", None) diff --git a/satpy/composites/config_loader.py b/satpy/composites/config_loader.py index e5165ada01..a33b1bfd06 100644 --- a/satpy/composites/config_loader.py +++ b/satpy/composites/config_loader.py @@ -28,6 +28,7 @@ from yaml import UnsafeLoader import satpy +import satpy._instruments as inst_utils from satpy import DataID, DataQuery from satpy._config import config_search_paths, get_entry_points_config_dirs, glob_config from satpy.dataset.dataid import minimal_default_keys_config @@ -268,7 +269,7 @@ def load_compositor_configs_for_sensor(sensor_name: str) -> tuple[dict[str, dict DataID key -> key properties """ - config_filename = sensor_name + ".yaml" + config_filename = inst_utils.wmo_to_internal(sensor_name) + ".yaml" logger.debug("Looking for composites config file %s", config_filename) paths = get_entry_points_config_dirs("satpy.composites") composite_configs = config_search_paths( diff --git a/satpy/composites/core.py b/satpy/composites/core.py index 1b124eb88a..e3b4b733e0 100644 --- a/satpy/composites/core.py +++ b/satpy/composites/core.py @@ -26,6 +26,7 @@ import numpy as np import xarray as xr +import satpy._instruments as inst_utils from satpy.dataset import DataID, combine_metadata from satpy.dataset.dataid import minimal_default_keys_config from satpy.utils import unify_chunks @@ -433,20 +434,11 @@ def _concat_datasets(self, projectables, mode): return data - def _get_sensors(self, projectables): - sensor = set() + def _get_sensors(self, projectables) -> set[str]: + sensors = set() for projectable in projectables: - current_sensor = projectable.attrs.get("sensor", None) - if current_sensor: - if isinstance(current_sensor, (str, bytes)): - sensor.add(current_sensor) - else: - sensor |= current_sensor - if len(sensor) == 0: - sensor = None - elif len(sensor) == 1: - sensor = list(sensor)[0] - return sensor + sensors.update(inst_utils.get_instruments_from_attrs(projectable.attrs)) + return sensors def __call__( self, @@ -512,7 +504,7 @@ def _get_updated_attrs(self, datasets, attrs, mode): new_attrs.update(self.attrs) if resolution is not None: new_attrs["resolution"] = resolution - new_attrs["sensor"] = self._get_sensors(datasets) + inst_utils.set_instruments_attr(new_attrs, self._get_sensors(datasets)) new_attrs["mode"] = mode return new_attrs diff --git a/satpy/composites/fill.py b/satpy/composites/fill.py index bd5b51577e..81e02750a1 100644 --- a/satpy/composites/fill.py +++ b/satpy/composites/fill.py @@ -25,6 +25,7 @@ import numpy as np import xarray as xr +import satpy._instruments as inst_utils from satpy.dataset import combine_metadata from .core import ( @@ -406,9 +407,10 @@ def _combine_metadata_with_mode_and_sensor(self, # 'mode' is no longer valid after we've remove the 'A' # let the base class __call__ determine mode attrs.pop("mode", None) - if attrs.get("sensor") is None: + if not inst_utils.get_instruments_from_attrs(attrs): # sensor can be a set - attrs["sensor"] = self._get_sensors([foreground, background]) + instruments = self._get_sensors([foreground, background]) + inst_utils.set_instruments_attr(attrs, instruments) return attrs @staticmethod diff --git a/satpy/composites/glm.py b/satpy/composites/glm.py index 866e952698..c7591e9de3 100644 --- a/satpy/composites/glm.py +++ b/satpy/composites/glm.py @@ -21,6 +21,7 @@ import xarray as xr +import satpy._instruments as inst_utils from satpy.enhancements.enhancer import get_enhanced_image from .core import GenericCompositor @@ -96,9 +97,7 @@ def _update_attrs(self, new_data, background_layer, highlight_layer): new_data.attrs = background_layer.attrs.copy() new_data.attrs["units"] = 1 new_sensors = self._get_sensors((highlight_layer, background_layer)) - new_data.attrs.update({ - "sensor": new_sensors, - }) + inst_utils.set_instruments_attr(new_data.attrs, new_sensors) def __call__(self, projectables, optional_datasets=None, **attrs): """Create RGBA image with highlighted pixels.""" diff --git a/satpy/dependency_tree.py b/satpy/dependency_tree.py index 97777a71e9..19348633a8 100644 --- a/satpy/dependency_tree.py +++ b/satpy/dependency_tree.py @@ -23,6 +23,7 @@ import numpy as np +import satpy._instruments as inst_utils from satpy import DataID, DatasetDict from satpy.dataset import ModifierTuple, create_filtered_query from satpy.dataset.data_dict import TooManyResults, get_key @@ -504,6 +505,7 @@ def get_compositor(self, key): def get_modifier(self, comp_id): """Get a modifer.""" # create a DataID for the compositor we are generating + instr_key = inst_utils.get_instruments_key() modifier = comp_id["modifiers"][-1] for sensor_name in sorted(self.modifiers): modifiers = self.modifiers[sensor_name] @@ -514,7 +516,7 @@ def get_modifier(self, comp_id): mloader, moptions = modifiers[modifier] moptions = moptions.copy() moptions.update(comp_id.to_dict()) - moptions["sensor"] = sensor_name + moptions[instr_key] = sensor_name compositors[comp_id] = mloader(_satpy_id=comp_id, **moptions) return compositors[comp_id] diff --git a/satpy/enhancements/enhancer.py b/satpy/enhancements/enhancer.py index b196c63e28..fb2299b660 100644 --- a/satpy/enhancements/enhancer.py +++ b/satpy/enhancements/enhancer.py @@ -22,9 +22,13 @@ import yaml from yaml import UnsafeLoader +import satpy._instruments as inst_utils from satpy._config import config_search_paths, get_entry_points_config_dirs from satpy.decision_tree import DecisionTree -from satpy.utils import get_logger, recursive_dict_update +from satpy.utils import ( + get_logger, + recursive_dict_update, +) LOG = get_logger(__name__) @@ -122,26 +126,23 @@ def __init__(self, enhancement_config_file=None): self.sensor_enhancement_configs = [] - def get_sensor_enhancement_config(self, sensor): + def get_sensor_enhancement_config(self, sensors: set[str]): """Get the sensor-specific config.""" - if isinstance(sensor, str): - # one single sensor - sensor = [sensor] - paths = get_entry_points_config_dirs("satpy.enhancements") - for sensor_name in sensor: - config_fn = os.path.join("enhancements", sensor_name + ".yaml") + for sensor_name in sensors: + basename = inst_utils.wmo_to_internal(sensor_name) + ".yaml" + config_fn = os.path.join("enhancements", basename) config_files = config_search_paths(config_fn, search_dirs=paths) # Note: Enhancement configuration files can't overwrite individual # options, only entire sections are overwritten for config_file in config_files: yield config_file - def add_sensor_enhancements(self, sensor): + def add_sensor_enhancements(self, sensors: set[str]): """Add sensor-specific enhancements.""" # XXX: Should we just load all enhancements from the base directory? new_configs = [] - for config_file in self.get_sensor_enhancement_config(sensor): + for config_file in self.get_sensor_enhancement_config(sensors): if config_file not in self.sensor_enhancement_configs: self.sensor_enhancement_configs.append(config_file) new_configs.append(config_file) @@ -209,10 +210,11 @@ def get_enhanced_image(dataset, enhance=None, overlay=None, decorate=None, if enhancer is None or enhancer.enhancement_tree is None: LOG.debug("No enhancement being applied to dataset") else: - if dataset.attrs.get("sensor", None): - enhancer.add_sensor_enhancements(dataset.attrs["sensor"]) - - enhancer.apply(img, **dataset.attrs) + sensors = inst_utils.get_instruments_from_attrs(dataset.attrs, to_internal=True) + if sensors: + enhancer.add_sensor_enhancements(sensors) + dataset_attrs = _get_dataset_attrs_for_enh(dataset, sensors) + enhancer.apply(img, **dataset_attrs) if overlay is not None: from satpy.enhancements.overlays import add_overlay @@ -225,3 +227,14 @@ def get_enhanced_image(dataset, enhance=None, overlay=None, decorate=None, img = add_decorate(img, fill_value=fill_value, **decorate) return img + + +def _get_dataset_attrs_for_enh(dataset, instruments_int): + """Get dataset attributes for applying enhancement. + + In particular, use instrument names in internal format so that + they match the enhancement definition in the YAML. + """ + attrs = dataset.attrs.copy() + inst_utils.set_instruments_attr(attrs, instruments_int) + return attrs diff --git a/satpy/modifiers/_crefl_utils.py b/satpy/modifiers/_crefl_utils.py index 7e6ca0c50a..69322d6a11 100644 --- a/satpy/modifiers/_crefl_utils.py +++ b/satpy/modifiers/_crefl_utils.py @@ -69,6 +69,7 @@ import numpy as np import xarray as xr +import satpy._instruments as inst_utils from satpy.dataset.dataid import WavelengthRange LOG = logging.getLogger(__name__) @@ -282,7 +283,8 @@ def run_crefl(refl, :param avg_elevation: average elevation (usually pre-calculated and stored in CMGDEM.hdf) """ - runner_cls = _runner_class_for_sensor(refl.attrs["sensor"]) + sensor = inst_utils.get_one_instrument_from_attrs(refl.attrs) + runner_cls = _runner_class_for_sensor(sensor) runner = runner_cls(refl) corr_refl = runner(sensor_azimuth, sensor_zenith, solar_azimuth, solar_zenith, avg_elevation) return corr_refl @@ -348,8 +350,9 @@ def _run_crefl(self, mus, muv, phi, solar_zenith, sensor_zenith, height, coeffs) class _VIIRSMODISCREFLRunner(_CREFLRunner): def _run_crefl(self, mus, muv, phi, solar_zenith, sensor_zenith, height, coeffs): + instrument = inst_utils.get_one_instrument_from_attrs(self._refl.attrs) return da.map_blocks(_run_crefl, self._refl.data, mus.data, muv.data, phi.data, - height, self._refl.attrs.get("sensor"), *coeffs, + height, instrument, *coeffs, meta=np.ndarray((), dtype=self._refl.dtype), chunks=self._refl.chunks, dtype=self._refl.dtype, ) @@ -384,7 +387,7 @@ def _run_crefl(self, mus, muv, phi, solar_zenith, sensor_zenith, height, coeffs) def _runner_class_for_sensor(sensor_name: str) -> Type[_CREFLRunner]: try: - return _SENSOR_TO_RUNNER[sensor_name] + return _SENSOR_TO_RUNNER[inst_utils.wmo_to_internal(sensor_name)] except KeyError: raise NotImplementedError(f"Don't know how to apply CREFL to data from sensor {sensor_name}.") diff --git a/satpy/modifiers/atmosphere.py b/satpy/modifiers/atmosphere.py index c7144c27ca..0a93418c70 100644 --- a/satpy/modifiers/atmosphere.py +++ b/satpy/modifiers/atmosphere.py @@ -23,6 +23,7 @@ import numpy as np import xarray as xr +import satpy._instruments as inst_utils from satpy.modifiers import ModifierBase from satpy.modifiers._crefl import ReflectanceCorrector # noqa from satpy.modifiers.angles import compute_relative_azimuth, get_angles, get_satellite_zenith_angle @@ -104,7 +105,10 @@ def __call__(self, projectables, optional_datasets=None, **info): logger.info("Removing Rayleigh scattering with atmosphere '%s' and " "aerosol type '%s' for '%s'", atmosphere, aerosol_type, vis.attrs["name"]) - corrector = Rayleigh(vis.attrs["platform_name"], vis.attrs["sensor"], + sensor = inst_utils.get_pyspectral_instrument_name( + inst_utils.get_one_instrument_from_attrs(vis.attrs) + ) + corrector = Rayleigh(vis.attrs["platform_name"], sensor, atmosphere=atmosphere, aerosol_type=aerosol_type) @@ -158,8 +162,11 @@ def __call__(self, projectables, optional_datasets=None, **info): satz = satz.data # get dask array underneath logger.info("Correction for limb cooling") + sensor = inst_utils.get_pyspectral_instrument_name( + inst_utils.get_one_instrument_from_attrs(band.attrs) + ) corrector = AtmosphericalCorrection(band.attrs["platform_name"], - band.attrs["sensor"]) + sensor) atm_corr = da.map_blocks(_call_mapped_correction, satz, band.data, corrector=corrector, diff --git a/satpy/modifiers/spectral.py b/satpy/modifiers/spectral.py index 402b5606d4..7c990417e8 100644 --- a/satpy/modifiers/spectral.py +++ b/satpy/modifiers/spectral.py @@ -21,6 +21,7 @@ import xarray as xr +import satpy._instruments as inst_utils from satpy.modifiers import ModifierBase try: @@ -131,8 +132,10 @@ def _init_reflectance_calculator(self, metadata): if not Calculator: logger.info("Couldn't load pyspectral") raise ImportError("No module named pyspectral.near_infrared_reflectance") - - reflectance_3x_calculator = Calculator(metadata["platform_name"], metadata["sensor"], metadata["name"], + sensor = inst_utils.get_pyspectral_instrument_name( + inst_utils.get_one_instrument_from_attrs(metadata) + ) + reflectance_3x_calculator = Calculator(metadata["platform_name"], sensor, metadata["name"], sunz_threshold=self.sun_zenith_threshold, masking_limit=self.masking_limit) return reflectance_3x_calculator diff --git a/satpy/readers/abi_l1b.py b/satpy/readers/abi_l1b.py index 48e82f6968..b144529ebe 100644 --- a/satpy/readers/abi_l1b.py +++ b/satpy/readers/abi_l1b.py @@ -73,7 +73,7 @@ def get_dataset(self, key, info): def _adjust_attrs(self, data, key): data.attrs.update({"platform_name": self.platform_name, - "sensor": self.sensor}) + "sensor": {self.sensor}}) # Add orbital parameters projection = self.nc["goes_imager_projection"] data.attrs["orbital_parameters"] = { diff --git a/satpy/readers/core/file_handlers.py b/satpy/readers/core/file_handlers.py index ccf4f60bc5..54b57ebf1d 100644 --- a/satpy/readers/core/file_handlers.py +++ b/satpy/readers/core/file_handlers.py @@ -156,7 +156,7 @@ def end_time(self): return self.filename_info.get("end_time", self.start_time) @property - def sensor_names(self): + def sensor_names(self) -> set: """List of sensors represented in this file.""" raise NotImplementedError diff --git a/satpy/scene.py b/satpy/scene.py index 8693cc77b4..796f72fd49 100644 --- a/satpy/scene.py +++ b/satpy/scene.py @@ -29,6 +29,7 @@ from pyresample.geometry import AreaDefinition, BaseDefinition, CoordinateDefinition, SwathDefinition from xarray import DataArray +import satpy._instruments as inst_utils from satpy.area import get_area_def from satpy.composites.config_loader import load_compositor_configs_for_sensors from satpy.composites.core import IncompatibleAreas @@ -36,7 +37,10 @@ from satpy.dependency_tree import DependencyTree from satpy.node import CompositorNode, MissingDependencies, ReaderNode from satpy.readers.core.loading import load_readers -from satpy.utils import convert_remote_files_to_fsspec, get_storage_options_from_reader_kwargs +from satpy.utils import ( + convert_remote_files_to_fsspec, + get_storage_options_from_reader_kwargs, +) LOG = logging.getLogger(__name__) @@ -179,7 +183,7 @@ def _create_reader_instances(self, @property def sensor_names(self) -> set[str]: - """Return sensor names for the data currently contained in this Scene. + """Return WMO sensor names for the data currently contained in this Scene. Sensor information is collected from data contained in the Scene whether loaded from a reader or generated as a composite with @@ -190,21 +194,29 @@ def sensor_names(self) -> set[str]: """ contained_sensor_names = self._contained_sensor_names() - reader_sensor_names = set([sensor for reader_instance in self._readers.values() - for sensor in reader_instance.sensor_names]) + reader_sensor_names = self._reader_sensor_names() return contained_sensor_names | reader_sensor_names def _contained_sensor_names(self) -> set[str]: sensor_names = set() for data_arr in self.values(): - if "sensor" not in data_arr.attrs: - continue - if isinstance(data_arr.attrs["sensor"], str): - sensor_names.add(data_arr.attrs["sensor"]) - elif isinstance(data_arr.attrs["sensor"], set): - sensor_names.update(data_arr.attrs["sensor"]) + sensor_names.update(inst_utils.get_instruments_from_attrs(data_arr.attrs)) return sensor_names + def _reader_sensor_names(self) -> set[str]: + """Get WMO instrument names from readers.""" + instruments = set( + [ + instrument + for reader_instance in self._readers.values() + for instrument in reader_instance.sensor_names + ] + ) + return { + inst_utils.internal_to_wmo(inst) + for inst in instruments + } + @property def start_time(self): """Return the start time of the contained data. @@ -829,8 +841,36 @@ def __getitem__(self, key): """Get a dataset or create a new 'slice' of the Scene.""" if isinstance(key, tuple): return self.slice(key) + # 8< v1.0 + data_array = self._datasets[key] + if self._should_add_sensor_attribute(data_array): + self._set_sensor_attribute(data_array) + return data_array + # >8 v1.0 return self._datasets[key] + # 8< v1.0 + def _should_add_sensor_attribute(self, data_array: xr.DataArray) -> bool: + return "instruments" in data_array.attrs and "sensor" not in data_array.attrs + + def _set_sensor_attribute(self, data_array: xr.DataArray) -> None: + instruments = data_array.attrs["instruments"] + if self._should_convert_to_string(instruments): + data_array.attrs["sensor"] = inst_utils.wmo_to_internal(list(instruments)[0]) + else: + data_array.attrs["sensor"] = {inst_utils.wmo_to_internal(inst) for inst in instruments} + + def _should_convert_to_string(self, instruments): + # In satpy < v1.0 single sensors are provided as string by most readers + has_one_element = len(instruments) == 1 + readers_providing_set = [ + "seadas_l2", + "oci_l2_bgc" + ] + is_exception = any([reader in self._readers for reader in readers_providing_set]) + return has_one_element and not is_exception + # >8 v1.0 + def __setitem__(self, key, value): """Add the item to the scene.""" self._datasets[key] = value diff --git a/satpy/tests/compositor_tests/test_aux_data.py b/satpy/tests/compositor_tests/test_aux_data.py index 255231703e..d9e3657c66 100644 --- a/satpy/tests/compositor_tests/test_aux_data.py +++ b/satpy/tests/compositor_tests/test_aux_data.py @@ -83,7 +83,7 @@ def load(self, arg): filenames=[IMAGE_FILENAME]) register.assert_not_called() retrieve.assert_not_called() - assert res.attrs["sensor"] is None + assert res.attrs["sensor"] == set() assert "modifiers" not in res.attrs assert "calibration" not in res.attrs @@ -95,7 +95,7 @@ def load(self, arg): res = comp() Scene.assert_called_once_with(reader="generic_image", filenames=[os.path.join("data_dir", "foo.tif")]) - assert res.attrs["sensor"] is None + assert res.attrs["sensor"] == set() assert "modifiers" not in res.attrs assert "calibration" not in res.attrs diff --git a/satpy/tests/compositor_tests/test_core.py b/satpy/tests/compositor_tests/test_core.py index 315431aa4c..55f699aa42 100644 --- a/satpy/tests/compositor_tests/test_core.py +++ b/satpy/tests/compositor_tests/test_core.py @@ -277,13 +277,13 @@ def test_concat_datasets(self): def test_get_sensors(self): """Test getting sensors from the dataset attributes.""" res = self.comp._get_sensors([self.all_valid]) - assert res is None + assert res == set() dset1 = self.all_valid dset1.attrs["sensor"] = "foo" res = self.comp._get_sensors([dset1]) - assert res == "foo" + assert res == {"foo"} dset2 = self.first_invalid - dset2.attrs["sensor"] = "bar" + dset2.attrs["sensor"] = {"bar"} res = self.comp._get_sensors([dset1, dset2]) assert "foo" in res assert "bar" in res @@ -327,12 +327,12 @@ def test_call(self): """Test calling generic compositor.""" # Multiple datasets with extra attributes all_valid = self.all_valid - all_valid.attrs["sensor"] = "foo" + all_valid.attrs["sensor"] = {"foo"} attrs = {"foo": "bar", "resolution": 333} self.comp.attrs["resolution"] = None res = self.comp([self.all_valid, self.first_invalid], **attrs) # Verify attributes - assert res.attrs.get("sensor") == "foo" + assert res.attrs.get("sensor") == {"foo"} assert "foo" in res.attrs assert res.attrs.get("foo") == "bar" assert "units" not in res.attrs diff --git a/satpy/tests/reader_tests/test_abi_l1b.py b/satpy/tests/reader_tests/test_abi_l1b.py index bcb6d35041..855265dd21 100644 --- a/satpy/tests/reader_tests/test_abi_l1b.py +++ b/satpy/tests/reader_tests/test_abi_l1b.py @@ -371,7 +371,7 @@ def test_get_dataset(self, c01_data_arr): "scan_mode": "M4", "scene_abbr": "C", "scene_id": None, - "sensor": "abi", + "sensor": {"abi"}, "timeline_ID": None, "suffix": "suffix", "units": "W m-2 um-1 sr-1", diff --git a/satpy/tests/reader_tests/test_iasi_l2.py b/satpy/tests/reader_tests/test_iasi_l2.py index 3f0e618119..6c92077a50 100644 --- a/satpy/tests/reader_tests/test_iasi_l2.py +++ b/satpy/tests/reader_tests/test_iasi_l2.py @@ -157,7 +157,7 @@ def test_scene(test_data): assert scn.start_time is not None assert scn.end_time is not None assert scn.sensor_names - assert "iasi" in scn.sensor_names + assert "IASI" in scn.sensor_names def test_scene_load_available_datasets(test_data): diff --git a/satpy/tests/reader_tests/test_iasi_l2_so2_bufr.py b/satpy/tests/reader_tests/test_iasi_l2_so2_bufr.py index b75ac67dee..e9f4c0180a 100644 --- a/satpy/tests/reader_tests/test_iasi_l2_so2_bufr.py +++ b/satpy/tests/reader_tests/test_iasi_l2_so2_bufr.py @@ -366,7 +366,7 @@ def test_scene(self): assert scn.start_time is not None assert scn.end_time is not None assert scn.sensor_names - assert "iasi" in scn.sensor_names + assert "IASI" in scn.sensor_names @unittest.skipIf(sys.platform.startswith("win"), "'eccodes' not supported on Windows") def test_scene_load_available_datasets(self): diff --git a/satpy/tests/reader_tests/test_seviri_l1b_native.py b/satpy/tests/reader_tests/test_seviri_l1b_native.py index c2a5ffee9e..aa8f83ed5b 100644 --- a/satpy/tests/reader_tests/test_seviri_l1b_native.py +++ b/satpy/tests/reader_tests/test_seviri_l1b_native.py @@ -1301,7 +1301,7 @@ def test_read_physical_seviri_nat_file(full_path): """ scene = scene_from_physical_seviri_nat_file(full_path) - assert scene.sensor_names == {"seviri"} + assert scene.sensor_names == {"SEVIRI"} assert len(scene.available_dataset_ids()) == 36 assert set(scene.available_dataset_names()) == set(CHANNEL_INDEX_LIST) diff --git a/satpy/tests/scene_tests/test_data_access.py b/satpy/tests/scene_tests/test_data_access.py index 60ada6c78b..f2463abf54 100644 --- a/satpy/tests/scene_tests/test_data_access.py +++ b/satpy/tests/scene_tests/test_data_access.py @@ -393,3 +393,36 @@ def test_chunk_pass_through(self): scene.load(["ds1"]) scene = scene.chunk(chunks=2) assert scene["ds1"].data.chunksize == (2, 2) + + +# 8< v1.0 +class TestLegacySensorAttribute: + """Tests for legacy sensor attribute.""" + + @pytest.mark.parametrize( + ("reader", "expected"), + [ + ("most-readers", "inst1"), + ("seadas_l2", {"inst1"}), + ("oci_l2_bgc", {"inst1"}), + ] + ) + def test_set_single_sensor(self, reader, expected): + """Test setting sensor attribute with a single sensor.""" + scene = Scene() + scene._readers[reader] = "dummy" + scene["ds"] = xr.DataArray(attrs={"instruments": {"inst1"}}) + assert scene["ds"].attrs["sensor"] == expected + + def test_set_multi_sensor(self): + """Test setting sensor attribute with multiple sensors.""" + scene = Scene() + scene["ds"] = xr.DataArray(attrs={"instruments": {"inst1", "inst2"}}) + assert scene["ds"].attrs["sensor"] == {"inst1", "inst2"} + + def test_set_no_sensor(self): + """Test sensor attribute is set only if instruments present.""" + scene = Scene() + scene["ds"] = xr.DataArray() + assert "sensor" not in scene["ds"].attrs +# >8 v1.0 diff --git a/satpy/tests/scene_tests/test_load.py b/satpy/tests/scene_tests/test_load.py index f3e862a46c..f39b416c8b 100644 --- a/satpy/tests/scene_tests/test_load.py +++ b/satpy/tests/scene_tests/test_load.py @@ -141,7 +141,7 @@ def test_available_when_sensor_none_in_preloaded_dataarrays(self): doesn't break available composite IDs. """ - scene = _scene_with_data_array_none_sensor() + scene = _scene_with_data_array_empty_sensor() available_comp_ids = scene.available_composite_ids() assert make_cid(name="static_image") in available_comp_ids @@ -645,31 +645,31 @@ def test_load_too_many(self): assert len(avail_comps) == 1 pytest.raises(KeyError, scene.load, [0.21]) - def test_load_when_sensor_none_in_preloaded_dataarrays(self): - """Test Scene loading when existing loaded arrays have sensor set to None. + def test_load_when_sensor_empty_in_preloaded_dataarrays(self): + """Test Scene loading when existing loaded arrays have empty sensor set. Some readers or composites (ex. static images) don't have a sensor and - developers choose to set it to `None`. This test makes sure this + developers choose to set it to `set()`. This test makes sure this doesn't break loading. """ - scene = _scene_with_data_array_none_sensor() + scene = _scene_with_data_array_empty_sensor() scene.load(["static_image"]) assert "static_image" in scene assert "my_data" in scene -def _scene_with_data_array_none_sensor(): +def _scene_with_data_array_empty_sensor(): scene = Scene(filenames=["fake1_1.txt"], reader="fake1") - scene["my_data"] = _data_array_none_sensor("my_data") + scene["my_data"] = _data_array_with_empty_sensor("my_data") return scene -def _data_array_none_sensor(name: str) -> xr.DataArray: - """Create a DataArray with sensor set to ``None``.""" +def _data_array_with_empty_sensor(name: str) -> xr.DataArray: + """Create a DataArray with empty sensor set.""" return xr.DataArray( da.zeros((2, 2)), attrs={ "name": name, - "sensor": None, + "sensor": set(), }) diff --git a/satpy/tests/test_instruments.py b/satpy/tests/test_instruments.py new file mode 100644 index 0000000000..1229a822c8 --- /dev/null +++ b/satpy/tests/test_instruments.py @@ -0,0 +1,110 @@ +# Copyright (c) 2026 Satpy developers +# +# This file is part of satpy. +# +# satpy is free software: you can redistribute it and/or modify it under the +# terms of the GNU General Public License as published by the Free Software +# Foundation, either version 3 of the License, or (at your option) any later +# version. +# +# satpy is distributed in the hope that it will be useful, but WITHOUT ANY +# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR +# A PARTICULAR PURPOSE. See the GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along with +# satpy. If not, see . +"""Unit tests for instrument helpers.""" + +import pytest + +import satpy +import satpy._instruments as inst_utils + + +@pytest.mark.parametrize( + ("attrs", "to_internal", "expected"), + [ + ({"instruments": {"AVHRR/3"}}, False, {"AVHRR/3"}), + ({"instruments": {"AVHRR/3"}}, True, {"avhrr-3"}), + ({}, False, set()), + ] +) +def test_get_instruments_from_attrs(attrs, to_internal, expected): + """Test getting instruments from dataset attributes.""" + assert inst_utils.get_instruments_from_attrs(attrs, to_internal) == expected + + +# 8< v1.0 +@pytest.mark.parametrize( + ("attrs", "expected"), + [ + ({"sensor": "myinstr"}, {"myinstr"}), + ({"sensor": {"myinstr"}}, {"myinstr"}), + ({"instruments": "myinstr"}, {"myinstr"}), + ] +) +def test_get_instruments_from_attrs_with_warning(attrs, expected): + """Test deprecation warnings when getting instruments.""" + with pytest.warns(DeprecationWarning, match="v1.0"): + assert inst_utils.get_instruments_from_attrs(attrs) == expected +# >8 v1.0 + + +@pytest.mark.parametrize( + ("instrument", "expected"), + [ + ("AVHRR/3", "avhrr-3"), + ("IMAGER (GOES 8-11)", "imager_goes_8-11"), + ("MERSI-1", "mersi-1"), + ("MSU-GS/A", "msu-gs-a"), + ] +) +def test_wmo_to_internal(instrument, expected): + """Test conversion to internal instrument name.""" + assert inst_utils.wmo_to_internal(instrument) == expected + + +def test_join_instruments(): + """Test joining a set of instruments.""" + instruments = {"mersi-1", "abi"} + expected = "abi-mersi-1" + assert inst_utils.join_instrument_names(instruments) == expected + + +def test_set_instruments_attr(): + """Test setting instruments attribute.""" + attrs = {"instruments": {"myinstrument"}} + new_instruments = {"i1", "i2"} + with satpy.config.set(instruments_key="instruments"): + inst_utils.set_instruments_attr(attrs, new_instruments) + assert attrs["instruments"] == new_instruments + + +def test_get_one_instrument_from_attrs(): + """Test getting a single instrument from dataset attributes.""" + attrs = {"instruments": {"i1"}} + with satpy.config.set(instruments_key="instruments"): + assert inst_utils.get_one_instrument_from_attrs(attrs) == "i1" + + +def test_get_one_instrument_from_attrs_with_warning(caplog): + """Test warnings when getting a single instrument.""" + attrs = {"instruments": {"i1", "i2"}} + with satpy.config.set(instruments_key="instruments"): + inst_utils.get_one_instrument_from_attrs(attrs) + assert "More than one" in caplog.text + with pytest.raises(KeyError): + inst_utils.get_one_instrument_from_attrs({}) + + + +@pytest.mark.parametrize( + ("instrument", "expected"), + [ + ("abi", "ABI"), + ("ABI", "ABI"), + ] +) +def test_internal_to_wmo(instrument, expected): + """Test conversion to WMO instrument name.""" + assert inst_utils.internal_to_wmo(instrument) == expected diff --git a/satpy/writers/core/base.py b/satpy/writers/core/base.py index e0a53d0f7e..11b81708ed 100644 --- a/satpy/writers/core/base.py +++ b/satpy/writers/core/base.py @@ -21,6 +21,7 @@ import typing import warnings +import satpy._instruments as inst_utils from satpy.aux_download import DataDownloadMixin from satpy.plugin_base import Plugin from satpy.writers.core.compute import compute_writer_results, split_results @@ -136,8 +137,9 @@ def create_filename_parser(self, base_dir): @staticmethod def _prepare_metadata_for_filename_formatting(attrs): - if isinstance(attrs.get("sensor"), set): - attrs["sensor"] = "-".join(sorted(attrs["sensor"])) + instruments = inst_utils.get_instruments_from_attrs(attrs, to_internal=True) + joined = inst_utils.join_instrument_names(instruments) + inst_utils.set_instruments_attr(attrs, joined) def get_filename(self, **kwargs): """Create a filename where output data will be saved. diff --git a/satpy/writers/mitiff.py b/satpy/writers/mitiff.py index 7788d4b78a..25d9656ed3 100644 --- a/satpy/writers/mitiff.py +++ b/satpy/writers/mitiff.py @@ -26,6 +26,7 @@ import numpy as np from PIL import Image, ImagePalette +import satpy._instruments as inst_utils from satpy.dataset import DataID, DataQuery from satpy.enhancements.enhancer import get_enhanced_image from satpy.writers.core.image import ImageWriter @@ -53,12 +54,9 @@ def _adjust_kwargs(dataset, kwargs): if "start_time" not in kwargs: kwargs["start_time"] = dataset.attrs["start_time"] if "sensor" not in kwargs: - kwargs["sensor"] = dataset.attrs["sensor"] - # Sensor attrs could be set. MITIFFs needing to handle sensor can only have one sensor - # Assume the first value of set as the sensor. - if isinstance(kwargs["sensor"], set): - LOG.warning("Sensor is set, will use the first value: %s", kwargs["sensor"]) - kwargs["sensor"] = (list(kwargs["sensor"]))[0] + # MITIFFs needing to handle sensor can only have one sensor + # Assume the first value of set as the sensor. + kwargs["sensor"] = inst_utils.get_one_instrument_from_attrs(dataset.attrs) class MITIFFWriter(ImageWriter):