diff --git a/CHANGELOG.md b/CHANGELOG.md index d4413ed0e..fad65f155 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,12 @@ - `hdmf.utils.get_docval` now also works on functions with type-hinted signatures: docval-compatible argument specs are synthesized from the signature, type hints, and Google-style docstring. This keeps the downstream splice pattern `@docval(*get_docval(Parent.__init__, ...))` working as HDMF (and downstream libraries) migrate off `@docval` module by module. Existing `@docval`-decorated functions are unaffected. @bendichter [#TBD] - New required dependencies: `beartype`, `numpydantic`, and `docstring_parser`. @bendichter [#TBD] - Migrated `hdmf.data_utils`, `hdmf.query`, `hdmf.validate.validator`, `hdmf.common.sparse`, `hdmf.common.resources`, and `hdmf.common.hierarchicaltable` off `@docval` to type-hinted signatures with `@validated` (includes `GenericDataChunkIterator`, `DataChunkIterator`, `DataChunk`, `DataIO`, `CSRMatrix`, `HERD`, and the validator classes). Downstream code is unaffected: `get_docval` on these functions returns equivalent specs synthesized from the type hints, so patterns like `@docval(*get_docval(DataIO.__init__), ...)` (used by `H5DataIO` and extensions) keep working, positional/keyword calling conventions are unchanged, and error messages keep the docval format. @bendichter [#TBD] +- Migrated `hdmf.common.table` (`VectorData`, `VectorIndex`, `ElementIdentifiers`, `DynamicTable`, `DynamicTableRegion`, `EnumData`, `MeaningsTable`), `hdmf.common.alignedtable`, and `hdmf.common.multi` off `@docval` to type-hinted signatures. The three `AlignedDynamicTable` methods that splice parent specs via `@docval(*get_docval(...))` intentionally remain on `@docval` and consume specs synthesized from the migrated parents. @bendichter [#TBD] +- Migrated the `hdmf.spec` subsystem (`spec.py`, `namespace.py`, `catalog.py`, `write.py`) off `@docval` to type-hinted signatures. The shared module-level spec-argument lists (`_attr_args`, `_group_args`, ...) are inlined into the signatures and removed. Note that error messages for *missing or unrecognized arguments* on migrated functions now use Python's native wording (e.g. "missing a required argument: 'doc'" instead of "missing argument 'doc'"); type and value error messages are unchanged. @bendichter [#TBD] +- Migrated `hdmf.container` (`AbstractContainer`, `Container`, `Data`, `DataRegion`, `Table` base classes) off `@docval` to type-hinted signatures. The dynamically composed decorators in `MultiContainerInterface` (`__make_get`/`__make_add`/`__make_create`/...) and the `Row`/`Table` row-class machinery intentionally remain on `@docval` until it is removed. @bendichter [#TBD] +- Migrated the remaining runtime modules off `@docval`: `hdmf.build` (`builders`, `manager`, `objectmapper`, `errors`), `hdmf.backends` (`io`, `utils`, `hdf5.h5_utils` including `H5DataIO`, `hdf5.h5tools` including `HDF5IO`), `hdmf.common.__init__`, `hdmf.common.io.table`, `hdmf.validate.errors`, and `hdmf.term_set`. The only remaining `@docval` uses are the spec-splicing composition sites (which consume synthesized specs) and `build/classgenerator.py`, which generates constructors for schema-derived classes and migrates in a later phase. @bendichter [#TBD] +- Schema-generated classes (from `get_class`/`TypeMap.get_dt_container_cls`, including all classes auto-generated by extensions) and the dynamically composed `MultiContainerInterface` methods (`add_*`, `create_*`, `get_*`, `__init__`) now have **real type-hinted signatures** instead of `(self, **kwargs)`, visible to IDEs, `help()`, and `inspect.signature`. They are built by the new `hdmf.typing.signature_function`, which converts docval-style argument specs into signatures at runtime (falling back to legacy `@docval` for non-identifier argument names). Their docstrings no longer include docval's synthetic signature line. @bendichter [#TBD] +- Added a `py.typed` marker: hdmf now declares inline type information to static type checkers. Annotation coverage is partial and will grow. @bendichter [#TBD] ## HDMF 6.1.0 (June 25, 2026) diff --git a/docs/source/conf.py b/docs/source/conf.py index f35ef939a..684c1b277 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -101,6 +101,8 @@ # Is[is_array_data], Is[has_shape_...], etc., which are not documented targets ('py:class', r'beartype\..*'), ('py:class', r'(is|has_shape)_\w*'), + ('py:class', r'h5py\._hl\..*'), + ('py:class', r'types\.(Lambda|Function)Type'), ] suppress_warnings = ["config.cache"] diff --git a/src/hdmf/backends/hdf5/h5_utils.py b/src/hdmf/backends/hdf5/h5_utils.py index 878ebf089..c198ed948 100644 --- a/src/hdmf/backends/hdf5/h5_utils.py +++ b/src/hdmf/backends/hdf5/h5_utils.py @@ -19,7 +19,9 @@ from ...data_utils import DataIO, AbstractDataChunkIterator, append_data from ...query import HDMFDataset, ReferenceResolver, ContainerResolver, BuilderResolver from ...spec import SpecWriter, SpecReader -from ...utils import docval, getargs, popargs, get_docval, get_data_shape +from ...utils import get_docval, get_data_shape +from typing import Any +from ...typing import Bool, Int, TypeName, validated class HDF5IODataChunkIteratorQueue(deque): @@ -83,12 +85,16 @@ def append(self, dataset, data): class H5Dataset(HDMFDataset): - @docval({'name': 'dataset', 'type': Dataset, 'doc': 'the HDF5 file lazily evaluate'}, - {'name': 'io', 'type': 'hdmf.backends.hdf5.h5tools.HDF5IO', - 'doc': 'the IO object that was used to read the underlying dataset'}) - def __init__(self, **kwargs): - self.__io = popargs('io', kwargs) - super().__init__(**kwargs) + @validated + def __init__(self, dataset: Dataset, io: TypeName['hdmf.backends.hdf5.h5tools.HDF5IO']): # noqa: F821 + """Initialize this object. + + Args: + dataset: the HDF5 file lazily evaluate + io: the IO object that was used to read the underlying dataset + """ + self.__io = io + super().__init__(dataset=dataset) @property def io(self): @@ -183,14 +189,16 @@ def get_object(self, h5obj): class AbstractH5TableDataset(DatasetOfReferences): - @docval({'name': 'dataset', 'type': Dataset, 'doc': 'the HDF5 file lazily evaluate'}, - {'name': 'io', 'type': 'hdmf.backends.hdf5.h5tools.HDF5IO', - 'doc': 'the IO object that was used to read the underlying dataset'}, - {'name': 'types', 'type': (list, tuple), - 'doc': 'the IO object that was used to read the underlying dataset'}) - def __init__(self, **kwargs): - types = popargs('types', kwargs) - super().__init__(**kwargs) + @validated + def __init__(self, dataset: Dataset, io: TypeName['hdmf.backends.hdf5.h5tools.HDF5IO'], types: list | tuple): # noqa: F821 + """Initialize this object. + + Args: + dataset: the HDF5 file lazily evaluate + io: the IO object that was used to read the underlying dataset + types: the types of the elements in the dataset + """ + super().__init__(dataset=dataset, io=io) self.__refgetters = dict() for i, t in enumerate(types): if t is Reference: @@ -317,9 +325,14 @@ class H5SpecWriter(SpecWriter): __str_type = special_dtype(vlen=str) - @docval({'name': 'group', 'type': Group, 'doc': 'the HDF5 file to write specs to'}) - def __init__(self, **kwargs): - self.__group = getargs('group', kwargs) + @validated + def __init__(self, group: Group): + """Initialize this object. + + Args: + group: the HDF5 file to write specs to + """ + self.__group = group @staticmethod def stringify(spec): @@ -344,9 +357,14 @@ def write_namespace(self, namespace, path): class H5SpecReader(SpecReader): """Class that reads cached JSON-formatted namespace and spec data from an HDF5 group.""" - @docval({'name': 'group', 'type': Group, 'doc': 'the HDF5 group to read specs from'}) - def __init__(self, **kwargs): - self.__group = popargs('group', kwargs) + @validated + def __init__(self, group: Group): + """Initialize this object. + + Args: + group: the HDF5 group to read specs from + """ + self.__group = group source = "%s:%s" % (os.path.abspath(self.__group.file.name), self.__group.name) super().__init__(source=source) self.__cache = None @@ -378,78 +396,56 @@ class H5DataIO(DataIO): for data arrays. """ - @docval({'name': 'data', - 'type': (np.ndarray, list, tuple, Dataset, Iterable), - 'doc': 'the data to be written. NOTE: If an h5py.Dataset is used, all other settings but link_data' + - ' will be ignored as the dataset will either be linked to or copied as is in H5DataIO.', - 'default': None}, - {'name': 'maxshape', - 'type': tuple, - 'doc': 'Dataset will be resizable up to this shape (Tuple). Automatically enables chunking.' + - 'Use None for the axes you want to be unlimited.', - 'default': None}, - {'name': 'chunks', - 'type': (bool, tuple), - 'doc': 'Chunk shape or True to enable auto-chunking', - 'default': None}, - {'name': 'compression', - 'type': (str, bool, int), - 'doc': 'Compression strategy. If a bool is given, then gzip compression will be used by default.' + - 'http://docs.h5py.org/en/latest/high/dataset.html#dataset-compression', - 'default': None}, - {'name': 'compression_opts', - 'type': (int, tuple), - 'doc': 'Parameter for compression filter', - 'default': None}, - {'name': 'fillvalue', - 'type': None, - 'doc': 'Value to be returned when reading uninitialized parts of the dataset', - 'default': None}, - {'name': 'shuffle', - 'type': bool, - 'doc': 'Enable shuffle I/O filter. http://docs.h5py.org/en/latest/high/dataset.html#dataset-shuffle', - 'default': None}, - {'name': 'fletcher32', - 'type': bool, - 'doc': 'Enable fletcher32 checksum. http://docs.h5py.org/en/latest/high/dataset.html#dataset-fletcher32', - 'default': None}, - {'name': 'link_data', - 'type': bool, - 'doc': 'If data is an h5py.Dataset should it be linked to or copied. NOTE: This parameter is only ' + - 'allowed if data is an h5py.Dataset', - 'default': False}, - {'name': 'allow_plugin_filters', - 'type': bool, - 'doc': 'Enable passing dynamically loaded filters as compression parameter', - 'default': False}, - {'name': 'shape', - 'type': tuple, - 'doc': 'the shape of the new dataset, used only if data is None', - 'default': None}, - {'name': 'dtype', - 'type': (str, type, np.dtype), - 'doc': 'the data type of the new dataset, used only if data is None', - 'default': None} - ) - def __init__(self, **kwargs): - # Get the list of I/O options that user has passed in - ioarg_names = [name for name in kwargs.keys() if name not in ['data', 'link_data', 'allow_plugin_filters', - 'dtype', 'shape']] - - # Remove the ioargs from kwargs - ioarg_values = [popargs(argname, kwargs) for argname in ioarg_names] + @validated + def __init__(self, + data: np.ndarray | list | tuple | Dataset | Iterable | None = None, + maxshape: tuple | None = None, + chunks: Bool | tuple | None = None, + compression: str | Bool | Int | None = None, + compression_opts: Int | tuple | None = None, + fillvalue: Any = None, + shuffle: Bool | None = None, + fletcher32: Bool | None = None, + link_data: Bool = False, + allow_plugin_filters: Bool = False, + shape: tuple | None = None, + dtype: str | type | np.dtype | None = None): + """Initialize this object. + + Args: + data: the data to be written. NOTE: If an h5py.Dataset is used, all other settings but link_data + will be ignored as the dataset will either be linked to or copied as is in H5DataIO. + maxshape: Dataset will be resizable up to this shape (Tuple). Automatically enables chunking. + Use None for the axes you want to be unlimited. + chunks: Chunk shape or True to enable auto-chunking + compression: Compression strategy. If a bool is given, then gzip compression will be used by + default. http://docs.h5py.org/en/latest/high/dataset.html#dataset-compression + compression_opts: Parameter for compression filter + fillvalue: Value to be returned when reading uninitialized parts of the dataset + shuffle: Enable shuffle I/O filter. http://docs.h5py.org/en/latest/high/dataset.html#dataset-shuffle + fletcher32: Enable fletcher32 checksum. http://docs.h5py.org/en/latest/high/dataset.html#dataset-fletcher32 + link_data: If data is an h5py.Dataset should it be linked to or copied. NOTE: This parameter is + only allowed if data is an h5py.Dataset + allow_plugin_filters: Enable passing dynamically loaded filters as compression parameter + shape: the shape of the new dataset, used only if data is None + dtype: the data type of the new dataset, used only if data is None + """ + # the h5py io settings (everything except the DataIO/link args) + ioargs = dict(maxshape=maxshape, chunks=chunks, compression=compression, + compression_opts=compression_opts, fillvalue=fillvalue, shuffle=shuffle, + fletcher32=fletcher32) # Consume link_data parameter - self.__link_data = popargs('link_data', kwargs) + self.__link_data = link_data # Consume allow_plugin_filters parameter - self.__allow_plugin_filters = popargs('allow_plugin_filters', kwargs) + self.__allow_plugin_filters = allow_plugin_filters # Check for possible collision with other parameters - if not isinstance(getargs('data', kwargs), Dataset) and self.__link_data: + if not isinstance(data, Dataset) and self.__link_data: self.__link_data = False warnings.warn('link_data parameter in H5DataIO will be ignored', stacklevel=3) # Call the super constructor and consume the data parameter - super().__init__(**kwargs) + super().__init__(data=data, shape=shape, dtype=dtype) # Construct the dict with the io args, ignoring all options that were set to None - self.__iosettings = {k: v for k, v in zip(ioarg_names, ioarg_values) if v is not None} + self.__iosettings = {k: v for k, v in ioargs.items() if v is not None} if self.data is None: self.__iosettings['dtype'] = self.dtype self.__iosettings['shape'] = self.shape diff --git a/src/hdmf/backends/hdf5/h5tools.py b/src/hdmf/backends/hdf5/h5tools.py index bd2a3eaa9..bc4e1fcb4 100644 --- a/src/hdmf/backends/hdf5/h5tools.py +++ b/src/hdmf/backends/hdf5/h5tools.py @@ -19,9 +19,10 @@ from ...container import Container from ...data_utils import AbstractDataChunkIterator from ...spec import RefSpec, DtypeSpec, NamespaceCatalog -from ...utils import (docval, getargs, popargs, get_data_shape, get_docval, StrDataset, is_zarr_array, +from ...utils import (docval, get_data_shape, get_docval, StrDataset, is_zarr_array, get_basic_array_info, generate_array_html_repr, _is_collection, _get_length) from ..utils import NamespaceToBuilderHelper, WriteStatusTracker +from ...typing import Bool, TypeName, validated ROOT_NAME = 'root' SPEC_LOC_ATTR = '.specloc' @@ -49,35 +50,31 @@ def can_read(path): except IOError: return False - @docval({'name': 'path', 'type': (str, Path), 'doc': 'the path to the HDF5 file', 'default': None}, - {'name': 'mode', 'type': str, - 'doc': ('the mode to open the HDF5 file with, one of ("w", "r", "r+", "a", "w-", "x"). ' - 'See `h5py.File `_ for ' - 'more details.'), - 'default': 'r'}, - {'name': 'manager', 'type': (TypeMap, BuildManager), - 'doc': 'the BuildManager or a TypeMap to construct a BuildManager to use for I/O', 'default': None}, - {'name': 'comm', 'type': 'Intracomm', - 'doc': 'the MPI communicator to use for parallel I/O', 'default': None}, - {'name': 'file', 'type': [File, "S3File", "RemFile"], - 'doc': 'a pre-existing h5py.File, S3File, or RemFile object', 'default': None}, - {'name': 'driver', 'type': str, 'doc': 'driver for h5py to use when opening HDF5 file', 'default': None}, - { - 'name': 'aws_region', - 'type': str, - 'doc': 'If driver is ros3, then specify the aws region of the url.', - 'default': None - }, - {'name': 'herd_path', 'type': str, - 'doc': 'The path to read/write the HERD file', 'default': None},) - def __init__(self, **kwargs): + @validated + def __init__(self, + path: str | Path | None = None, + mode: str = 'r', + manager: TypeMap | BuildManager | None = None, + comm: TypeName['Intracomm'] | None = None, # noqa: F821 + file: File | TypeName['S3File'] | TypeName['RemFile'] | None = None, # noqa: F821 + driver: str | None = None, + aws_region: str | None = None, + herd_path: str | None = None): """Open an HDF5 file for IO. + + Args: + path: the path to the HDF5 file + mode: the mode to open the HDF5 file with, one of ("w", "r", "r+", "a", "w-", "x"). See `h5py.File + `_ for more details. + manager: the BuildManager or a TypeMap to construct a BuildManager to use for I/O + comm: the MPI communicator to use for parallel I/O + file: a pre-existing h5py.File, S3File, or RemFile object + driver: driver for h5py to use when opening HDF5 file + aws_region: If driver is ros3, then specify the aws region of the url. + herd_path: The path to read/write the HERD file """ self.logger = logging.getLogger('%s.%s' % (self.__class__.__module__, self.__class__.__qualname__)) - path, manager, mode, comm, file_obj, driver, aws_region, herd_path = popargs('path', 'manager', 'mode', - 'comm', 'file', 'driver', - 'aws_region', 'herd_path', - kwargs) + file_obj = file self.__open_links = [] # keep track of other files opened from links in this file self.__file = None # This will be set below, but set to None first in case an error occurs and we need to close @@ -159,27 +156,14 @@ def __resolve_file_obj(cls, path, file_obj, driver, aws_region=None): return file_obj @classmethod - @docval( - { - 'name': 'namespace_catalog', - 'type': (NamespaceCatalog, TypeMap), - 'doc': 'the NamespaceCatalog or TypeMap to load namespaces into' - }, - {'name': 'path', 'type': (str, Path), 'doc': 'the path to the HDF5 file', 'default': None}, - {'name': 'namespaces', 'type': list, 'doc': 'the namespaces to load', 'default': None}, - {'name': 'file', 'type': File, 'doc': 'a pre-existing h5py.File object', 'default': None}, - {'name': 'driver', 'type': str, 'doc': 'driver for h5py to use when opening HDF5 file', 'default': None}, - { - 'name': 'aws_region', - 'type': str, - 'doc': 'If driver is ros3, then specify the aws region of the url.', - 'default': None - }, - returns=("dict mapping the names of the loaded namespaces to a dict mapping included namespace names and " - "the included data types"), - rtype=dict - ) - def load_namespaces(cls, **kwargs): + @validated + def load_namespaces(cls, + namespace_catalog: NamespaceCatalog | TypeMap, + path: str | Path | None = None, + namespaces: list | None = None, + file: File | None = None, + driver: str | None = None, + aws_region: str | None = None) -> dict: """Load cached namespaces from a file into the provided NamespaceCatalog or TypeMap. If `file` is not supplied, then an :py:class:`h5py.File` object will be opened for the given `path`, the @@ -187,9 +171,20 @@ def load_namespaces(cls, **kwargs): the given File object will be read from and not closed. :raises ValueError: if both `path` and `file` are supplied but `path` is not the same as the path of `file` + + Args: + namespace_catalog: the NamespaceCatalog or TypeMap to load namespaces into + path: the path to the HDF5 file + namespaces: the namespaces to load + file: a pre-existing h5py.File object + driver: driver for h5py to use when opening HDF5 file + aws_region: If driver is ros3, then specify the aws region of the url. + + Returns: + dict mapping the names of the loaded namespaces to a dict mapping included namespace names and + the included data types """ - namespace_catalog, path, namespaces, file_obj, driver, aws_region = popargs( - 'namespace_catalog', 'path', 'namespaces', 'file', 'driver', 'aws_region', kwargs) + file_obj = file open_file_obj = cls.__resolve_file_obj(path, file_obj, driver, aws_region=aws_region) if file_obj is None: # need to close the file object that we just opened @@ -197,17 +192,14 @@ def load_namespaces(cls, **kwargs): return cls.__load_namespaces(namespace_catalog, namespaces, open_file_obj) return cls.__load_namespaces(namespace_catalog, namespaces, open_file_obj) - @docval( - { - 'name': 'namespace_catalog', - 'type': (NamespaceCatalog, TypeMap), - 'doc': 'the NamespaceCatalog or TypeMap to load namespaces into' - }, - {'name': 'namespaces', 'type': list, 'doc': 'the namespaces to load', 'default': None} - ) - def load_namespaces_io(self, **kwargs): - """Load cached namespaces from this HDF5IO object into the provided NamespaceCatalog or TypeMap.""" - namespace_catalog, namespaces = getargs('namespace_catalog', 'namespaces', kwargs) + @validated + def load_namespaces_io(self, namespace_catalog: NamespaceCatalog | TypeMap, namespaces: list | None = None): + """Load cached namespaces from this HDF5IO object into the provided NamespaceCatalog or TypeMap. + + Args: + namespace_catalog: the NamespaceCatalog or TypeMap to load namespaces into + namespaces: the namespaces to load + """ if not self.__file: raise UnsupportedOperation("Cannot load namespaces from closed HDF5 file '%s'" % self.source) return self.__load_namespaces(namespace_catalog, namespaces, self.__file) @@ -237,13 +229,12 @@ def __check_specloc(cls, file_obj): return SPEC_LOC_ATTR in file_obj.attrs @classmethod - @docval({'name': 'path', 'type': (str, Path), 'doc': 'the path to the HDF5 file', 'default': None}, - {'name': 'file', 'type': File, 'doc': 'a pre-existing h5py.File object', 'default': None}, - {'name': 'driver', 'type': str, 'doc': 'driver for h5py to use when opening HDF5 file', 'default': None}, - {'name': 'aws_region', 'type': str, 'doc': 'If driver is ros3, then specify the aws region of the url.', - 'default': None}, - returns="dict mapping names to versions of the namespaces in the file", rtype=dict) - def get_namespaces(cls, **kwargs): + @validated + def get_namespaces(cls, + path: str | Path | None = None, + file: File | None = None, + driver: str | None = None, + aws_region: str | None = None) -> dict: """Get the names and versions of the cached namespaces from a file. If ``file`` is not supplied, then an :py:class:`h5py.File` object will be opened for the given ``path``, the @@ -254,8 +245,17 @@ def get_namespaces(cls, **kwargs): ordering) is returned. This is the version of the namespace that is loaded by HDF5IO.load_namespaces(...). :raises ValueError: if both `path` and `file` are supplied but `path` is not the same as the path of `file`. + + Args: + path: the path to the HDF5 file + file: a pre-existing h5py.File object + driver: driver for h5py to use when opening HDF5 file + aws_region: If driver is ros3, then specify the aws region of the url. + + Returns: + dict mapping names to versions of the namespaces in the file """ - path, file_obj, driver, aws_region = popargs('path', 'file', 'driver', 'aws_region', kwargs) + path, file_obj, driver, aws_region = path, file, driver, aws_region open_file_obj = cls.__resolve_file_obj(path, file_obj, driver, aws_region=aws_region) if file_obj is None: # need to close the file object that we just opened @@ -295,37 +295,37 @@ def __get_namespaces(cls, file_obj): return used_version_names - @docval({'name': 'container', 'type': Container, 'doc': 'the Container object to write'}, - {'name': 'cache_spec', 'type': bool, - 'doc': ('If True (default), cache specification to file (highly recommended). If False, do not cache ' - 'specification to file. The appropriate specification will then need to be loaded prior to ' - 'reading the file.'), - 'default': True}, - {'name': 'link_data', 'type': bool, - 'doc': 'If True (default), create external links to HDF5 Datasets. If False, copy HDF5 Datasets.', - 'default': True}, - {'name': 'exhaust_dci', 'type': bool, - 'doc': 'If True (default), exhaust DataChunkIterators one at a time. If False, exhaust them concurrently.', - 'default': True}, - {'name': 'herd', 'type': 'hdmf.common.resources.HERD', - 'doc': 'A HERD object to populate with references.', - 'default': None}, - {'name': 'expandable', 'type': (list, tuple), - 'default': ("VectorData", "ElementIdentifiers"), - 'doc': ('A list of data type names whose datasets (and subclasses) will be created as ' - 'expandable — maxshape is set based on the matching shape defined in the spec. ' - 'Default is ("VectorData", "ElementIdentifiers"), so only DynamicTable columns ' - 'and id are expandable. Pass an empty list/tuple to disable automatic expansion ' - 'entirely.')}) - def write(self, **kwargs): - """Write the container to an HDF5 file.""" + @validated + def write(self, + container: Container, + cache_spec: Bool = True, + link_data: Bool = True, + exhaust_dci: Bool = True, + herd: TypeName['hdmf.common.resources.HERD'] | None = None, # noqa: F821 + expandable: list | tuple = ('VectorData', 'ElementIdentifiers')): + """Write the container to an HDF5 file. + + Args: + container: the Container object to write + cache_spec: If True (default), cache specification to file (highly recommended). If False, do not cache + specification to file. The appropriate specification will then need to be loaded prior to reading the + file. + link_data: If True (default), create external links to HDF5 Datasets. If False, copy HDF5 Datasets. + exhaust_dci: If True (default), exhaust DataChunkIterators one at a time. If False, exhaust them + concurrently. + herd: A HERD object to populate with references. + expandable: A list of data type names whose datasets (and subclasses) will be created as expandable — + maxshape is set based on the matching shape defined in the spec. Default is ("VectorData", + "ElementIdentifiers"), so only DynamicTable columns and id are expandable. Pass an empty list/tuple to + disable automatic expansion entirely. + """ if self.__mode == 'r': raise UnsupportedOperation(("Cannot write to file %s in mode '%s'. " "Please use mode 'r+', 'w', 'w-', 'x', or 'a'") % (self.source, self.__mode)) - cache_spec = popargs('cache_spec', kwargs) - super().write(**kwargs) + super().write(container, herd=herd, link_data=link_data, exhaust_dci=exhaust_dci, + expandable=expandable) if cache_spec: self.__cache_spec() @@ -349,33 +349,28 @@ def __cache_spec(self): writer = H5SpecWriter(ns_group) ns_builder.export(self.__ns_spec_path, writer=writer) - _export_args = ( - {'name': 'src_io', 'type': 'hdmf.backends.io.HDMFIO', - 'doc': 'the HDMFIO object for reading the data to export'}, - {'name': 'container', 'type': Container, - 'doc': ('the Container object to export. If None, then the entire contents of the HDMFIO object will be ' - 'exported'), - 'default': None}, - {'name': 'write_args', 'type': dict, 'doc': 'arguments to pass to :py:meth:`write_builder`', - 'default': None}, - {'name': 'cache_spec', 'type': bool, 'doc': 'whether to cache the specification to file', - 'default': True} - # clear_cache is an arg on HDMFIO.export but it is intended for internal usage - # so it is not available on HDF5IO - ) - - @docval(*_export_args) - def export(self, **kwargs): + + @validated + def export(self, + src_io: TypeName['hdmf.backends.io.HDMFIO'], # noqa: F821 + container: Container | None = None, + write_args: dict | None = None, + cache_spec: Bool = True): """Export data read from a file from any backend to HDF5. See :py:meth:`hdmf.backends.io.HDMFIO.export` for more details. + + Args: + src_io: the HDMFIO object for reading the data to export + container: the Container object to export. If None, then the entire contents of the HDMFIO + object will be exported + write_args: arguments to pass to :py:meth:`write_builder` + cache_spec: whether to cache the specification to file """ if self.__mode != 'w': raise UnsupportedOperation("Cannot export to file %s in mode '%s'. Please use mode 'w'." % (self.source, self.__mode)) - src_io = getargs('src_io', kwargs) - write_args, cache_spec = popargs('write_args', 'cache_spec', kwargs) if write_args is None: write_args = dict() @@ -384,8 +379,7 @@ def export(self, **kwargs): "link_data=True." % src_io.__class__.__name__) write_args['export_source'] = os.path.abspath(src_io.source) if src_io.source is not None else None - ckwargs = kwargs.copy() - ckwargs['write_args'] = write_args + ckwargs = dict(src_io=src_io, container=container, write_args=write_args) if not write_args.get('link_data', True): ckwargs['clear_cache'] = True super().export(**ckwargs) @@ -400,11 +394,14 @@ def export(self, **kwargs): self.__cache_spec() @classmethod - @docval({'name': 'path', 'type': str, 'doc': 'the path to the destination HDF5 file'}, - {'name': 'comm', 'type': 'Intracomm', 'doc': 'the MPI communicator to use for parallel I/O', - 'default': None}, - *_export_args) # NOTE: src_io is required and is the second positional argument - def export_io(self, **kwargs): + @validated + def export_io(cls, + path: str, + src_io: TypeName['hdmf.backends.io.HDMFIO'], # noqa: F821 + comm: TypeName['Intracomm'] | None = None, # noqa: F821 + container: Container | None = None, + write_args: dict | None = None, + cache_spec: Bool = True): """Export from one backend to HDF5 (class method). Convenience function for :py:meth:`export` where you do not need to @@ -420,10 +417,9 @@ def export_io(self, **kwargs): See :py:meth:`export` for more details. """ - path, comm = popargs('path', 'comm', kwargs) - with HDF5IO(path=path, comm=comm, mode='w') as write_io: - write_io.export(**kwargs) + write_io.export(src_io=src_io, container=container, write_args=write_args, + cache_spec=cache_spec) def read(self, **kwargs): if self.__mode == 'w' or self.__mode == 'w-' or self.__mode == 'x': @@ -436,14 +432,16 @@ def read(self, **kwargs): raise UnsupportedOperation("Cannot read data from file %s in mode '%s'. There are no values." % (self.source, self.__mode)) - @docval(returns='a GroupBuilder representing the data object', rtype=GroupBuilder) - def read_builder(self): - """ - Read data and return the GroupBuilder representing it. + @validated + def read_builder(self) -> GroupBuilder: + """Read data and return the GroupBuilder representing it. NOTE: On read, the Builder.source may will usually not be set of the Builders. NOTE: The Builder.location is used internally to ensure correct handling of links (in particular on export) and should be set on read for all GroupBuilder, DatasetBuilder, and LinkBuilder objects. + + Returns: + a GroupBuilder representing the data object """ if not self.__file: raise UnsupportedOperation("Cannot read data from closed HDF5 file '%s'" % self.source) @@ -507,15 +505,15 @@ def __get_built(self, fpath, id): else: return None - @docval({'name': 'h5obj', 'type': (Dataset, Group), - 'doc': 'the HDF5 object to the corresponding Builder object for'}) - def get_builder(self, **kwargs): - """ - Get the builder for the corresponding h5py Group or Dataset + @validated + def get_builder(self, h5obj: Dataset | Group): + """Get the builder for the corresponding h5py Group or Dataset :raises ValueError: When no builder has been constructed yet for the given h5py object + + Args: + h5obj: the HDF5 object to the corresponding Builder object for """ - h5obj = getargs('h5obj', kwargs) fpath = h5obj.file.filename builder = self.__get_built(fpath, h5obj.id) if builder is None: @@ -523,15 +521,15 @@ def get_builder(self, **kwargs): raise ValueError(msg) return builder - @docval({'name': 'h5obj', 'type': (Dataset, Group), - 'doc': 'the HDF5 object to the corresponding Container/Data object for'}) - def get_container(self, **kwargs): - """ - Get the container for the corresponding h5py Group or Dataset + @validated + def get_container(self, h5obj: Dataset | Group): + """Get the container for the corresponding h5py Group or Dataset :raises ValueError: When no builder has been constructed yet for the given h5py object + + Args: + h5obj: the HDF5 object to the corresponding Container/Data object for """ - h5obj = getargs('h5obj', kwargs) builder = self.get_builder(h5obj) container = self.manager.construct(builder) return container @@ -755,23 +753,28 @@ def close_linked_files(self): finally: self.__open_links = [] - @docval({'name': 'builder', 'type': GroupBuilder, 'doc': 'the GroupBuilder object representing the HDF5 file'}, - {'name': 'link_data', 'type': bool, - 'doc': 'If not specified otherwise link (True) or copy (False) HDF5 Datasets', 'default': True}, - {'name': 'exhaust_dci', 'type': bool, - 'doc': 'exhaust DataChunkIterators one at a time. If False, exhaust them concurrently', - 'default': True}, - {'name': 'export_source', 'type': str, - 'doc': 'The source of the builders when exporting', 'default': None}, - {'name': 'expandable', 'type': (list, tuple), - 'default': ("VectorData", "ElementIdentifiers"), - 'doc': ('A list of data type names whose datasets (and subclasses) will be created as ' - 'expandable — maxshape is set based on the matching shape defined in the spec. ' - 'Default is ("VectorData", "ElementIdentifiers"), so only DynamicTable columns ' - 'and id are expandable. Pass an empty list/tuple to disable automatic expansion ' - 'entirely.')}) - def write_builder(self, **kwargs): - f_builder = popargs('builder', kwargs) + @validated + def write_builder(self, + builder: GroupBuilder, + link_data: Bool = True, + exhaust_dci: Bool = True, + export_source: str | None = None, + expandable: list | tuple = ('VectorData', 'ElementIdentifiers')): + """write_builder + + Args: + builder: the GroupBuilder object representing the HDF5 file + link_data: If not specified otherwise link (True) or copy (False) HDF5 Datasets + exhaust_dci: exhaust DataChunkIterators one at a time. If False, exhaust them concurrently + export_source: The source of the builders when exporting + expandable: A list of data type names whose datasets (and subclasses) will be created as expandable — + maxshape is set based on the matching shape defined in the spec. Default is ("VectorData", + "ElementIdentifiers"), so only DynamicTable columns and id are expandable. Pass an empty list/tuple to + disable automatic expansion entirely. + """ + kwargs = dict(link_data=link_data, exhaust_dci=exhaust_dci, export_source=export_source, + expandable=expandable) + f_builder = builder self.logger.debug("Writing GroupBuilder '%s' to path '%s' with kwargs=%s" % (f_builder.name, self.source, kwargs)) for gbldr in f_builder.groups.values(): @@ -889,12 +892,14 @@ def __resolve_dtype_helper__(cls, dtype): else: return np.dtype([(x['name'], cls.__resolve_dtype_helper__(x['dtype'])) for x in dtype]) - @docval({'name': 'obj', 'type': (Group, Dataset), 'doc': 'the HDF5 object to add attributes to'}, - {'name': 'attributes', - 'type': dict, - 'doc': 'a dict containing the attributes on the Group or Dataset, indexed by attribute name'}) - def set_attributes(self, **kwargs): - obj, attributes = getargs('obj', 'attributes', kwargs) + @validated + def set_attributes(self, obj: Group | Dataset, attributes: dict): + """set_attributes + + Args: + obj: the HDF5 object to add attributes to + attributes: a dict containing the attributes on the Group or Dataset, indexed by attribute name + """ for key, value in attributes.items(): try: if isinstance(value, (set, list, tuple)): @@ -938,25 +943,32 @@ def _filler(): obj.attrs[key] = self.__get_ref(value) return _filler - @docval({'name': 'parent', 'type': Group, 'doc': 'the parent HDF5 object'}, - {'name': 'builder', 'type': GroupBuilder, 'doc': 'the GroupBuilder to write'}, - {'name': 'link_data', 'type': bool, - 'doc': 'If not specified otherwise link (True) or copy (False) HDF5 Datasets', 'default': True}, - {'name': 'exhaust_dci', 'type': bool, - 'doc': 'exhaust DataChunkIterators one at a time. If False, exhaust them concurrently', - 'default': True}, - {'name': 'export_source', 'type': str, - 'doc': 'The source of the builders when exporting', 'default': None}, - {'name': 'expandable', 'type': (list, tuple), - 'default': ("VectorData", "ElementIdentifiers"), - 'doc': ('A list of data type names whose datasets (and subclasses) will be created as ' - 'expandable — maxshape is set based on the matching shape defined in the spec. ' - 'Default is ("VectorData", "ElementIdentifiers"), so only DynamicTable columns ' - 'and id are expandable. Pass an empty list/tuple to disable automatic expansion ' - 'entirely.')}, - returns='the Group that was created', rtype=Group) - def write_group(self, **kwargs): - parent, builder = popargs('parent', 'builder', kwargs) + @validated + def write_group(self, + parent: Group, + builder: GroupBuilder, + link_data: Bool = True, + exhaust_dci: Bool = True, + export_source: str | None = None, + expandable: list | tuple = ('VectorData', 'ElementIdentifiers')) -> Group: + """write_group + + Args: + parent: the parent HDF5 object + builder: the GroupBuilder to write + link_data: If not specified otherwise link (True) or copy (False) HDF5 Datasets + exhaust_dci: exhaust DataChunkIterators one at a time. If False, exhaust them concurrently + export_source: The source of the builders when exporting + expandable: A list of data type names whose datasets (and subclasses) will be created as expandable — + maxshape is set based on the matching shape defined in the spec. Default is ("VectorData", + "ElementIdentifiers"), so only DynamicTable columns and id are expandable. Pass an empty list/tuple to + disable automatic expansion entirely. + + Returns: + the Group that was created + """ + kwargs = dict(link_data=link_data, exhaust_dci=exhaust_dci, export_source=export_source, + expandable=expandable) self.logger.debug("Writing GroupBuilder '%s' to parent group '%s'" % (builder.name, parent.name)) if self.get_written(builder): self.logger.debug(" GroupBuilder '%s' is already written" % builder.name) @@ -1005,13 +1017,21 @@ def __get_path(self, builder): path = "%s%s" % (delim, delim.join(reversed(names))) return path - @docval({'name': 'parent', 'type': Group, 'doc': 'the parent HDF5 object'}, - {'name': 'builder', 'type': LinkBuilder, 'doc': 'the LinkBuilder to write'}, - {'name': 'export_source', 'type': str, - 'doc': 'The source of the builders when exporting', 'default': None}, - returns='the Link that was created', rtype=(SoftLink, ExternalLink)) - def write_link(self, **kwargs): - parent, builder, export_source = getargs('parent', 'builder', 'export_source', kwargs) + @validated + def write_link(self, + parent: Group, + builder: LinkBuilder, + export_source: str | None = None) -> (SoftLink, ExternalLink): + """write_link + + Args: + parent: the parent HDF5 object + builder: the LinkBuilder to write + export_source: The source of the builders when exporting + + Returns: + the Link that was created + """ self.logger.debug("Writing LinkBuilder '%s' to parent group '%s'" % (builder.name, parent.name)) if self.get_written(builder): self.logger.debug(" LinkBuilder '%s' is already written" % builder.name) @@ -1045,31 +1065,33 @@ def write_link(self, **kwargs): self.__set_written(builder) return link_obj - @docval({'name': 'parent', 'type': Group, 'doc': 'the parent HDF5 object'}, - {'name': 'builder', 'type': DatasetBuilder, 'doc': 'the DatasetBuilder to write'}, - {'name': 'link_data', 'type': bool, - 'doc': 'If not specified otherwise link (True) or copy (False) HDF5 Datasets', 'default': True}, - {'name': 'exhaust_dci', 'type': bool, - 'doc': 'exhaust DataChunkIterators one at a time. If False, exhaust them concurrently', - 'default': True}, - {'name': 'export_source', 'type': str, - 'doc': 'The source of the builders when exporting', 'default': None}, - {'name': 'expandable', 'type': (list, tuple), - 'default': ("VectorData", "ElementIdentifiers"), - 'doc': ('A list of data type names whose datasets (and subclasses) will be created as ' - 'expandable — maxshape is set based on the matching shape defined in the spec. ' - 'Default is ("VectorData", "ElementIdentifiers"), so only DynamicTable columns ' - 'and id are expandable. Pass an empty list/tuple to disable automatic expansion ' - 'entirely.')}, - returns='the Dataset that was created', rtype=Dataset) - def write_dataset(self, **kwargs): # noqa: C901 - """ Write a dataset to HDF5 + @validated + def write_dataset(self, # noqa: C901 + parent: Group, + builder: DatasetBuilder, + link_data: Bool = True, + exhaust_dci: Bool = True, + export_source: str | None = None, + expandable: list | tuple = ('VectorData', 'ElementIdentifiers')) -> Dataset: + """Write a dataset to HDF5 The function uses other dataset-dependent write functions, e.g, ``__scalar_fill__``, ``__list_fill__``, and ``__setup_chunked_dset__`` to write the data. + + Args: + parent: the parent HDF5 object + builder: the DatasetBuilder to write + link_data: If not specified otherwise link (True) or copy (False) HDF5 Datasets + exhaust_dci: exhaust DataChunkIterators one at a time. If False, exhaust them concurrently + export_source: The source of the builders when exporting + expandable: A list of data type names whose datasets (and subclasses) will be created as expandable — + maxshape is set based on the matching shape defined in the spec. Default is ("VectorData", + "ElementIdentifiers"), so only DynamicTable columns and id are expandable. Pass an empty list/tuple to + disable automatic expansion entirely. + + Returns: + the Dataset that was created """ - parent, builder, expandable = popargs('parent', 'builder', 'expandable', kwargs) - link_data, exhaust_dci, export_source = getargs('link_data', 'exhaust_dci', 'export_source', kwargs) self.logger.debug("Writing DatasetBuilder '%s' to parent group '%s'" % (builder.name, parent.name)) if self.get_written(builder): self.logger.debug(" DatasetBuilder '%s' is already written" % builder.name) @@ -1453,11 +1475,16 @@ def __list_fill__(cls, parent, name, data, options=None): raise e return dset - @docval({'name': 'container', 'type': (Builder, Container, ReferenceBuilder), 'doc': 'the object to reference', - 'default': None}, - returns='the reference', rtype=Reference) - def __get_ref(self, **kwargs): - container = getargs('container', kwargs) + @validated + def __get_ref(self, container: Builder | Container | ReferenceBuilder | None = None) -> Reference: + """__get_ref + + Args: + container: the object to reference + + Returns: + the reference + """ if container is None: return None if isinstance(container, Builder): @@ -1477,11 +1504,17 @@ def __get_ref(self, **kwargs): self.logger.debug("Getting reference at path '%s'" % path) return self.__file[path].ref - @docval({'name': 'container', 'type': (Builder, Container, ReferenceBuilder), 'doc': 'the object to reference', - 'default': None}, - returns='the reference', rtype=Reference) - def _create_ref(self, **kwargs): - return self.__get_ref(**kwargs) + @validated + def _create_ref(self, container: Builder | Container | ReferenceBuilder | None = None) -> Reference: + """_create_ref + + Args: + container: the object to reference + + Returns: + the reference + """ + return self.__get_ref(container) @staticmethod def compute_default_chunk_shape(data_shape, dtype, target_chunk_bytes=4 * 1024 * 1024, neurodata_type=None): @@ -1584,6 +1617,7 @@ def mode(self): return self.__mode @classmethod + # intentionally on @docval: splices H5DataIO argument specs; migrates with docval removal @docval(*get_docval(H5DataIO.__init__)) def set_dataio(cls, **kwargs): """ diff --git a/src/hdmf/backends/io.py b/src/hdmf/backends/io.py index 23319628f..b54489ccd 100644 --- a/src/hdmf/backends/io.py +++ b/src/hdmf/backends/io.py @@ -5,21 +5,26 @@ from ..build import BuildManager, GroupBuilder, TypeMap from ..container import Container, HERDManager from .errors import UnsupportedOperation -from ..utils import docval, getargs, popargs, get_basic_array_info, generate_array_html_repr +from ..utils import get_basic_array_info, generate_array_html_repr from ..spec import NamespaceCatalog from warnings import warn +from ..typing import Bool, TypeName, validated class HDMFIO(metaclass=ABCMeta): - @docval({'name': 'manager', 'type': BuildManager, - 'doc': 'the BuildManager to use for I/O', 'default': None}, - {"name": "source", "type": (str, Path), - "doc": "the source of container being built i.e. file path", 'default': None}, - {'name': 'herd_path', 'type': str, - 'doc': 'The path to read/write the HERD file', 'default': None},) - def __init__(self, **kwargs): - manager, source, herd_path = getargs('manager', 'source', 'herd_path', kwargs) + @validated + def __init__(self, + manager: BuildManager | None = None, + source: str | Path | None = None, + herd_path: str | None = None): + """Initialize this object. + + Args: + manager: the BuildManager to use for I/O + source: the source of container being built i.e. file path + herd_path: The path to read/write the HERD file + """ if isinstance(source, Path): source = source.resolve() elif (isinstance(source, str) and @@ -44,9 +49,13 @@ def source(self): '''The source of the container being read/written i.e. file path''' return self.__source - @docval(returns='the Container object that was read in', rtype=Container) - def read(self, **kwargs): - """Read a container from the IO source.""" + @validated + def read(self) -> Container: + """Read a container from the IO source. + + Returns: + the Container object that was read in + """ f_builder = self.read_builder() if all(len(v) == 0 for v in f_builder.values()): # TODO also check that the keys are appropriate. print a better error message @@ -68,14 +77,14 @@ def read(self, **kwargs): return container - @docval({'name': 'container', 'type': Container, 'doc': 'the Container object to write'}, - {'name': 'herd', 'type': 'hdmf.common.resources.HERD', - 'doc': 'A HERD object to populate with references.', - 'default': None}, allow_extra=True) - def write(self, **kwargs): - container = popargs('container', kwargs) - herd = popargs('herd', kwargs) + @validated + def write(self, container: Container, herd: TypeName['hdmf.common.resources.HERD'] | None = None, **kwargs): # noqa: F821 + """write + Args: + container: the Container object to write + herd: A HERD object to populate with references. + """ """Optional: Write HERD.""" if self.herd_path is not None: # If HERD is not provided, create a new one, else extend existing one @@ -92,17 +101,12 @@ def write(self, **kwargs): f_builder = self.__manager.build(container, source=self.__source, root=True) self.write_builder(f_builder, **kwargs) - @docval({'name': 'src_io', 'type': 'hdmf.backends.io.HDMFIO', - 'doc': 'the HDMFIO object for reading the data to export'}, - {'name': 'container', 'type': Container, - 'doc': ('the Container object to export. If None, then the entire contents of the HDMFIO object will be ' - 'exported'), - 'default': None}, - {'name': 'write_args', 'type': dict, 'doc': 'arguments to pass to :py:meth:`write_builder`', - 'default': dict()}, - {'name': 'clear_cache', 'type': bool, 'doc': 'whether to clear the build manager cache', - 'default': False}) - def export(self, **kwargs): + @validated + def export(self, + src_io: TypeName['hdmf.backends.io.HDMFIO'], # noqa: F821 + container: Container | None = None, + write_args: dict | None = None, + clear_cache: Bool = False): """Export from one backend to the backend represented by this class. If `container` is provided, then the build manager of `src_io` is used to build the container, and the resulting @@ -128,8 +132,16 @@ def export(self, **kwargs): on the Builders. As such, when writing LinkBuilders we need to determine if LinkBuilder.source and LinkBuilder.builder.source are the same, and if so the link should be internal to the current file (even if the Builder.source points to a different location). + + Args: + src_io: the HDMFIO object for reading the data to export + container: the Container object to export. If None, then the entire contents of the HDMFIO object will be + exported + write_args: arguments to pass to :py:meth:`write_builder` + clear_cache: whether to clear the build manager cache """ - src_io, container, write_args, clear_cache = getargs('src_io', 'container', 'write_args', 'clear_cache', kwargs) + if write_args is None: + write_args = dict() if container is None and clear_cache: # clear all containers and builders from cache so that they can all get rebuilt with export=True. # constructing the container is not efficient but there is no elegant way to trigger a @@ -160,16 +172,23 @@ def export(self, **kwargs): self.write_builder(builder=bldr, **write_args) @abstractmethod - @docval(returns='a GroupBuilder representing the read data', rtype='GroupBuilder') - def read_builder(self): - ''' Read data and return the GroupBuilder representing it ''' + @validated + def read_builder(self) -> 'GroupBuilder': + """Read data and return the GroupBuilder representing it + + Returns: + a GroupBuilder representing the read data + """ pass @abstractmethod - @docval({'name': 'builder', 'type': GroupBuilder, 'doc': 'the GroupBuilder object representing the Container'}, - allow_extra=True) - def write_builder(self, **kwargs): - ''' Write a GroupBuilder representing an Container object ''' + @validated + def write_builder(self, builder: GroupBuilder, **kwargs): + """Write a GroupBuilder representing an Container object + + Args: + builder: the GroupBuilder object representing the Container + """ pass @abstractmethod diff --git a/src/hdmf/backends/utils.py b/src/hdmf/backends/utils.py index a20e50c10..82c87cba6 100644 --- a/src/hdmf/backends/utils.py +++ b/src/hdmf/backends/utils.py @@ -1,7 +1,7 @@ """Module with utility functions and classes used for implementation of I/O backends""" import os from ..spec import NamespaceCatalog, NamespaceBuilder -from ..utils import docval, popargs +from ..typing import validated class WriteStatusTracker(dict): @@ -47,12 +47,14 @@ class NamespaceToBuilderHelper(object): """Helper class used to convert a namespace to a builder for I/O""" @classmethod - @docval({'name': 'ns_catalog', 'type': NamespaceCatalog, 'doc': 'the namespace catalog with the specs'}, - {'name': 'namespace', 'type': str, 'doc': 'the name of the namespace to be converted to a builder'}, - rtype=NamespaceBuilder) - def convert_namespace(cls, **kwargs): - """Convert a namespace to a builder""" - ns_catalog, namespace = popargs('ns_catalog', 'namespace', kwargs) + @validated + def convert_namespace(cls, ns_catalog: NamespaceCatalog, namespace: str) -> NamespaceBuilder: + """Convert a namespace to a builder + + Args: + ns_catalog: the namespace catalog with the specs + namespace: the name of the namespace to be converted to a builder + """ ns = ns_catalog.get_namespace(namespace) builder = NamespaceBuilder(ns.doc, ns.name, full_name=ns.full_name, diff --git a/src/hdmf/build/builders.py b/src/hdmf/build/builders.py index 228aacb0f..4e6fd9f97 100644 --- a/src/hdmf/build/builders.py +++ b/src/hdmf/build/builders.py @@ -8,17 +8,24 @@ import numpy as np from ..utils import docval, getargs, get_docval +from typing import Any +from ..typing import AnyData, ArrayData, Bool, Int, ScalarData, TypeName, validated class Builder(dict, metaclass=ABCMeta): - @docval({'name': 'name', 'type': str, 'doc': 'the name of the group'}, - {'name': 'parent', 'type': 'hdmf.build.builders.Builder', 'doc': 'the parent builder of this Builder', - 'default': None}, - {'name': 'source', 'type': str, - 'doc': 'the source of the data in this builder e.g. file name', 'default': None}) - def __init__(self, **kwargs): - name, parent, source = getargs('name', 'parent', 'source', kwargs) + @validated + def __init__(self, + name: str, + parent: TypeName['Builder'] | None = None, # noqa: F821 + source: str | None = None): + """Initialize this object. + + Args: + name: the name of the group + parent: the parent builder of this Builder + source: the source of the data in this builder e.g. file name + """ super().__init__() self.__name = name self.__parent = parent @@ -92,15 +99,22 @@ def __repr__(self): class BaseBuilder(Builder, metaclass=ABCMeta): __attribute = 'attributes' # self dictionary key for attributes - @docval({'name': 'name', 'type': str, 'doc': 'The name of the builder.'}, - {'name': 'attributes', 'type': dict, 'doc': 'A dictionary of attributes to create in this builder.', - 'default': dict()}, - {'name': 'parent', 'type': 'hdmf.build.builders.GroupBuilder', 'doc': 'The parent builder of this builder.', - 'default': None}, - {'name': 'source', 'type': str, - 'doc': 'The source of the data represented in this builder', 'default': None}) - def __init__(self, **kwargs): - name, attributes, parent, source = getargs('name', 'attributes', 'parent', 'source', kwargs) + @validated + def __init__(self, + name: str, + attributes: dict | None = None, + parent: TypeName['GroupBuilder'] | None = None, # noqa: F821 + source: str | None = None): + """Initialize this object. + + Args: + name: The name of the builder. + attributes: A dictionary of attributes to create in this builder. + parent: The parent builder of this builder. + source: The source of the data represented in this builder + """ + if attributes is None: + attributes = dict() super().__init__(name, parent, source) super().__setitem__(BaseBuilder.__attribute, dict()) for name, val in attributes.items(): @@ -121,11 +135,14 @@ def attributes(self): """The attributes stored in this Builder object.""" return super().__getitem__(BaseBuilder.__attribute) - @docval({'name': 'name', 'type': str, 'doc': 'The name of the attribute.'}, - {'name': 'value', 'type': None, 'doc': 'The attribute value.'}) - def set_attribute(self, **kwargs): - """Set an attribute for this group.""" - name, value = getargs('name', 'value', kwargs) + @validated + def set_attribute(self, name: str, value: Any): + """Set an attribute for this group. + + Args: + name: The name of the attribute. + value: The attribute value. + """ self.attributes[name] = value @@ -136,29 +153,32 @@ class GroupBuilder(BaseBuilder): __link = 'links' __attribute = 'attributes' - @docval({'name': 'name', 'type': str, 'doc': 'The name of the group.'}, - {'name': 'groups', 'type': (dict, list), - 'doc': ('A dictionary or list of subgroups to add to this group. If a dict is provided, only the ' - 'values are used.'), - 'default': dict()}, - {'name': 'datasets', 'type': (dict, list), - 'doc': ('A dictionary or list of datasets to add to this group. If a dict is provided, only the ' - 'values are used.'), - 'default': dict()}, - {'name': 'attributes', 'type': dict, 'doc': 'A dictionary of attributes to create in this group.', - 'default': dict()}, - {'name': 'links', 'type': (dict, list), - 'doc': ('A dictionary or list of links to add to this group. If a dict is provided, only the ' - 'values are used.'), - 'default': dict()}, - {'name': 'parent', 'type': 'hdmf.build.builders.GroupBuilder', 'doc': 'The parent builder of this builder.', - 'default': None}, - {'name': 'source', 'type': str, - 'doc': 'The source of the data represented in this builder.', 'default': None}) - def __init__(self, **kwargs): - """Create a builder object for a group.""" - name, groups, datasets, links, attributes, parent, source = getargs( - 'name', 'groups', 'datasets', 'links', 'attributes', 'parent', 'source', kwargs) + @validated + def __init__(self, + name: str, + groups: dict | list | None = None, + datasets: dict | list | None = None, + attributes: dict | None = None, + links: dict | list | None = None, + parent: TypeName['GroupBuilder'] | None = None, # noqa: F821 + source: str | None = None): + """Create a builder object for a group. + + Args: + name: The name of the group. + groups: A dictionary or list of subgroups to add to this group. If a dict is provided, only the values are + used. + datasets: A dictionary or list of datasets to add to this group. If a dict is provided, only the values + are used. + attributes: A dictionary of attributes to create in this group. + links: A dictionary or list of links to add to this group. If a dict is provided, only the values are used. + parent: The parent builder of this builder. + source: The source of the data represented in this builder. + """ + groups = groups if groups is not None else dict() + datasets = datasets if datasets is not None else dict() + attributes = attributes if attributes is not None else dict() + links = links if links is not None else dict() # NOTE: if groups, datasets, or links are dicts, their keys are unused groups = self.__to_list(groups) datasets = self.__to_list(datasets) @@ -217,6 +237,7 @@ def links(self): """The links contained in this group.""" return super().__getitem__(GroupBuilder.__link) + # intentionally on @docval: splices parent argument specs; migrates with docval removal @docval(*get_docval(BaseBuilder.set_attribute)) def set_attribute(self, **kwargs): """Set an attribute for this group.""" @@ -231,25 +252,31 @@ def __check_obj_type(self, name, obj_type): raise ValueError("'%s' already exists in %s.%s, cannot set in %s." % (name, self.name, self.obj_type[name], obj_type)) - @docval({'name': 'builder', 'type': 'hdmf.build.builders.GroupBuilder', - 'doc': 'The GroupBuilder to add to this group.'}) - def set_group(self, **kwargs): - """Add a subgroup to this group.""" - builder = getargs('builder', kwargs) + @validated + def set_group(self, builder: TypeName['GroupBuilder']): # noqa: F821 + """Add a subgroup to this group. + + Args: + builder: The GroupBuilder to add to this group. + """ self.__set_builder(builder, GroupBuilder.__group) - @docval({'name': 'builder', 'type': 'hdmf.build.builders.DatasetBuilder', - 'doc': 'The DatasetBuilder to add to this group.'}) - def set_dataset(self, **kwargs): - """Add a dataset to this group.""" - builder = getargs('builder', kwargs) + @validated + def set_dataset(self, builder: TypeName['DatasetBuilder']): # noqa: F821 + """Add a dataset to this group. + + Args: + builder: The DatasetBuilder to add to this group. + """ self.__set_builder(builder, GroupBuilder.__dataset) - @docval({'name': 'builder', 'type': 'hdmf.build.builders.LinkBuilder', - 'doc': 'The LinkBuilder to add to this group.'}) - def set_link(self, **kwargs): - """Add a link to this group.""" - builder = getargs('builder', kwargs) + @validated + def set_link(self, builder: TypeName['LinkBuilder']): # noqa: F821 + """Add a link to this group. + + Args: + builder: The LinkBuilder to add to this group. + """ self.__set_builder(builder, GroupBuilder.__link) def __set_builder(self, builder, obj_type): @@ -336,33 +363,37 @@ def values(self): class DatasetBuilder(BaseBuilder): OBJECT_REF_TYPE = 'object' - @docval({'name': 'name', 'type': str, 'doc': 'The name of the dataset.'}, - {'name': 'data', - 'type': ('array_data', 'scalar_data', 'data', 'DatasetBuilder', Iterable, datetime, date), - 'doc': 'The data in this dataset.', 'default': None}, - {'name': 'dtype', 'type': (type, np.dtype, str, list), - 'doc': 'The datatype of this dataset.', 'default': None}, - {'name': 'attributes', 'type': dict, - 'doc': 'A dictionary of attributes to create in this dataset.', 'default': dict()}, - {'name': 'matched_spec_shape', 'type': tuple, - 'doc': ('The shape defined in the spec that matches the shape of this dataset. Currently this is ' - 'supplied only on build.'), - 'default': None}, - {'name': 'dimension_labels', 'type': tuple, - 'doc': ('A list of labels for each dimension of this dataset from the spec. Currently this is ' - 'supplied only on build.'), - 'default': None}, - {'name': 'maxshape', 'type': (int, tuple), - 'doc': 'The shape of this dataset. Use None for scalars.', 'default': None}, - {'name': 'chunks', 'type': bool, 'doc': 'Whether or not to chunk this dataset.', 'default': False}, - {'name': 'parent', 'type': GroupBuilder, 'doc': 'The parent builder of this builder.', 'default': None}, - {'name': 'source', 'type': str, 'doc': 'The source of the data in this builder.', 'default': None}) - def __init__(self, **kwargs): - """ Create a Builder object for a dataset """ - name, data, dtype, attributes, matched_spec_shape, dimension_labels = getargs( - 'name', 'data', 'dtype', 'attributes', 'matched_spec_shape', 'dimension_labels', kwargs - ) - maxshape, chunks, parent, source = getargs('maxshape', 'chunks', 'parent', 'source', kwargs) + @validated + def __init__(self, + name: str, + data: ArrayData | ScalarData | AnyData | TypeName['DatasetBuilder'] # noqa: F821 + | Iterable | datetime | date | None = None, + dtype: type | np.dtype | str | list | None = None, + attributes: dict | None = None, + matched_spec_shape: tuple | None = None, + dimension_labels: tuple | None = None, + maxshape: Int | tuple | None = None, + chunks: Bool = False, + parent: GroupBuilder | None = None, + source: str | None = None): + """Create a Builder object for a dataset + + Args: + name: The name of the dataset. + data: The data in this dataset. + dtype: The datatype of this dataset. + attributes: A dictionary of attributes to create in this dataset. + matched_spec_shape: The shape defined in the spec that matches the shape of this dataset. Currently this + is supplied only on build. + dimension_labels: A list of labels for each dimension of this dataset from the spec. Currently this is + supplied only on build. + maxshape: The shape of this dataset. Use None for scalars. + chunks: Whether or not to chunk this dataset. + parent: The parent builder of this builder. + source: The source of the data in this builder. + """ + if attributes is None: + attributes = dict() super().__init__(name, attributes, parent, source) self['data'] = data self['attributes'] = _copy.copy(attributes) @@ -421,14 +452,20 @@ def dtype(self, val): class LinkBuilder(Builder): - @docval({'name': 'builder', 'type': (DatasetBuilder, GroupBuilder), - 'doc': 'The target group or dataset of this link.'}, - {'name': 'name', 'type': str, 'doc': 'The name of the link', 'default': None}, - {'name': 'parent', 'type': GroupBuilder, 'doc': 'The parent builder of this builder', 'default': None}, - {'name': 'source', 'type': str, 'doc': 'The source of the data in this builder', 'default': None}) - def __init__(self, **kwargs): - """Create a builder object for a link.""" - name, builder, parent, source = getargs('name', 'builder', 'parent', 'source', kwargs) + @validated + def __init__(self, + builder: DatasetBuilder | GroupBuilder, + name: str | None = None, + parent: GroupBuilder | None = None, + source: str | None = None): + """Create a builder object for a link. + + Args: + builder: The target group or dataset of this link. + name: The name of the link + parent: The parent builder of this builder + source: The source of the data in this builder + """ if name is None: name = builder.name super().__init__(name, parent, source) @@ -442,11 +479,13 @@ def builder(self): class ReferenceBuilder(dict): - @docval({'name': 'builder', 'type': (DatasetBuilder, GroupBuilder), - 'doc': 'The group or dataset this reference applies to.'}) - def __init__(self, **kwargs): - """Create a builder object for a reference.""" - builder = getargs('builder', kwargs) + @validated + def __init__(self, builder: DatasetBuilder | GroupBuilder): + """Create a builder object for a reference. + + Args: + builder: The group or dataset this reference applies to. + """ self['builder'] = builder @property diff --git a/src/hdmf/build/classgenerator.py b/src/hdmf/build/classgenerator.py index 944fded7d..f1f350bf9 100644 --- a/src/hdmf/build/classgenerator.py +++ b/src/hdmf/build/classgenerator.py @@ -7,6 +7,7 @@ from ..container import Container, Data, MultiContainerInterface from ..spec import AttributeSpec, LinkSpec, RefSpec, GroupSpec, DatasetSpec from ..spec.spec import BaseStorageSpec, ZERO_OR_MANY, ONE_OR_MANY +from ..typing import signature_function from ..utils import docval, getargs, ExtenderMeta, get_docval, popargs, AllowPositional @@ -377,12 +378,8 @@ def set_init(cls, classdict, bases, docval_args, not_inherited_fields, name): if item['name'] == 'skip_post_init': docval_args.remove(item) - @docval(*docval_args, - {'name': 'skip_post_init', 'type': bool, 'default': False, - 'doc': 'bool to skip post_init'}, - allow_positional=AllowPositional.WARNING) - def __init__(self, **kwargs): - skip_post_init = popargs('skip_post_init', kwargs) + def _init_body(self, kwargs): + skip_post_init = kwargs.pop('skip_post_init') original_kwargs = dict(kwargs) if name is not None: # force container name to be the fixed name in the spec @@ -413,7 +410,14 @@ def __init__(self, **kwargs): if self.post_init_method is not None and not skip_post_init: self.post_init_method(**original_kwargs) - classdict['__init__'] = __init__ + classdict['__init__'] = signature_function( + '__init__', + list(docval_args) + [{'name': 'skip_post_init', 'type': bool, 'default': False, + 'doc': 'bool to skip post_init'}], + _init_body, + doc='Initialize the container, setting all fields defined in the schema.', + allow_positional=AllowPositional.WARNING, + ) class MCIClassGenerator(CustomClassGenerator): @@ -483,8 +487,7 @@ def set_init(cls, classdict, bases, docval_args, not_inherited_fields, name): if '__clsconf__' in classdict: previous_init = classdict['__init__'] - @docval(*docval_args, allow_positional=AllowPositional.WARNING) - def __init__(self, **kwargs): + def _mci_init_body(self, kwargs): # store the values passed to init for each MCI attribute so that they can be added # after calling __init__ original_kwargs = dict(kwargs) @@ -497,7 +500,7 @@ def __init__(self, **kwargs): add_method_name = field_clsconf['add'] new_kwarg = dict( attr_name=attr_name, - value=popargs(attr_name, kwargs), + value=kwargs.pop(attr_name), add_method_name=add_method_name ) new_kwargs.append(new_kwarg) @@ -520,4 +523,8 @@ def __init__(self, **kwargs): self.post_init_method(**original_kwargs) # override __init__ - classdict['__init__'] = __init__ + classdict['__init__'] = signature_function( + '__init__', list(docval_args), _mci_init_body, + doc='Initialize the container, setting all fields defined in the schema.', + allow_positional=AllowPositional.WARNING, + ) diff --git a/src/hdmf/build/errors.py b/src/hdmf/build/errors.py index ec31ef5ba..41991badf 100644 --- a/src/hdmf/build/errors.py +++ b/src/hdmf/build/errors.py @@ -1,28 +1,37 @@ """Module for build error definitions""" from .builders import Builder from ..container import AbstractContainer -from ..utils import docval, getargs +from ..typing import validated class BuildError(Exception): """Error raised when building a container into a builder.""" - @docval({'name': 'builder', 'type': Builder, 'doc': 'the builder that cannot be built'}, - {'name': 'reason', 'type': str, 'doc': 'the reason for the error'}) - def __init__(self, **kwargs): - self.__builder = getargs('builder', kwargs) - self.__reason = getargs('reason', kwargs) + @validated + def __init__(self, builder: Builder, reason: str): + """Initialize this object. + + Args: + builder: the builder that cannot be built + reason: the reason for the error + """ + self.__builder = builder + self.__reason = reason self.__message = "%s (%s): %s" % (self.__builder.name, self.__builder.path, self.__reason) super().__init__(self.__message) class OrphanContainerBuildError(BuildError): - @docval({'name': 'builder', 'type': Builder, 'doc': 'the builder containing the broken link'}, - {'name': 'container', 'type': AbstractContainer, 'doc': 'the container that has no parent'}) - def __init__(self, **kwargs): - builder = getargs('builder', kwargs) - self.__container = getargs('container', kwargs) + @validated + def __init__(self, builder: Builder, container: AbstractContainer): + """Initialize this object. + + Args: + builder: the builder containing the broken link + container: the container that has no parent + """ + self.__container = container reason = ("Linked %s '%s' has no parent. Remove the link or ensure the linked container is added properly." % (self.__container.__class__.__name__, self.__container.name)) super().__init__(builder=builder, reason=reason) @@ -30,11 +39,15 @@ def __init__(self, **kwargs): class ReferenceTargetNotBuiltError(BuildError): - @docval({'name': 'builder', 'type': Builder, 'doc': 'the builder containing the reference that cannot be found'}, - {'name': 'container', 'type': AbstractContainer, 'doc': 'the container that is not built yet'}) - def __init__(self, **kwargs): - builder = getargs('builder', kwargs) - self.__container = getargs('container', kwargs) + @validated + def __init__(self, builder: Builder, container: AbstractContainer): + """Initialize this object. + + Args: + builder: the builder containing the reference that cannot be found + container: the container that is not built yet + """ + self.__container = container reason = ("Could not find already-built Builder for %s '%s' in BuildManager" % (self.__container.__class__.__name__, self.__container.name)) super().__init__(builder=builder, reason=reason) diff --git a/src/hdmf/build/manager.py b/src/hdmf/build/manager.py index 97487786a..8a3c988c9 100644 --- a/src/hdmf/build/manager.py +++ b/src/hdmf/build/manager.py @@ -12,7 +12,8 @@ from ..term_set import TypeConfigurator from ..spec import DatasetSpec, GroupSpec, NamespaceCatalog, RefSpec from ..spec.spec import BaseStorageSpec -from ..utils import docval, getargs, ExtenderMeta, get_docval +from ..utils import docval, ExtenderMeta, get_docval +from ..typing import Bool, validated class Proxy: @@ -53,16 +54,25 @@ def data_type(self): """The data_type of Container that should match this Proxy""" return self.__data_type - @docval({"name": "object", "type": (BaseBuilder, Container), "doc": "the container or builder to get a proxy for"}) - def matches(self, **kwargs): - obj = getargs('object', kwargs) + @validated + def matches(self, object: BaseBuilder | Container): + """matches + + Args: + object: the container or builder to get a proxy for + """ + obj = object if not isinstance(obj, Proxy): obj = self.__manager.get_proxy(obj) return self == obj - @docval({"name": "container", "type": Container, "doc": "the Container to add as a candidate match"}) - def add_candidate(self, **kwargs): - container = getargs('container', kwargs) + @validated + def add_candidate(self, container: Container): + """add_candidate + + Args: + container: the Container to add as a candidate match + """ self.__candidates.append(container) def resolve(self): @@ -110,12 +120,15 @@ def type_map(self): def in_export_mode(self): return self.__in_export_mode - @docval({"name": "object", "type": (BaseBuilder, AbstractContainer), - "doc": "the container or builder to get a proxy for"}, - {"name": "source", "type": str, - "doc": "the source of container being built i.e. file path", 'default': None}) - def get_proxy(self, **kwargs): - obj = getargs('object', kwargs) + @validated + def get_proxy(self, object: BaseBuilder | AbstractContainer, source: str | None = None): + """get_proxy + + Args: + object: the container or builder to get a proxy for + source: the source of container being built i.e. file path + """ + obj = object if isinstance(obj, BaseBuilder): return self._get_proxy_builder(obj) elif isinstance(obj, AbstractContainer): @@ -146,21 +159,23 @@ def _get_proxy_container(self, container): loc = "/".join(reversed(stack)) return Proxy(self, container.container_source, loc, ns, dt) - @docval({"name": "container", "type": AbstractContainer, "doc": "the container to convert to a Builder"}, - {"name": "source", "type": str, - "doc": "the source of container being built i.e. file path", 'default': None}, - {"name": "matched_spec", "type": BaseStorageSpec, - "doc": ("the position-resolved subspec for this container in its parent's spec tree. Threaded through " - "to ObjectMapper.build, which sets it on the resulting builder as `builder.matched_spec`."), - 'default': None}, - {"name": "export", "type": bool, "doc": "whether this build is for exporting", - 'default': False}, - {"name": "root", "type": bool, "doc": "whether the container is the root of the build process", - 'default': False}) - def build(self, **kwargs): - """ Build the GroupBuilder/DatasetBuilder for the given AbstractContainer""" - container, export = getargs('container', 'export', kwargs) - source, matched_spec, root = getargs('source', 'matched_spec', 'root', kwargs) + @validated + def build(self, + container: AbstractContainer, + source: str | None = None, + matched_spec: BaseStorageSpec | None = None, + export: Bool = False, + root: Bool = False): + """Build the GroupBuilder/DatasetBuilder for the given AbstractContainer + + Args: + container: the container to convert to a Builder + source: the source of container being built i.e. file path + matched_spec: the position-resolved subspec for this container in its parent's spec tree. Threaded through + to ObjectMapper.build, which sets it on the resulting builder as `builder.matched_spec`. + export: whether this build is for exporting + root: whether the container is the root of the build process + """ if export: self.__in_export_mode = True result = self.get_builder(container) @@ -205,12 +220,14 @@ def build(self, **kwargs): self.__in_export_mode = False return result - @docval({"name": "container", "type": AbstractContainer, "doc": "the AbstractContainer to save as prebuilt"}, - {'name': 'builder', 'type': (DatasetBuilder, GroupBuilder), - 'doc': 'the Builder representation of the given container'}) - def prebuilt(self, **kwargs): - ''' Save the Builder for a given AbstractContainer for future use ''' - container, builder = getargs('container', 'builder', kwargs) + @validated + def prebuilt(self, container: AbstractContainer, builder: DatasetBuilder | GroupBuilder): + """Save the Builder for a given AbstractContainer for future use + + Args: + container: the AbstractContainer to save as prebuilt + builder: the Builder representation of the given container + """ container_id = self.__conthash__(container) self.__builders[container_id] = builder builder_id = self.__bldrhash__(builder) @@ -272,19 +289,24 @@ def clear_cache(self): self.__builders.clear() self.__containers.clear() - @docval({"name": "container", "type": AbstractContainer, "doc": "the container to get the builder for"}) - def get_builder(self, **kwargs): - """Return the prebuilt builder for the given container or None if it does not exist.""" - container = getargs('container', kwargs) + @validated + def get_builder(self, container: AbstractContainer): + """Return the prebuilt builder for the given container or None if it does not exist. + + Args: + container: the container to get the builder for + """ container_id = self.__conthash__(container) result = self.__builders.get(container_id) return result - @docval({'name': 'builder', 'type': (DatasetBuilder, GroupBuilder), - 'doc': 'the builder to construct the AbstractContainer from'}) - def construct(self, **kwargs): - """ Construct the AbstractContainer represented by the given builder """ - builder = getargs('builder', kwargs) + @validated + def construct(self, builder: DatasetBuilder | GroupBuilder): + """Construct the AbstractContainer represented by the given builder + + Args: + builder: the builder to construct the AbstractContainer from + """ if isinstance(builder, LinkBuilder): builder = builder.target builder_id = self.__bldrhash__(builder) @@ -329,54 +351,69 @@ def __get_parent_dt_builder(self, builder): # *** The following methods just delegate calls to self.__type_map *** - @docval({'name': 'builder', 'type': Builder, 'doc': 'the Builder to get the class object for'}) - def get_cls(self, **kwargs): - ''' Get the class object for the given Builder ''' - builder = getargs('builder', kwargs) + @validated + def get_cls(self, builder: Builder): + """Get the class object for the given Builder + + Args: + builder: the Builder to get the class object for + """ return self.__type_map.get_cls(builder) - @docval({"name": "container", "type": AbstractContainer, "doc": "the container to convert to a Builder"}, - returns='The name a Builder should be given when building this container', rtype=str) - def get_builder_name(self, **kwargs): - ''' Get the name a Builder should be given ''' - container = getargs('container', kwargs) + @validated + def get_builder_name(self, container: AbstractContainer) -> str: + """Get the name a Builder should be given + + Args: + container: the container to convert to a Builder + + Returns: + The name a Builder should be given when building this container + """ return self.__type_map.get_builder_name(container) - @docval({'name': 'spec', 'type': (DatasetSpec, GroupSpec), 'doc': 'the parent spec to search'}, - {'name': 'builder', 'type': (DatasetBuilder, GroupBuilder, LinkBuilder), - 'doc': 'the builder to get the sub-specification for'}) - def get_subspec(self, **kwargs): - ''' Get the specification from this spec that corresponds to the given builder ''' - spec, builder = getargs('spec', 'builder', kwargs) + @validated + def get_subspec(self, spec: DatasetSpec | GroupSpec, builder: DatasetBuilder | GroupBuilder | LinkBuilder): + """Get the specification from this spec that corresponds to the given builder + + Args: + spec: the parent spec to search + builder: the builder to get the sub-specification for + """ return self.__type_map.get_subspec(spec, builder) - @docval({'name': 'builder', 'type': (DatasetBuilder, GroupBuilder, LinkBuilder), - 'doc': 'the builder to get the sub-specification for'}) - def get_builder_ns(self, **kwargs): - ''' Get the namespace of a builder ''' - builder = getargs('builder', kwargs) + @validated + def get_builder_ns(self, builder: DatasetBuilder | GroupBuilder | LinkBuilder): + """Get the namespace of a builder + + Args: + builder: the builder to get the sub-specification for + """ return self.__type_map.get_builder_ns(builder) - @docval({'name': 'builder', 'type': (DatasetBuilder, GroupBuilder, LinkBuilder), - 'doc': 'the builder to get the data_type for'}) - def get_builder_dt(self, **kwargs): - ''' - Get the data_type of a builder - ''' - builder = getargs('builder', kwargs) + @validated + def get_builder_dt(self, builder: DatasetBuilder | GroupBuilder | LinkBuilder): + """Get the data_type of a builder + + Args: + builder: the builder to get the data_type for + """ return self.__type_map.get_builder_dt(builder) - @docval({'name': 'builder', 'type': (GroupBuilder, DatasetBuilder, AbstractContainer), - 'doc': 'the builder or container to check'}, - {'name': 'parent_data_type', 'type': str, - 'doc': 'the potential parent data_type that refers to a data_type'}, - returns="True if data_type of *builder* is a sub-data_type of *parent_data_type*, False otherwise", - rtype=bool) - def is_sub_data_type(self, **kwargs): - ''' - Return whether or not data_type of *builder* is a sub-data_type of *parent_data_type* - ''' - builder, parent_dt = getargs('builder', 'parent_data_type', kwargs) + @validated + def is_sub_data_type(self, + builder: GroupBuilder | DatasetBuilder | AbstractContainer, + parent_data_type: str) -> bool: + """Return whether or not data_type of *builder* is a sub-data_type of *parent_data_type* + + Args: + builder: the builder or container to check + parent_data_type: the potential parent data_type that refers to a data_type + + Returns: + True if data_type of *builder* is a sub-data_type of *parent_data_type*, False otherwise + """ + builder, parent_dt = builder, parent_data_type if isinstance(builder, (GroupBuilder, DatasetBuilder)): ns = self.get_builder_ns(builder) dt = self.get_builder_dt(builder) @@ -419,12 +456,18 @@ class TypeMap: A class to maintain the map between ObjectMappers and AbstractContainer classes """ - @docval({'name': 'namespaces', 'type': NamespaceCatalog, 'doc': 'the NamespaceCatalog to use', 'default': None}, - {'name': 'mapper_cls', 'type': type, 'doc': 'the ObjectMapper class to use', 'default': None}, - {'name': 'type_config', 'type': TypeConfigurator, 'doc': 'The TypeConfigurator to use.', - 'default': None}) - def __init__(self, **kwargs): - namespaces, mapper_cls, type_config = getargs('namespaces', 'mapper_cls', 'type_config', kwargs) + @validated + def __init__(self, + namespaces: NamespaceCatalog | None = None, + mapper_cls: type | None = None, + type_config: TypeConfigurator | None = None): + """Initialize this object. + + Args: + namespaces: the NamespaceCatalog to use + mapper_cls: the ObjectMapper class to use + type_config: The TypeConfigurator to use. + """ if namespaces is None: namespaces = NamespaceCatalog() if mapper_cls is None: @@ -503,12 +546,16 @@ def merge(self, type_map: "TypeMap", ns_catalog: bool = False): self.register_generator(custom_generators) # NOTE: the type config is not merged from the input type map to the new one. add if there is a clear use case - @docval({"name": "generator", "type": type, "doc": "the CustomClassGenerator class to register"}) - def register_generator(self, **kwargs): - """Add a custom class generator.""" - generator = getargs('generator', kwargs) + @validated + def register_generator(self, generator: type): + """Add a custom class generator. + + Args: + generator: the CustomClassGenerator class to register + """ self.__class_generator_manager.register_generator(generator) + # intentionally on @docval: splices NamespaceCatalog argument specs; migrates with docval removal @docval(*get_docval(NamespaceCatalog.load_namespaces), returns="the namespaces loaded from the given file", rtype=dict) def load_namespaces(self, **kwargs): @@ -544,23 +591,28 @@ def load_namespaces(self, **kwargs): # where some functions allow either a NamespaceCatalog or a TypeMap to be passed in return types - @docval({"name": "data_type", "type": str, "doc": "the data type to create a AbstractContainer class for"}, - {"name": "namespace", "type": str, "doc": "the namespace containing the data_type", "default": None}, - {'name': 'post_init_method', 'type': Callable, 'default': None, - 'doc': 'The function used as a post_init method to validate the class generation.'}, - {"name": "autogen", "type": bool, "doc": "autogenerate class if one does not exist", "default": True}, - returns='the class for the given namespace and data_type', rtype=type) - def get_dt_container_cls(self, **kwargs): + @validated + def get_dt_container_cls(self, + data_type: str, + namespace: str | None = None, + post_init_method: Callable | None = None, + autogen: Bool = True) -> type: """Get the container class from data type specification. If no class has been associated with the ``data_type`` from ``namespace``, a class will be dynamically created and returned. Namespace is optional. If namespace is unknown, it will be looked up from all namespaces. + + Args: + data_type: the data type to create a AbstractContainer class for + namespace: the namespace containing the data_type + post_init_method: The function used as a post_init method to validate the class generation. + autogen: autogenerate class if one does not exist + + Returns: + the class for the given namespace and data_type """ - namespace, data_type, post_init_method, autogen = getargs( - 'namespace', 'data_type', 'post_init_method', 'autogen', kwargs - ) # namespace is unknown, so look it up if namespace is None: @@ -681,12 +733,13 @@ def __get_parent_cls(self, namespace: str, data_type: str, spec: Union[GroupSpec raise ValueError(f"Parent class {parent_cls} is not of type ExtenderMeta: {type(parent_cls)}") return parent_cls - @docval({'name': 'obj', 'type': (GroupBuilder, DatasetBuilder, LinkBuilder, GroupSpec, DatasetSpec), - 'doc': 'the object to get the type key for'}) - def __type_key(self, obj): - """ - A wrapper function to simplify the process of getting a type_key for an object. + @validated + def __type_key(self, obj: GroupBuilder | DatasetBuilder | LinkBuilder | GroupSpec | DatasetSpec): + """A wrapper function to simplify the process of getting a type_key for an object. The type_key is used to get the data_type from a Builder's attributes. + + Args: + obj: the object to get the type key for """ if isinstance(obj, LinkBuilder): obj = obj.builder @@ -695,13 +748,13 @@ def __type_key(self, obj): else: return self.__ns_catalog.dataset_spec_cls.type_key() - @docval({'name': 'builder', 'type': (DatasetBuilder, GroupBuilder, LinkBuilder), - 'doc': 'the builder to get the data_type for'}) - def get_builder_dt(self, **kwargs): - ''' - Get the data_type of a builder - ''' - builder = getargs('builder', kwargs) + @validated + def get_builder_dt(self, builder: DatasetBuilder | GroupBuilder | LinkBuilder): + """Get the data_type of a builder + + Args: + builder: the builder to get the data_type for + """ ret = None if isinstance(builder, LinkBuilder): builder = builder.builder @@ -713,21 +766,25 @@ def get_builder_dt(self, **kwargs): ret = ret.decode('UTF-8') return ret - @docval({'name': 'builder', 'type': (DatasetBuilder, GroupBuilder, LinkBuilder), - 'doc': 'the builder to get the sub-specification for'}) - def get_builder_ns(self, **kwargs): - ''' Get the namespace of a builder ''' - builder = getargs('builder', kwargs) + @validated + def get_builder_ns(self, builder: DatasetBuilder | GroupBuilder | LinkBuilder): + """Get the namespace of a builder + + Args: + builder: the builder to get the sub-specification for + """ if isinstance(builder, LinkBuilder): builder = builder.builder ret = builder.attributes.get('namespace') return ret - @docval({'name': 'builder', 'type': Builder, - 'doc': 'the Builder object to get the corresponding AbstractContainer class for'}) - def get_cls(self, **kwargs): - ''' Get the class object for the given Builder ''' - builder = getargs('builder', kwargs) + @validated + def get_cls(self, builder: Builder): + """Get the class object for the given Builder + + Args: + builder: the Builder object to get the corresponding AbstractContainer class for + """ data_type = self.get_builder_dt(builder) if data_type is None: raise ValueError("No data_type found for builder %s" % builder.path) @@ -736,12 +793,14 @@ def get_cls(self, **kwargs): raise ValueError("No namespace found for builder %s" % builder.path) return self.get_dt_container_cls(data_type, namespace) - @docval({'name': 'spec', 'type': (DatasetSpec, GroupSpec), 'doc': 'the parent spec to search'}, - {'name': 'builder', 'type': (DatasetBuilder, GroupBuilder, LinkBuilder), - 'doc': 'the builder to get the sub-specification for'}) - def get_subspec(self, **kwargs): - ''' Get the specification from this spec that corresponds to the given builder ''' - spec, builder = getargs('spec', 'builder', kwargs) + @validated + def get_subspec(self, spec: DatasetSpec | GroupSpec, builder: DatasetBuilder | GroupBuilder | LinkBuilder): + """Get the specification from this spec that corresponds to the given builder + + Args: + spec: the parent spec to search + builder: the builder to get the sub-specification for + """ if isinstance(builder, LinkBuilder): builder_type = type(builder.builder) # TODO consider checking against spec.get_link @@ -782,20 +841,28 @@ def get_container_cls_dt(self, cls): return ret return ret - @docval({'name': 'namespace', 'type': str, - 'doc': 'the namespace to get the container classes for', 'default': None}) - def get_container_classes(self, **kwargs): - namespace = getargs('namespace', kwargs) + @validated + def get_container_classes(self, namespace: str | None = None): + """get_container_classes + + Args: + namespace: the namespace to get the container classes for + """ ret = (k for k in self.__container_cls_to_ns_dt if not isinstance(k, TypeSource)) if namespace is not None: ret = filter(lambda x: self.__container_cls_to_ns_dt[x][0] == namespace, ret) return list(ret) - @docval({'name': 'obj', 'type': (AbstractContainer, Builder), 'doc': 'the object to get the ObjectMapper for'}, - returns='the ObjectMapper to use for mapping the given object', rtype='ObjectMapper') - def get_map(self, **kwargs): - """ Return the ObjectMapper object that should be used for the given container """ - obj = getargs('obj', kwargs) + @validated + def get_map(self, obj: AbstractContainer | Builder) -> 'ObjectMapper': # noqa: F821 + """Return the ObjectMapper object that should be used for the given container + + Args: + obj: the object to get the ObjectMapper for + + Returns: + the ObjectMapper to use for mapping the given object + """ # get the container class, and namespace/data_type if isinstance(obj, AbstractContainer): container_cls = obj.__class__ @@ -820,12 +887,15 @@ def get_map(self, **kwargs): self.__mappers[container_cls] = mapper return mapper - @docval({"name": "namespace", "type": str, "doc": "the namespace containing the data_type to map the class to"}, - {"name": "data_type", "type": str, "doc": "the data_type to map the class to"}, - {"name": "container_cls", "type": (TypeSource, type), "doc": "the class to map to the specified data_type"}) - def register_container_type(self, **kwargs): - ''' Map a container class to a data_type ''' - namespace, data_type, container_cls = getargs('namespace', 'data_type', 'container_cls', kwargs) + @validated + def register_container_type(self, namespace: str, data_type: str, container_cls: TypeSource | type): + """Map a container class to a data_type + + Args: + namespace: the namespace containing the data_type to map the class to + data_type: the data_type to map the class to + container_cls: the class to map to the specified data_type + """ spec = self.__ns_catalog.get_spec(namespace, data_type) # make sure the spec exists self.__ns_dt_to_container_cls.setdefault(namespace, dict()) previous_cls = self.__ns_dt_to_container_cls[namespace].get(data_type) @@ -846,31 +916,35 @@ def register_container_type(self, **kwargs): setattr(container_cls, spec.type_key(), data_type) setattr(container_cls, 'namespace', namespace) - @docval({"name": "container_cls", "type": type, - "doc": "the AbstractContainer class for which the given ObjectMapper class gets used for"}, - {"name": "mapper_cls", "type": type, "doc": "the ObjectMapper class to use to map"}) - def register_map(self, **kwargs): - ''' Map a container class to an ObjectMapper class ''' - container_cls, mapper_cls = getargs('container_cls', 'mapper_cls', kwargs) + @validated + def register_map(self, container_cls: type, mapper_cls: type): + """Map a container class to an ObjectMapper class + + Args: + container_cls: the AbstractContainer class for which the given ObjectMapper class gets used for + mapper_cls: the ObjectMapper class to use to map + """ if self.get_container_cls_dt(container_cls) == (None, None): raise ValueError('cannot register map for type %s - no data_type found' % container_cls) self.__mapper_cls[container_cls] = mapper_cls - @docval({"name": "container", "type": AbstractContainer, "doc": "the container to convert to a Builder"}, - {"name": "manager", "type": BuildManager, - "doc": "the BuildManager to use for managing this build", 'default': None}, - {"name": "source", "type": str, - "doc": "the source of container being built i.e. file path", 'default': None}, - {"name": "builder", "type": BaseBuilder, "doc": "the Builder to build on", 'default': None}, - {"name": "matched_spec", "type": BaseStorageSpec, - "doc": ("the position-resolved subspec for this container in its parent's spec tree. Forwarded to " - "ObjectMapper.build, which records it on the resulting builder as `builder.matched_spec`."), - 'default': None}, - ) - def build(self, **kwargs): - """Build the GroupBuilder/DatasetBuilder for the given AbstractContainer""" - container, manager, builder = getargs('container', 'manager', 'builder', kwargs) - source, matched_spec = getargs('source', 'matched_spec', kwargs) + @validated + def build(self, + container: AbstractContainer, + manager: BuildManager | None = None, + source: str | None = None, + builder: BaseBuilder | None = None, + matched_spec: BaseStorageSpec | None = None): + """Build the GroupBuilder/DatasetBuilder for the given AbstractContainer + + Args: + container: the container to convert to a Builder + manager: the BuildManager to use for managing this build + source: the source of container being built i.e. file path + builder: the Builder to build on + matched_spec: the position-resolved subspec for this container in its parent's spec tree. Forwarded to + ObjectMapper.build, which records it on the resulting builder as `builder.matched_spec`. + """ # get the ObjectMapper to map between Spec objects and AbstractContainer attributes obj_mapper = self.get_map(container) @@ -889,15 +963,18 @@ def build(self, **kwargs): builder.set_attribute(obj_mapper.spec.id_key(), container.object_id) return builder - @docval({'name': 'builder', 'type': (DatasetBuilder, GroupBuilder), - 'doc': 'the builder to construct the AbstractContainer from'}, - {'name': 'build_manager', 'type': BuildManager, - 'doc': 'the BuildManager for constructing', 'default': None}, - {'name': 'parent', 'type': (Proxy, Container), - 'doc': 'the parent Container/Proxy for the Container being built', 'default': None}) - def construct(self, **kwargs): - """ Construct the AbstractContainer represented by the given builder """ - builder, build_manager, parent = getargs('builder', 'build_manager', 'parent', kwargs) + @validated + def construct(self, + builder: DatasetBuilder | GroupBuilder, + build_manager: BuildManager | None = None, + parent: Proxy | Container | None = None): + """Construct the AbstractContainer represented by the given builder + + Args: + builder: the builder to construct the AbstractContainer from + build_manager: the BuildManager for constructing + parent: the parent Container/Proxy for the Container being built + """ if build_manager is None: build_manager = BuildManager(self) obj_mapper = self.get_map(builder) @@ -907,11 +984,16 @@ def construct(self, **kwargs): else: return obj_mapper.construct(builder, build_manager, parent) - @docval({"name": "container", "type": AbstractContainer, "doc": "the container to convert to a Builder"}, - returns='The name a Builder should be given when building this container', rtype=str) - def get_builder_name(self, **kwargs): - ''' Get the name a Builder should be given ''' - container = getargs('container', kwargs) + @validated + def get_builder_name(self, container: AbstractContainer) -> str: + """Get the name a Builder should be given + + Args: + container: the container to convert to a Builder + + Returns: + The name a Builder should be given when building this container + """ obj_mapper = self.get_map(container) if obj_mapper is None: raise ValueError('No ObjectMapper found for container of type %s' % str(container.__class__.__name__)) diff --git a/src/hdmf/build/objectmapper.py b/src/hdmf/build/objectmapper.py index a180b086a..2475510eb 100644 --- a/src/hdmf/build/objectmapper.py +++ b/src/hdmf/build/objectmapper.py @@ -23,23 +23,25 @@ from ..query import ReferenceResolver from ..spec import Spec, AttributeSpec, DatasetSpec, GroupSpec, LinkSpec, RefSpec from ..spec.spec import BaseStorageSpec -from ..utils import docval, getargs, ExtenderMeta, get_docval, get_data_shape, is_array_like, is_zarr_array, StrDataset +from ..utils import ExtenderMeta, get_docval, get_data_shape, is_array_like, is_zarr_array, StrDataset +from ..typing import validated _const_arg = '__constructor_arg' -@docval({'name': 'name', 'type': str, 'doc': 'the name of the constructor argument'}, - is_method=False) -def _constructor_arg(**kwargs): - '''Decorator to override the default mapping scheme for a given constructor argument. +@validated +def _constructor_arg(name: str): + """Decorator to override the default mapping scheme for a given constructor argument. Decorate ObjectMapper methods with this function when extending ObjectMapper to override the default scheme for mapping between AbstractContainer and Builder objects. The decorated method should accept as its first argument the Builder object that is being mapped. The method should return the value to be passed to the target AbstractContainer class constructor argument given by *name*. - ''' - name = getargs('name', kwargs) + + Args: + name: the name of the constructor argument + """ def _dec(func): setattr(func, _const_arg, name) @@ -51,18 +53,19 @@ def _dec(func): _obj_attr = '__object_attr' -@docval({'name': 'name', 'type': str, 'doc': 'the name of the constructor argument'}, - is_method=False) -def _object_attr(**kwargs): - '''Decorator to override the default mapping scheme for a given object attribute. +@validated +def _object_attr(name: str): + """Decorator to override the default mapping scheme for a given object attribute. Decorate ObjectMapper methods with this function when extending ObjectMapper to override the default scheme for mapping between AbstractContainer and Builder objects. The decorated method should accept as its first argument the AbstractContainer object that is being mapped. The method should return the child Builder object (or scalar if the object attribute corresponds to an AttributeSpec) that represents the attribute given by *name*. - ''' - name = getargs('name', kwargs) + + Args: + name: the name of the constructor argument + """ def _dec(func): setattr(func, _obj_attr, name) @@ -438,34 +441,36 @@ def __check_edgecases(cls, spec, value): # noqa: C901 _const_arg = '__constructor_arg' @staticmethod - @docval({'name': 'name', 'type': str, 'doc': 'the name of the constructor argument'}, - is_method=False) - def constructor_arg(**kwargs): - '''Decorator to override the default mapping scheme for a given constructor argument. + @validated + def constructor_arg(name: str): + """Decorator to override the default mapping scheme for a given constructor argument. Decorate ObjectMapper methods with this function when extending ObjectMapper to override the default scheme for mapping between AbstractContainer and Builder objects. The decorated method should accept as its first argument the Builder object that is being mapped. The method should return the value to be passed to the target AbstractContainer class constructor argument given by *name*. - ''' - name = getargs('name', kwargs) + + Args: + name: the name of the constructor argument + """ return _constructor_arg(name) _obj_attr = '__object_attr' @staticmethod - @docval({'name': 'name', 'type': str, 'doc': 'the name of the constructor argument'}, - is_method=False) - def object_attr(**kwargs): - '''Decorator to override the default mapping scheme for a given object attribute. + @validated + def object_attr(name: str): + """Decorator to override the default mapping scheme for a given object attribute. Decorate ObjectMapper methods with this function when extending ObjectMapper to override the default scheme for mapping between AbstractContainer and Builder objects. The decorated method should accept as its first argument the AbstractContainer object that is being mapped. The method should return the child Builder object (or scalar if the object attribute corresponds to an AttributeSpec) that represents the attribute given by *name*. - ''' - name = getargs('name', kwargs) + + Args: + name: the name of the constructor argument + """ return _object_attr(name) @staticmethod @@ -500,12 +505,14 @@ def __gather_procedures(cls, name, bases, classdict): elif cls.__is_attr(func): cls.obj_attrs[cls.__get_obj_attr(func)] = getattr(cls, name) - @docval({'name': 'spec', 'type': (DatasetSpec, GroupSpec), - 'doc': 'The specification for mapping objects to builders'}) - def __init__(self, **kwargs): - """ Create a map from AbstractContainer attributes to specifications """ + @validated + def __init__(self, spec: DatasetSpec | GroupSpec): + """Create a map from AbstractContainer attributes to specifications + + Args: + spec: The specification for mapping objects to builders + """ self.logger = logging.getLogger('%s.%s' % (self.__class__.__module__, self.__class__.__qualname__)) - spec = getargs('spec', kwargs) self.__spec = spec self.__data_type_key = spec.type_key() self.__spec2attr = dict() @@ -525,10 +532,13 @@ def get_container_name(self, *args): return builder.name @classmethod - @docval({'name': 'spec', 'type': Spec, 'doc': 'the specification to get the name for'}) - def convert_dt_name(cls, **kwargs): - '''Construct the attribute name corresponding to a specification''' - spec = getargs('spec', kwargs) + @validated + def convert_dt_name(cls, spec: Spec): + """Construct the attribute name corresponding to a specification + + Args: + spec: the specification to get the name for + """ name = cls.__get_data_type(spec) s1 = re.sub('(.)([A-Z][a-z]+)', r'\1_\2', name) name = re.sub('([a-z0-9])([A-Z])', r'\1_\2', s1).lower() @@ -563,10 +573,13 @@ def __get_fields(cls, name_stack, all_names, spec): name_stack.pop() @classmethod - @docval({'name': 'spec', 'type': Spec, 'doc': 'the specification to get the object attribute names for'}) - def get_attr_names(cls, **kwargs): - '''Get the attribute names for each subspecification in a Spec''' - spec = getargs('spec', kwargs) + @validated + def get_attr_names(cls, spec: Spec): + """Get the attribute names for each subspecification in a Spec + + Args: + spec: the specification to get the object attribute names for + """ names = OrderedDict() for subspec in spec.attributes: cls.__get_fields(list(), names, subspec) @@ -584,46 +597,64 @@ def __map_spec(self, spec): for k, v in attr_names.items(): self.map_spec(k, v) - @docval({"name": "attr_name", "type": str, "doc": "the name of the object to map"}, - {"name": "spec", "type": Spec, "doc": "the spec to map the attribute to"}) - def map_attr(self, **kwargs): - """ Map an attribute to spec. Use this to override default behavior """ - attr_name, spec = getargs('attr_name', 'spec', kwargs) + @validated + def map_attr(self, attr_name: str, spec: Spec): + """Map an attribute to spec. Use this to override default behavior + + Args: + attr_name: the name of the object to map + spec: the spec to map the attribute to + """ self.__spec2attr[spec] = attr_name self.__attr2spec[attr_name] = spec - @docval({"name": "attr_name", "type": str, "doc": "the name of the attribute"}) - def get_attr_spec(self, **kwargs): - """ Return the Spec for a given attribute """ - attr_name = getargs('attr_name', kwargs) + @validated + def get_attr_spec(self, attr_name: str): + """Return the Spec for a given attribute + + Args: + attr_name: the name of the attribute + """ return self.__attr2spec.get(attr_name) - @docval({"name": "carg_name", "type": str, "doc": "the name of the constructor argument"}) - def get_carg_spec(self, **kwargs): - """ Return the Spec for a given constructor argument """ - carg_name = getargs('carg_name', kwargs) + @validated + def get_carg_spec(self, carg_name: str): + """Return the Spec for a given constructor argument + + Args: + carg_name: the name of the constructor argument + """ return self.__carg2spec.get(carg_name) - @docval({"name": "const_arg", "type": str, "doc": "the name of the constructor argument to map"}, - {"name": "spec", "type": Spec, "doc": "the spec to map the attribute to"}) - def map_const_arg(self, **kwargs): - """ Map an attribute to spec. Use this to override default behavior """ - const_arg, spec = getargs('const_arg', 'spec', kwargs) + @validated + def map_const_arg(self, const_arg: str, spec: Spec): + """Map an attribute to spec. Use this to override default behavior + + Args: + const_arg: the name of the constructor argument to map + spec: the spec to map the attribute to + """ self.__spec2carg[spec] = const_arg self.__carg2spec[const_arg] = spec - @docval({"name": "spec", "type": Spec, "doc": "the spec to map the attribute to"}) - def unmap(self, **kwargs): - """ Removing any mapping for a specification. Use this to override default mapping """ - spec = getargs('spec', kwargs) + @validated + def unmap(self, spec: Spec): + """Removing any mapping for a specification. Use this to override default mapping + + Args: + spec: the spec to map the attribute to + """ self.__spec2attr.pop(spec, None) self.__spec2carg.pop(spec, None) - @docval({"name": "attr_carg", "type": str, "doc": "the constructor argument/object attribute to map this spec to"}, - {"name": "spec", "type": Spec, "doc": "the spec to map the attribute to"}) - def map_spec(self, **kwargs): - """ Map the given specification to the construct argument and object attribute """ - spec, attr_carg = getargs('spec', 'attr_carg', kwargs) + @validated + def map_spec(self, attr_carg: str, spec: Spec): + """Map the given specification to the construct argument and object attribute + + Args: + attr_carg: the constructor argument/object attribute to map this spec to + spec: the spec to map the attribute to + """ self.map_const_arg(attr_carg, spec) self.map_attr(attr_carg, spec) @@ -643,21 +674,31 @@ def __get_override_attr(self, name, container, manager): return func(self, container, manager) return None - @docval({"name": "spec", "type": Spec, "doc": "the spec to get the attribute for"}, - returns='the attribute name', rtype=str) - def get_attribute(self, **kwargs): - ''' Get the object attribute name for the given Spec ''' - spec = getargs('spec', kwargs) + @validated + def get_attribute(self, spec: Spec) -> str: + """Get the object attribute name for the given Spec + + Args: + spec: the spec to get the attribute for + + Returns: + the attribute name + """ val = self.__spec2attr.get(spec, None) return val - @docval({"name": "spec", "type": Spec, "doc": "the spec to get the attribute value for"}, - {"name": "container", "type": AbstractContainer, "doc": "the container to get the attribute value from"}, - {"name": "manager", "type": BuildManager, "doc": "the BuildManager used for managing this build"}, - returns='the value of the attribute') - def get_attr_value(self, **kwargs): - ''' Get the value of the attribute corresponding to this spec from the given container ''' - spec, container, manager = getargs('spec', 'container', 'manager', kwargs) + @validated + def get_attr_value(self, spec: Spec, container: AbstractContainer, manager: BuildManager): + """Get the value of the attribute corresponding to this spec from the given container + + Args: + spec: the spec to get the attribute value for + container: the container to get the attribute value from + manager: the BuildManager used for managing this build + + Returns: + the value of the attribute + """ attr_name = self.get_attribute(spec) if attr_name is None: return None @@ -798,32 +839,43 @@ def __check_quantity(self, attr_value, spec, container): warnings.warn(msg, IncorrectQuantityBuildWarning) self.logger.debug('IncorrectQuantityBuildWarning: ' + msg) - @docval({"name": "spec", "type": Spec, "doc": "the spec to get the constructor argument for"}, - returns="the name of the constructor argument", rtype=str) - def get_const_arg(self, **kwargs): - ''' Get the constructor argument for the given Spec ''' - spec = getargs('spec', kwargs) + @validated + def get_const_arg(self, spec: Spec) -> str: + """Get the constructor argument for the given Spec + + Args: + spec: the spec to get the constructor argument for + + Returns: + the name of the constructor argument + """ return self.__spec2carg.get(spec, None) - @docval({"name": "container", "type": AbstractContainer, "doc": "the container to convert to a Builder"}, - {"name": "manager", "type": BuildManager, "doc": "the BuildManager to use for managing this build"}, - {"name": "parent", "type": GroupBuilder, "doc": "the parent of the resulting Builder", 'default': None}, - {"name": "source", "type": str, - "doc": "the source of container being built i.e. file path", 'default': None}, - {"name": "builder", "type": BaseBuilder, "doc": "the Builder to build on", 'default': None}, - {"name": "matched_spec", "type": BaseStorageSpec, - "doc": ("the position-resolved subspec for this container in its parent's spec tree, used to compute " - "dtype/shape for the new dataset builder before it exists. Stored on the resulting builder as " - "`builder.matched_spec` so post-creation consumers can read it without re-deriving the match."), - 'default': None}, - returns="the Builder representing the given AbstractContainer", rtype=Builder) - def build(self, **kwargs): - '''Convert an AbstractContainer to a Builder representation. + @validated + def build(self, + container: AbstractContainer, + manager: BuildManager, + parent: GroupBuilder | None = None, + source: str | None = None, + builder: BaseBuilder | None = None, + matched_spec: BaseStorageSpec | None = None) -> Builder: + """Convert an AbstractContainer to a Builder representation. References are not added but are queued to be added in the BuildManager. - ''' - container, manager, parent, source = getargs('container', 'manager', 'parent', 'source', kwargs) - builder, matched_spec = getargs('builder', 'matched_spec', kwargs) + + Args: + container: the container to convert to a Builder + manager: the BuildManager to use for managing this build + parent: the parent of the resulting Builder + source: the source of container being built i.e. file path + builder: the Builder to build on + matched_spec: the position-resolved subspec for this container in its parent's spec tree, used to compute + dtype/shape for the new dataset builder before it exists. Stored on the resulting builder as + `builder.matched_spec` so post-creation consumers can read it without re-deriving the match. + + Returns: + the Builder representing the given AbstractContainer + """ name = manager.get_builder_name(container) if isinstance(self.__spec, GroupSpec): self.logger.debug("Building %s '%s' as a group (source: %s)" @@ -1564,14 +1616,18 @@ def __flatten(self, sub_builder, subspec, manager): tmp = tmp[0] return tmp - @docval({'name': 'builder', 'type': (DatasetBuilder, GroupBuilder), - 'doc': 'the builder to construct the AbstractContainer from'}, - {'name': 'manager', 'type': BuildManager, 'doc': 'the BuildManager for this build'}, - {'name': 'parent', 'type': (Proxy, AbstractContainer), - 'doc': 'the parent AbstractContainer/Proxy for the AbstractContainer being built', 'default': None}) - def construct(self, **kwargs): - ''' Construct an AbstractContainer from the given Builder ''' - builder, manager, parent = getargs('builder', 'manager', 'parent', kwargs) + @validated + def construct(self, + builder: DatasetBuilder | GroupBuilder, + manager: BuildManager, + parent: Proxy | AbstractContainer | None = None): + """Construct an AbstractContainer from the given Builder + + Args: + builder: the builder to construct the AbstractContainer from + manager: the BuildManager for this build + parent: the parent AbstractContainer/Proxy for the AbstractContainer being built + """ cls = manager.get_cls(builder) # gather all subspecs subspecs = self.__get_subspec_values(builder, self.spec, manager) @@ -1626,11 +1682,13 @@ def __new_container__(self, cls, container_source, parent, object_id, **kwargs): obj._in_construct_mode = False # reset to False to indicate that the construction of the object is complete return obj - @docval({'name': 'container', 'type': AbstractContainer, - 'doc': 'the AbstractContainer to get the Builder name for'}) - def get_builder_name(self, **kwargs): - '''Get the name of a Builder that represents a AbstractContainer''' - container = getargs('container', kwargs) + @validated + def get_builder_name(self, container: AbstractContainer): + """Get the name of a Builder that represents a AbstractContainer + + Args: + container: the AbstractContainer to get the Builder name for + """ if self.__spec.name is not None: ret = self.__spec.name else: diff --git a/src/hdmf/common/__init__.py b/src/hdmf/common/__init__.py index 20d39905a..5deab3eeb 100644 --- a/src/hdmf/common/__init__.py +++ b/src/hdmf/common/__init__.py @@ -17,30 +17,33 @@ from ..validate import ValidatorMap # noqa: E402 from ..build import BuildManager, TypeMap # noqa: E402 from ..container import _set_exp # noqa: E402 +from typing import Any # noqa: E402 +from ..typing import Bool, Int, TypeName, validated # noqa: E402 # a global type map global __TYPE_MAP -@docval({'name': 'config_path', 'type': str, 'doc': 'Path to the configuration file.'}, - {'name': 'type_map', 'type': TypeMap, 'doc': 'The TypeMap.', 'default': None}, - is_method=False) -def load_type_config(**kwargs): - """ - This method will either load the config at the given path into either the global type map or a specific type map. +@validated +def load_type_config(config_path: str, type_map: TypeMap | None = None): + """This method will either load the config at the given path into either the global type map or a specific type map. + + Args: + config_path: Path to the configuration file. + type_map: The TypeMap. """ - config_path = kwargs['config_path'] - type_map = kwargs['type_map'] or __TYPE_MAP + type_map = type_map or __TYPE_MAP type_map.type_config.load_type_config(config_path) -@docval({'name': 'type_map', 'type': TypeMap, 'doc': 'The TypeMap.', 'default': None}, - is_method=False) -def get_loaded_type_config(**kwargs): - """ - This method returns a dictionary with the configuration for each namespace and data type. +@validated +def get_loaded_type_config(type_map: TypeMap | None = None): + """This method returns a dictionary with the configuration for each namespace and data type. + + Args: + type_map: The TypeMap. """ - type_map = kwargs['type_map'] or __TYPE_MAP + type_map = type_map or __TYPE_MAP if type_map.type_config.config is None: msg = "No configuration is loaded." @@ -48,28 +51,29 @@ def get_loaded_type_config(**kwargs): return type_map.type_config.config -@docval({'name': 'type_map', 'type': TypeMap, 'doc': 'The TypeMap.', 'default': None}, - is_method=False) -def unload_type_config(**kwargs): - """ - Unload all type configurations from the global type map or a specific type map. +@validated +def unload_type_config(type_map: TypeMap | None = None): + """Unload all type configurations from the global type map or a specific type map. + + Args: + type_map: The TypeMap. """ - type_map = kwargs['type_map'] or __TYPE_MAP + type_map = type_map or __TYPE_MAP return type_map.type_config.unload_type_config() # a function to register a container classes with the global map -@docval({'name': 'data_type', 'type': str, 'doc': 'the data_type to get the spec for'}, - {'name': 'namespace', 'type': str, 'doc': 'the name of the namespace', 'default': CORE_NAMESPACE}, - {"name": "container_cls", "type": type, - "doc": "the class to map to the specified data_type", 'default': None}, - is_method=False) -def register_class(**kwargs): +@validated +def register_class(data_type: str, namespace: str = CORE_NAMESPACE, container_cls: type | None = None): """Register an Container class to use for reading and writing a data_type from a specification If container_cls is not specified, returns a decorator for registering an Container subclass as the class for data_type in namespace. + + Args: + data_type: the data_type to get the spec for + namespace: the name of the namespace + container_cls: the class to map to the specified data_type """ - data_type, namespace, container_cls = getargs('data_type', 'namespace', 'container_cls', kwargs) if namespace == EXP_NAMESPACE: def _dec(cls): _set_exp(cls) @@ -87,16 +91,16 @@ def _dec(cls): # a function to register an object mapper for a container class -@docval({"name": "container_cls", "type": type, - "doc": "the Container class for which the given ObjectMapper class gets used for"}, - {"name": "mapper_cls", "type": type, "doc": "the ObjectMapper class to use to map", 'default': None}, - is_method=False) -def register_map(**kwargs): +@validated +def register_map(container_cls: type, mapper_cls: type | None = None): """Register an ObjectMapper to use for a Container class type If mapper_cls is not specified, returns a decorator for registering an ObjectMapper class as the mapper for container_cls. If mapper_cls specified, register the class as the mapper for container_cls + + Args: + container_cls: the Container class for which the given ObjectMapper class gets used for + mapper_cls: the ObjectMapper class to use to map """ - container_cls, mapper_cls = getargs('container_cls', 'mapper_cls', kwargs) def _dec(cls): __TYPE_MAP.register_map(container_cls, cls) @@ -124,15 +128,16 @@ def _get_resources(): return __get_resources() -@docval({'name': 'namespace_path', 'type': str, - 'doc': 'the path to the YAML with the namespace definition'}, - returns="the namespaces loaded from the given file", rtype=tuple, - is_method=False) -def load_namespaces(**kwargs): - ''' - Load namespaces from file - ''' - namespace_path = getargs('namespace_path', kwargs) +@validated +def load_namespaces(namespace_path: str) -> tuple: + """Load namespaces from file + + Args: + namespace_path: the path to the YAML with the namespace definition + + Returns: + the namespaces loaded from the given file + """ return __TYPE_MAP.load_namespaces(namespace_path) @@ -141,13 +146,8 @@ def available_namespaces(): # a function to get the container class for a give type -@docval({'name': 'data_type', 'type': str, - 'doc': 'the data_type to get the Container class for'}, - {'name': 'namespace', 'type': str, 'doc': 'the namespace the data_type is defined in'}, - {'name': 'post_init_method', 'type': Callable, 'default': None, - 'doc': 'The function used as a post_init method to validate the class generation.'}, - is_method=False) -def get_class(**kwargs): +@validated +def get_class(data_type: str, namespace: str, post_init_method: Callable | None = None): """Get the class object of the Container subclass corresponding to a given neurdata_type. For developers: @@ -163,25 +163,27 @@ def get_class(**kwargs): Remember that the generation of a class means the __init__ is being created for you. You don't ever see it. The generation also builds the docval for the __init__ and prepares the __fields__ dict for creating setters, which are handled in AbstractContainer. + + Args: + data_type: the data_type to get the Container class for + namespace: the namespace the data_type is defined in + post_init_method: The function used as a post_init method to validate the class generation. """ - data_type, namespace, post_init_method = getargs('data_type', 'namespace', 'post_init_method', kwargs) return __TYPE_MAP.get_dt_container_cls(data_type, namespace, post_init_method) -@docval({ - 'name': 'copy', 'type': bool, - 'doc': 'Whether to return a deepcopy of the TypeMap. ' - 'If False, a direct reference may be returned (use with caution).', - 'default': True - }, - allow_positional=AllowPositional.ERROR, - returns="the namespaces loaded from the given file", rtype=tuple, - is_method=False) -def get_type_map(**kwargs): - ''' - Get a BuildManager to use for I/O using the core namespace. - ''' - copy_map = getargs('copy', kwargs) +@validated(allow_positional=AllowPositional.ERROR) +def get_type_map(copy: Bool = True) -> tuple: + """Get a BuildManager to use for I/O using the core namespace. + + Args: + copy: Whether to return a deepcopy of the TypeMap. If False, a direct reference may be returned (use with + caution). + + Returns: + the namespaces loaded from the given file + """ + copy_map = copy if copy_map: type_map = deepcopy(__TYPE_MAP) else: @@ -189,6 +191,7 @@ def get_type_map(**kwargs): return type_map +# intentionally on @docval: splices get_type_map argument specs; migrates with docval removal @docval(*get_docval(get_type_map), returns="a build manager with namespaces loaded from the given file", rtype=BuildManager, is_method=False) @@ -201,17 +204,18 @@ def get_manager(**kwargs): return BuildManager(type_map) -@docval({'name': 'io', 'type': HDMFIO, - 'doc': 'the HDMFIO object to read from'}, - {'name': 'namespace', 'type': str, - 'doc': 'the namespace to validate against', 'default': CORE_NAMESPACE}, - {'name': 'experimental', 'type': bool, - 'doc': 'data type is an experimental data type', 'default': False}, - returns="errors in the file", rtype=list, - is_method=False) -def validate(**kwargs): - """Validate an file against a namespace""" - io, namespace, experimental = getargs('io', 'namespace', 'experimental', kwargs) +@validated +def validate(io: HDMFIO, namespace: str = CORE_NAMESPACE, experimental: Bool = False) -> list: + """Validate an file against a namespace + + Args: + io: the HDMFIO object to read from + namespace: the namespace to validate against + experimental: data type is an experimental data type + + Returns: + errors in the file + """ if experimental: namespace = EXP_NAMESPACE builder = io.read_builder() @@ -219,6 +223,7 @@ def validate(**kwargs): return validator.validate(builder) +# intentionally on @docval: splices HDF5IO argument specs; migrates with docval removal @docval(*get_docval(HDF5IO.__init__), is_method=False) def get_hdf5io(**kwargs): """ diff --git a/src/hdmf/common/alignedtable.py b/src/hdmf/common/alignedtable.py index 8625fbd00..2a1512680 100644 --- a/src/hdmf/common/alignedtable.py +++ b/src/hdmf/common/alignedtable.py @@ -7,7 +7,8 @@ import pandas as pd from . import register_class -from .table import DynamicTable +from .table import DynamicTable, MeaningsTable +from ..typing import Bool, Int, validated from ..utils import docval, getargs, popargs, get_docval, AllowPositional @@ -30,6 +31,8 @@ class AlignedDynamicTable(DynamicTable): """ __fields__ = ({'name': 'category_tables', 'child': True}, ) + # this decorator intentionally stays on @docval: it splices parent argument specs + # (get_docval works on the type-hinted parent); it migrates when docval is removed @docval(*get_docval(DynamicTable.__init__), {'name': 'category_tables', 'type': list, 'doc': 'List of DynamicTables to be added to the container. NOTE - Only regular ' @@ -130,10 +133,9 @@ def categories(self): """ return list(self.category_tables.keys()) - @docval({'name': 'category', 'type': DynamicTable, 'doc': 'Add a new DynamicTable category'},) - def add_category(self, **kwargs): - """ - Add a new DynamicTable to the AlignedDynamicTable to create a new category in the table. + @validated + def add_category(self, category: DynamicTable): + """Add a new DynamicTable to the AlignedDynamicTable to create a new category in the table. NOTE: The table must align with (i.e, have the same number of rows as) the main data table (and other category tables). I.e., if the AlignedDynamicTable is already populated with data @@ -141,8 +143,10 @@ def add_category(self, **kwargs): :raises: ValueError is raised if the input table does not have the same number of rows as the main table. ValueError is raised if the table is an AlignedDynamicTable instead of regular DynamicTable. + + Args: + category: Add a new DynamicTable category """ - category = getargs('category', kwargs) if len(category) != len(self): raise ValueError('New category DynamicTable does not align, it has %i rows expected %i' % (len(category), len(self))) @@ -154,14 +158,20 @@ def add_category(self, **kwargs): self.category_tables[category.name] = category category.parent = self - @docval({'name': 'name', 'type': str, 'doc': 'Name of the category we want to retrieve', 'default': None}) - def get_category(self, **kwargs): - name = popargs('name', kwargs) + @validated + def get_category(self, name: str | None = None): + """get_category + + Args: + name: Name of the category we want to retrieve + """ if name is None or (name not in self.category_tables and name == self.name): return self else: return self.category_tables[name] + # this decorator intentionally stays on @docval: it splices parent argument specs + # (get_docval works on the type-hinted parent); it migrates when docval is removed @docval(*get_docval(DynamicTable.add_column), {'name': 'category', 'type': str, 'doc': 'The category the column should be added to', 'default': None}) @@ -184,16 +194,16 @@ def add_column(self, **kwargs): raise KeyError("Category %s not in table" % category_name) category.add_column(**kwargs) - @docval({'name': 'data', 'type': dict, 'doc': 'the data to put in this row', 'default': None}, - {'name': 'id', 'type': int, 'doc': 'the ID for the row', 'default': None}, - {'name': 'enforce_unique_id', 'type': bool, 'doc': 'enforce that the id in the table must be unique', - 'default': False}, - allow_extra=True) - def add_row(self, **kwargs): - """ - We can either provide the row data as a single dict or by specifying a dict for each category + @validated + def add_row(self, data: dict | None = None, id: Int | None = None, enforce_unique_id: Bool = False, **kwargs): + """We can either provide the row data as a single dict or by specifying a dict for each category + + Args: + data: the data to put in this row + id: the ID for the row + enforce_unique_id: enforce that the id in the table must be unique """ - data, row_id, enforce_unique_id = popargs('data', 'id', 'enforce_unique_id', kwargs) + row_id = id data = data if data is not None else kwargs # extract the category data @@ -217,21 +227,21 @@ def add_row(self, **kwargs): for category, values in category_data.items(): self.category_tables[category].add_row(**values) - @docval({'name': 'include_category_tables', 'type': bool, - 'doc': "Ignore sub-category tables and just look at the main table", 'default': False}, - {'name': 'ignore_category_ids', 'type': bool, - 'doc': "Ignore id columns of sub-category tables", 'default': False}) - def get_colnames(self, **kwargs): + @validated + def get_colnames(self, include_category_tables: Bool = False, ignore_category_ids: Bool = False): """Get the full list of names of columns for this table :returns: List of tuples (str, str) where the first string is the name of the DynamicTable that contains the column and the second string is the name of the column. If include_category_tables is False, then a list of column names is returned. + + Args: + include_category_tables: Ignore sub-category tables and just look at the main table + ignore_category_ids: Ignore id columns of sub-category tables """ - if not getargs('include_category_tables', kwargs): + if not include_category_tables: return self.colnames else: - ignore_category_ids = getargs('ignore_category_ids', kwargs) columns = [(self.name, c) for c in self.colnames] for category in self.category_tables.values(): if not ignore_category_ids: @@ -239,12 +249,15 @@ def get_colnames(self, **kwargs): columns += [(category.name, c) for c in category.colnames] return columns - @docval({'name': 'ignore_category_ids', 'type': bool, - 'doc': "Ignore id columns of sub-category tables", 'default': False}) - def to_dataframe(self, **kwargs): - """Convert the collection of tables to a single pandas DataFrame""" + @validated + def to_dataframe(self, ignore_category_ids: Bool = False): + """Convert the collection of tables to a single pandas DataFrame + + Args: + ignore_category_ids: Ignore id columns of sub-category tables + """ dfs = [super().to_dataframe().reset_index(), ] - if getargs('ignore_category_ids', kwargs): + if ignore_category_ids: dfs += [category.to_dataframe() for category in self.category_tables.values()] else: dfs += [category.to_dataframe().reset_index() for category in self.category_tables.values()] @@ -359,16 +372,15 @@ def get(self, item, **kwargs): "[row, (category, column)] or a tuple of length 3 of the form " "[category, column, row], [row, category, column]") - @docval({'name': 'ignore_category_tables', 'type': bool, - 'doc': "Ignore the category tables and only check in the main table columns", 'default': False}, - allow_extra=False) - def has_foreign_columns(self, **kwargs): - """ - Does the table contain DynamicTableRegion columns + @validated + def has_foreign_columns(self, ignore_category_tables: Bool = False): + """Does the table contain DynamicTableRegion columns :returns: True if the table or any of the category tables contains a DynamicTableRegion column, else False + + Args: + ignore_category_tables: Ignore the category tables and only check in the main table columns """ - ignore_category_tables = getargs('ignore_category_tables', kwargs) if super().has_foreign_columns(): return True if not ignore_category_tables: @@ -377,26 +389,27 @@ def has_foreign_columns(self, **kwargs): return True return False - @docval({'name': 'ignore_category_tables', 'type': bool, - 'doc': "Ignore the category tables and only check in the main table columns", 'default': False}, - allow_extra=False) - def get_foreign_columns(self, **kwargs): - """ - Determine the names of all columns that link to another DynamicTable, i.e., + @validated + def get_foreign_columns(self, ignore_category_tables: Bool = False): + """Determine the names of all columns that link to another DynamicTable, i.e., find all DynamicTableRegion type columns. Similar to a foreign key in a database, a DynamicTableRegion column references elements in another table. :returns: List of tuples (str, str) where the first string is the name of the category table (or None if the column is in the main table) and the second string is the column name. + + Args: + ignore_category_tables: Ignore the category tables and only check in the main table columns """ - ignore_category_tables = getargs('ignore_category_tables', kwargs) col_names = [(None, col_name) for col_name in super().get_foreign_columns()] if not ignore_category_tables: for table in self.category_tables.values(): col_names += [(table.name, col_name) for col_name in table.get_foreign_columns()] return col_names + # this decorator intentionally stays on @docval: it splices parent argument specs + # (get_docval works on the type-hinted parent); it migrates when docval is removed @docval(*get_docval(DynamicTable.get_linked_tables), {'name': 'ignore_category_tables', 'type': bool, 'doc': "Ignore the category tables and only check in the main table columns", 'default': False}, @@ -418,14 +431,17 @@ def get_linked_tables(self, **kwargs): other_tables = None if ignore_category_tables else list(self.category_tables.values()) return super().get_linked_tables(other_tables=other_tables) - @docval({'name': 'col_name', 'type': str, - 'doc': 'The name of the column to get the MeaningsTable for.'}, - {'name': 'category', 'type': str, - 'doc': 'The category the column belongs to.', 'default': None}, - returns='the MeaningsTable for the given column', rtype='MeaningsTable') - def get_meanings_for_column(self, **kwargs): - """Get a MeaningsTable for a column in this DynamicTable.""" - col_name, category = getargs('col_name', 'category', kwargs) + @validated + def get_meanings_for_column(self, col_name: str, category: str | None = None) -> MeaningsTable: + """Get a MeaningsTable for a column in this DynamicTable. + + Args: + col_name: The name of the column to get the MeaningsTable for. + category: The category the column belongs to. + + Returns: + the MeaningsTable for the given column + """ if category is not None: category_table = self.get_category(category) meanings_table_name = f"{col_name}_meanings" diff --git a/src/hdmf/common/io/table.py b/src/hdmf/common/io/table.py index 34ad0b707..e295f37a6 100644 --- a/src/hdmf/common/io/table.py +++ b/src/hdmf/common/io/table.py @@ -2,7 +2,7 @@ from ..table import DynamicTable, VectorData, VectorIndex, DynamicTableRegion from ...build import ObjectMapper, BuildManager, CustomClassGenerator from ...spec import Spec -from ...utils import docval, getargs +from ...typing import validated @register_map(DynamicTable) @@ -23,13 +23,18 @@ def attr_columns(self, container, manager): return tuple() return container.colnames - @docval({"name": "spec", "type": Spec, "doc": "the spec to get the attribute value for"}, - {"name": "container", "type": DynamicTable, "doc": "the container to get the attribute value from"}, - {"name": "manager", "type": BuildManager, "doc": "the BuildManager used for managing this build"}, - returns='the value of the attribute') - def get_attr_value(self, **kwargs): - ''' Get the value of the attribute corresponding to this spec from the given container ''' - spec, container, manager = getargs('spec', 'container', 'manager', kwargs) + @validated + def get_attr_value(self, spec: Spec, container: DynamicTable, manager: BuildManager): + """Get the value of the attribute corresponding to this spec from the given container + + Args: + spec: the spec to get the attribute value for + container: the container to get the attribute value from + manager: the BuildManager used for managing this build + + Returns: + the value of the attribute + """ attr_value = super().get_attr_value(spec, container, manager) if attr_value is None and spec.name in container: if spec.data_type_inc == 'VectorData': diff --git a/src/hdmf/common/multi.py b/src/hdmf/common/multi.py index 598b3cc93..c5ebda9a0 100644 --- a/src/hdmf/common/multi.py +++ b/src/hdmf/common/multi.py @@ -1,6 +1,7 @@ from . import register_class from ..container import Container, Data, MultiContainerInterface -from ..utils import docval, popargs, AllowPositional +from ..typing import validated +from ..utils import AllowPositional @register_class('SimpleMultiContainer') @@ -13,11 +14,13 @@ class SimpleMultiContainer(MultiContainerInterface): 'get': 'get_container', } - @docval({'name': 'name', 'type': str, 'doc': 'the name of this container'}, - {'name': 'containers', 'type': (list, tuple), 'default': None, - 'doc': 'the Container or Data objects in this file'}, - allow_positional=AllowPositional.WARNING) - def __init__(self, **kwargs): - containers = popargs('containers', kwargs) - super().__init__(**kwargs) + @validated(allow_positional=AllowPositional.WARNING) + def __init__(self, name: str, containers: list | tuple | None = None): + """Initialize the SimpleMultiContainer. + + Args: + name: the name of this container + containers: the Container or Data objects in this file + """ + super().__init__(name=name) self.containers = containers diff --git a/src/hdmf/common/table.py b/src/hdmf/common/table.py index 8e4c4c4f2..69965c06a 100644 --- a/src/hdmf/common/table.py +++ b/src/hdmf/common/table.py @@ -15,9 +15,11 @@ from . import register_class, EXP_NAMESPACE from ..container import Container, Data from ..data_utils import DataIO, AbstractDataChunkIterator -from ..utils import (docval, getargs, ExtenderMeta, popargs, pystr, AllowPositional, check_type, is_ragged, +from ..typing import AnyData, ArrayData, Bool, Int, Shaped, TypeName, validated +from ..utils import (ExtenderMeta, pystr, AllowPositional, check_type, is_ragged, _is_collection, _get_length, _unwrap_scalar) from ..term_set import TermSetWrapper +from typing import Any def _flatten_one_ragged_level(data): @@ -48,20 +50,27 @@ class VectorData(Data): __fields__ = ("description",) - @docval({'name': 'name', 'type': str, 'doc': 'the name of this VectorData'}, - {'name': 'description', 'type': str, 'doc': 'a description for this column'}, - {'name': 'data', 'type': ('array_data', 'data'), - 'doc': 'a dataset where the first dimension is a concatenation of multiple vectors', 'default': list()}, - allow_positional=AllowPositional.WARNING) - def __init__(self, **kwargs): - description = popargs('description', kwargs) - super().__init__(**kwargs) + @validated(allow_positional=AllowPositional.WARNING) + def __init__(self, name: str, description: str, data: ArrayData | AnyData | None = None): + """Initialize this container. + + Args: + name: the name of this VectorData + description: a description for this column + data: a dataset where the first dimension is a concatenation of multiple vectors + """ + if data is None: + data = list() + super().__init__(name=name, data=data) self.description = description - @docval({'name': 'val', 'type': None, 'doc': 'the value to add to this column'}) - def add_row(self, **kwargs): - """Append a data value to this VectorData column""" - val = getargs('val', kwargs) + @validated + def add_row(self, val: Any): + """Append a data value to this VectorData column + + Args: + val: the value to add to this column + """ self.append(val) def get(self, key, **kwargs): @@ -120,16 +129,17 @@ class VectorIndex(VectorData): __fields__ = ("target",) - @docval({'name': 'name', 'type': str, 'doc': 'the name of this VectorIndex'}, - {'name': 'data', 'type': ('array_data', 'data'), - 'doc': 'a 1D dataset containing indexes that apply to VectorData object'}, - {'name': 'target', 'type': VectorData, - 'doc': 'the target dataset that this index applies to'}, - allow_positional=AllowPositional.WARNING) - def __init__(self, **kwargs): - target = popargs('target', kwargs) - kwargs['description'] = "Index for VectorData '%s'" % target.name - super().__init__(**kwargs) + @validated(allow_positional=AllowPositional.WARNING) + def __init__(self, name: str, data: ArrayData | AnyData, target: VectorData): + """Initialize this container. + + Args: + name: the name of this VectorIndex + data: a 1D dataset containing indexes that apply to VectorData object + target: the target dataset that this index applies to + """ + super().__init__(name=name, data=data, + description="Index for VectorData '%s'" % target.name) self.target = target self.__uint = np.uint8 self.__maxval = 255 @@ -262,25 +272,31 @@ class ElementIdentifiers(Data): Data container with a list of unique identifiers for values within a dataset, e.g. rows of a DynamicTable. """ - @docval({'name': 'name', 'type': str, 'doc': 'the name of this ElementIdentifiers'}, - {'name': 'data', 'type': ('array_data', 'data'), 'doc': 'a 1D dataset containing integer identifiers', - 'default': list(), 'shape': (None,)}, - allow_positional=AllowPositional.WARNING) - def __init__(self, **kwargs): - super().__init__(**kwargs) - - @docval({'name': 'other', 'type': (Data, np.ndarray, list, tuple, int), - 'doc': 'List of ids to search for in this ElementIdentifer object'}, - rtype=np.ndarray, - returns='Array with the list of indices where the elements in the list where found.' - 'Note, the elements in the returned list are ordered in increasing index' - 'of the found elements, rather than in the order in which the elements' - 'where given for the search. Also the length of the result may be different from the length' - 'of the input array. E.g., if our ids are [1,2,3] and we are search for [3,1,5] the ' - 'result would be [0,2] and NOT [2,0,None]') - def __eq__(self, other): + @validated(allow_positional=AllowPositional.WARNING) + def __init__(self, name: str, data: Shaped[ArrayData | AnyData, (None,)] | None = None): + """Initialize this container. + + Args: + name: the name of this ElementIdentifiers + data: a 1D dataset containing integer identifiers """ - Given a list of ids return the indices in the ElementIdentifiers array where the indices are found. + if data is None: + data = list() + super().__init__(name=name, data=data) + + @validated + def __eq__(self, other: Data | np.ndarray | list | tuple | Int) -> np.ndarray: + """Given a list of ids return the indices in the ElementIdentifiers array where the indices are found. + + Args: + other: List of ids to search for in this ElementIdentifer object + + Returns: + Array with the list of indices where the elements in the list where found.Note, the elements in the + returned list are ordered in increasing indexof the found elements, rather than in the order in which the + elementswhere given for the search. Also the length of the result may be different from the lengthof the + input array. E.g., if our ids are [1,2,3] and we are search for [3,1,5] the result would be [0,2] and NOT + [2,0,None] """ # Determine the ids we want to find search_ids = other if not isinstance(other, Data) else other.data @@ -362,32 +378,31 @@ def __gather_columns(cls, name, bases, classdict): except AttributeError: # raises error when "__columns__" is not an attr of item continue - @docval({'name': 'name', 'type': str, 'doc': 'the name of this table'}, - {'name': 'description', 'type': str, 'doc': 'a description of what is in this table'}, - {'name': 'id', 'type': ('array_data', 'data', ElementIdentifiers), 'doc': 'the identifiers for this table', - 'default': None}, - {'name': 'columns', 'type': (tuple, list), 'doc': 'the columns in this table', 'default': None}, - {'name': 'colnames', 'type': 'array_data', - 'doc': 'the ordered names of the columns in this table. columns must also be provided.', - 'default': None}, - {'name': 'target_tables', - 'doc': ('dict mapping DynamicTableRegion column name to the table that the DTR points to. The column is ' - 'added to the table if it is not already present (i.e., when it is optional).'), - 'type': dict, - 'default': None}, - {'name': 'meanings_tables', - 'doc': ('MeaningsTable objects that provide meanings for values in VectorData columns. ' - 'Each MeaningsTable must have a name of "{column_name}_meanings" where column_name ' - 'is the name of the target column in this DynamicTable. The target column must ' - 'exist in this table.'), - 'type': (tuple, list), - 'default': None}, - allow_positional=AllowPositional.WARNING) - def __init__(self, **kwargs): # noqa: C901 - id, columns, desc, colnames = popargs('id', 'columns', 'description', 'colnames', kwargs) - target_tables = popargs('target_tables', kwargs) - meanings_tables = popargs('meanings_tables', kwargs) - super().__init__(**kwargs) + @validated(allow_positional=AllowPositional.WARNING) + def __init__(self, # noqa: C901 + name: str, + description: str, + id: ArrayData | AnyData | ElementIdentifiers | None = None, + columns: tuple | list | None = None, + colnames: ArrayData | None = None, + target_tables: dict | None = None, + meanings_tables: tuple | list | None = None): + """Initialize this container. + + Args: + name: the name of this table + description: a description of what is in this table + id: the identifiers for this table + columns: the columns in this table + colnames: the ordered names of the columns in this table. columns must also be provided. + target_tables: dict mapping DynamicTableRegion column name to the table that the DTR points to. The column + is added to the table if it is not already present (i.e., when it is optional). + meanings_tables: MeaningsTable objects that provide meanings for values in VectorData columns. Each + MeaningsTable must have a name of "{column_name}_meanings" where column_name is the name of the target + column in this DynamicTable. The target column must exist in this table. + """ + desc = description + super().__init__(name=name) self.description = desc # hold names of optional columns that are defined in __columns__ that are not yet initialized @@ -572,20 +587,26 @@ def meanings_tables(self): return self.__meanings_tables @meanings_tables.setter - @docval({'name': 'val', 'type': (tuple, list), 'doc': 'The MeaningsTable objects to set', 'default': None}) - def meanings_tables(self, val): - """Set the MeaningsTable objects in this DynamicTable.""" + @validated + def meanings_tables(self, val: tuple | list | None = None): + """Set the MeaningsTable objects in this DynamicTable. + + Args: + val: The MeaningsTable objects to set + """ self.__meanings_tables = dict() if val is not None: for mt in val: self.add_meanings_table(mt) - @docval({'name': 'meanings_table', 'type': 'MeaningsTable', - 'doc': 'The MeaningsTable to add. Its name must be "{column_name}_meanings" where ' - 'column_name is the name of a column in this DynamicTable.'}) - def add_meanings_table(self, **kwargs): - """Add a MeaningsTable to this DynamicTable.""" - meanings_table = getargs('meanings_table', kwargs) + @validated + def add_meanings_table(self, meanings_table: TypeName['MeaningsTable']): + """Add a MeaningsTable to this DynamicTable. + + Args: + meanings_table: The MeaningsTable to add. Its name must be "{column_name}_meanings" where column_name is + the name of a column in this DynamicTable. + """ if meanings_table.name in self.__meanings_tables: raise ValueError(f"MeaningsTable '{meanings_table.name}' already exists in this DynamicTable") # Check that the target is a column of this DynamicTable @@ -598,22 +619,30 @@ def add_meanings_table(self, **kwargs): self.set_modified() self.__meanings_tables[meanings_table.name] = meanings_table - @docval({'name': 'name', 'type': str, - 'doc': 'The name of the MeaningsTable to get.'}, - returns='the MeaningsTable with the given name', rtype='MeaningsTable') - def get_meanings_table(self, **kwargs): - """Get a MeaningsTable from this DynamicTable by name.""" - name = getargs('name', kwargs) + @validated + def get_meanings_table(self, name: str) -> 'MeaningsTable': + """Get a MeaningsTable from this DynamicTable by name. + + Args: + name: The name of the MeaningsTable to get. + + Returns: + the MeaningsTable with the given name + """ if name not in self.__meanings_tables: raise KeyError(f"MeaningsTable '{name}' not found in DynamicTable '{self.name}'") return self.__meanings_tables[name] - @docval({'name': 'col_name', 'type': str, - 'doc': 'The name of the column to get the MeaningsTable for.'}, - returns='the MeaningsTable for the given column', rtype='MeaningsTable') - def get_meanings_for_column(self, **kwargs): - """Get a MeaningsTable for a column in this DynamicTable.""" - col_name = getargs('col_name', kwargs) + @validated + def get_meanings_for_column(self, col_name: str) -> 'MeaningsTable': + """Get a MeaningsTable for a column in this DynamicTable. + + Args: + col_name: The name of the column to get the MeaningsTable for. + + Returns: + the MeaningsTable for the given column + """ meanings_table_name = f"{col_name}_meanings" if meanings_table_name not in self.__meanings_tables: raise KeyError(f"No MeaningsTable found for column '{col_name}' in DynamicTable '{self.name}'") @@ -808,20 +837,23 @@ def _add_extra_predefined_columns(self, data: dict): ]) ) - @docval({'name': 'data', 'type': dict, 'doc': 'the data to put in this row', 'default': None}, - {'name': 'id', 'type': int, 'doc': 'the ID for the row', 'default': None}, - {'name': 'enforce_unique_id', 'type': bool, 'doc': 'enforce that the id in the table must be unique', - 'default': False}, - {'name': 'check_ragged', 'type': bool, 'default': True, - 'doc': ('whether or not to check for ragged arrays when adding data to the table. ' - 'Set to False to avoid checking every element if performance issues occur.')}, - allow_extra=True) - def add_row(self, **kwargs): - """ - Add a row to the table. If *id* is not provided, it will auto-increment. + @validated + def add_row(self, + data: dict | None = None, + id: Int | None = None, + enforce_unique_id: Bool = False, + check_ragged: Bool = True, + **kwargs): + """Add a row to the table. If *id* is not provided, it will auto-increment. + + Args: + data: the data to put in this row + id: the ID for the row + enforce_unique_id: enforce that the id in the table must be unique + check_ragged: whether or not to check for ragged arrays when adding data to the table. Set to False to + avoid checking every element if performance issues occur. """ - data, row_id, enforce_unique_id, check_ragged = popargs('data', 'id', 'enforce_unique_id', 'check_ragged', - kwargs) + row_id = id data = data if data is not None else kwargs self._validate_new_row(data) @@ -870,42 +902,46 @@ def __eq__(self, other): return False return self.to_dataframe().equals(other.to_dataframe()) - @docval({'name': 'name', 'type': str, 'doc': 'the name of this VectorData'}, - {'name': 'description', 'type': str, 'doc': 'a description for this column'}, - {'name': 'data', 'type': ('array_data', 'data'), - 'doc': 'a dataset where the first dimension is a concatenation of multiple vectors', 'default': list()}, - {'name': 'table', 'type': (bool, 'DynamicTable'), - 'doc': 'whether or not this is a table region or the table the region applies to', 'default': False}, - {'name': 'index', 'type': (bool, VectorIndex, 'array_data', int), - 'doc': ' * ``False`` (default): do not generate a VectorIndex\n\n' - ' * ``True``: generate one empty VectorIndex \n\n' - ' * ``VectorIndex``: Use the supplied VectorIndex \n\n' - ' * array-like of ints: Create a VectorIndex and use these values as the data \n\n' - ' * ``int``: Recursively create `n` VectorIndex objects for a multi-ragged array \n', - 'default': False}, - {'name': 'enum', 'type': (bool, 'array_data'), 'default': False, - 'doc': ('whether or not this column contains data from a fixed set of elements')}, - {'name': 'col_cls', 'type': type, 'default': None, - 'doc': ('class to use to represent the column data. If table=True, this field is ignored and a ' - 'DynamicTableRegion object is used. If enum=True, this field is ignored and a EnumData ' - 'object is used.')}, - {'name': 'check_ragged', 'type': bool, 'default': True, - 'doc': ('whether or not to check for ragged arrays when adding data to the table. ' - 'Set to False to avoid checking every element if performance issues occur.')}, - allow_extra=True) - def add_column(self, **kwargs): # noqa: C901 - """ - Add a column to this table. + @validated + def add_column(self, # noqa: C901 + name: str, + description: str, + data: ArrayData | AnyData | None = None, + table: Bool | TypeName['DynamicTable'] = False, + index: Bool | VectorIndex | ArrayData | Int = False, + enum: Bool | ArrayData = False, + col_cls: type | None = None, + check_ragged: Bool = True, + **kwargs): + """Add a column to this table. If data is provided, it must contain the same number of rows as the current state of the table. Extra keyword arguments will be passed to the constructor of the column class ("col_cls"). :raises ValueError: if the column has already been added to the table - """ - name, data = getargs('name', 'data', kwargs) - index, table, enum, col_cls, check_ragged = popargs('index', 'table', 'enum', 'col_cls', 'check_ragged', kwargs) + Args: + name: the name of this VectorData + description: a description for this column + data: a dataset where the first dimension is a concatenation of multiple vectors + table: whether or not this is a table region or the table the region applies to + index: how to index this column. One of: + + * ``False`` (default): do not generate a VectorIndex + * ``True``: generate one empty VectorIndex + * ``VectorIndex``: use the supplied VectorIndex + * array-like of ints: create a VectorIndex and use these values as the data + * ``int``: recursively create ``n`` VectorIndex objects for a multi-ragged array + + enum: whether or not this column contains data from a fixed set of elements + col_cls: class to use to represent the column data. If table=True, this field is ignored and a + DynamicTableRegion object is used. If enum=True, this field is ignored and a EnumData object is used. + check_ragged: whether or not to check for ragged arrays when adding data to the table. Set to False to + avoid checking every element if performance issues occur. + """ + if data is None: + data = list() if isinstance(index, VectorIndex): msg = "Passing a VectorIndex may lead to unexpected behavior. This functionality is not supported." raise ValueError(msg) @@ -937,7 +973,8 @@ def add_column(self, **kwargs): # noqa: C901 % (name, self.__class__.__name__, spec_index)) warn(msg, stacklevel=3) - ckwargs = dict(kwargs) + # column-constructor kwargs: the named args plus any extra keyword arguments + ckwargs = dict(name=name, description=description, data=data, **kwargs) # Add table if it's been specified if table and enum: @@ -1079,24 +1116,23 @@ def __add_column_index_helper(self, col_index): if col_index in self.__uninit_cols: self.__uninit_cols.pop(col_index) - @docval({'name': 'name', 'type': str, 'doc': 'the name of the DynamicTableRegion object'}, - {'name': 'region', 'type': (slice, list, tuple), 'doc': 'the indices of the table'}, - {'name': 'description', 'type': str, 'doc': 'a brief description of what the region is'}) - def create_region(self, **kwargs): - """ - Create a DynamicTableRegion selecting a region (i.e., rows) in this DynamicTable. + @validated + def create_region(self, name: str, region: slice | list | tuple, description: str): + """Create a DynamicTableRegion selecting a region (i.e., rows) in this DynamicTable. :raises: IndexError if the provided region contains invalid indices + Args: + name: the name of the DynamicTableRegion object + region: the indices of the table + description: a brief description of what the region is """ - region = getargs('region', kwargs) if isinstance(region, slice): if (region.start is not None and region.start < 0) or (region.stop is not None and region.stop > len(self)): msg = 'region slice %s is out of range for this DynamicTable of length %d' % (str(region), len(self)) raise IndexError(msg) region = list(range(*region.indices(len(self)))) - desc = getargs('description', kwargs) - name = getargs('name', kwargs) + desc = description return DynamicTableRegion(name=name, data=region, description=desc, table=self) def __getitem__(self, key): @@ -1291,14 +1327,9 @@ def has_foreign_columns(self): return True return False - @docval({'name': 'other_tables', 'type': (list, tuple, set), - 'doc': "List of additional tables to consider in the search. Usually this " - "parameter is used for internal purposes, e.g., when we need to " - "consider AlignedDynamicTable", 'default': None}, - allow_extra=False) - def get_linked_tables(self, **kwargs): - """ - Get a list of the full list of all tables that are being linked to directly or indirectly + @validated + def get_linked_tables(self, other_tables: list | tuple | set | None = None): + """Get a list of the full list of all tables that are being linked to directly or indirectly from this table via foreign DynamicTableColumns included in this table or in any table that can be reached through DynamicTableRegion columns @@ -1306,13 +1337,16 @@ def get_linked_tables(self, **kwargs): * 'source_table' : The source table containing the DynamicTableRegion column * 'source_column' : The relevant DynamicTableRegion column in the 'source_table' * 'target_table' : The target DynamicTable; same as source_column.table. + + Args: + other_tables: List of additional tables to consider in the search. Usually this parameter is used for + internal purposes, e.g., when we need to consider AlignedDynamicTable """ link_type = NamedTuple('DynamicTableLink', [('source_table', DynamicTable), ('source_column', DynamicTableRegion | VectorIndex), ('target_table', DynamicTable)]) curr_tables = [self, ] # Set of tables - other_tables = getargs('other_tables', kwargs) if other_tables is not None: curr_tables += other_tables curr_index = 0 @@ -1332,23 +1366,21 @@ def get_linked_tables(self, **kwargs): curr_index += 1 return foreign_cols - @docval({'name': 'exclude', 'type': set, 'doc': 'Set of column names to exclude from the dataframe', - 'default': None}, - {'name': 'index', 'type': bool, - 'doc': ('Whether to return indices for a DynamicTableRegion column. If False, nested dataframes will be ' - 'returned.'), - 'default': False} - ) - def to_dataframe(self, **kwargs): - """ - Produce a pandas DataFrame containing this table's data. + @validated + def to_dataframe(self, exclude: set | None = None, index: Bool = False): + """Produce a pandas DataFrame containing this table's data. If this table contains a DynamicTableRegion, by default, If exclude is None, this is equivalent to table.get(slice(None, None, None), index=False). + + Args: + exclude: Set of column names to exclude from the dataframe + index: Whether to return indices for a DynamicTableRegion column. If False, nested dataframes will be + returned. """ arg = slice(None, None, None) # select all rows - sel = self.__get_selection_as_dict(arg, df=True, **kwargs) + sel = self.__get_selection_as_dict(arg, df=True, exclude=exclude, index=index) ret = self.__get_selection_as_df(sel) return ret @@ -1408,32 +1440,15 @@ def generate_html_repr(self, level: int = 0, access_code: str = "", nrows: int = return out @classmethod - @docval( - {'name': 'df', 'type': pd.DataFrame, 'doc': 'source DataFrame'}, - {'name': 'name', 'type': str, 'doc': 'the name of this table'}, - { - 'name': 'index_column', - 'type': str, - 'doc': 'if provided, this column will become the table\'s index', - 'default': None - }, - { - 'name': 'table_description', - 'type': str, - 'doc': 'a description of what is in the resulting table', - 'default': '' - }, - { - 'name': 'columns', - 'type': (list, tuple), - 'doc': 'a list/tuple of dictionaries specifying columns in the table', - 'default': None - }, - allow_extra=True - ) - def from_dataframe(cls, **kwargs): - ''' - Construct an instance of DynamicTable (or a subclass) from a pandas DataFrame. + @validated + def from_dataframe(cls, + df: pd.DataFrame, + name: str, + index_column: str | None = None, + table_description: str = '', + columns: list | tuple | None = None, + **kwargs): + """Construct an instance of DynamicTable (or a subclass) from a pandas DataFrame. The columns of the resulting table are defined by the columns of the dataframe and the index by the dataframe's index (make sure it has a @@ -1442,13 +1457,15 @@ def from_dataframe(cls, **kwargs): dictionaries containing the name and description of the column- to help others understand the contents of your table. See :py:class:`~hdmf.common.table.DynamicTable` for more details on *columns*. - ''' - columns = kwargs.pop('columns') - df = kwargs.pop('df') - name = kwargs.pop('name') - index_column = kwargs.pop('index_column') - table_description = kwargs.pop('table_description') + Args: + df: source DataFrame + name: the name of this table + index_column: if provided, this column will become the table's index + table_description: a description of what is in the resulting table + columns: a list/tuple of dictionaries specifying columns in the table + """ + column_descriptions = kwargs.pop('column_descriptions', dict()) supplied_columns = dict() @@ -1512,23 +1529,27 @@ class DynamicTableRegion(VectorData): 'table', ) - @docval({'name': 'name', 'type': str, 'doc': 'the name of this VectorData'}, - {'name': 'data', 'type': ('array_data', 'data'), - 'doc': 'a dataset where the first dimension is a concatenation of multiple vectors'}, - {'name': 'description', 'type': str, 'doc': 'a description of what this region represents'}, - {'name': 'table', 'type': DynamicTable, - 'doc': 'the DynamicTable this region applies to', 'default': None}, - {'name': 'validate_data', 'type': bool, - 'doc': 'whether to validate the data is in bounds of the linked table', 'default': True}, - allow_positional=AllowPositional.WARNING) - def __init__(self, **kwargs): - table, validate_data = popargs('table', 'validate_data', kwargs) - data = getargs('data', kwargs) + @validated(allow_positional=AllowPositional.WARNING) + def __init__(self, + name: str, + data: ArrayData | AnyData, + description: str, + table: DynamicTable | None = None, + validate_data: Bool = True): + """Initialize this container. + + Args: + name: the name of this VectorData + data: a dataset where the first dimension is a concatenation of multiple vectors + description: a description of what this region represents + table: the DynamicTable this region applies to + validate_data: whether to validate the data is in bounds of the linked table + """ self._validate_data = validate_data if self._validate_data: self._validate_index_in_range(data, table) - super().__init__(**kwargs) + super().__init__(name=name, data=data, description=description) if table is not None: # set the table attribute using fields to avoid another validation in the setter self.fields['table'] = table @@ -1605,10 +1626,13 @@ def validate_data(self): return self._validate_data @validate_data.setter - @docval({'name': 'val', 'type': bool, 'doc': 'whether to validate data is in bounds of the linked table'}) - def validate_data(self, **kwargs): - """Set whether to validate data is in bounds of the linked table.""" - val = getargs('val', kwargs) + @validated + def validate_data(self, val: Bool): + """Set whether to validate data is in bounds of the linked table. + + Args: + val: whether to validate data is in bounds of the linked table + """ self._validate_data = val def extend(self, arg): @@ -1805,16 +1829,25 @@ class EnumData(VectorData): __fields__ = ('elements', ) - @docval({'name': 'name', 'type': str, 'doc': 'the name of this column'}, - {'name': 'description', 'type': str, 'doc': 'a description for this column'}, - {'name': 'data', 'type': ('array_data', 'data'), - 'doc': 'integers that index into elements for the value of each row', 'default': list()}, - {'name': 'elements', 'type': ('array_data', 'data', VectorData), 'default': list(), - 'doc': 'lookup values for each integer in ``data``'}, - allow_positional=AllowPositional.WARNING) - def __init__(self, **kwargs): - elements = popargs('elements', kwargs) - super().__init__(**kwargs) + @validated(allow_positional=AllowPositional.WARNING) + def __init__(self, + name: str, + description: str, + data: ArrayData | AnyData | None = None, + elements: ArrayData | AnyData | VectorData | None = None): + """Initialize this container. + + Args: + name: the name of this column + description: a description for this column + data: integers that index into elements for the value of each row + elements: lookup values for each integer in ``data`` + """ + if data is None: + data = list() + if elements is None: + elements = list() + super().__init__(name=name, description=description, data=data) if not isinstance(elements, VectorData): elements = VectorData(name='%s_elements' % self.name, data=elements, description='fixed set of elements referenced by %s' % self.name) @@ -1884,16 +1917,17 @@ def get(self, arg, index=False, join=False, **kwargs): idx = self.data[arg] return self._get_helper(idx, index=index, join=join, **kwargs) - @docval({'name': 'val', 'type': None, 'doc': 'the value to add to this column'}, - {'name': 'index', 'type': bool, 'doc': 'whether or not the value being added is an index', - 'default': False}) - def add_row(self, **kwargs): + @validated + def add_row(self, val: Any, index: Bool = False): """Append a data value to this EnumData column If an element is provided for *val* (i.e. *index* is False), the correct index value will be determined. Otherwise, *val* will be added as provided. + + Args: + val: the value to add to this column + index: whether or not the value being added is an index """ - val, index = getargs('val', 'index', kwargs) if not index: val = self.__add_term(val) super().append(val) @@ -1922,32 +1956,36 @@ class MeaningsTable(DynamicTable): {'name': 'meaning', 'description': 'The meaning of the value.', 'required': True}, ) - @docval({'name': 'target', 'type': VectorData, - 'doc': 'the VectorData object for which this table provides meanings'}, - {'name': 'description', 'type': str, - 'doc': 'a description of what is in this table', 'default': None}, - {'name': 'id', 'type': ('array_data', 'data', ElementIdentifiers), - 'doc': 'the identifiers for this table', 'default': None}, - {'name': 'columns', 'type': (tuple, list), 'doc': 'the columns in this table', 'default': None}, - {'name': 'colnames', 'type': 'array_data', - 'doc': 'the ordered names of the columns in this table. columns must also be provided.', - 'default': None}, - allow_positional=AllowPositional.WARNING) - def __init__(self, **kwargs): - target = popargs('target', kwargs) - kwargs['name'] = f"{target.name}_meanings" - description = kwargs.get('description') + @validated(allow_positional=AllowPositional.WARNING) + def __init__(self, + target: VectorData, + description: str | None = None, + id: ArrayData | AnyData | ElementIdentifiers | None = None, + columns: tuple | list | None = None, + colnames: ArrayData | None = None): + """Initialize this container. + + Args: + target: the VectorData object for which this table provides meanings + description: a description of what is in this table + id: the identifiers for this table + columns: the columns in this table + colnames: the ordered names of the columns in this table. columns must also be provided. + """ + name = f"{target.name}_meanings" if description is None: - kwargs['description'] = f"Meanings for values in '{target.name}'" - super().__init__(**kwargs) + description = f"Meanings for values in '{target.name}'" + super().__init__(name=name, description=description, id=id, columns=columns, colnames=colnames) self.target = target - @docval({'name': 'value', 'type': None, 'doc': 'the value in the linked VectorData object'}, - {'name': 'meaning', 'type': str, 'doc': 'the meaning of the value'}, - {'name': 'id', 'type': int, 'doc': 'the ID for the row', 'default': None}, - {'name': 'enforce_unique_id', 'type': bool, 'doc': 'enforce that the id in the table must be unique', - 'default': False}, - allow_extra=True) - def add_row(self, **kwargs): - """Add a row to the table mapping a value to its meaning.""" - super().add_row(**kwargs) + @validated + def add_row(self, value: Any, meaning: str, id: Int | None = None, enforce_unique_id: Bool = False, **kwargs): + """Add a row to the table mapping a value to its meaning. + + Args: + value: the value in the linked VectorData object + meaning: the meaning of the value + id: the ID for the row + enforce_unique_id: enforce that the id in the table must be unique + """ + super().add_row(value=value, meaning=meaning, id=id, enforce_unique_id=enforce_unique_id, **kwargs) diff --git a/src/hdmf/container.py b/src/hdmf/container.py index 9a414e37b..16e13208d 100644 --- a/src/hdmf/container.py +++ b/src/hdmf/container.py @@ -11,11 +11,12 @@ import pandas as pd from .data_utils import DataIO, append_data, extend_data, AbstractDataChunkIterator -from .utils import (docval, get_docval, getargs, ExtenderMeta, get_data_shape, popargs, LabelledDict, +from .utils import (get_docval, ExtenderMeta, get_data_shape, LabelledDict, get_basic_array_info, generate_array_html_repr, _is_collection, _get_length, _unwrap_scalar, coerce_pandas_data) from .term_set import TermSet, TermSetWrapper +from .typing import AnyData, ArrayData, Bool, ScalarData, signature_function, validated def _set_exp(cls): @@ -353,9 +354,13 @@ def __new__(cls, *args, **kwargs): inst.parent = kwargs.pop('parent', None) return inst - @docval({'name': 'name', 'type': str, 'doc': 'the name of this container'}) - def __init__(self, **kwargs): - name = getargs('name', kwargs) + @validated + def __init__(self, name: str): + """Initialize this container. + + Args: + name: the name of this container + """ if ('/' in name or ':' in name) and not self._in_construct_mode: raise ValueError(f"name '{name}' cannot contain a '/' or ':'") self.__name = name @@ -419,12 +424,13 @@ def name(self): ''' return self.__name - @docval({'name': 'data_type', 'type': str, 'doc': 'the data_type to search for', 'default': None}) - def get_ancestor(self, **kwargs): - """ - Traverse parent hierarchy and return first instance of the specified data_type + @validated + def get_ancestor(self, data_type: str | None = None): + """Traverse parent hierarchy and return first instance of the specified data_type + + Args: + data_type: the data_type to search for """ - data_type = getargs('data_type', kwargs) if data_type is None: return self.parent p = self.parent @@ -465,8 +471,10 @@ def all_objects(self): self.all_children() return self.__obj - @docval() - def get_ancestors(self, **kwargs): + @validated + def get_ancestors(self): + """get_ancestors + """ p = self.parent ret = [] while p is not None: @@ -493,25 +501,30 @@ def object_id(self): self.__object_id = str(uuid4()) return self.__object_id - @docval({'name': 'recurse', 'type': bool, - 'doc': "whether or not to change the object ID of this container's children", 'default': True}) - def generate_new_id(self, **kwargs): - """Changes the object ID of this Container and all of its children to a new UUID string.""" - recurse = getargs('recurse', kwargs) + @validated + def generate_new_id(self, recurse: Bool = True): + """Changes the object ID of this Container and all of its children to a new UUID string. + + Args: + recurse: whether or not to change the object ID of this container's children + """ self.__object_id = str(uuid4()) self.set_modified() if recurse: for c in self.children: - c.generate_new_id(**kwargs) + c.generate_new_id(recurse=recurse) @property def modified(self): return self.__modified - @docval({'name': 'modified', 'type': bool, - 'doc': 'whether or not this Container has been modified', 'default': True}) - def set_modified(self, **kwargs): - modified = getargs('modified', kwargs) + @validated + def set_modified(self, modified: Bool = True): + """set_modified + + Args: + modified: whether or not this Container has been modified + """ self.__modified = modified if modified and isinstance(self.parent, Container): self.parent.set_modified() @@ -986,11 +999,15 @@ class Data(AbstractContainer): """ A class for representing dataset containers """ - @docval({'name': 'name', 'type': str, 'doc': 'the name of this container'}, - {'name': 'data', 'type': ('scalar_data', 'array_data', 'data'), 'doc': 'the source of the data'}) - def __init__(self, **kwargs): - data = popargs('data', kwargs) - super().__init__(**kwargs) + @validated + def __init__(self, name: str, data: ScalarData | ArrayData | AnyData): + """Initialize this container. + + Args: + name: the name of this container + data: the source of the data + """ + super().__init__(name=name) data = coerce_pandas_data(data) self._validate_new_data(data) @@ -1041,15 +1058,16 @@ def set_data_io( data = data_chunk_iterator_class(data=data, **data_chunk_iterator_kwargs) self.__data = data_io_class(data=data, **data_io_kwargs) - @docval({'name': 'func', 'type': types.FunctionType, 'doc': 'a function to transform *data*'}) - def transform(self, **kwargs): - """ - Transform data from the current underlying state. + @validated + def transform(self, func: types.FunctionType): + """Transform data from the current underlying state. This function can be used to permanently load data from disk, or convert to a different representation, such as a torch.Tensor + + Args: + func: a function to transform *data* """ - func = getargs('func', kwargs) self.__data = func(self.__data) return self @@ -1167,12 +1185,8 @@ def tostr(x): def __make_get(cls, func_name, attr_name, container_type): doc = "Get %s from this %s" % (cls.__add_article(container_type), cls.__name__) - @docval({'name': 'name', 'type': str, 'doc': 'the name of the %s' % cls.__join(container_type), - 'default': None}, - rtype=container_type, returns='the %s with the given name' % cls.__join(container_type), - func_name=func_name, doc=doc) - def _func(self, **kwargs): - name = getargs('name', kwargs) + def _body(self, kwargs): + name = kwargs['name'] d = getattr(self, attr_name) ret = None if name is None: @@ -1193,19 +1207,19 @@ def _func(self, **kwargs): raise KeyError(msg) return ret - return _func + return signature_function( + func_name, + [{'name': 'name', 'type': str, 'doc': 'the name of the %s' % cls.__join(container_type), + 'default': None}], + _body, doc=doc, returns='the %s with the given name' % cls.__join(container_type)) @classmethod def __make_getitem(cls, attr_name, container_type): doc = "Get %s from this %s" % (cls.__add_article(container_type), cls.__name__) - @docval({'name': 'name', 'type': str, 'doc': 'the name of the %s' % cls.__join(container_type), - 'default': None}, - rtype=container_type, returns='the %s with the given name' % cls.__join(container_type), - func_name='__getitem__', doc=doc) - def _func(self, **kwargs): + def _body(self, kwargs): # NOTE this is the same code as the getter but with different error messages - name = getargs('name', kwargs) + name = kwargs['name'] d = getattr(self, attr_name) ret = None if name is None: @@ -1226,17 +1240,18 @@ def _func(self, **kwargs): raise KeyError(msg) return ret - return _func + return signature_function( + '__getitem__', + [{'name': 'name', 'type': str, 'doc': 'the name of the %s' % cls.__join(container_type), + 'default': None}], + _body, doc=doc, returns='the %s with the given name' % cls.__join(container_type)) @classmethod def __make_add(cls, func_name, attr_name, container_type): doc = "Add one or multiple %s objects to this %s" % (cls.__join(container_type), cls.__name__) - @docval({'name': attr_name, 'type': (list, tuple, dict, container_type), - 'doc': 'one or multiple %s objects to add to this %s' % (cls.__join(container_type), cls.__name__)}, - func_name=func_name, doc=doc) - def _func(self, **kwargs): - container = getargs(attr_name, kwargs) + def _body(self, kwargs): + container = kwargs[attr_name] if isinstance(container, container_type): containers = [container] elif isinstance(container, dict): @@ -1259,20 +1274,25 @@ def _func(self, **kwargs): d[tmp.name] = tmp return container - return _func + return signature_function( + func_name, + [{'name': attr_name, 'type': (list, tuple, dict, container_type), + 'doc': 'one or multiple %s objects to add to this %s' + % (cls.__join(container_type), cls.__name__)}], + _body, doc=doc) @classmethod def __make_create(cls, func_name, add_name, container_type): doc = "Create %s object and add it to this %s" % (cls.__add_article(container_type), cls.__name__) - @docval(*get_docval(container_type.__init__), func_name=func_name, doc=doc, - returns="the %s object that was created" % cls.__join(container_type), rtype=container_type) - def _func(self, **kwargs): + def _body(self, kwargs): ret = container_type(**kwargs) getattr(self, add_name)(ret) return ret - return _func + return signature_function( + func_name, list(get_docval(container_type.__init__)), _body, doc=doc, + returns="the %s object that was created" % cls.__join(container_type)) @classmethod def __make_constructor(cls, clsconf): @@ -1285,17 +1305,17 @@ def __make_constructor(cls, clsconf): args.append({'name': 'name', 'type': str, 'doc': 'the name of this container', 'default': cls.__name__}) - @docval(*args, func_name='__init__') - def _func(self, **kwargs): + def _body(self, kwargs): super().__init__(name=kwargs['name']) for conf in clsconf: attr_name = conf['attr'] add_name = conf['add'] - container = popargs(attr_name, kwargs) + container = kwargs[attr_name] add = getattr(self, add_name) add(container) - return _func + return signature_function('__init__', args, _body, + doc='Initialize the %s.' % cls.__name__) @classmethod def __make_getter(cls, attr): @@ -1319,9 +1339,13 @@ def _remove_child(child): def __make_setter(cls, add_name): """Make a setter function for creating a :py:func:`property`""" - @docval({'name': 'val', 'type': (list, tuple, dict), 'doc': 'the sub items to add', 'default': None}) - def _func(self, **kwargs): - val = getargs('val', kwargs) + @validated + def _func(self, val: list | tuple | dict | None = None): + """_func + + Args: + val: the sub items to add + """ if val is None: return getattr(self, add_name)(val) @@ -1510,10 +1534,10 @@ def __build_row_class(cls, name, bases, classdict): func_args.append({'name': 'idx', 'type': int, 'default': None, 'help': 'the index for this row'}) - @docval(*func_args) - def __init__(self, **kwargs): + def _row_init_body(self, kwargs): super(cls, self).__init__() - table, idx = popargs('table', 'idx', kwargs) + table = kwargs.pop('table') + idx = kwargs.pop('idx') self.__keys = list() self.__idx = None self.__table = None @@ -1523,7 +1547,8 @@ def __init__(self, **kwargs): self.idx = idx self.table = table - setattr(cls, '__init__', __init__) + setattr(cls, '__init__', signature_function( + '__init__', func_args, _row_init_body, doc='Initialize the row.')) def todict(self): return {k: getattr(self, k) for k in self.__keys} @@ -1606,41 +1631,52 @@ def __build_table_class(cls, name, bases, classdict): if defname is not None: name['default'] = defname # override the name with the default name if present - @docval(name, - {'name': 'data', 'type': ('array_data', 'data'), 'doc': 'the data in this table', - 'default': list()}) - def __init__(self, **kwargs): - name, data = getargs('name', 'data', kwargs) + def _table_init_body(self, kwargs): colnames = [i['name'] for i in columns] - super(cls, self).__init__(colnames, name, data) + super(cls, self).__init__(colnames, kwargs['name'], kwargs['data']) - setattr(cls, '__init__', __init__) + setattr(cls, '__init__', signature_function( + '__init__', + [name, {'name': 'data', 'type': ('array_data', 'data'), + 'doc': 'the data in this table', 'default': list()}], + _table_init_body, doc='Initialize the table.')) if cls.add_row == bases[-1].add_row: # check if add_row is overridden - @docval(*columns) - def add_row(self, **kwargs): + def _add_row_body(self, kwargs): return super(cls, self).add_row(kwargs) - setattr(cls, 'add_row', add_row) + setattr(cls, 'add_row', signature_function( + 'add_row', columns, _add_row_body, doc='Add a row to the table.')) - @docval({'name': 'columns', 'type': (list, tuple), 'doc': 'a list of the columns in this table'}, - {'name': 'name', 'type': str, 'doc': 'the name of this container'}, - {'name': 'data', 'type': ('array_data', 'data'), 'doc': 'the source of the data', 'default': list()}) - def __init__(self, **kwargs): - self.__columns = tuple(popargs('columns', kwargs)) - self.__col_index = {name: idx for idx, name in enumerate(self.__columns)} + @validated + def __init__(self, columns: list | tuple, name: str, data: ArrayData | AnyData | None = None): + """Initialize this container. + + Args: + columns: a list of the columns in this table + name: the name of this container + data: the source of the data + """ + if data is None: + data = list() + self.__columns = tuple(columns) + self.__col_index = {col_name: idx for idx, col_name in enumerate(self.__columns)} if getattr(self, '__rowclass__') is not None: self.row = RowGetter(self) - super().__init__(**kwargs) + super().__init__(name=name, data=data) @property def columns(self): return self.__columns - @docval({'name': 'values', 'type': dict, 'doc': 'the values for each column'}) - def add_row(self, **kwargs): - values = getargs('values', kwargs) + @validated + def add_row(self, values: dict): + """add_row + + Args: + values: the values for each column + """ if not isinstance(self.data, list): msg = 'Cannot append row to %s' % type(self.data) raise ValueError(msg) @@ -1700,22 +1736,17 @@ def to_dataframe(self): return pd.DataFrame(data) @classmethod - @docval( - {'name': 'df', 'type': pd.DataFrame, 'doc': 'input data'}, - {'name': 'name', 'type': str, 'doc': 'the name of this container', 'default': None}, - { - 'name': 'extra_ok', - 'type': bool, - 'doc': 'accept (and ignore) unexpected columns on the input dataframe', - 'default': False - }, - ) - def from_dataframe(cls, **kwargs): - '''Construct an instance of Table (or a subclass) from a pandas DataFrame. The columns of the dataframe + @validated + def from_dataframe(cls, df: pd.DataFrame, name: str | None = None, extra_ok: Bool = False): + """Construct an instance of Table (or a subclass) from a pandas DataFrame. The columns of the dataframe should match the columns defined on the Table subclass. - ''' - df, name, extra_ok = getargs('df', 'name', 'extra_ok', kwargs) + Args: + df: input data + name: the name of this container + extra_ok: accept (and ignore) unexpected columns on the input dataframe + """ + cls_cols = list([col['name'] for col in getattr(cls, '__columns__')]) df_cols = list(df.columns) diff --git a/src/hdmf/py.typed b/src/hdmf/py.typed new file mode 100644 index 000000000..e69de29bb diff --git a/src/hdmf/spec/catalog.py b/src/hdmf/spec/catalog.py index c40d2130c..7883ced7a 100644 --- a/src/hdmf/spec/catalog.py +++ b/src/hdmf/spec/catalog.py @@ -3,7 +3,7 @@ from collections import OrderedDict from .spec import BaseStorageSpec, GroupSpec -from ..utils import docval, getargs +from ..typing import Bool, validated class SpecCatalog: @@ -27,14 +27,14 @@ def __init__(self): self.__hierarchy = dict() self.__spec_source_files = dict() - @docval({'name': 'spec', 'type': BaseStorageSpec, 'doc': 'a Spec object'}, - {'name': 'source_file', 'type': str, - 'doc': 'path to the source file from which the spec was loaded', 'default': None}) - def register_spec(self, **kwargs): - ''' - Associate a specified object type with a specification - ''' - spec, source_file = getargs('spec', 'source_file', kwargs) + @validated + def register_spec(self, spec: BaseStorageSpec, source_file: str | None = None): + """Associate a specified object type with a specification + + Args: + spec: a Spec object + source_file: path to the source file from which the spec was loaded + """ ndt = spec.data_type_inc ndt_def = spec.data_type_def if ndt_def is None: @@ -55,46 +55,52 @@ def register_spec(self, **kwargs): self.__specs[type_name] = spec self.__spec_source_files[type_name] = source_file - @docval({'name': 'data_type', 'type': str, 'doc': 'the data_type to get the Spec for'}, - returns="the specification for writing the given object type to HDF5 ", rtype=BaseStorageSpec) - def get_spec(self, **kwargs): - ''' - Get the Spec object for the given type - ''' - data_type = getargs('data_type', kwargs) + @validated + def get_spec(self, data_type: str) -> BaseStorageSpec: + """Get the Spec object for the given type + + Args: + data_type: the data_type to get the Spec for + + Returns: + the specification for writing the given object type to HDF5 + """ return self.__specs.get(data_type, None) - @docval(rtype=tuple) - def get_registered_types(self, **kwargs): - ''' - Return all registered specifications - ''' - # kwargs is not used here but is used by docval + def get_registered_types(self) -> tuple: + """Return all registered specifications. + + Returns: + all registered specifications + """ return tuple(self.__specs.keys()) - @docval({'name': 'data_type', 'type': str, 'doc': 'the data_type of the spec to get the source file for'}, - returns="the path to source specification file from which the spec was originally loaded or None ", - rtype='str') - def get_spec_source_file(self, **kwargs): - ''' - Return the path to the source file from which the spec for the given + @validated + def get_spec_source_file(self, data_type: str) -> 'str': + """Return the path to the source file from which the spec for the given type was loaded from. None is returned if no file path is available for the spec. Note: The spec in the file may not be identical to the object in case the spec is modified after load. - ''' - data_type = getargs('data_type', kwargs) + + Args: + data_type: the data_type of the spec to get the source file for + + Returns: + the path to source specification file from which the spec was originally loaded or None + """ return self.__spec_source_files.get(data_type, None) - @docval({'name': 'spec', 'type': BaseStorageSpec, 'doc': 'the Spec object to register'}, - {'name': 'source_file', - 'type': str, - 'doc': 'path to the source file from which the spec was loaded', 'default': None}, - rtype=tuple, returns='the types that were registered with this spec') - def auto_register(self, **kwargs): - ''' - Register this specification and all sub-specification using data_type as object type name - ''' - spec, source_file = getargs('spec', 'source_file', kwargs) + @validated + def auto_register(self, spec: BaseStorageSpec, source_file: str | None = None) -> tuple: + """Register this specification and all sub-specification using data_type as object type name + + Args: + spec: the Spec object to register + source_file: path to the source file from which the spec was loaded + + Returns: + the types that were registered with this spec + """ ndt = spec.data_type_def ret = list() if ndt is not None: @@ -110,18 +116,19 @@ def auto_register(self, **kwargs): ret.extend(self.auto_register(group_spec, source_file)) return tuple(ret) - @docval({'name': 'data_type', 'type': (str, type), - 'doc': 'the data_type to get the hierarchy of'}, - returns="Tuple of strings with the names of the types the given data_type inherits from.", - rtype=tuple) - def get_hierarchy(self, **kwargs): - """ - For a given type get the type inheritance hierarchy for that type. + @validated + def get_hierarchy(self, data_type: str | type) -> tuple: + """For a given type get the type inheritance hierarchy for that type. E.g., if we have a type MyContainer that inherits from BaseContainer then the result will be a tuple with the strings ('MyContainer', 'BaseContainer') + + Args: + data_type: the data_type to get the hierarchy of + + Returns: + Tuple of strings with the names of the types the given data_type inherits from. """ - data_type = getargs('data_type', kwargs) if isinstance(data_type, type): data_type = data_type.__name__ ret = self.__hierarchy.get(data_type) @@ -142,12 +149,13 @@ def get_hierarchy(self, **kwargs): tmp_hier = tmp_hier[1:] return tuple(ret) - @docval(returns="Hierarchically nested OrderedDict with the hierarchy of all the types", - rtype=OrderedDict) - def get_full_hierarchy(self): - """ - Get the complete hierarchy of all types. The function attempts to sort types by name using + @validated + def get_full_hierarchy(self) -> OrderedDict: + """Get the complete hierarchy of all types. The function attempts to sort types by name using standard Python sorted. + + Returns: + Hierarchically nested OrderedDict with the hierarchy of all the types """ # Get the list of all types registered_types = self.get_registered_types() @@ -169,16 +177,9 @@ def get_type_hierarchy(data_type, spec_catalog): return type_hierarchy - @docval({'name': 'data_type', 'type': (str, type), - 'doc': 'the data_type to get the subtypes for'}, - {'name': 'recursive', 'type': bool, - 'doc': 'recursively get all subtypes. Set to False to only get the direct subtypes', - 'default': True}, - returns="Tuple of strings with the names of all types of the given data_type.", - rtype=tuple) - def get_subtypes(self, **kwargs): - """ - For a given data type recursively find all the subtypes that inherit from it. + @validated + def get_subtypes(self, data_type: str | type, recursive: Bool = True) -> tuple: + """For a given data type recursively find all the subtypes that inherit from it. E.g., assume we have the following inheritance hierarchy:: @@ -188,8 +189,14 @@ def get_subtypes(self, **kwargs): In this case, the subtypes of BaseContainer would be (AContainer, ADContainer, BContainer), the subtypes of AContainer would be (ADContainer), and the subtypes of BContainer would be empty (). + + Args: + data_type: the data_type to get the subtypes for + recursive: recursively get all subtypes. Set to False to only get the direct subtypes + + Returns: + Tuple of strings with the names of all types of the given data_type. """ - data_type, recursive = getargs('data_type', 'recursive', kwargs) curr_spec = self.get_spec(data_type) if isinstance(curr_spec, BaseStorageSpec): # Only BaseStorageSpec have data_type_inc/def keys subtypes = [] diff --git a/src/hdmf/spec/namespace.py b/src/hdmf/spec/namespace.py index 923d9e6b5..7a5a3f859 100644 --- a/src/hdmf/spec/namespace.py +++ b/src/hdmf/spec/namespace.py @@ -9,7 +9,8 @@ from .catalog import SpecCatalog from .spec import DatasetSpec, GroupSpec -from ..utils import docval, getargs, popargs, get_docval, is_newer_version +from ..typing import Bool, validated +from ..utils import get_docval, is_newer_version _namespace_args = [ {'name': 'doc', 'type': str, 'doc': 'a description about what this namespace represents'}, @@ -36,10 +37,30 @@ class SpecNamespace(dict): UNVERSIONED = None # value representing missing version - @docval(*_namespace_args) - def __init__(self, **kwargs): - doc, full_name, name, version, date, author, contact, schema, catalog = \ - popargs('doc', 'full_name', 'name', 'version', 'date', 'author', 'contact', 'schema', 'catalog', kwargs) + @validated + def __init__(self, + doc: str, + name: str, + schema: list, + full_name: str | None = None, + version: str | tuple | list | None = None, + date: datetime | str | None = None, + author: str | list | None = None, + contact: str | list | None = None, + catalog: SpecCatalog | None = None): + """Initialize this spec. + + Args: + doc: a description about what this namespace represents + name: the name of this namespace + schema: location of schema specification files or other Namespaces + full_name: extended full name of this namespace + version: Version number of the namespace + date: Date last modified or released. Formatting is %Y-%m-%d %H:%M:%S, e.g, 2017-04-25 17:14:13 + author: Author or list of authors. + contact: List of emails. Ordering should be the same as for author + catalog: The SpecCatalog object for this SpecNamespace + """ super().__init__() self['doc'] = doc self['schema'] = schema @@ -124,13 +145,17 @@ def get_source_files(self): """ return [item['source'] for item in self.schema if 'source' in item] - @docval({'name': 'sourcefile', 'type': str, 'doc': 'Name of the source file'}, - returns='Dict with the source file documentation', rtype=dict) - def get_source_description(self, sourcefile): - """ - Get the description of a source file as described in the namespace. The result is a + @validated + def get_source_description(self, sourcefile: str) -> dict: + """Get the description of a source file as described in the namespace. The result is a dict which contains the 'source' and optionally 'title', 'doc' and 'data_types' imported from the source file + + Args: + sourcefile: Name of the source file + + Returns: + Dict with the source file documentation """ for item in self.schema: if item.get('source', None) == sourcefile: @@ -141,25 +166,37 @@ def catalog(self): """The SpecCatalog containing all the Specs""" return self.__catalog - @docval({'name': 'data_type', 'type': (str, type), 'doc': 'the data_type to get the spec for'}) - def get_spec(self, **kwargs): - """Get the Spec object for the given data type""" - data_type = getargs('data_type', kwargs) + @validated + def get_spec(self, data_type: str | type): + """Get the Spec object for the given data type + + Args: + data_type: the data_type to get the spec for + """ spec = self.__catalog.get_spec(data_type) if spec is None: raise ValueError("No specification for '%s' in namespace '%s'" % (data_type, self.name)) return spec - @docval(returns="the a tuple of the available data types", rtype=tuple) - def get_registered_types(self, **kwargs): - """Get the available types in this namespace""" + @validated + def get_registered_types(self) -> tuple: + """Get the available types in this namespace + + Returns: + the a tuple of the available data types + """ return self.__catalog.get_registered_types() - @docval({'name': 'data_type', 'type': (str, type), 'doc': 'the data_type to get the hierarchy of'}, - returns="a tuple with the type hierarchy", rtype=tuple) - def get_hierarchy(self, **kwargs): - ''' Get the extension hierarchy for the given data_type in this namespace''' - data_type = getargs('data_type', kwargs) + @validated + def get_hierarchy(self, data_type: str | type) -> tuple: + """Get the extension hierarchy for the given data_type in this namespace + + Args: + data_type: the data_type to get the hierarchy of + + Returns: + a tuple with the type hierarchy + """ return self.__catalog.get_hierarchy(data_type) @classmethod @@ -174,9 +211,14 @@ def build_namespace(cls, **spec_dict): class SpecReader(metaclass=ABCMeta): - @docval({'name': 'source', 'type': str, 'doc': 'the source from which this reader reads from'}) - def __init__(self, **kwargs): - self.__source = getargs('source', kwargs) + @validated + def __init__(self, source: str): + """Initialize this spec. + + Args: + source: the source from which this reader reads from + """ + self.__source = source @property def source(self): @@ -193,9 +235,14 @@ def read_namespace(self): class YAMLSpecReader(SpecReader): - @docval({'name': 'indir', 'type': str, 'doc': 'the path spec files are relative to', 'default': '.'}) - def __init__(self, **kwargs): - super().__init__(source=kwargs['indir']) + @validated + def __init__(self, indir: str = '.'): + """Initialize this spec. + + Args: + indir: the path spec files are relative to + """ + super().__init__(source=indir) def read_namespace(self, namespace_path): namespaces = None @@ -224,22 +271,27 @@ def __get_spec_path(self, spec_path): class NamespaceCatalog: - @docval({'name': 'group_spec_cls', 'type': type, - 'doc': 'the class to use for group specifications', 'default': GroupSpec}, - {'name': 'dataset_spec_cls', 'type': type, - 'doc': 'the class to use for dataset specifications', 'default': DatasetSpec}, - {'name': 'spec_namespace_cls', 'type': type, - 'doc': 'the class to use for specification namespaces', 'default': SpecNamespace}, - {'name': 'core_namespaces', 'type': list, - 'doc': 'the names of the core namespaces', 'default': list()}) - def __init__(self, **kwargs): - """Create a catalog for storing multiple Namespaces""" + @validated + def __init__(self, + group_spec_cls: type = GroupSpec, + dataset_spec_cls: type = DatasetSpec, + spec_namespace_cls: type = SpecNamespace, + core_namespaces: list | None = None): + """Create a catalog for storing multiple Namespaces + + Args: + group_spec_cls: the class to use for group specifications + dataset_spec_cls: the class to use for dataset specifications + spec_namespace_cls: the class to use for specification namespaces + core_namespaces: the names of the core namespaces + """ + if core_namespaces is None: + core_namespaces = list() self.__namespaces = OrderedDict() - self.__dataset_spec_cls = getargs('dataset_spec_cls', kwargs) - self.__group_spec_cls = getargs('group_spec_cls', kwargs) - self.__spec_namespace_cls = getargs('spec_namespace_cls', kwargs) + self.__dataset_spec_cls = dataset_spec_cls + self.__group_spec_cls = group_spec_cls + self.__spec_namespace_cls = spec_namespace_cls - core_namespaces = getargs('core_namespaces', kwargs) self.__core_namespaces = core_namespaces # keep track of all spec objects ever loaded, so we don't have @@ -273,9 +325,13 @@ def merge(self, ns_catalog): self.__core_namespaces.extend(ns_catalog.__core_namespaces) @property - @docval(returns='a tuple of the available namespaces', rtype=tuple) - def namespaces(self): - """The namespaces in this NamespaceCatalog""" + @validated + def namespaces(self) -> tuple: + """The namespaces in this NamespaceCatalog + + Returns: + a tuple of the available namespaces + """ return tuple(self.__namespaces.keys()) @property @@ -306,11 +362,14 @@ def get_source_types(self, ns_name): """ return self.__source_types.get(ns_name, ()) - @docval({'name': 'name', 'type': str, 'doc': 'the name of this namespace'}, - {'name': 'namespace', 'type': SpecNamespace, 'doc': 'the SpecNamespace object'}) - def add_namespace(self, **kwargs): - """Add a namespace to this catalog""" - name, namespace = getargs('name', 'namespace', kwargs) + @validated + def add_namespace(self, name: str, namespace: SpecNamespace): + """Add a namespace to this catalog + + Args: + name: the name of this namespace + namespace: the SpecNamespace object + """ if name in self.__namespaces: raise KeyError("namespace '%s' already exists" % name) self.__namespaces[name] = namespace @@ -346,76 +405,90 @@ def add_namespace(self, **kwargs): elif isinstance(spec, DatasetSpec): self.__unresolved_spec_dicts[source]['datasets'].append(spec) - @docval({'name': 'name', 'type': str, 'doc': 'the name of this namespace'}, - returns="the SpecNamespace with the given name", rtype=SpecNamespace) - def get_namespace(self, **kwargs): - """Get the a SpecNamespace""" - name = getargs('name', kwargs) + @validated + def get_namespace(self, name: str) -> SpecNamespace: + """Get the a SpecNamespace + + Args: + name: the name of this namespace + + Returns: + the SpecNamespace with the given name + """ ret = self.__namespaces.get(name) if ret is None: raise KeyError("'%s' not a namespace" % name) return ret - @docval({'name': 'namespace', 'type': str, 'doc': 'the name of the namespace'}, - {'name': 'data_type', 'type': (str, type), 'doc': 'the data_type to get the spec for'}, - returns="the specification for writing the given object type to HDF5 ", rtype='Spec') - def get_spec(self, **kwargs): - ''' - Get the Spec object for the given type from the given Namespace - ''' - namespace, data_type = getargs('namespace', 'data_type', kwargs) + @validated + def get_spec(self, namespace: str, data_type: str | type) -> DatasetSpec | GroupSpec: + """Get the Spec object for the given type from the given Namespace + + Args: + namespace: the name of the namespace + data_type: the data_type to get the spec for + + Returns: + the specification for writing the given object type to HDF5 + """ if namespace not in self.__namespaces: raise KeyError("'%s' not a namespace" % namespace) return self.__namespaces[namespace].get_spec(data_type) - @docval({'name': 'namespace', 'type': str, 'doc': 'the name of the namespace'}, - {'name': 'data_type', 'type': (str, type), 'doc': 'the data_type to get the spec for'}, - returns="a tuple with the type hierarchy", rtype=tuple) - def get_hierarchy(self, **kwargs): - ''' - Get the type hierarchy for a given data_type in a given namespace - ''' - namespace, data_type = getargs('namespace', 'data_type', kwargs) + @validated + def get_hierarchy(self, namespace: str, data_type: str | type) -> tuple: + """Get the type hierarchy for a given data_type in a given namespace + + Args: + namespace: the name of the namespace + data_type: the data_type to get the spec for + + Returns: + a tuple with the type hierarchy + """ spec_ns = self.__namespaces.get(namespace) if spec_ns is None: raise KeyError("'%s' not a namespace" % namespace) return spec_ns.get_hierarchy(data_type) - @docval({'name': 'namespace', 'type': str, 'doc': 'the name of the namespace containing the data_type'}, - {'name': 'data_type', 'type': str, 'doc': 'the data_type to check'}, - {'name': 'parent_data_type', 'type': str, 'doc': 'the potential parent data_type'}, - returns="True if *data_type* is a sub `data_type` of *parent_data_type*, False otherwise", rtype=bool) - def is_sub_data_type(self, **kwargs): - ''' - Return whether or not *data_type* is a sub `data_type` of *parent_data_type* - ''' - ns, dt, parent_dt = getargs('namespace', 'data_type', 'parent_data_type', kwargs) + @validated + def is_sub_data_type(self, namespace: str, data_type: str, parent_data_type: str) -> bool: + """Return whether or not *data_type* is a sub `data_type` of *parent_data_type* + + Args: + namespace: the name of the namespace containing the data_type + data_type: the data_type to check + parent_data_type: the potential parent data_type + + Returns: + True if *data_type* is a sub `data_type` of *parent_data_type*, False otherwise + """ + ns, dt, parent_dt = namespace, data_type, parent_data_type hier = self.get_hierarchy(ns, dt) return parent_dt in hier - @docval(rtype=tuple) - def get_sources(self, **kwargs): - ''' - Get all the source specification files that were loaded in this catalog - ''' + @validated + def get_sources(self) -> tuple: + """Get all the source specification files that were loaded in this catalog + """ return tuple(self.__loaded_specs.keys()) - @docval({'name': 'namespace', 'type': str, 'doc': 'the name of the namespace'}, - rtype=tuple) - def get_namespace_sources(self, **kwargs): - ''' - Get all the source specifications that were loaded for a given namespace - ''' - namespace = getargs('namespace', kwargs) + @validated + def get_namespace_sources(self, namespace: str) -> tuple: + """Get all the source specifications that were loaded for a given namespace + + Args: + namespace: the name of the namespace + """ return tuple(self.__included_sources[namespace]) - @docval({'name': 'source', 'type': str, 'doc': 'the name of the source'}, - rtype=tuple) - def get_types(self, **kwargs): - ''' - Get the types that were loaded from a given source - ''' - source = getargs('source', kwargs) + @validated + def get_types(self, source: str) -> tuple: + """Get the types that were loaded from a given source + + Args: + source: the name of the source + """ ret = self.__loaded_specs.get(source) if ret is not None: ret = tuple(ret) @@ -423,16 +496,16 @@ def get_types(self, **kwargs): ret = tuple() return ret - @docval({'name': 'source', 'type': str, 'doc': 'the name of the source file'}, - rtype=dict) - def get_spec_source_dict(self, **kwargs): - ''' - Get the unresolved specs for a given source file. + @validated + def get_spec_source_dict(self, source: str) -> dict: + """Get the unresolved specs for a given source file. Returns a dict with 'datasets' and 'groups' keys containing lists of unresolved Spec objects (before resolution of inherited fields). Returns None if the source is not found. - ''' - source = getargs('source', kwargs) + + Args: + source: the name of the source file + """ return self.__unresolved_spec_dicts.get(source, None) def __load_spec_file(self, reader, spec_source, catalog, types_to_load): @@ -701,20 +774,22 @@ def __register_dependent_types_helper(child_spec): __register_dependent_types_helper(child_spec) - @docval({'name': 'namespace_path', 'type': str, 'doc': 'the path to the file containing the namespaces(s) to load'}, - {'name': 'resolve', - 'type': bool, - 'doc': ('whether or not to include objects from included/parent spec objects. In practice, this is ' - 'False when generating documentation where it is useful to show the unresolved specs'), - 'default': True}, - {'name': 'reader', - 'type': (SpecReader, dict), - 'doc': 'the SpecReader or dict of SpecReader classes to use for reading specifications', - 'default': None}, - returns='a dictionary describing the dependencies of loaded namespaces', rtype=dict) - def load_namespaces(self, **kwargs): - """Load the namespaces in the given file""" - namespace_path, resolve, reader = getargs('namespace_path', 'resolve', 'reader', kwargs) + @validated + def load_namespaces(self, + namespace_path: str, + resolve: Bool = True, + reader: SpecReader | dict | None = None) -> dict: + """Load the namespaces in the given file + + Args: + namespace_path: the path to the file containing the namespaces(s) to load + resolve: whether or not to include objects from included/parent spec objects. In practice, this is False + when generating documentation where it is useful to show the unresolved specs + reader: the SpecReader or dict of SpecReader classes to use for reading specifications + + Returns: + a dictionary describing the dependencies of loaded namespaces + """ # determine which readers and order of readers to use for loading specs if reader is None: diff --git a/src/hdmf/spec/spec.py b/src/hdmf/spec/spec.py index 71817888f..0e3395594 100644 --- a/src/hdmf/spec/spec.py +++ b/src/hdmf/spec/spec.py @@ -5,7 +5,10 @@ from typing import TYPE_CHECKING from warnings import warn -from ..utils import docval, getargs, popargs, get_docval +from typing import Any + +from ..typing import Bool, Int, TypeName, validated +from ..utils import get_docval if TYPE_CHECKING: from .namespace import SpecNamespace # noqa: F401 @@ -255,12 +258,20 @@ class Spec(ConstructableDict): ''' A base specification class ''' - @docval({'name': 'doc', 'type': str, 'doc': 'a description about what this specification represents'}, - {'name': 'name', 'type': str, 'doc': 'The name of this attribute', 'default': None}, - {'name': 'required', 'type': bool, 'doc': 'whether or not this attribute is required', 'default': True}, - {'name': 'parent', 'type': 'hdmf.spec.spec.Spec', 'doc': 'the parent of this spec', 'default': None}) - def __init__(self, **kwargs): - name, doc, required, parent = getargs('name', 'doc', 'required', 'parent', kwargs) + @validated + def __init__(self, + doc: str, + name: str | None = None, + required: Bool = True, + parent: TypeName['Spec'] | None = None): # noqa: F821 + """Initialize this spec. + + Args: + doc: a description about what this specification represents + name: The name of this attribute + required: whether or not this attribute is required + parent: the parent of this spec + """ super().__init__() self['doc'] = doc if name is not None: @@ -321,19 +332,18 @@ def path(self): _target_type_key = 'target_type' -_ref_args = [ - {'name': _target_type_key, 'type': str, 'doc': 'the target type GroupSpec or DatasetSpec'}, - {'name': 'reftype', 'type': str, - 'doc': 'the type of reference this is. only "object" is supported currently.'}, -] - class RefSpec(ConstructableDict): __allowable_types = ('object', ) - @docval(*_ref_args) - def __init__(self, **kwargs): - target_type, reftype = getargs(_target_type_key, 'reftype', kwargs) + @validated + def __init__(self, target_type: str, reftype: str): + """Initialize the RefSpec. + + Args: + target_type: the target type GroupSpec or DatasetSpec + reftype: the type of reference this is. only "object" is supported currently. + """ self[_target_type_key] = target_type if reftype not in self.__allowable_types: msg = "reftype must be one of the following: %s" % ", ".join(self.__allowable_types) @@ -351,28 +361,34 @@ def reftype(self): return self['reftype'] -_attr_args = [ - {'name': 'name', 'type': str, 'doc': 'The name of this attribute'}, - {'name': 'doc', 'type': str, 'doc': 'a description about what this specification represents'}, - {'name': 'dtype', 'type': (str, RefSpec), 'doc': 'The data type of this attribute'}, - {'name': 'shape', 'type': (list, tuple), 'doc': 'the shape of this dataset', 'default': None}, - {'name': 'dims', 'type': (list, tuple), 'doc': 'the dimensions of this dataset', 'default': None}, - {'name': 'required', 'type': bool, - 'doc': 'whether or not this attribute is required. ignored when "value" is specified', 'default': True}, - {'name': 'parent', 'type': 'hdmf.spec.spec.BaseStorageSpec', 'doc': 'the parent of this spec', 'default': None}, - {'name': 'value', 'type': None, 'doc': 'a constant value for this attribute', 'default': None}, - {'name': 'default_value', 'type': None, 'doc': 'a default value for this attribute', 'default': None} -] - - class AttributeSpec(Spec): ''' Specification for attributes ''' - @docval(*_attr_args) - def __init__(self, **kwargs): - name, dtype, doc, dims, shape, required, parent, value, default_value = getargs( - 'name', 'dtype', 'doc', 'dims', 'shape', 'required', 'parent', 'value', 'default_value', kwargs) + @validated + def __init__(self, + name: str, + doc: str, + dtype: str | RefSpec, + shape: list | tuple | None = None, + dims: list | tuple | None = None, + required: Bool = True, + parent: TypeName['BaseStorageSpec'] | None = None, # noqa: F821 + value: Any = None, + default_value: Any = None): + """Initialize this spec. + + Args: + name: The name of this attribute + doc: a description about what this specification represents + dtype: The data type of this attribute + shape: the shape of this dataset + dims: the dimensions of this dataset + required: whether or not this attribute is required. ignored when "value" is specified + parent: the parent of this spec + value: a constant value for this attribute + default_value: a default value for this attribute + """ super().__init__(doc, name=name, required=required, parent=parent) self['dtype'] = DtypeHelper.check_dtype(dtype) if value is not None: @@ -436,21 +452,6 @@ def build_const_args(cls, spec_dict): return ret -_attrbl_args = [ - {'name': 'doc', 'type': str, 'doc': 'a description about what this specification represents'}, - {'name': 'name', 'type': str, - 'doc': 'the name of this base storage container, allowed only if quantity is not \'%s\' or \'%s\'' - % (ONE_OR_MANY, ZERO_OR_MANY), 'default': None}, - {'name': 'default_name', 'type': str, - 'doc': 'The default name of this base storage container, used only if name is None', 'default': None}, - {'name': 'attributes', 'type': list, 'doc': 'the attributes on this group', 'default': list()}, - {'name': 'linkable', 'type': bool, 'doc': 'whether or not this group can be linked', 'default': True}, - {'name': 'quantity', 'type': (str, int), 'doc': 'the required number of allowed instance', 'default': 1}, - {'name': 'data_type_def', 'type': str, 'doc': 'the data type this specification represents', 'default': None}, - {'name': 'data_type_inc', 'type': str, 'doc': 'the data type this specification extends', 'default': None}, -] - - class BaseStorageSpec(Spec): ''' A specification for any object that can hold attributes. ''' @@ -459,17 +460,36 @@ class BaseStorageSpec(Spec): __type_key = 'data_type' __id_key = 'object_id' - @docval(*_attrbl_args) - def __init__(self, **kwargs): - name, doc, quantity, attributes, linkable, data_type_def, data_type_inc = \ - getargs('name', 'doc', 'quantity', 'attributes', 'linkable', 'data_type_def', 'data_type_inc', kwargs) + @validated + def __init__(self, + doc: str, + name: str | None = None, + default_name: str | None = None, + attributes: list | None = None, + linkable: Bool = True, + quantity: str | Int = 1, + data_type_def: str | None = None, + data_type_inc: str | None = None): + """Initialize this spec. + + Args: + doc: a description about what this specification represents + name: the name of this base storage container, allowed only if quantity is not '*' or '+' + default_name: The default name of this base storage container, used only if name is None + attributes: the attributes on this group + linkable: whether or not this group can be linked + quantity: the required number of allowed instance + data_type_def: the data type this specification represents + data_type_inc: the data type this specification extends + """ + if attributes is None: + attributes = list() if name is not None and "/" in name: raise ValueError(f"Name '{name}' is invalid. Names of Groups and Datasets cannot contain '/'") if name is None and data_type_def is None and data_type_inc is None: raise ValueError("Cannot create Group or Dataset spec with no name " "without specifying '%s' and/or '%s'." % (self.def_key(), self.inc_key())) super().__init__(doc, name=name) - default_name = getargs('default_name', kwargs) if default_name: if "/" in default_name: raise ValueError( @@ -562,50 +582,54 @@ def resolve_inc_spec(self, inc_spec: 'BaseStorageSpec', namespace: 'SpecNamespac self.set_attribute(inc_spec_attribute) self.__inc_spec_resolved = True - @docval({'name': 'spec', 'type': Spec, 'doc': 'the specification to check'}) - def is_inherited_spec(self, **kwargs): - ''' - Return True if this spec was inherited from the parent type, False otherwise. + @validated + def is_inherited_spec(self, spec: Spec): + """Return True if this spec was inherited from the parent type, False otherwise. Returns False if the spec is not found. - ''' - spec = getargs('spec', kwargs) + + Args: + spec: the specification to check + """ if spec.parent is self and spec.name in self.__attributes: return self.is_inherited_attribute(spec.name) return False - @docval({'name': 'spec', 'type': Spec, 'doc': 'the specification to check'}) - def is_overridden_spec(self, **kwargs): - ''' - Return True if this spec overrides a specification from the parent type, False otherwise. + @validated + def is_overridden_spec(self, spec: Spec): + """Return True if this spec overrides a specification from the parent type, False otherwise. Returns False if the spec is not found. - ''' - spec = getargs('spec', kwargs) + + Args: + spec: the specification to check + """ if spec.parent is self and spec.name in self.__attributes: return self.is_overridden_attribute(spec.name) return False - @docval({'name': 'name', 'type': str, 'doc': 'the name of the attribute to check'}) - def is_inherited_attribute(self, **kwargs): - ''' - Return True if the attribute was inherited from the parent type, False otherwise. + @validated + def is_inherited_attribute(self, name: str): + """Return True if the attribute was inherited from the parent type, False otherwise. Raises a ValueError if the spec is not found. - ''' - name = getargs('name', kwargs) + + Args: + name: the name of the attribute to check + """ if name not in self.__attributes: raise ValueError("Attribute '%s' not found" % name) return name not in self.__new_attributes - @docval({'name': 'name', 'type': str, 'doc': 'the name of the attribute to check'}) - def is_overridden_attribute(self, **kwargs): - ''' - Return True if the given attribute overrides the specification from the parent, False otherwise. + @validated + def is_overridden_attribute(self, name: str): + """Return True if the given attribute overrides the specification from the parent, False otherwise. Raises a ValueError if the spec is not found. - ''' - name = getargs('name', kwargs) + + Args: + name: the name of the attribute to check + """ if name not in self.__attributes: raise ValueError("Attribute '%s' not found" % name) return name in self.__overridden_attributes @@ -693,20 +717,45 @@ def quantity(self): ''' The number of times the object being specified should be present ''' return self.get('quantity', DEF_QUANTITY) - @docval(*_attr_args) - def add_attribute(self, **kwargs): - ''' Add an attribute to this specification ''' + @validated + def add_attribute(self, + name: str, + doc: str, + dtype: str | RefSpec, + shape: list | tuple | None = None, + dims: list | tuple | None = None, + required: Bool = True, + parent: TypeName['BaseStorageSpec'] | None = None, # noqa: F821 + value: Any = None, + default_value: Any = None): + """Add an attribute to this specification + + Args: + name: The name of this attribute + doc: a description about what this specification represents + dtype: The data type of this attribute + shape: the shape of this dataset + dims: the dimensions of this dataset + required: whether or not this attribute is required. ignored when "value" is specified + parent: the parent of this spec + value: a constant value for this attribute + default_value: a default value for this attribute + """ warn("BaseStorageSpec.add_attribute is deprecated and will be removed in HDMF 6.0. " "Use BaseStorageSpec.set_attribute instead.", DeprecationWarning, stacklevel=2) - spec = AttributeSpec(**kwargs) + spec = AttributeSpec(name=name, doc=doc, dtype=dtype, shape=shape, dims=dims, + required=required, parent=parent, value=value, default_value=default_value) self.set_attribute(spec) return spec - @docval({'name': 'spec', 'type': AttributeSpec, 'doc': 'the specification for the attribute to add'}) - def set_attribute(self, **kwargs): - ''' Set an attribute on this specification ''' - spec = kwargs.get('spec') + @validated + def set_attribute(self, spec: AttributeSpec): + """Set an attribute on this specification + + Args: + spec: the specification for the attribute to add + """ attributes = self.setdefault('attributes', list()) if spec.parent is not None: spec = AttributeSpec.build_spec(spec) @@ -732,10 +781,13 @@ def set_attribute(self, **kwargs): self.__attributes[spec.name] = spec spec.parent = self - @docval({'name': 'name', 'type': str, 'doc': 'the name of the attribute to the Spec for'}) - def get_attribute(self, **kwargs): - ''' Get an attribute on this specification ''' - name = getargs('name', kwargs) + @validated + def get_attribute(self, name: str): + """Get an attribute on this specification + + Args: + name: the name of the attribute to the Spec for + """ return self.__attributes.get(name) @classmethod @@ -747,19 +799,18 @@ def build_const_args(cls, spec_dict): return ret -_dt_args = [ - {'name': 'name', 'type': str, 'doc': 'the name of this column'}, - {'name': 'doc', 'type': str, 'doc': 'a description about what this data type is'}, - {'name': 'dtype', 'type': (str, list, RefSpec), 'doc': 'the data type of this column'}, -] - - class DtypeSpec(ConstructableDict): '''A class for specifying a component of a compound type''' - @docval(*_dt_args) - def __init__(self, **kwargs): - doc, name, dtype = getargs('doc', 'name', 'dtype', kwargs) + @validated + def __init__(self, name: str, doc: str, dtype: str | list | RefSpec): + """Initialize this spec. + + Args: + name: the name of this column + doc: a description about what this data type is + dtype: the data type of this column + """ self['doc'] = doc self['name'] = name self.check_valid_dtype(dtype) @@ -796,9 +847,13 @@ def check_valid_dtype(dtype): return True @staticmethod - @docval({'name': 'spec', 'type': (str, dict), 'doc': 'the spec object to check'}, is_method=False) - def is_ref(**kwargs): - spec = getargs('spec', kwargs) + @validated + def is_ref(spec: str | dict): + """is_ref + + Args: + spec: the spec object to check + """ spec_is_ref = False if isinstance(spec, dict): if _target_type_key in spec: @@ -818,35 +873,44 @@ def build_const_args(cls, spec_dict): return ret -_dataset_args = [ - {'name': 'doc', 'type': str, 'doc': 'a description about what this specification represents'}, - {'name': 'dtype', 'type': (str, list, RefSpec), - 'doc': 'The data type of this attribute. Use a list of DtypeSpecs to specify a compound data type.', - 'default': None}, - {'name': 'name', 'type': str, 'doc': 'The name of this dataset', 'default': None}, - {'name': 'default_name', 'type': str, 'doc': 'The default name of this dataset', 'default': None}, - {'name': 'shape', 'type': (list, tuple), 'doc': 'the shape of this dataset', 'default': None}, - {'name': 'dims', 'type': (list, tuple), 'doc': 'the dimensions of this dataset', 'default': None}, - {'name': 'attributes', 'type': list, 'doc': 'the attributes on this group', 'default': list()}, - {'name': 'linkable', 'type': bool, 'doc': 'whether or not this group can be linked', 'default': True}, - {'name': 'quantity', 'type': (str, int), 'doc': 'the required number of allowed instance', 'default': 1}, - {'name': 'default_value', 'type': None, 'doc': 'a default value for this dataset', 'default': None}, - {'name': 'value', 'type': None, 'doc': 'a fixed value for this dataset', 'default': None}, - {'name': 'data_type_def', 'type': str, 'doc': 'the data type this specification represents', 'default': None}, - {'name': 'data_type_inc', 'type': str, 'doc': 'the data type this specification extends', 'default': None}, -] - - class DatasetSpec(BaseStorageSpec): ''' Specification for datasets To specify a table-like dataset i.e. a compound data type. ''' - @docval(*_dataset_args) - def __init__(self, **kwargs): - doc, shape, dims, dtype = popargs('doc', 'shape', 'dims', 'dtype', kwargs) - default_value, value = popargs('default_value', 'value', kwargs) + @validated + def __init__(self, + doc: str, + dtype: str | list | RefSpec | None = None, + name: str | None = None, + default_name: str | None = None, + shape: list | tuple | None = None, + dims: list | tuple | None = None, + attributes: list | None = None, + linkable: Bool = True, + quantity: str | Int = 1, + default_value: Any = None, + value: Any = None, + data_type_def: str | None = None, + data_type_inc: str | None = None): + """Initialize this spec. + + Args: + doc: a description about what this specification represents + dtype: The data type of this attribute. Use a list of DtypeSpecs to specify a compound data type. + name: The name of this dataset + default_name: The default name of this dataset + shape: the shape of this dataset + dims: the dimensions of this dataset + attributes: the attributes on this group + linkable: whether or not this group can be linked + quantity: the required number of allowed instance + default_value: a default value for this dataset + value: a fixed value for this dataset + data_type_def: the data type this specification represents + data_type_inc: the data type this specification extends + """ if shape is not None: if len(shape) == 0: raise ValueError("'shape' must not be empty") @@ -870,7 +934,9 @@ def __init__(self, **kwargs): else: DtypeHelper.check_dtype(dtype) self['dtype'] = dtype - super().__init__(doc, **kwargs) + super().__init__(doc, name=name, default_name=default_name, attributes=attributes, + linkable=linkable, quantity=quantity, data_type_def=data_type_def, + data_type_inc=data_type_inc) if default_value is not None: self['default_value'] = default_value if value is not None: @@ -947,20 +1013,23 @@ def build_const_args(cls, spec_dict): return ret -_link_args = [ - {'name': 'doc', 'type': str, 'doc': 'a description about what this link represents'}, - {'name': _target_type_key, 'type': (str, BaseStorageSpec), 'doc': 'the target type GroupSpec or DatasetSpec'}, - {'name': 'quantity', 'type': (str, int), 'doc': 'the required number of allowed instance', 'default': 1}, - {'name': 'name', 'type': str, 'doc': 'the name of this link', 'default': None} -] - - class LinkSpec(Spec): - @docval(*_link_args) - def __init__(self, **kwargs): - doc, target_type, name, quantity = popargs('doc', _target_type_key, 'name', 'quantity', kwargs) - super().__init__(doc, name, **kwargs) + @validated + def __init__(self, + doc: str, + target_type: str | BaseStorageSpec, + quantity: str | Int = 1, + name: str | None = None): + """Initialize the LinkSpec. + + Args: + doc: a description about what this link represents + target_type: the target type GroupSpec or DatasetSpec + quantity: the required number of allowed instance + name: the name of this link + """ + super().__init__(doc, name) if isinstance(target_type, BaseStorageSpec): if target_type.data_type_def is None: msg = ("'%s' must be a string or a GroupSpec or DatasetSpec with a '%s' key." @@ -996,44 +1065,47 @@ def required(self): return self.quantity not in (ZERO_OR_ONE, ZERO_OR_MANY) -_group_args = [ - {'name': 'doc', 'type': str, 'doc': 'a description about what this specification represents'}, - { - 'name': 'name', - 'type': str, - 'doc': 'the name of the Group that is written to the file. If this argument is omitted, users will be ' - 'required to enter a ``name`` field when creating instances of this data type in the API. Another ' - 'option is to specify ``default_name``, in which case this name will be used as the name of the Group ' - 'if no other name is provided.', - 'default': None, - }, - {'name': 'default_name', 'type': str, 'doc': 'The default name of this group', 'default': None}, - {'name': 'groups', 'type': list, 'doc': 'the subgroups in this group', 'default': list()}, - {'name': 'datasets', 'type': list, 'doc': 'the datasets in this group', 'default': list()}, - {'name': 'attributes', 'type': list, 'doc': 'the attributes on this group', 'default': list()}, - {'name': 'links', 'type': list, 'doc': 'the links in this group', 'default': list()}, - {'name': 'linkable', 'type': bool, 'doc': 'whether or not this group can be linked', 'default': True}, - { - 'name': 'quantity', - 'type': (str, int), - 'doc': "the allowable number of instance of this group in a certain location. See table of options " - "`here `_. Note that if you" - "specify ``name``, ``quantity`` cannot be ``'*'``, ``'+'``, or an integer greater that 1, because you " - "cannot have more than one group of the same name in the same parent group.", - 'default': 1, - }, - {'name': 'data_type_def', 'type': str, 'doc': 'the data type this specification represents', 'default': None}, - {'name': 'data_type_inc', 'type': str, 'doc': 'the data type this specification extends', 'default': None}, -] - - class GroupSpec(BaseStorageSpec): ''' Specification for groups ''' - @docval(*_group_args) - def __init__(self, **kwargs): - doc, groups, datasets, links = popargs('doc', 'groups', 'datasets', 'links', kwargs) + @validated + def __init__(self, + doc: str, + name: str | None = None, + default_name: str | None = None, + groups: list | None = None, + datasets: list | None = None, + attributes: list | None = None, + links: list | None = None, + linkable: Bool = True, + quantity: str | Int = 1, + data_type_def: str | None = None, + data_type_inc: str | None = None): + """Initialize this spec. + + Args: + doc: a description about what this specification represents + name: the name of the Group that is written to the file. If this argument is omitted, users will be + required to enter a ``name`` field when creating instances of this data type in the API. Another + option is to specify ``default_name``, in which case this name will be used as the name of the Group + if no other name is provided. + default_name: The default name of this group + groups: the subgroups in this group + datasets: the datasets in this group + attributes: the attributes on this group + links: the links in this group + linkable: whether or not this group can be linked + quantity: the allowable number of instance of this group in a certain location. See table of options `here + `_. Note that if + youspecify ``name``, ``quantity`` cannot be ``'*'``, ``'+'``, or an integer greater that 1, because + you cannot have more than one group of the same name in the same parent group. + data_type_def: the data type this specification represents + data_type_inc: the data type this specification extends + """ + groups = groups if groups is not None else list() + datasets = datasets if datasets is not None else list() + links = links if links is not None else list() self.__data_types = dict() # for GroupSpec/DatasetSpec data_type_def/inc self.__target_types = dict() # for LinkSpec target_types self.__groups = dict() @@ -1053,7 +1125,9 @@ def __init__(self, **kwargs): self.__overridden_links = set() self.__new_groups = set(self.__groups.keys()) self.__overridden_groups = set() - super().__init__(doc, **kwargs) + super().__init__(doc, name=name, default_name=default_name, attributes=attributes, + linkable=linkable, quantity=quantity, data_type_def=data_type_def, + data_type_inc=data_type_inc) def resolve_inc_spec(self, inc_spec: 'GroupSpec', namespace: 'SpecNamespace'): # noqa: C901 """Add groups, datasets, links, and attributes from the inc_spec to this spec and track which ones are new and @@ -1165,64 +1239,79 @@ def resolve_inc_spec(self, inc_spec: 'GroupSpec', namespace: 'SpecNamespace'): self.set_link(link_spec) super().resolve_inc_spec(inc_spec, namespace) - @docval({'name': 'name', 'type': str, 'doc': 'the name of the dataset'}, - raises="ValueError, if 'name' is not part of this spec") - def is_inherited_dataset(self, **kwargs): - '''Return true if a dataset with the given name was inherited''' - name = getargs('name', kwargs) + @validated + def is_inherited_dataset(self, name: str): + """Return true if a dataset with the given name was inherited + + Args: + name: the name of the dataset + """ if name not in self.__datasets: raise ValueError("Dataset '%s' not found in spec" % name) return name not in self.__new_datasets - @docval({'name': 'name', 'type': str, 'doc': 'the name of the dataset'}, - raises="ValueError, if 'name' is not part of this spec") - def is_overridden_dataset(self, **kwargs): - '''Return true if a dataset with the given name overrides a specification from the parent type''' - name = getargs('name', kwargs) + @validated + def is_overridden_dataset(self, name: str): + """Return true if a dataset with the given name overrides a specification from the parent type + + Args: + name: the name of the dataset + """ if name not in self.__datasets: raise ValueError("Dataset '%s' not found in spec" % name) return name in self.__overridden_datasets - @docval({'name': 'name', 'type': str, 'doc': 'the name of the group'}, - raises="ValueError, if 'name' is not part of this spec") - def is_inherited_group(self, **kwargs): - '''Return true if a group with the given name was inherited''' - name = getargs('name', kwargs) + @validated + def is_inherited_group(self, name: str): + """Return true if a group with the given name was inherited + + Args: + name: the name of the group + """ if name not in self.__groups: raise ValueError("Group '%s' not found in spec" % name) return name not in self.__new_groups - @docval({'name': 'name', 'type': str, 'doc': 'the name of the group'}, - raises="ValueError, if 'name' is not part of this spec") - def is_overridden_group(self, **kwargs): - '''Return true if a group with the given name overrides a specification from the parent type''' - name = getargs('name', kwargs) + @validated + def is_overridden_group(self, name: str): + """Return true if a group with the given name overrides a specification from the parent type + + Args: + name: the name of the group + """ if name not in self.__groups: raise ValueError("Group '%s' not found in spec" % name) return name in self.__overridden_groups - @docval({'name': 'name', 'type': str, 'doc': 'the name of the link'}, - raises="ValueError, if 'name' is not part of this spec") - def is_inherited_link(self, **kwargs): - '''Return true if a link with the given name was inherited''' - name = getargs('name', kwargs) + @validated + def is_inherited_link(self, name: str): + """Return true if a link with the given name was inherited + + Args: + name: the name of the link + """ if name not in self.__links: raise ValueError("Link '%s' not found in spec" % name) return name not in self.__new_links - @docval({'name': 'name', 'type': str, 'doc': 'the name of the link'}, - raises="ValueError, if 'name' is not part of this spec") - def is_overridden_link(self, **kwargs): - '''Return true if a link with the given name overrides a specification from the parent type''' - name = getargs('name', kwargs) + @validated + def is_overridden_link(self, name: str): + """Return true if a link with the given name overrides a specification from the parent type + + Args: + name: the name of the link + """ if name not in self.__links: raise ValueError("Link '%s' not found in spec" % name) return name in self.__overridden_links - @docval({'name': 'spec', 'type': Spec, 'doc': 'the specification to check'}) - def is_inherited_spec(self, **kwargs): - ''' Returns 'True' if specification was inherited from a parent type ''' - spec = getargs('spec', kwargs) + @validated + def is_inherited_spec(self, spec: Spec): + """Returns 'True' if specification was inherited from a parent type + + Args: + spec: the specification to check + """ spec_name = spec.name if spec_name is None and hasattr(spec, 'data_type_def'): spec_name = spec.data_type_def @@ -1259,10 +1348,13 @@ def is_inherited_spec(self, **kwargs): return True return False - @docval({'name': 'spec', 'type': Spec, 'doc': 'the specification to check'}) - def is_overridden_spec(self, **kwargs): - ''' Returns 'True' if specification overrides a specification from the parent type ''' - spec = getargs('spec', kwargs) + @validated + def is_overridden_spec(self, spec: Spec): + """Returns 'True' if specification overrides a specification from the parent type + + Args: + spec: the specification to check + """ spec_name = spec.name if spec_name is None: if isinstance(spec, LinkSpec): # unnamed LinkSpec cannot be overridden @@ -1300,35 +1392,47 @@ def is_overridden_spec(self, **kwargs): return True return False - @docval({'name': 'spec', 'type': (BaseStorageSpec, str), 'doc': 'the specification to check'}) - def is_inherited_type(self, **kwargs): - ''' Returns True if `spec` represents a data type that was inherited ''' - spec = getargs('spec', kwargs) + @validated + def is_inherited_type(self, spec: BaseStorageSpec | str): + """Returns True if `spec` represents a data type that was inherited + + Args: + spec: the specification to check + """ if isinstance(spec, BaseStorageSpec): if spec.data_type_def is None: # why not also check data_type_inc? raise ValueError('cannot check if something was inherited if it does not have a %s' % self.def_key()) spec = spec.data_type_def return spec not in self.__new_data_types - @docval({'name': 'spec', 'type': (BaseStorageSpec, str), 'doc': 'the specification to check'}, - raises="ValueError, if 'name' is not part of this spec") - def is_overridden_type(self, **kwargs): - ''' Returns True if `spec` represents a data type that overrides a specification from a parent type ''' - return self.is_inherited_type(**kwargs) + @validated + def is_overridden_type(self, spec: BaseStorageSpec | str): + """Returns True if `spec` represents a data type that overrides a specification from a parent type + + Args: + spec: the specification to check + """ + return self.is_inherited_type(spec) - @docval({'name': 'spec', 'type': (LinkSpec, str), 'doc': 'the specification to check'}) - def is_inherited_target_type(self, **kwargs): - ''' Returns True if `spec` represents a target type that was inherited ''' - spec = getargs('spec', kwargs) + @validated + def is_inherited_target_type(self, spec: LinkSpec | str): + """Returns True if `spec` represents a target type that was inherited + + Args: + spec: the specification to check + """ if isinstance(spec, LinkSpec): spec = spec.target_type return spec not in self.__new_target_types - @docval({'name': 'spec', 'type': (LinkSpec, str), 'doc': 'the specification to check'}, - raises="ValueError, if 'name' is not part of this spec") - def is_overridden_target_type(self, **kwargs): - ''' Returns True if `spec` represents a target type that overrides a specification from a parent type ''' - return self.is_inherited_target_type(**kwargs) + @validated + def is_overridden_target_type(self, spec: LinkSpec | str): + """Returns True if `spec` represents a target type that overrides a specification from a parent type + + Args: + spec: the specification to check + """ + return self.is_inherited_target_type(spec) def __add_data_type_inc(self, spec): # update the __data_types dict with the given groupspec/datasetspec so that: @@ -1425,9 +1529,9 @@ def __add_target_type(self, spec): else: self.__target_types[dt] = spec - @docval({'name': 'data_type', 'type': str, 'doc': 'the data_type to retrieve'}) - def get_data_type(self, **kwargs): - ''' Get a specification by "data_type" + @validated + def get_data_type(self, data_type: str): + """Get a specification by "data_type" NOTE: If there is only one spec for a given data type, then it is returned. If there are multiple specs for a given data type and they are all named, then they are returned in a list. @@ -1435,13 +1539,16 @@ def get_data_type(self, **kwargs): The other named specs can be returned using get_group or get_dataset. NOTE: this method looks for an exact match of the data type and does not consider the type hierarchy. - ''' - ndt = getargs('data_type', kwargs) + + Args: + data_type: the data_type to retrieve + """ + ndt = data_type return self.__data_types.get(ndt, None) - @docval({'name': 'target_type', 'type': str, 'doc': 'the target_type to retrieve'}) - def get_target_type(self, **kwargs): - ''' Get a specification by "target_type" + @validated + def get_target_type(self, target_type: str): + """Get a specification by "target_type" NOTE: If there is only one spec for a given target type, then it is returned. If there are multiple specs for a given target type and they are all named, then they are returned in a list. @@ -1449,8 +1556,11 @@ def get_target_type(self, **kwargs): The other named specs can be returned using get_link. NOTE: this method looks for an exact match of the target type and does not consider the type hierarchy. - ''' - ndt = getargs('target_type', kwargs) + + Args: + target_type: the target_type to retrieve + """ + ndt = target_type return self.__target_types.get(ndt, None) @property @@ -1468,19 +1578,56 @@ def links(self): ''' The links specified in this GroupSpec ''' return tuple(self.get('links', tuple())) - @docval(*_group_args) - def add_group(self, **kwargs): - ''' Add a new specification for a subgroup to this group specification ''' + @validated + def add_group(self, + doc: str, + name: str | None = None, + default_name: str | None = None, + groups: list | None = None, + datasets: list | None = None, + attributes: list | None = None, + links: list | None = None, + linkable: Bool = True, + quantity: str | Int = 1, + data_type_def: str | None = None, + data_type_inc: str | None = None): + """Add a new specification for a subgroup to this group specification + + Args: + doc: a description about what this specification represents + name: the name of the Group that is written to the file. If this argument is omitted, users will be + required to enter a ``name`` field when creating instances of this data type in the API. Another + option is to specify ``default_name``, in which case this name will be used as the name of the Group + if no other name is provided. + default_name: The default name of this group + groups: the subgroups in this group + datasets: the datasets in this group + attributes: the attributes on this group + links: the links in this group + linkable: whether or not this group can be linked + quantity: the allowable number of instance of this group in a certain location. See table of options `here + `_. Note that if + youspecify ``name``, ``quantity`` cannot be ``'*'``, ``'+'``, or an integer greater that 1, because + you cannot have more than one group of the same name in the same parent group. + data_type_def: the data type this specification represents + data_type_inc: the data type this specification extends + """ warn("GroupSpec.add_group is deprecated and will be removed in HDMF 6.0. Use GroupSpec.set_group instead.", DeprecationWarning, stacklevel=2) - spec = self.__class__(**kwargs) + spec = self.__class__(doc, name=name, default_name=default_name, groups=groups, + datasets=datasets, attributes=attributes, links=links, + linkable=linkable, quantity=quantity, + data_type_def=data_type_def, data_type_inc=data_type_inc) self.set_group(spec) return spec - @docval({'name': 'spec', 'type': ('GroupSpec'), 'doc': 'the specification for the subgroup'}) - def set_group(self, **kwargs): - ''' Add the given specification for a subgroup to this group specification ''' - spec = getargs('spec', kwargs) + @validated + def set_group(self, spec: TypeName['GroupSpec']): + """Add the given specification for a subgroup to this group specification + + Args: + spec: the specification for the subgroup + """ if spec.parent is not None: spec = self.build_spec(spec) if spec.name is None: @@ -1497,25 +1644,64 @@ def set_group(self, **kwargs): self.setdefault('groups', list()).append(spec) spec.parent = self - @docval({'name': 'name', 'type': str, 'doc': 'the name of the group to the Spec for'}) - def get_group(self, **kwargs): - ''' Get a specification for a subgroup to this group specification ''' - name = getargs('name', kwargs) + @validated + def get_group(self, name: str): + """Get a specification for a subgroup to this group specification + + Args: + name: the name of the group to the Spec for + """ return self.__groups.get(name, self.__links.get(name)) - @docval(*_dataset_args) - def add_dataset(self, **kwargs): - ''' Add a new specification for a dataset to this group specification ''' + @validated + def add_dataset(self, + doc: str, + dtype: str | list | RefSpec | None = None, + name: str | None = None, + default_name: str | None = None, + shape: list | tuple | None = None, + dims: list | tuple | None = None, + attributes: list | None = None, + linkable: Bool = True, + quantity: str | Int = 1, + default_value: Any = None, + value: Any = None, + data_type_def: str | None = None, + data_type_inc: str | None = None): + """Add a new specification for a dataset to this group specification + + Args: + doc: a description about what this specification represents + dtype: The data type of this attribute. Use a list of DtypeSpecs to specify a compound data type. + name: The name of this dataset + default_name: The default name of this dataset + shape: the shape of this dataset + dims: the dimensions of this dataset + attributes: the attributes on this group + linkable: whether or not this group can be linked + quantity: the required number of allowed instance + default_value: a default value for this dataset + value: a fixed value for this dataset + data_type_def: the data type this specification represents + data_type_inc: the data type this specification extends + """ warn("GroupSpec.add_dataset is deprecated and will be removed in HDMF 6.0. Use GroupSpec.set_dataset instead.", DeprecationWarning, stacklevel=2) - spec = self.dataset_spec_cls()(**kwargs) + spec = self.dataset_spec_cls()(doc, name=name, default_name=default_name, dtype=dtype, + shape=shape, dims=dims, attributes=attributes, + linkable=linkable, quantity=quantity, + default_value=default_value, value=value, + data_type_def=data_type_def, data_type_inc=data_type_inc) self.set_dataset(spec) return spec - @docval({'name': 'spec', 'type': 'hdmf.spec.spec.DatasetSpec', 'doc': 'the specification for the dataset'}) - def set_dataset(self, **kwargs): - ''' Add the given specification for a dataset to this group specification ''' - spec = getargs('spec', kwargs) + @validated + def set_dataset(self, spec: TypeName['DatasetSpec']): # noqa: F821 + """Add the given specification for a dataset to this group specification + + Args: + spec: the specification for the dataset + """ if spec.parent is not None: spec = self.dataset_spec_cls().build_spec(spec) if spec.name is None: @@ -1532,25 +1718,42 @@ def set_dataset(self, **kwargs): self.setdefault('datasets', list()).append(spec) spec.parent = self - @docval({'name': 'name', 'type': str, 'doc': 'the name of the dataset to the Spec for'}) - def get_dataset(self, **kwargs): - ''' Get a specification for a dataset to this group specification ''' - name = getargs('name', kwargs) + @validated + def get_dataset(self, name: str): + """Get a specification for a dataset to this group specification + + Args: + name: the name of the dataset to the Spec for + """ return self.__datasets.get(name, self.__links.get(name)) - @docval(*_link_args) - def add_link(self, **kwargs): - ''' Add a new specification for a link to this group specification ''' + @validated + def add_link(self, + doc: str, + target_type: str | BaseStorageSpec, + quantity: str | Int = 1, + name: str | None = None): + """Add a new specification for a link to this group specification. + + Args: + doc: a description about what this link represents + target_type: the target type GroupSpec or DatasetSpec + quantity: the required number of allowed instance + name: the name of this link + """ warn("GroupSpec.add_link is deprecated and will be removed in HDMF 6.0. Use GroupSpec.set_link instead.", DeprecationWarning, stacklevel=2) - spec = self.link_spec_cls()(**kwargs) + spec = self.link_spec_cls()(doc=doc, target_type=target_type, quantity=quantity, name=name) self.set_link(spec) return spec - @docval({'name': 'spec', 'type': 'hdmf.spec.spec.LinkSpec', 'doc': 'the specification for the object to link to'}) - def set_link(self, **kwargs): - ''' Add a given specification for a link to this group specification ''' - spec = getargs('spec', kwargs) + @validated + def set_link(self, spec: TypeName['LinkSpec']): # noqa: F821 + """Add a given specification for a link to this group specification + + Args: + spec: the specification for the object to link to + """ if spec.parent is not None: spec = self.link_spec_cls().build_spec(spec) # NOTE named specs can be present in both __links and __target_types @@ -1560,10 +1763,13 @@ def set_link(self, **kwargs): self.setdefault('links', list()).append(spec) spec.parent = self - @docval({'name': 'name', 'type': str, 'doc': 'the name of the link to the Spec for'}) - def get_link(self, **kwargs): - ''' Get a specification for a link to this group specification ''' - name = getargs('name', kwargs) + @validated + def get_link(self, name: str): + """Get a specification for a link to this group specification + + Args: + name: the name of the link to the Spec for + """ return self.__links.get(name) @classmethod diff --git a/src/hdmf/spec/write.py b/src/hdmf/spec/write.py index da2aa97e2..5b94fc6a5 100644 --- a/src/hdmf/spec/write.py +++ b/src/hdmf/spec/write.py @@ -10,7 +10,7 @@ from .catalog import SpecCatalog from .namespace import SpecNamespace from .spec import GroupSpec, DatasetSpec -from ..utils import docval, getargs, popargs +from ..typing import validated class SpecWriter(metaclass=ABCMeta): @@ -26,11 +26,14 @@ def write_namespace(self, namespace, path): class YAMLSpecWriter(SpecWriter): - @docval({'name': 'outdir', - 'type': str, - 'doc': 'the path to write the directory to output the namespace and specs too', 'default': '.'}) - def __init__(self, **kwargs): - self.__outdir = getargs('outdir', kwargs) + @validated + def __init__(self, outdir: str = '.'): + """Initialize this spec. + + Args: + outdir: the path to write the directory to output the namespace and specs too + """ + self.__outdir = outdir def __dump_spec(self, specs, stream): specs_plain_dict = json.loads(json.dumps(specs)) @@ -105,45 +108,62 @@ def my_represent_none(self, data): class NamespaceBuilder: ''' A class for building namespace and spec files ''' - @docval({'name': 'doc', 'type': str, 'doc': 'Description about what the namespace represents'}, - {'name': 'name', 'type': str, 'doc': 'Name of the namespace'}, - {'name': 'full_name', 'type': str, 'doc': 'Extended full name of the namespace', 'default': None}, - {'name': 'version', 'type': (str, tuple, list), 'doc': 'Version number of the namespace', 'default': None}, - {'name': 'author', 'type': (str, list), 'doc': 'Author or list of authors.', 'default': None}, - {'name': 'contact', 'type': (str, list), - 'doc': 'List of emails. Ordering should be the same as for author', 'default': None}, - {'name': 'date', 'type': (datetime, str), - 'doc': "Date last modified or released. Formatting is %Y-%m-%d %H:%M:%S, e.g, 2017-04-25 17:14:13", - 'default': None}, - {'name': 'namespace_cls', 'type': type, 'doc': 'the SpecNamespace type', 'default': SpecNamespace}) - def __init__(self, **kwargs): - ns_cls = popargs('namespace_cls', kwargs) - if kwargs['version'] is None: + @validated + def __init__(self, + doc: str, + name: str, + full_name: str | None = None, + version: str | tuple | list | None = None, + author: str | list | None = None, + contact: str | list | None = None, + date: datetime | str | None = None, + namespace_cls: type = SpecNamespace): + """Initialize this spec. + + Args: + doc: Description about what the namespace represents + name: Name of the namespace + full_name: Extended full name of the namespace + version: Version number of the namespace + author: Author or list of authors. + contact: List of emails. Ordering should be the same as for author + date: Date last modified or released. Formatting is %Y-%m-%d %H:%M:%S, e.g, 2017-04-25 17:14:13 + namespace_cls: the SpecNamespace type + """ + ns_cls = namespace_cls + if version is None: # version is required on write as of HDMF 1.5. this check should prevent the writing of namespace files # without a version raise ValueError("Namespace '%s' missing key 'version'. Please specify a version for the extension." - % kwargs['name']) - self.__ns_args = copy.deepcopy(kwargs) + % name) + self.__ns_args = copy.deepcopy(dict(doc=doc, name=name, full_name=full_name, version=version, + author=author, contact=contact, date=date)) self.__namespaces = OrderedDict() self.__sources = OrderedDict() self.__catalog = SpecCatalog() self.__dt_key = ns_cls.types_key() - @docval({'name': 'source', 'type': str, 'doc': 'the path to write the spec to'}, - {'name': 'spec', 'type': (GroupSpec, DatasetSpec), 'doc': 'the Spec to add'}) - def add_spec(self, **kwargs): - ''' Add a Spec to the namespace ''' - source, spec = getargs('source', 'spec', kwargs) + @validated + def add_spec(self, source: str, spec: GroupSpec | DatasetSpec): + """Add a Spec to the namespace + + Args: + source: the path to write the spec to + spec: the Spec to add + """ self.__catalog.auto_register(spec, source) self.add_source(source) self.__sources[source].setdefault(self.__dt_key, list()).append(spec) - @docval({'name': 'source', 'type': str, 'doc': 'the path to write the spec to'}, - {'name': 'doc', 'type': str, 'doc': 'additional documentation for the source file', 'default': None}, - {'name': 'title', 'type': str, 'doc': 'optional heading to be used for the source', 'default': None}) - def add_source(self, **kwargs): - ''' Add a source file to the namespace ''' - source, doc, title = getargs('source', 'doc', 'title', kwargs) + @validated + def add_source(self, source: str, doc: str | None = None, title: str | None = None): + """Add a source file to the namespace + + Args: + source: the path to write the spec to + doc: additional documentation for the source file + title: optional heading to be used for the source + """ if '/' in source or source[0] == '.': raise ValueError('source must be a base file') source_dict = {'source': source} @@ -154,13 +174,16 @@ def add_source(self, **kwargs): if title is not None: self.__sources[source]['title'] = doc - @docval({'name': 'data_type', 'type': str, 'doc': 'the data type to include'}, - {'name': 'source', 'type': str, 'doc': 'the source file to include the type from', 'default': None}, - {'name': 'namespace', 'type': str, - 'doc': 'the namespace from which to include the data type', 'default': None}) - def include_type(self, **kwargs): - ''' Include a data type from an existing namespace or source ''' - dt, src, ns = getargs('data_type', 'source', 'namespace', kwargs) + @validated + def include_type(self, data_type: str, source: str | None = None, namespace: str | None = None): + """Include a data type from an existing namespace or source + + Args: + data_type: the data type to include + source: the source file to include the type from + namespace: the namespace from which to include the data type + """ + dt, src, ns = data_type, source, namespace if src is not None: self.add_source(src) self.__sources[src].setdefault(self.__dt_key, list()).append(dt) @@ -170,28 +193,30 @@ def include_type(self, **kwargs): else: raise ValueError("must specify 'source' or 'namespace' when including type") - @docval({'name': 'namespace', 'type': str, 'doc': 'the namespace to include'}) - def include_namespace(self, **kwargs): - ''' Include an entire namespace ''' - namespace = getargs('namespace', kwargs) + @validated + def include_namespace(self, namespace: str): + """Include an entire namespace + + Args: + namespace: the namespace to include + """ self.__namespaces.setdefault(namespace, {'namespace': namespace}) - @docval({'name': 'path', 'type': str, 'doc': 'the path to write the spec to'}, - {'name': 'outdir', - 'type': str, - 'doc': 'the path to write the directory to output the namespace and specs too', 'default': '.'}, - {'name': 'writer', - 'type': SpecWriter, - 'doc': 'the SpecWriter to use to write the namespace', 'default': None}) - def export(self, **kwargs): - ''' Export the namespace to the given path. + @validated + def export(self, path: str, outdir: str = '.', writer: SpecWriter | None = None): + """Export the namespace to the given path. All new specification source files will be written in the same directory as the given path. - ''' - ns_path, writer = getargs('path', 'writer', kwargs) + + Args: + path: the path to write the spec to + outdir: the path to write the directory to output the namespace and specs too + writer: the SpecWriter to use to write the namespace + """ + ns_path = path if writer is None: - writer = YAMLSpecWriter(outdir=getargs('outdir', kwargs)) + writer = YAMLSpecWriter(outdir=outdir) ns_args = copy.copy(self.__ns_args) ns_args['schema'] = list() for ns, info in self.__namespaces.items(): @@ -226,9 +251,13 @@ def name(self): class SpecFileBuilder(dict): - @docval({'name': 'spec', 'type': (GroupSpec, DatasetSpec), 'doc': 'the Spec to add'}) - def add_spec(self, **kwargs): - spec = getargs('spec', kwargs) + @validated + def add_spec(self, spec: GroupSpec | DatasetSpec): + """add_spec + + Args: + spec: the Spec to add + """ if isinstance(spec, GroupSpec): self.setdefault('groups', list()).append(spec) elif isinstance(spec, DatasetSpec): diff --git a/src/hdmf/term_set.py b/src/hdmf/term_set.py index 628fb4d7a..78359161f 100644 --- a/src/hdmf/term_set.py +++ b/src/hdmf/term_set.py @@ -1,11 +1,11 @@ import glob import os from collections import namedtuple -from .utils import docval import warnings import numpy as np from .data_utils import append_data, extend_data from ruamel.yaml import YAML +from .typing import validated class TermSet: @@ -117,12 +117,13 @@ def __perm_value_key_info(self, perm_values_dict: dict, key: str): return info_tuple(enum_meaning, description, meaning) - @docval({'name': 'term', 'type': str, 'doc': "term to be validated"}) - def validate(self, **kwargs): - """ - Validate term in dataset towards a termset. + @validated + def validate(self, term: str): + """Validate term in dataset towards a termset. + + Args: + term: term to be validated """ - term = kwargs['term'] try: self[term] return True @@ -219,19 +220,18 @@ class TermSetWrapper: """ This class allows any HDF5 dataset or attribute to have a TermSet. """ - @docval({'name': 'termset', - 'type': TermSet, - 'doc': 'The TermSet to be used.'}, - {'name': 'value', - 'type': (list, np.ndarray, dict, str, tuple), - 'doc': 'The target item that is wrapped, either data or attribute.'}, - {'name': 'field', 'type': str, 'default': None, - 'doc': 'The field within a compound array.'} - ) - def __init__(self, **kwargs): - self.__value = kwargs['value'] - self.__termset = kwargs['termset'] - self.__field = kwargs['field'] + @validated + def __init__(self, termset: TermSet, value: list | np.ndarray | dict | str | tuple, field: str | None = None): + """Initialize this object. + + Args: + termset: The TermSet to be used. + value: The target item that is wrapped, either data or attribute. + field: The field within a compound array. + """ + self.__value = value + self.__termset = termset + self.__field = field self.__validate() def __validate(self): @@ -355,21 +355,26 @@ class TypeConfigurator: When toggled on, every instance of a configuration file supported data type will be validated according to the corresponding TermSet. """ - @docval({'name': 'paths', 'type': list, 'doc': 'Paths to configuration files.', 'default': None}) - def __init__(self, **kwargs): + @validated + def __init__(self, paths: list | None = None): + """Initialize this object. + + Args: + paths: Paths to configuration files. + """ self.config = None self.paths = [] - if kwargs["paths"]: - for p in kwargs["paths"]: + if paths: + for p in paths: self.load_type_config(p) - @docval({'name': 'data_type', 'type': str, - 'doc': 'The desired data type within the configuration file.'}, - {'name': 'namespace', 'type': str, - 'doc': 'The namespace for the data type.'}) - def get_config(self, data_type, namespace): - """ - Return the config for that data type in the given namespace. + @validated + def get_config(self, data_type: str, namespace: str): + """Return the config for that data type in the given namespace. + + Args: + data_type: The desired data type within the configuration file. + namespace: The namespace for the data type. """ try: namespace_config = self.config['namespaces'][namespace] @@ -384,10 +389,12 @@ def get_config(self, data_type, namespace): msg = '%s was not found within the configuration for that namespace.' % data_type raise ValueError(msg) - @docval({'name': 'config_path', 'type': str, 'doc': 'Path to the configuration file.'}) - def load_type_config(self, config_path): - """ - Load the configuration file for validation on the fields defined for the objects within the file. + @validated + def load_type_config(self, config_path: str): + """Load the configuration file for validation on the fields defined for the objects within the file. + + Args: + config_path: Path to the configuration file. """ with open(config_path, 'r') as config: yaml = YAML(typ='safe') diff --git a/src/hdmf/typing/__init__.py b/src/hdmf/typing/__init__.py index e9444ee7d..d302cc082 100644 --- a/src/hdmf/typing/__init__.py +++ b/src/hdmf/typing/__init__.py @@ -25,6 +25,7 @@ """ from ..utils import AllowPositional # re-export: used as a @validated option +from ._build import hint_from_spec, signature_function from ._compat import map_hint, synthesize_docval from ._decorator import set_type_checking, validated from ._shapes import Shaped @@ -51,8 +52,10 @@ 'Shaped', 'TypeName', 'UInt', + 'hint_from_spec', 'map_hint', 'register_macro', + 'signature_function', 'set_type_checking', 'synthesize_docval', 'validated', diff --git a/src/hdmf/typing/_build.py b/src/hdmf/typing/_build.py new file mode 100644 index 000000000..667229c6c --- /dev/null +++ b/src/hdmf/typing/_build.py @@ -0,0 +1,131 @@ +"""Runtime construction of real-signature functions from docval-style argument specs. + +HDMF generates constructors and convenience methods at runtime: `hdmf.build.classgenerator` +synthesizes ``__init__`` for schema-derived classes, and `MultiContainerInterface` +composes ``add_*``/``create_*``/``get_*`` methods at class-definition time. Historically +these were assembled with ``@docval(*spec_dicts)``. :func:`signature_function` builds the +same functions with real type-hinted signatures instead, so generated classes look like +ordinary Python to IDEs, ``inspect.signature``, and ``get_docval`` alike. +""" + +import copy +import typing +from typing import Any, Literal + +from ..utils import AllowPositional +from ._decorator import validated +from ._shapes import Shaped +from ._types import AnyData, ArrayData, Bool, Float, Int, ScalarData, TypeName, UInt + +_NUMERIC_HINTS = {'int': Int, 'uint': UInt, 'float': Float, 'bool': Bool} +_MACRO_HINTS = {'array_data': ArrayData, 'scalar_data': ScalarData, 'data': AnyData} +# docval widened the bare python numeric classes exactly like the corresponding strings +_WIDENED_CLASSES = {int: Int, float: Float, bool: Bool} + + +def _hint_from_type(t): + """Map a docval type expression (type, name string, macro, or tuple) to a type hint.""" + if t is None: + return Any + if isinstance(t, str): + if t in _NUMERIC_HINTS: + return _NUMERIC_HINTS[t] + if t in _MACRO_HINTS: + return _MACRO_HINTS[t] + return TypeName[t] + if isinstance(t, type): + return _WIDENED_CLASSES.get(t, t) + if isinstance(t, (list, tuple)): + members = tuple(_hint_from_type(x) for x in t) + if len(members) == 1: + return members[0] + return typing.Union[members] + raise TypeError(f"cannot map docval type expression to a hint: {t!r}") + + +def hint_from_spec(spec): + """Map one docval argument spec dict to the equivalent type hint.""" + if 'enum' in spec: + hint = Literal[tuple(spec['enum'])] + else: + hint = _hint_from_type(spec.get('type')) + if spec.get('shape') is not None: + hint = Shaped[hint, tuple(spec['shape'])] + if 'default' in spec and (spec['default'] is None or spec.get('allow_none', False)) and hint is not Any: + hint = typing.Optional[hint] + return hint + + +def _google_docstring(intro, arg_specs, returns=None): + lines = [intro or '', ''] + if arg_specs: + lines.append('Args:') + for spec in arg_specs: + lines.append(f" {spec['name']}: {spec.get('doc') or spec.get('help', '')}") + if returns: + lines.append('') + lines.append('Returns:') + lines.append(f' {returns}') + return '\n'.join(lines).strip() + '\n' + + +def signature_function(func_name, arg_specs, body, *, doc=None, returns=None, + allow_positional=AllowPositional.ALLOWED, validate=True): + """Build a real-signature function from docval-style argument spec dicts. + + Args: + func_name: the ``__name__`` for the built function + arg_specs: docval-style argument spec dicts (name/type/doc/default/shape/enum/allow_none) + body: callable invoked as ``body(self, kwargs)``, where ``kwargs`` maps every + declared argument name to its value (as ``@docval`` used to provide) + doc: description for the generated docstring + returns: description of the return value for the generated docstring + allow_positional: positional-argument policy for the ``@validated`` wrapper + validate: whether to wrap the function with ``@validated`` + + Returns: + the built function, with type-hinted signature, Google-style docstring, and + (by default) ``@validated`` runtime checking + """ + required = [a for a in arg_specs if 'default' not in a] + optional = [a for a in arg_specs if 'default' in a] + ordered = required + optional + + if not func_name.isidentifier() or any(not a['name'].isidentifier() for a in ordered): + # python signatures cannot express non-identifier names; keep the legacy + # docval behavior for these exotic cases (removed together with docval) + from ..utils import docval + + def _adapter(self, **kwargs): + return body(self, kwargs) + + return docval(*arg_specs, func_name=func_name, doc=doc, + allow_positional=allow_positional, enforce_type=validate)(_adapter) + + namespace = {'__body__': body, '__deepcopy__': copy.deepcopy} + sig_parts = ['self'] + dict_parts = [] + for i, spec in enumerate(ordered): + name = spec['name'] + if 'default' in spec: + default_ref = f'__dflt_{i}__' + namespace[default_ref] = spec['default'] + sig_parts.append(f'{name}={default_ref}') + if isinstance(spec['default'], (list, dict, set)): + # docval deepcopied defaults on every call; preserve that for mutables + dict_parts.append(f'{name!r}: ({name} if {name} is not {default_ref} ' + f'else __deepcopy__({default_ref}))') + continue + else: + sig_parts.append(name) + dict_parts.append(f'{name!r}: {name}') + + src = (f"def {func_name}({', '.join(sig_parts)}):\n" + f" return __body__(self, {{{', '.join(dict_parts)}}})\n") + exec(src, namespace) # noqa: S102 + func = namespace[func_name] + func.__annotations__ = {spec['name']: hint_from_spec(spec) for spec in ordered} + func.__doc__ = _google_docstring(doc, ordered, returns) + if validate: + func = validated(func, allow_positional=allow_positional) + return func diff --git a/src/hdmf/typing/_compat.py b/src/hdmf/typing/_compat.py index 0e55efb8e..f62e88411 100644 --- a/src/hdmf/typing/_compat.py +++ b/src/hdmf/typing/_compat.py @@ -12,6 +12,7 @@ import inspect import types import typing +import warnings from typing import Annotated, Any, Literal from ._docstrings import parse_docstring @@ -112,6 +113,15 @@ def map_hint(hint): # noqa: C901 # unresolved forward reference: docval matches the string against the MRO if isinstance(hint, str): + if not hint.replace('.', '').isidentifier(): + # a compound annotation string (e.g. "A | None") that could not be + # resolved; matching it against MRO names would never succeed + warnings.warn( + f"could not resolve string annotation {hint!r}; it will not be validated. " + "Use hdmf.typing.TypeName[...] for forward references to later-defined names.", + stacklevel=2, + ) + return MappedHint({'type': None}, exact=False) return MappedHint({'type': hint}) if isinstance(hint, typing.ForwardRef): return MappedHint({'type': hint.__forward_arg__}) diff --git a/src/hdmf/typing/migrate.py b/src/hdmf/typing/migrate.py index 57880a2e0..431bb795e 100644 --- a/src/hdmf/typing/migrate.py +++ b/src/hdmf/typing/migrate.py @@ -192,16 +192,24 @@ def _convert_arg(self, spec_node): arg.default_src = default_src return arg - def _convert_function(self, node, dec): + def _convert_function(self, node, dec, module_spec_lists=None): """Convert one @docval function. Returns a MigratedFunction, or None to skip.""" func = MigratedFunction(node=node, decorator=dec, args=[], options={}, returns_doc=None, rtype_hint=None, allow_extra=False) + spec_nodes = [] for spec_node in dec.args: if isinstance(spec_node, ast.Starred): + # resolve splices of module-level literal spec lists, e.g. @docval(*_attr_args) + if (isinstance(spec_node.value, ast.Name) + and (module_spec_lists or {}).get(spec_node.value.id) is not None): + spec_nodes.extend(module_spec_lists[spec_node.value.id]) + continue func.todos.append( f"decorator splices other functions' specs ({ast.unparse(spec_node)}); " "convert by hand") return None + spec_nodes.append(spec_node) + for spec_node in spec_nodes: arg = self._convert_arg(spec_node) if arg is None: func.todos.append(f"non-literal argument spec: {ast.unparse(spec_node)}") @@ -337,14 +345,18 @@ def rewrite(logical, lead): out.extend(buffer if replacement is None else replacement) buffer = [] out.extend(buffer) # unbalanced trailing lines, if any - leftover_kwargs = any('kwargs' in line for line in out) and not func.allow_extra + leftover_kwargs = any('kwargs' in line for line in out) todos = list(func.todos) for arg in func.args: todos.extend(f"{arg.name}: {t}" for t in arg.todos) - if leftover_kwargs: + if leftover_kwargs and not func.allow_extra: todos.append("body still references `kwargs`, which no longer exists; rewrite " "remaining uses (e.g. multi-line getargs/popargs, " "super().__init__(**kwargs) -> explicit keywords)") + elif leftover_kwargs: + todos.append("`kwargs` now holds ONLY extra keyword arguments (docval's kwargs held " + "ALL parsed args) — audit each use; named parameters must be passed " + "explicitly (e.g. f(**kwargs) -> f(a=a, b=b, **kwargs))") todo_lines = [f"{indent} # TODO(migrate): {t}" for t in todos] return todo_lines + out @@ -356,12 +368,21 @@ def migrate_source(self, source): targets = self._find_docval_functions(tree) if not targets: return source, 0, 0 + # module-level `NAME = [ {...}, ... ]` spec lists, resolvable in *NAME splices + module_spec_lists = {} + for stmt in tree.body: + if (isinstance(stmt, ast.Assign) and len(stmt.targets) == 1 + and isinstance(stmt.targets[0], ast.Name) + and isinstance(stmt.value, (ast.List, ast.Tuple)) + and stmt.value.elts + and all(isinstance(e, (ast.Dict, ast.Call)) for e in stmt.value.elts)): + module_spec_lists[stmt.targets[0].id] = stmt.value.elts lines = source.splitlines() n_converted = 0 n_skipped = 0 # bottom-up so earlier line numbers stay valid for node, dec in sorted(targets, key=lambda t: t[0].lineno, reverse=True): - func = self._convert_function(node, dec) + func = self._convert_function(node, dec, module_spec_lists) if func is None: n_skipped += 1 dec_line = dec.lineno - 1 diff --git a/src/hdmf/validate/errors.py b/src/hdmf/validate/errors.py index c1b4360a1..0b307c125 100644 --- a/src/hdmf/validate/errors.py +++ b/src/hdmf/validate/errors.py @@ -1,7 +1,7 @@ from numpy import dtype from ..spec.spec import DtypeHelper -from ..utils import docval, getargs +from ..typing import Int, validated __all__ = [ "Error", @@ -18,13 +18,18 @@ class Error: - @docval({'name': 'name', 'type': str, 'doc': 'the name of the component that is erroneous'}, - {'name': 'reason', 'type': str, 'doc': 'the reason for the error'}, - {'name': 'location', 'type': str, 'doc': 'the location of the error', 'default': None}) - def __init__(self, **kwargs): - self.__name = getargs('name', kwargs) - self.__reason = getargs('reason', kwargs) - self.__location = getargs('location', kwargs) + @validated + def __init__(self, name: str, reason: str, location: str | None = None): + """Initialize this object. + + Args: + name: the name of the component that is erroneous + reason: the reason for the error + location: the location of the error + """ + self.__name = name + self.__reason = reason + self.__location = location @property def name(self): @@ -87,44 +92,58 @@ def __eq__(self, other): class DtypeError(Error): - @docval({'name': 'name', 'type': str, 'doc': 'the name of the component that is erroneous'}, - {'name': 'expected', 'type': (dtype, type, str, list), 'doc': 'the expected dtype'}, - {'name': 'received', 'type': (dtype, type, str, list), 'doc': 'the received dtype'}, - {'name': 'location', 'type': str, 'doc': 'the location of the error', 'default': None}) - def __init__(self, **kwargs): - name = getargs('name', kwargs) - expected = getargs('expected', kwargs) - received = getargs('received', kwargs) + @validated + def __init__(self, + name: str, + expected: dtype | type | str | list, + received: dtype | type | str | list, + location: str | None = None): + """Initialize this object. + + Args: + name: the name of the component that is erroneous + expected: the expected dtype + received: the received dtype + location: the location of the error + """ if isinstance(expected, list): expected = DtypeHelper.simplify_cpd_type(expected) reason = "incorrect type - expected '%s', got '%s'" % (expected, received) - loc = getargs('location', kwargs) + loc = location super().__init__(name, reason, location=loc) class MissingError(Error): - @docval({'name': 'name', 'type': str, 'doc': 'the name of the component that is erroneous'}, - {'name': 'location', 'type': str, 'doc': 'the location of the error', 'default': None}) - def __init__(self, **kwargs): - name = getargs('name', kwargs) + @validated + def __init__(self, name: str, location: str | None = None): + """Initialize this object. + + Args: + name: the name of the component that is erroneous + location: the location of the error + """ reason = "argument missing" - loc = getargs('location', kwargs) + loc = location super().__init__(name, reason, location=loc) class MissingDataType(Error): - @docval({'name': 'name', 'type': str, 'doc': 'the name of the component that is erroneous'}, - {'name': 'data_type', 'type': str, 'doc': 'the missing data type'}, - {'name': 'location', 'type': str, 'doc': 'the location of the error', 'default': None}, - {'name': 'missing_dt_name', 'type': str, 'doc': 'the name of the missing data type', 'default': None}) - def __init__(self, **kwargs): - name, data_type, missing_dt_name = getargs('name', 'data_type', 'missing_dt_name', kwargs) + @validated + def __init__(self, name: str, data_type: str, location: str | None = None, missing_dt_name: str | None = None): + """Initialize this object. + + Args: + name: the name of the component that is erroneous + data_type: the missing data type + location: the location of the error + missing_dt_name: the name of the missing data type + """ self.__data_type = data_type if missing_dt_name is not None: reason = "missing data type %s (%s)" % (self.__data_type, missing_dt_name) else: reason = "missing data type %s" % self.__data_type - loc = getargs('location', kwargs) + loc = location super().__init__(name, reason, location=loc) @property @@ -134,50 +153,63 @@ def data_type(self): class IncorrectQuantityError(Error): """A validation error indicating that a child group/dataset/link has the incorrect quantity of matching elements""" - @docval({'name': 'name', 'type': str, 'doc': 'the name of the component that is erroneous'}, - {'name': 'data_type', 'type': str, 'doc': 'the data type which has the incorrect quantity'}, - {'name': 'expected', 'type': (str, int), 'doc': 'the expected quantity'}, - {'name': 'received', 'type': (str, int), 'doc': 'the received quantity'}, - {'name': 'location', 'type': str, 'doc': 'the location of the error', 'default': None}) - def __init__(self, **kwargs): - name, data_type, expected, received = getargs('name', 'data_type', 'expected', 'received', kwargs) + @validated + def __init__(self, + name: str, + data_type: str, + expected: str | Int, + received: str | Int, + location: str | None = None): + """Initialize this object. + + Args: + name: the name of the component that is erroneous + data_type: the data type which has the incorrect quantity + expected: the expected quantity + received: the received quantity + location: the location of the error + """ reason = "expected a quantity of %s for data type %s, received %s" % (str(expected), data_type, str(received)) - loc = getargs('location', kwargs) + loc = location super().__init__(name, reason, location=loc) class ExpectedArrayError(Error): - @docval({'name': 'name', 'type': str, 'doc': 'the name of the component that is erroneous'}, - {'name': 'expected', 'type': (tuple, list), 'doc': 'the expected shape'}, - {'name': 'received', 'type': str, 'doc': 'the received data'}, - {'name': 'location', 'type': str, 'doc': 'the location of the error', 'default': None}) - def __init__(self, **kwargs): - name = getargs('name', kwargs) - expected = getargs('expected', kwargs) - received = getargs('received', kwargs) + @validated + def __init__(self, name: str, expected: tuple | list, received: str, location: str | None = None): + """Initialize this object. + + Args: + name: the name of the component that is erroneous + expected: the expected shape + received: the received data + location: the location of the error + """ reason = "incorrect shape - expected an array of shape '%s', got non-array data '%s'" % (expected, received) - loc = getargs('location', kwargs) + loc = location super().__init__(name, reason, location=loc) class ShapeError(Error): - @docval({'name': 'name', 'type': str, 'doc': 'the name of the component that is erroneous'}, - {'name': 'expected', 'type': (tuple, list), 'doc': 'the expected shape'}, - {'name': 'received', 'type': (tuple, list), 'doc': 'the received shape'}, - {'name': 'location', 'type': str, 'doc': 'the location of the error', 'default': None}) - def __init__(self, **kwargs): - name = getargs('name', kwargs) - expected = getargs('expected', kwargs) - received = getargs('received', kwargs) + @validated + def __init__(self, name: str, expected: tuple | list, received: tuple | list, location: str | None = None): + """Initialize this object. + + Args: + name: the name of the component that is erroneous + expected: the expected shape + received: the received shape + location: the location of the error + """ if isinstance(expected, (list, tuple)) and all(isinstance(e, (list, tuple)) for e in expected): allowable_shapes_str = " or ".join(map(str, expected)) else: allowable_shapes_str = str(expected) allowable_shapes_str = allowable_shapes_str.replace("None", "*") reason = "incorrect shape - expected '%s', got '%s'" % (allowable_shapes_str, received) - loc = getargs('location', kwargs) + loc = location super().__init__(name, reason, location=loc) @@ -187,12 +219,16 @@ class IllegalLinkError(Error): (i.e. a dataset or a group) must be used """ - @docval({'name': 'name', 'type': str, 'doc': 'the name of the component that is erroneous'}, - {'name': 'location', 'type': str, 'doc': 'the location of the error', 'default': None}) - def __init__(self, **kwargs): - name = getargs('name', kwargs) + @validated + def __init__(self, name: str, location: str | None = None): + """Initialize this object. + + Args: + name: the name of the component that is erroneous + location: the location of the error + """ reason = "illegal use of link (linked object will not be validated)" - loc = getargs('location', kwargs) + loc = location super().__init__(name, reason, location=loc) @@ -201,14 +237,16 @@ class IncorrectDataType(Error): A validation error for indicating that the incorrect data_type (not dtype) was used. """ - @docval({'name': 'name', 'type': str, 'doc': 'the name of the component that is erroneous'}, - {'name': 'expected', 'type': str, 'doc': 'the expected data_type'}, - {'name': 'received', 'type': str, 'doc': 'the received data_type'}, - {'name': 'location', 'type': str, 'doc': 'the location of the error', 'default': None}) - def __init__(self, **kwargs): - name = getargs('name', kwargs) - expected = getargs('expected', kwargs) - received = getargs('received', kwargs) + @validated + def __init__(self, name: str, expected: str, received: str, location: str | None = None): + """Initialize this object. + + Args: + name: the name of the component that is erroneous + expected: the expected data_type + received: the received data_type + location: the location of the error + """ reason = "incorrect data_type - expected '%s', got '%s'" % (expected, received) - loc = getargs('location', kwargs) + loc = location super().__init__(name, reason, location=loc) diff --git a/tests/unit/build_tests/test_classgenerator.py b/tests/unit/build_tests/test_classgenerator.py index 5f666cb48..2e2b40891 100644 --- a/tests/unit/build_tests/test_classgenerator.py +++ b/tests/unit/build_tests/test_classgenerator.py @@ -501,7 +501,7 @@ def test_multi_container_spec_one_or_more_missing(self): self.spec_catalog.register_spec(multi_spec, 'extension.yaml') self.type_map.namespace_catalog.resolve_all_specs() Multi = self.type_map.get_dt_container_cls('Multi', CORE_NAMESPACE) - with self.assertRaisesWith(TypeError, "MCIClassGenerator.set_init..__init__: missing argument 'bars'"): + with self.assertRaisesWith(TypeError, "__init__: missing a required argument: 'bars'"): Multi( name='my_multi', attr3=5. @@ -726,7 +726,8 @@ def test_gen_parent_class(self): {'name': 'my_baz1', 'doc': 'A composition inside with a fixed name', 'type': baz1_cls}, {'name': 'my_baz2', 'doc': 'A composition inside with a fixed name', 'type': baz2_cls}, {'name': 'my_baz1_link', 'doc': 'A composition inside without a fixed name', 'type': baz1_cls}, - {'name': 'skip_post_init', 'type': bool, 'default': False, + # 'bool' (the docval type string) and the bool class validate identically + {'name': 'skip_post_init', 'type': 'bool', 'default': False, 'doc': 'bool to skip post_init'} )) diff --git a/tests/unit/spec_tests/test_attribute_spec.py b/tests/unit/spec_tests/test_attribute_spec.py index c9accb8fc..517f4f18e 100644 --- a/tests/unit/spec_tests/test_attribute_spec.py +++ b/tests/unit/spec_tests/test_attribute_spec.py @@ -98,7 +98,7 @@ def test_build_spec_reftype(self): def test_build_spec_no_doc(self): spec_dict = {'name': 'attribute1', 'dtype': 'text'} - msg = "AttributeSpec.__init__: missing argument 'doc'" + msg = "AttributeSpec.__init__: missing a required argument: 'doc'" with self.assertRaisesWith(TypeError, msg): AttributeSpec.build_spec(spec_dict) diff --git a/tests/unit/test_multicontainerinterface.py b/tests/unit/test_multicontainerinterface.py index 2a474b74c..201405f97 100644 --- a/tests/unit/test_multicontainerinterface.py +++ b/tests/unit/test_multicontainerinterface.py @@ -101,7 +101,8 @@ def test_init_docval(self): def test_add_docval(self): """Test that the docval for the add method is set correctly.""" - expected_doc = "add_container(containers)\n\nAdd one or multiple Container objects to this Foo" + # real signatures replaced docval's synthetic signature line in the docstring + expected_doc = "Add one or multiple Container objects to this Foo" self.assertTrue(Foo.add_container.__doc__.startswith(expected_doc)) dv = get_docval(Foo.add_container) self.assertEqual(dv[0]['name'], 'containers') diff --git a/tests/unit/typing_tests/test_build.py b/tests/unit/typing_tests/test_build.py new file mode 100644 index 000000000..70719976d --- /dev/null +++ b/tests/unit/typing_tests/test_build.py @@ -0,0 +1,89 @@ +"""Tests for hdmf.typing.signature_function (runtime signature builder).""" + +import inspect + +import numpy as np + +from hdmf.testing import TestCase +from hdmf.typing import signature_function +from hdmf.utils import AllowPositional, docval, get_docval + + +SPECS = [ + {'name': 'name', 'type': str, 'doc': 'the name'}, + {'name': 'data', 'type': ('array_data', 'data'), 'doc': 'the data', 'shape': (None,)}, + {'name': 'count', 'type': 'int', 'doc': 'how many', 'default': 1}, + {'name': 'tags', 'type': list, 'doc': 'tags', 'default': list()}, +] + + +def _body(self, kwargs): + return kwargs + + +class Dummy: + pass + + +class TestSignatureFunction(TestCase): + + def setUp(self): + self.func = signature_function('__init__', SPECS, _body, doc='Make a thing.') + self.obj = Dummy() + + def test_real_signature(self): + params = list(inspect.signature(self.func).parameters) + self.assertEqual(params, ['self', 'name', 'data', 'count', 'tags']) + + def test_call_and_defaults(self): + out = self.func(self.obj, name='n', data=[1, 2], count=np.int8(3)) + self.assertEqual(out['name'], 'n') + self.assertEqual(out['count'], 3) + self.assertEqual(out['tags'], []) + + def test_mutable_default_isolated(self): + out1 = self.func(self.obj, name='n', data=[1]) + out1['tags'].append('x') + out2 = self.func(self.obj, name='n', data=[1]) + self.assertEqual(out2['tags'], []) + + def test_validation(self): + with self.assertRaisesRegex(TypeError, "incorrect type for 'name'"): + self.func(self.obj, name=5, data=[1]) + with self.assertRaisesRegex(ValueError, "incorrect shape for data"): + self.func(self.obj, name='n', data=[[1, 2]]) + + def test_get_docval_round_trip_splices_into_docval(self): + specs = get_docval(self.func) + self.assertEqual([s['name'] for s in specs], ['name', 'data', 'count', 'tags']) + + @docval(*[dict(s) for s in specs], is_method=False) + def downstream(**kwargs): + return kwargs + + self.assertEqual(downstream(name='n', data=[1, 2, 3])['count'], 1) + with self.assertRaisesRegex(TypeError, "incorrect type for 'name'"): + downstream(name=5, data=[1]) + + def test_docstring(self): + self.assertIn('Make a thing.', self.func.__doc__) + self.assertIn('name: the name', self.func.__doc__) + + def test_non_identifier_names_fall_back_to_docval(self): + func = signature_function('...', [{'name': '...', 'type': str, 'doc': 'weird'}], _body) + self.assertTrue(hasattr(func, '__docval__')) # legacy path + self.assertEqual(func(self.obj, **{'...': 'x'})['...'], 'x') + + def test_positional_policy(self): + func = signature_function('f', [{'name': 'a', 'type': str, 'doc': 'a'}], _body, + allow_positional=AllowPositional.ERROR) + with self.assertRaisesRegex(SyntaxError, 'Only keyword arguments'): + func(self.obj, 'x') + self.assertEqual(func(self.obj, a='x')['a'], 'x') + + def test_enum_spec(self): + func = signature_function('f', [{'name': 'mode', 'type': str, 'doc': 'm', 'enum': ['r', 'w'], + 'default': 'r'}], _body) + self.assertEqual(func(self.obj, mode='w')['mode'], 'w') + with self.assertRaisesRegex(ValueError, "forbidden value for 'mode'"): + func(self.obj, mode='x')