diff --git a/.github/workflows/HDMF_dev.yaml b/.github/workflows/HDMF_dev.yaml index 119c7041..f61ae531 100644 --- a/.github/workflows/HDMF_dev.yaml +++ b/.github/workflows/HDMF_dev.yaml @@ -21,7 +21,7 @@ jobs: - name: Install HDMF_Zarr Requirements run: | - python -m pip install ".[test]" + python -m pip install --group test "." - name: Clone and Install HDMF Dev Branch run: | diff --git a/.github/workflows/check_external_links.yml b/.github/workflows/check_external_links.yml index 370e546b..cc2b4a17 100644 --- a/.github/workflows/check_external_links.yml +++ b/.github/workflows/check_external_links.yml @@ -23,7 +23,7 @@ jobs: - name: Install Sphinx dependencies and package run: | python -m pip install --upgrade pip - python -m pip install ".[test,docs,full]" + python -m pip install --group docs ".[full]" - name: Check Sphinx external links run: sphinx-build -W -b linkcheck ./docs/source ./test_build diff --git a/.github/workflows/run_coverage.yml b/.github/workflows/run_coverage.yml index 5d86177c..ac9370be 100644 --- a/.github/workflows/run_coverage.yml +++ b/.github/workflows/run_coverage.yml @@ -44,15 +44,16 @@ jobs: - name: Upgrade pip run: | python -m pip install --upgrade pip + python -m pip --version - name: Install package if: ${{ ! matrix.opt_req }} run: | - python -m pip install ".[test]" + python -m pip install --group test "." - name: Install package with optional dependencies if: ${{ matrix.opt_req }} - run: python -m pip install ".[test,full]" + run: python -m pip install --group test ".[full]" - name: Run tests and generate coverage report run: | diff --git a/.readthedocs.yaml b/.readthedocs.yaml index 569ac5ac..df0c621e 100644 --- a/.readthedocs.yaml +++ b/.readthedocs.yaml @@ -9,6 +9,10 @@ build: os: ubuntu-24.04 tools: python: '3.14' + jobs: + install: + - pip install -U pip + - pip install --group docs ".[full]" # Build documentation in the docs/ directory with Sphinx sphinx: @@ -16,8 +20,3 @@ sphinx: # Optionally build your docs in additional formats such as PDF and ePub formats: all - -# Optionally set the version of Python and requirements required to build your docs -python: - install: - - path: .[docs,full] # path to the package relative to the root diff --git a/CHANGELOG.md b/CHANGELOG.md index bc32aabe..4f586e6f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,8 +11,11 @@ - Fixed bug where the cached spec was not included in consolidated metadata when exporting using `ZarrIO`. @rly [#315](https://github.com/hdmf-dev/hdmf-zarr/pull/315) - Fixed bug where consolidated metadata was always written by `ZarrIO`. @rly [#315](https://github.com/hdmf-dev/hdmf-zarr/pull/315) - Fixed bug where the `specifications` group and its contents were read during `ZarrIO.read_builder`. @rly [#322](https://github.com/hdmf-dev/hdmf-zarr/pull/322) +- Fixed issue with `tox.ini` configuration. @rly [#326](https://github.com/hdmf-dev/hdmf-zarr/pull/326) +- Fixed `ZarrIO.is_remote()` returning `False` for remote stores using consolidated metadata, which caused `resolve_ref` to mangle HTTPS URLs and fail with `PathNotFoundError` when reading links. @rly [#328](https://github.com/hdmf-dev/hdmf-zarr/pull/328) ### Changed +- Replaced `requirements-min.txt` with `uv pip install --resolution lowest-direct` in tox and converted `test` and `docs` from optional dependencies to dependency groups (PEP 735), making the project compatible with uv. @h-mayorquin [#327](https://github.com/hdmf-dev/hdmf-zarr/pull/327) - Updated documentation about how the `zarr_dtype` attribute is used to specify a compound dtype (structured array) for a dataset. @rly [#313](https://github.com/hdmf-dev/hdmf-zarr/pull/313) - Dropped testing for Python 3.9. @rly [#298](https://github.com/hdmf-dev/hdmf-zarr/pull/298) - Added support for Python 3.14. @rly [#298](https://github.com/hdmf-dev/hdmf-zarr/pull/298) diff --git a/pyproject.toml b/pyproject.toml index a3b4a348..6c51e865 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -30,7 +30,7 @@ classifiers = [ "Topic :: Scientific/Engineering :: Medical Science Apps." ] dependencies = [ - "hdmf>=4.2.0", + "hdmf @ git+https://github.com/hdmf-dev/hdmf.git@remove_duck_typing_for_type", "zarr>=3.1.3", "numpy>=1.26.0", "numcodecs>=0.14.0", @@ -47,6 +47,10 @@ full = [ "s3fs", ] +# all runtime optional dependencies +all = ["hdmf-zarr[full]"] + +[dependency-groups] # development dependencies test = [ "black", @@ -57,28 +61,31 @@ test = [ "pytest-cov", "python-dateutil", # used in some tests "ruff", - "scipy", # used in some tests + "scipy>=1.7", # used in some tests "tox", ] # documentation dependencies docs = [ - "matplotlib", - "scipy", # used in some docs + "matplotlib>=3.6", + "scipy>=1.7", # used in some docs "sphinx>=4", # improved support for docutils>=0.17 "sphinx_rtd_theme>=1", # <1 does not work with docutils>=0.17 - "sphinx-gallery", - "sphinx-copybutton", + "sphinx-gallery>=0.16", + "sphinx-copybutton>=0.5", ] -# all possible dependencies -all = ["hdmf-zarr[full,test,docs]"] - [project.urls] "Homepage" = "https://github.com/hdmf-dev/hdmf-zarr" "Bug Tracker" = "https://github.com/hdmf-dev/hdmf-zarr/issues" +[tool.uv.sources] +hdmf = { git = "https://github.com/hdmf-dev/hdmf.git", branch = "remove_duck_typing_for_type" } + +[tool.hatch.metadata] +allow-direct-references = true + [tool.hatch.version] source = "vcs" diff --git a/requirements-min.txt b/requirements-min.txt deleted file mode 100644 index 4ea81510..00000000 --- a/requirements-min.txt +++ /dev/null @@ -1,8 +0,0 @@ -# minimum versions of package dependencies for installing HDMF -# NOTE: these should match the minimum bound for dependencies in pyproject.toml -hdmf==4.2.0 -zarr==3.1.3 -numpy==1.26.0 -numcodecs==0.14.0 -pynwb==2.8.3 -threadpoolctl==3.1.0 diff --git a/src/hdmf_zarr/backend.py b/src/hdmf_zarr/backend.py index a5b1aa5f..0db9e139 100644 --- a/src/hdmf_zarr/backend.py +++ b/src/hdmf_zarr/backend.py @@ -36,24 +36,6 @@ from pathlib import Path -# zarr v3 Array does not implement __len__; add it for compatibility with array-like interfaces -if not hasattr(Array, "__len__"): - Array.__len__ = lambda self: self.shape[0] - -# zarr v3 Array scalar indexing returns 0-d ndarrays instead of numpy scalars; -# patch to match zarr v2 / numpy behavior expected by hdmf type checks -_zarr_array_original_getitem = Array.__getitem__ - - -def _zarr_array_getitem_scalar_fix(self, key): - result = _zarr_array_original_getitem(self, key) - if isinstance(result, np.ndarray) and result.ndim == 0: - return result[()] - return result - - -Array.__getitem__ = _zarr_array_getitem_scalar_fix - # Module variables ROOT_NAME = "root" @@ -1400,7 +1382,7 @@ def write_dataset(self, **kwargs): # noqa: C901 ) self._written_builders.set_written(builder) # record that the builder has been written dset.attrs["zarr_dtype"] = type_str - if hasattr(refs, "__len__") and not isinstance(refs, dict): + if self._is_collection(refs) and not isinstance(refs, dict): json_refs = [json.dumps(dict(r)) for r in refs] for i, jr in enumerate(json_refs): dset[i] = jr @@ -1414,7 +1396,7 @@ def write_dataset(self, **kwargs): # noqa: C901 elif isinstance(data, AbstractDataChunkIterator): dset = self.__setup_chunked_dataset__(parent, name, data, options) self.__dci_queue.append(dataset=dset, data=data) - elif hasattr(data, "__len__"): + elif self._is_collection(data): dset = self.__list_fill__(parent, name, data, options) else: dset = self.__scalar_fill__(parent, name, data, options) @@ -1506,13 +1488,39 @@ def get_type(cls, data): return cls.__dtypes.get("str") elif isinstance(data, bytes): return cls.__dtypes.get("bytes") - elif not hasattr(data, "__len__"): + elif isinstance(data, np.ndarray) and data.ndim == 0: + return type(data.item()) + elif not cls._is_collection(data): return type(data) else: - if len(data) == 0: + if cls._get_length(data) == 0: raise ValueError("cannot determine type for empty data") return cls.get_type(data[0]) + @staticmethod + def _is_collection(data): + """Check if data is a collection (array-like with elements) vs a scalar. + + Uses ndim for array-like objects (numpy, zarr, h5py, dask) and falls back + to __len__ for plain Python containers (list, tuple). Strings and bytes + are treated as scalars. + """ + if isinstance(data, (str, bytes)): + return False + if hasattr(data, "ndim"): + return data.ndim > 0 + return hasattr(data, "__len__") + + @staticmethod + def _get_length(data): + """Get the length of the first dimension of a collection. + + Uses shape[0] for array-like objects and len() for plain containers. + """ + if hasattr(data, "shape") and data.shape is not None: + return data.shape[0] + return len(data) + __reserve_attribute = ("zarr_dtype", "zarr_link", SPEC_LOC_ATTR) def __list_fill__(self, parent, name, data, options=None): # noqa: C901 diff --git a/tests/unit/test_fsspec_streaming.py b/tests/unit/test_fsspec_streaming.py index 70650095..e69763c9 100644 --- a/tests/unit/test_fsspec_streaming.py +++ b/tests/unit/test_fsspec_streaming.py @@ -40,6 +40,20 @@ def test_s3_open_with_consolidated_(self): read_io.open() self.assertIsNotNone(read_io._file) + @unittest.skipIf(not HAVE_FSSPEC, "fsspec not installed") + def test_is_remote_with_consolidated(self): + """Test that is_remote() returns True for remote HTTPS stores with consolidated metadata.""" + with NWBZarrIO(self.https_s3_path, mode="r") as read_io: + read_io.open() + self.assertTrue(read_io.is_remote()) + + @unittest.skipIf(not HAVE_FSSPEC, "fsspec not installed") + def test_is_remote_without_consolidated(self): + """Test that is_remote() returns True for remote HTTPS stores without consolidated metadata.""" + with NWBZarrIO(self.https_s3_path, mode="-r") as read_io: + read_io.open() + self.assertTrue(read_io.is_remote()) + @unittest.skipIf(not HAVE_FSSPEC, "fsspec not installed") def test_fsspec_streaming_via_read_nwb(self): """ diff --git a/tests/unit/test_zarrio.py b/tests/unit/test_zarrio.py index 53f0aa2d..f7e0e725 100644 --- a/tests/unit/test_zarrio.py +++ b/tests/unit/test_zarrio.py @@ -198,6 +198,20 @@ def test_force_open_without_consolidated_fails(self): except ValueError as e: self.fail("ZarrIO.__open_file_consolidated raised an unexpected ValueError: {}".format(e)) + def test_is_remote_local_with_consolidated(self): + """Test that is_remote() returns False for local stores with consolidated metadata.""" + self.create_zarr(consolidate_metadata=True) + with ZarrIO(self.store_path, mode="r") as read_io: + read_io.open() + self.assertFalse(read_io.is_remote()) + + def test_is_remote_local_without_consolidated(self): + """Test that is_remote() returns False for local stores without consolidated metadata.""" + self.create_zarr() + with ZarrIO(self.store_path, mode="r-") as read_io: + read_io.open() + self.assertFalse(read_io.is_remote()) + class TestOverwriteExistingFile(ZarrStoreTestCase): def test_force_overwrite_when_file_exists(self): diff --git a/tox.ini b/tox.ini index 332326c2..21d2d792 100644 --- a/tox.ini +++ b/tox.ini @@ -16,7 +16,7 @@ setenv = PYTHONDONTWRITEBYTECODE = 1 install_command = python -m pip install -U {opts} {packages} -extras = test +dependency_groups = test commands = python -m pip list python -m pip check @@ -37,34 +37,41 @@ commands = {[testenv]commands} # Test with python 3.13 and all optional dependencies [testenv:py313-optional] -extras = {[testenv]extras}, full +extras = full +dependency_groups = {[testenv]dependency_groups} commands = {[testenv]commands} # Test with python 3.13 and all optional dependencies, using pre-release versions [testenv:py313-prerelease] install_command = python -m pip install -U --pre {opts} {packages} -extras = {[testenv]extras}, full +extras = full +dependency_groups = {[testenv]dependency_groups} commands = {[testenv]commands} # Test with python 3.14 and all optional dependencies [testenv:py314-optional] -extras = {[testenv]extras}, full +extras = full +dependency_groups = {[testenv]dependency_groups} commands = {[testenv]commands} # Test with python 3.14 and all optional dependencies, using pre-release versions [testenv:py314-prerelease] install_command = python -m pip install -U --pre {opts} {packages} -extras = {[testenv]extras}, full +extras = full +dependency_groups = {[testenv]dependency_groups} commands = {[testenv]commands} -# Test with python 3.10 and minimum dependencies +# Test with python 3.11 and minimum dependencies +# Uses uv with --resolution lowest-direct to install the lowest compatible +# versions of direct dependencies, replacing the old requirements-min.txt approach. [testenv:py311-minimum] -install_command = - python -m pip install {opts} {packages} -deps = - -r requirements-min.txt +skip_install = True +dependency_groups = test +commands_pre = + python -m pip install uv + python -m uv pip install --resolution lowest-direct -e . commands = {[testenv]commands} @@ -99,32 +106,37 @@ commands = {[testenv:gallery]commands} # Test with python 3.13 and all optional dependencies [testenv:gallery-py313-optional] -extras = {[testenv:gallery]extras}, full +extras = full +dependency_groups = docs commands = {[testenv:gallery]commands} # Test with python 3.13 and all optional dependencies, using pre-release versions [testenv:gallery-py313-prerelease] install_command = python -m pip install -U --pre {opts} {packages} -extras = {[testenv:gallery]extras}, full +extras = full +dependency_groups = docs commands = {[testenv:gallery]commands} # Test with python 3.14 and all optional dependencies [testenv:gallery-py314-optional] -extras = {[testenv:gallery]extras}, full +extras = full +dependency_groups = docs commands = {[testenv:gallery]commands} # Test with python 3.14 and all optional dependencies, using pre-release versions [testenv:gallery-py314-prerelease] install_command = python -m pip install -U --pre {opts} {packages} -extras = {[testenv:gallery]extras}, full +extras = full +dependency_groups = docs commands = {[testenv:gallery]commands} -# Test with python 3.10 and minimum dependencies +# Test with python 3.11 and minimum dependencies [testenv:gallery-py311-minimum] -install_command = - python -m pip install {opts} {packages} -deps = - -r requirements-min.txt +skip_install = True +dependency_groups = test +commands_pre = + python -m pip install uv + python -m uv pip install --resolution lowest-direct --group docs -e . commands = {[testenv:gallery]commands}