Migrate hdmf-zarr from zarr-python v2 to v3

.github/workflows/run_all_tests.yml

@@ -25,24 +25,21 @@ jobs:
       fail-fast: false
       matrix:
         include:
-          - { name: linux-python3.10-minimum     , test-tox-env: py310-minimum   , python-ver: "3.10", os: ubuntu-latest }
-          - { name: linux-python3.10             , test-tox-env: py310           , python-ver: "3.10", os: ubuntu-latest }
+          - { name: linux-python3.11-minimum     , test-tox-env: py311-minimum   , python-ver: "3.11", os: ubuntu-latest }
           - { name: linux-python3.11             , test-tox-env: py311           , python-ver: "3.11", os: ubuntu-latest }
           - { name: linux-python3.12             , test-tox-env: py312           , python-ver: "3.12", os: ubuntu-latest }
           - { name: linux-python3.13             , test-tox-env: py313           , python-ver: "3.13", os: ubuntu-latest }
           - { name: linux-python3.14             , test-tox-env: py314           , python-ver: "3.14", os: ubuntu-latest }
           - { name: linux-python3.14-optional    , test-tox-env: py314-optional  , python-ver: "3.14", os: ubuntu-latest }
           - { name: linux-python3.14-prerelease  , test-tox-env: py314-prerelease, python-ver: "3.14", os: ubuntu-latest }
-          - { name: windows-python3.10-minimum   , test-tox-env: py310-minimum   , python-ver: "3.10", os: windows-latest }
-          - { name: windows-python3.10           , test-tox-env: py310           , python-ver: "3.10", os: windows-latest }
+          - { name: windows-python3.11-minimum   , test-tox-env: py311-minimum   , python-ver: "3.11", os: windows-latest }
           - { name: windows-python3.11           , test-tox-env: py311           , python-ver: "3.11", os: windows-latest }
           - { name: windows-python3.12           , test-tox-env: py312           , python-ver: "3.12", os: windows-latest }
           - { name: windows-python3.13           , test-tox-env: py313           , python-ver: "3.13", os: windows-latest }
           - { name: windows-python3.14           , test-tox-env: py314           , python-ver: "3.14", os: windows-latest }
           - { name: windows-python3.14-optional  , test-tox-env: py314-optional  , python-ver: "3.14", os: windows-latest }
           - { name: windows-python3.14-prerelease, test-tox-env: py314-prerelease, python-ver: "3.14", os: windows-latest }
-          - { name: macos-python3.10-minimum     , test-tox-env: py310-minimum   , python-ver: "3.10", os: macos-15-intel }
-          - { name: macos-python3.10             , test-tox-env: py310           , python-ver: "3.10", os: macos-latest }
+          - { name: macos-python3.11-minimum     , test-tox-env: py311-minimum   , python-ver: "3.11", os: macos-15-intel }
           - { name: macos-python3.11             , test-tox-env: py311           , python-ver: "3.11", os: macos-latest }
           - { name: macos-python3.12             , test-tox-env: py312           , python-ver: "3.12", os: macos-latest }
           - { name: macos-python3.13             , test-tox-env: py313           , python-ver: "3.13", os: macos-latest }
@@ -100,13 +97,13 @@ jobs:
       fail-fast: false
       matrix:
         include:
-          - { name: linux-gallery-python3.10-minimum     , test-tox-env: gallery-py310-minimum   , python-ver: "3.10", os: ubuntu-latest }
+          - { name: linux-gallery-python3.11-minimum     , test-tox-env: gallery-py311-minimum   , python-ver: "3.11", os: ubuntu-latest }
           - { name: linux-gallery-python3.14-optional    , test-tox-env: gallery-py314-optional  , python-ver: "3.14", os: ubuntu-latest }
           - { name: linux-gallery-python3.14-prerelease  , test-tox-env: gallery-py314-prerelease, python-ver: "3.14", os: ubuntu-latest }
-          - { name: windows-gallery-python3.10-minimum   , test-tox-env: gallery-py310-minimum   , python-ver: "3.10", os: windows-latest }
+          - { name: windows-gallery-python3.11-minimum   , test-tox-env: gallery-py311-minimum   , python-ver: "3.11", os: windows-latest }
           - { name: windows-gallery-python3.14-optional  , test-tox-env: gallery-py314-optional  , python-ver: "3.14", os: windows-latest }
           - { name: windows-gallery-python3.14-prerelease, test-tox-env: gallery-py314-prerelease, python-ver: "3.14", os: windows-latest }
-          - { name: macos-gallery-python3.10-minimum     , test-tox-env: gallery-py310-minimum   , python-ver: "3.10", os: macos-15-intel }
+          - { name: macos-gallery-python3.11-minimum     , test-tox-env: gallery-py311-minimum   , python-ver: "3.11", os: macos-15-intel }
           - { name: macos-gallery-python3.13-optional    , test-tox-env: gallery-py313-optional  , python-ver: "3.13", os: macos-latest }
           - { name: macos-gallery-python3.13-prerelease  , test-tox-env: gallery-py313-prerelease, python-ver: "3.13", os: macos-latest }
           # No prebuilt wheels for numcodecs==0.15.1 for Python 3.14 on PyPI and cannot build numcodecs == 0.15.1 on macos-latest
@@ -147,8 +144,7 @@ jobs:
       fail-fast: false
       matrix:
         include:
-          - { name: conda-linux-python3.10-minimum   , test-tox-env: py310-minimum   , python-ver: "3.10", os: ubuntu-latest }
-          - { name: conda-linux-python3.10           , test-tox-env: py310           , python-ver: "3.10", os: ubuntu-latest }
+          - { name: conda-linux-python3.11-minimum   , test-tox-env: py311-minimum   , python-ver: "3.11", os: ubuntu-latest }
           - { name: conda-linux-python3.11           , test-tox-env: py311           , python-ver: "3.11", os: ubuntu-latest }
           - { name: conda-linux-python3.12           , test-tox-env: py312           , python-ver: "3.12", os: ubuntu-latest }
           - { name: conda-linux-python3.13           , test-tox-env: py313           , python-ver: "3.13", os: ubuntu-latest }

.github/workflows/run_tests.yml

@@ -19,13 +19,13 @@ jobs:
       fail-fast: false
       matrix:
         include:
-          - { name: linux-python3.10-minimum   , test-tox-env: py310-minimum , python-ver: "3.10", os: ubuntu-latest }
+          - { name: linux-python3.11-minimum   , test-tox-env: py311-minimum , python-ver: "3.11", os: ubuntu-latest }
           - { name: linux-python3.14           , test-tox-env: py314         , python-ver: "3.14", os: ubuntu-latest }
           - { name: linux-python3.14-optional  , test-tox-env: py314-optional, python-ver: "3.14", os: ubuntu-latest }
-          - { name: windows-python3.10-minimum , test-tox-env: py310-minimum , python-ver: "3.10", os: windows-latest }
+          - { name: windows-python3.11-minimum , test-tox-env: py311-minimum , python-ver: "3.11", os: windows-latest }
           - { name: windows-python3.14         , test-tox-env: py314         , python-ver: "3.14", os: windows-latest }
           - { name: windows-python3.14-optional, test-tox-env: py314-optional, python-ver: "3.14", os: windows-latest }
-          - { name: macos-python3.10-minimum   , test-tox-env: py310-minimum , python-ver: "3.10", os: macos-15-intel }
+          - { name: macos-python3.11-minimum   , test-tox-env: py311-minimum , python-ver: "3.11", os: macos-15-intel }
           - { name: macos-python3.13           , test-tox-env: py313         , python-ver: "3.13", os: macos-latest }
           - { name: macos-python3.13-optional  , test-tox-env: py313-optional, python-ver: "3.13", os: macos-latest }
           # No prebuilt wheels for numcodecs==0.15.1 for Python 3.14 on PyPI and cannot build numcodecs == 0.15.1 on macos-latest
@@ -79,9 +79,9 @@ jobs:
       fail-fast: false
       matrix:
         include:
-          - { name: linux-gallery-python3.10-minimum   , test-tox-env: gallery-py310-minimum , python-ver: "3.10", os: ubuntu-latest }
+          - { name: linux-gallery-python3.11-minimum   , test-tox-env: gallery-py311-minimum , python-ver: "3.11", os: ubuntu-latest }
           - { name: linux-gallery-python3.14-optional  , test-tox-env: gallery-py314-optional, python-ver: "3.14", os: ubuntu-latest }
-          - { name: windows-gallery-python3.10-minimum , test-tox-env: gallery-py310-minimum , python-ver: "3.10", os: windows-latest }
+          - { name: windows-gallery-python3.11-minimum , test-tox-env: gallery-py311-minimum , python-ver: "3.11", os: windows-latest }
           - { name: windows-gallery-python3.14-optional, test-tox-env: gallery-py314-optional, python-ver: "3.14", os: windows-latest }
     steps:
       - name: Checkout repo
@@ -117,7 +117,7 @@ jobs:
       fail-fast: false
       matrix:
         include:
-          - { name: conda-linux-python3.10-minimum , test-tox-env: py310-minimum , python-ver: "3.10", os: ubuntu-latest }
+          - { name: conda-linux-python3.11-minimum , test-tox-env: py311-minimum , python-ver: "3.11", os: ubuntu-latest }
           - { name: conda-linux-python3.14         , test-tox-env: py314         , python-ver: "3.14", os: ubuntu-latest }
           - { name: conda-linux-python3.14-optional, test-tox-env: py314-optional, python-ver: "3.14", os: ubuntu-latest }
     steps:

docs/gallery/plot_s3_streaming.py

@@ -142,7 +142,7 @@
 #     Zarr store. This step is critical before uploading modified files to S3.
 #
 #     For more details on consolidated metadata, see the
-#     :zarr-docs:`Zarr documentation <tutorial.html#consolidating-metadata>` and the
+#     `Zarr documentation <https://zarr.readthedocs.io/en/stable/user-guide/consolidated_metadata.html>`_ and the
 #     :ref:`sec-zarr-storage` section of the hdmf-zarr documentation.
 
 ###############################################################################

docs/gallery/plot_zarr_dataset_io.py

@@ -41,13 +41,13 @@
 # To define custom settings for write (e.g., for chunking and compression) we simply
 # wrap our data array using  :py:class:`~hdmf_zarr.utils.ZarrDataIO`.
 
-from numcodecs import Blosc
+from zarr.codecs import BloscCodec
 
 data_with_data_io = ZarrDataIO(
     data=data * 3,
     chunks=(10, 10),
     fillvalue=0,
-    compressor=Blosc(cname='zstd', clevel=1, shuffle=Blosc.SHUFFLE)
+    compressor=BloscCodec(cname='zstd', clevel=1, shuffle='shuffle')
 )
 
 ###############################################################################
@@ -103,10 +103,11 @@
 # Check dataset settings used.
 #
 for c in intable.columns:
-    print("Name=%s, Chunks=% s, Compressor=%s" %
+    compressors = c.data.compressors if hasattr(c.data, 'compressors') else []
+    print("Name=%s, Chunks=%s, Compressors=%s" %
           (c.name,
            str(c.data.chunks),
-           str(c.data.compressor)))
+           str(compressors)))
 
 ###############################################################################
 #

docs/gallery/plot_zarr_io.py

@@ -145,8 +145,7 @@
 # -----------------------------------
 #
 # :py:class:`~hdmf_zarr.backend.ZarrIO` supports a subset of data stores available
-# for Zarr, e.g., :py:class`~zarr.storage.DirectoryStore`, :py:class`~zarr.storage.TempStore`,
-# and :py:class`~zarr.storage.NestedDirectoryStore`. The supported stores are defined
+# for Zarr, e.g., :py:class:`~zarr.storage.LocalStore`. The supported stores are defined
 # in :py:attr:`~hdmf_zarr.backend.SUPPORTED_ZARR_STORES`. A main limitation to supporting
 # all possible Zarr stores in :py:class:`~hdmf_zarr.backend.ZarrIO` is due to the fact that
 # Zarr does not support links and references.
@@ -157,29 +156,29 @@
 #     new stores with :py:class:`~hdmf_zarr.backend.ZarrIO`.
 #
 # To use a store other than the default, we simply need to instantiate the store
-# and set pass it to :py:class:`~hdmf_zarr.backend.ZarrIO` via the ``path`` parameter.
-# Here we use a :py:class`~zarr.storage.NestedDirectoryStore` to write a simple
+# and pass it to :py:class:`~hdmf_zarr.backend.ZarrIO` via the ``path`` parameter.
+# Here we use a :py:class:`~zarr.storage.LocalStore` to write a simple
 # :py:class:`hdmf.common.CSRMatrix` container to disk.
 #
 
-from zarr.storage import NestedDirectoryStore
+from zarr.storage import LocalStore
 from hdmf.common import CSRMatrix
 
-zarr_nsd_dir = "example_nested_store.zarr"
-store = NestedDirectoryStore(zarr_dir)
+zarr_ls_dir = "example_local_store.zarr"
+store = LocalStore(zarr_ls_dir)
 csr_container = CSRMatrix(
     name=ROOT_NAME,
     data=[1, 2, 3, 4, 5, 6],
     indices=[0, 2, 2, 0, 1, 2],
     indptr=[0, 2, 3, 6],
     shape=(3, 3))
 
-# Write the csr_container to Zarr using a NestedDirectoryStore
-with ZarrIO(path=zarr_nsd_dir,  manager=get_manager(), mode='w') as zarr_io:
+# Write the csr_container to Zarr using a LocalStore
+with ZarrIO(path=zarr_ls_dir,  manager=get_manager(), mode='w') as zarr_io:
     zarr_io.write(csr_container)
 
 # Read the CSR matrix to confirm the data was written correctly
-with ZarrIO(path=zarr_nsd_dir, manager=get_manager(), mode='r') as zarr_io:
+with ZarrIO(path=zarr_ls_dir, manager=get_manager(), mode='r') as zarr_io:
     csr_read = zarr_io.read()
     print(" data=%s\n indices=%s\n indptr=%s\n shape=%s" %
           (str(csr_read.data), str(csr_read.indices), str(csr_read.indptr), str(csr_read.shape)))

docs/source/conf.py

@@ -83,14 +83,14 @@
     'pandas': ('https://pandas.pydata.org/pandas-docs/stable/', None),
     'hdmf': ('https://hdmf.readthedocs.io/en/stable/', None),
     'pynwb': ('https://pynwb.readthedocs.io/en/stable/', None),
-    'zarr': ('https://zarr.readthedocs.io/en/v2.18.4/', None)
+    'zarr': ('https://zarr.readthedocs.io/en/stable/', None)
 }
 
 # Use this for mapping to external links
 extlinks = {
     'pynwb-docs': ('https://pynwb.readthedocs.io/en/stable/%s', '%s'),
     'hdmf-docs': ('https://hdmf.readthedocs.io/en/stable/%s', '%s'),
-    'zarr-docs': ('https://zarr.readthedocs.io/en/v2.18.4/%s', '%s')
+    'zarr-docs': ('https://zarr.readthedocs.io/en/stable/%s', '%s')
 }
 
 # Add any paths that contain templates here, relative to this directory.

docs/source/index.rst

@@ -6,15 +6,14 @@
 Welcome to hdmf-zarr's documentation!
 =====================================
 
-**hdmf_zarr** implements a Zarr v2 backend for `HDMF <https://hdmf.readthedocs.io/en/stable/>`_ as well as
+**hdmf_zarr** implements a `Zarr <https://zarr.readthedocs.io/en/stable/>`_ backend for
+`HDMF <https://hdmf.readthedocs.io/en/stable/>`_ as well as
 convenience classes for integration of Zarr with `PyNWB <https://pynwb.readthedocs.io/en/stable/>`_ to
-support writing of NWB files to `Zarr v2 <https://zarr.readthedocs.io/en/v2.18.4/>`_.
+support writing of NWB files to Zarr.
 
 **Status:** The Zarr backend is **under development** and may still change. See the
 :ref:`sec-overview` section for a description of available features and known limitations of hdmf-zarr.
 
-Support for Zarr v3 is planned. You can track progress of the support in https://github.com/hdmf-dev/hdmf-zarr/issues/202.
-
 
 Citing hdmf-zarr
 ^^^^^^^^^^^^^^^^

docs/source/integrating_data_stores.rst

@@ -5,7 +5,7 @@ Integrating New Zarr Data Stores
 ================================
 
 :py:class:`~hdmf_zarr.backend.ZarrIO` by default uses the Zarr
-:zarr-docs:`DirectoryStore <api/storage.html#zarr.storage.DirectoryStore>` via
+:zarr-docs:`LocalStore <api/zarr/storage/index.html#zarr.storage.LocalStore>` via
 the :py:meth:`zarr.convenience.open` method. :py:class:`~hdmf_zarr.backend.ZarrIO` further
 supports all stores listed in :py:class:`~hdmf_zarr.backend.SUPPORTED_ZARR_STORES`.
 Users can specify a particular store using the ``path`` parameter when creating a new

docs/source/storage.rst

@@ -4,7 +4,7 @@
 Storage Specification
 =====================
 
-hdmf-zarr currently uses the Zarr :zarr-docs:`DirectoryStore <api/storage.html#zarr.storage.DirectoryStore>`,
+hdmf-zarr currently uses the Zarr :zarr-docs:`LocalStore <api/zarr/storage/index.html#zarr.storage.LocalStore>`,
 which uses directories and files on a standard file system to serialize data.
 
 Format Mapping
@@ -247,12 +247,9 @@ Zarr file. The individual object references are defined in the
 :py:class:`~hdmf_zarr.backend.ZarrIO` as py:class:`~hdmf_zarr.utils.ZarrReference` object created via
 the :py:meth:`~hdmf_zarr.backend.ZarrIO._create_ref` helper function.
 
-By default, :py:class:`~hdmf_zarr.backend.ZarrIO` uses the ``numcodecs.pickles.Pickle`` codec to
-encode object references defined as py:class:`~hdmf_zarr.utils.ZarrReference` dicts in datasets.
-Users may set the codec used to encode objects in Zarr datasets via the ``object_codec_class``
-parameter of the :py:func:`~hdmf_zarr.backend.ZarrIO.__init__` constructor of
-:py:class:`~hdmf_zarr.backend.ZarrIO`. E.g.,  we could use
-``ZarrIO( ... , object_codec_class=numcodecs.JSON)`` to serialize objects using JSON.
+In zarr v3, :py:class:`~hdmf_zarr.backend.ZarrIO` stores object references as JSON-serialized strings
+in variable-length string (``StringDType``) datasets. Each element is a JSON string encoding a
+py:class:`~hdmf_zarr.utils.ZarrReference` dict.
 
 Storing object references in Attributes
 ---------------------------------------

pyproject.toml

@@ -12,11 +12,10 @@ authors = [
 ]
 description = "A package defining a Zarr I/O backend for HDMF"
 readme = "README.rst"
-requires-python = ">=3.10"
+requires-python = ">=3.11"
 license = {text = "BSD"}
 classifiers = [
     "Programming Language :: Python",
-    "Programming Language :: Python :: 3.10",
     "Programming Language :: Python :: 3.11",
     "Programming Language :: Python :: 3.12",
     "Programming Language :: Python :: 3.13",
@@ -31,10 +30,10 @@ classifiers = [
     "Topic :: Scientific/Engineering :: Medical Science Apps."
 ]
 dependencies = [
-    "hdmf>=4.1.0",
-    "zarr>=2.18.0, <3.0",  # pin below 3.0 until HDMF-zarr supports zarr 3.0
-    "numpy>=1.24.0",
-    "numcodecs>=0.12.0, <0.16.0",  # numcodecs 0.16.0 is not compatible with some versions of zarr 2.18.* and those versions do not include this restriction
+    "hdmf>=4.2.0",
+    "zarr>=3.1.3",
+    "numpy>=1.26.0",
+    "numcodecs>=0.14.0",
     "pynwb>=2.8.3",
     "threadpoolctl>=3.1.0",
 ]

requirements-min.txt

@@ -1,8 +1,8 @@
 # minimum versions of package dependencies for installing HDMF
 # NOTE: these should match the minimum bound for dependencies in pyproject.toml
-hdmf==4.1.0
-zarr==2.18.0
-numpy==1.24.0
-numcodecs==0.12.0
+hdmf==4.2.0
+zarr==3.1.3
+numpy==1.26.0
+numcodecs==0.14.0
 pynwb==2.8.3
 threadpoolctl==3.1.0