Migrate hdmf-zarr from zarr-python v2 to v3

.github/workflows/run_all_tests.yml

@@ -25,24 +25,21 @@ jobs:
       fail-fast: false
       matrix:
         include:
-          - { name: linux-python3.10-minimum     , test-tox-env: py310-minimum   , python-ver: "3.10", os: ubuntu-latest }
-          - { name: linux-python3.10             , test-tox-env: py310           , python-ver: "3.10", os: ubuntu-latest }
+          - { name: linux-python3.11-minimum     , test-tox-env: py311-minimum   , python-ver: "3.11", os: ubuntu-latest }
           - { name: linux-python3.11             , test-tox-env: py311           , python-ver: "3.11", os: ubuntu-latest }
           - { name: linux-python3.12             , test-tox-env: py312           , python-ver: "3.12", os: ubuntu-latest }
           - { name: linux-python3.13             , test-tox-env: py313           , python-ver: "3.13", os: ubuntu-latest }
           - { name: linux-python3.14             , test-tox-env: py314           , python-ver: "3.14", os: ubuntu-latest }
           - { name: linux-python3.14-optional    , test-tox-env: py314-optional  , python-ver: "3.14", os: ubuntu-latest }
           - { name: linux-python3.14-prerelease  , test-tox-env: py314-prerelease, python-ver: "3.14", os: ubuntu-latest }
-          - { name: windows-python3.10-minimum   , test-tox-env: py310-minimum   , python-ver: "3.10", os: windows-latest }
-          - { name: windows-python3.10           , test-tox-env: py310           , python-ver: "3.10", os: windows-latest }
+          - { name: windows-python3.11-minimum   , test-tox-env: py311-minimum   , python-ver: "3.11", os: windows-latest }
           - { name: windows-python3.11           , test-tox-env: py311           , python-ver: "3.11", os: windows-latest }
           - { name: windows-python3.12           , test-tox-env: py312           , python-ver: "3.12", os: windows-latest }
           - { name: windows-python3.13           , test-tox-env: py313           , python-ver: "3.13", os: windows-latest }
           - { name: windows-python3.14           , test-tox-env: py314           , python-ver: "3.14", os: windows-latest }
           - { name: windows-python3.14-optional  , test-tox-env: py314-optional  , python-ver: "3.14", os: windows-latest }
           - { name: windows-python3.14-prerelease, test-tox-env: py314-prerelease, python-ver: "3.14", os: windows-latest }
-          - { name: macos-python3.10-minimum     , test-tox-env: py310-minimum   , python-ver: "3.10", os: macos-15-intel }
-          - { name: macos-python3.10             , test-tox-env: py310           , python-ver: "3.10", os: macos-latest }
+          - { name: macos-python3.11-minimum     , test-tox-env: py311-minimum   , python-ver: "3.11", os: macos-15-intel }
           - { name: macos-python3.11             , test-tox-env: py311           , python-ver: "3.11", os: macos-latest }
           - { name: macos-python3.12             , test-tox-env: py312           , python-ver: "3.12", os: macos-latest }
           - { name: macos-python3.13             , test-tox-env: py313           , python-ver: "3.13", os: macos-latest }
@@ -100,13 +97,13 @@ jobs:
       fail-fast: false
       matrix:
         include:
-          - { name: linux-gallery-python3.10-minimum     , test-tox-env: gallery-py310-minimum   , python-ver: "3.10", os: ubuntu-latest }
+          - { name: linux-gallery-python3.11-minimum     , test-tox-env: gallery-py311-minimum   , python-ver: "3.11", os: ubuntu-latest }
           - { name: linux-gallery-python3.14-optional    , test-tox-env: gallery-py314-optional  , python-ver: "3.14", os: ubuntu-latest }
           - { name: linux-gallery-python3.14-prerelease  , test-tox-env: gallery-py314-prerelease, python-ver: "3.14", os: ubuntu-latest }
-          - { name: windows-gallery-python3.10-minimum   , test-tox-env: gallery-py310-minimum   , python-ver: "3.10", os: windows-latest }
+          - { name: windows-gallery-python3.11-minimum   , test-tox-env: gallery-py311-minimum   , python-ver: "3.11", os: windows-latest }
           - { name: windows-gallery-python3.14-optional  , test-tox-env: gallery-py314-optional  , python-ver: "3.14", os: windows-latest }
           - { name: windows-gallery-python3.14-prerelease, test-tox-env: gallery-py314-prerelease, python-ver: "3.14", os: windows-latest }
-          - { name: macos-gallery-python3.10-minimum     , test-tox-env: gallery-py310-minimum   , python-ver: "3.10", os: macos-15-intel }
+          - { name: macos-gallery-python3.11-minimum     , test-tox-env: gallery-py311-minimum   , python-ver: "3.11", os: macos-15-intel }
           - { name: macos-gallery-python3.13-optional    , test-tox-env: gallery-py313-optional  , python-ver: "3.13", os: macos-latest }
           - { name: macos-gallery-python3.13-prerelease  , test-tox-env: gallery-py313-prerelease, python-ver: "3.13", os: macos-latest }
           # No prebuilt wheels for numcodecs==0.15.1 for Python 3.14 on PyPI and cannot build numcodecs == 0.15.1 on macos-latest
@@ -147,8 +144,7 @@ jobs:
       fail-fast: false
       matrix:
         include:
-          - { name: conda-linux-python3.10-minimum   , test-tox-env: py310-minimum   , python-ver: "3.10", os: ubuntu-latest }
-          - { name: conda-linux-python3.10           , test-tox-env: py310           , python-ver: "3.10", os: ubuntu-latest }
+          - { name: conda-linux-python3.11-minimum   , test-tox-env: py311-minimum   , python-ver: "3.11", os: ubuntu-latest }
           - { name: conda-linux-python3.11           , test-tox-env: py311           , python-ver: "3.11", os: ubuntu-latest }
           - { name: conda-linux-python3.12           , test-tox-env: py312           , python-ver: "3.12", os: ubuntu-latest }
           - { name: conda-linux-python3.13           , test-tox-env: py313           , python-ver: "3.13", os: ubuntu-latest }

.github/workflows/run_tests.yml

@@ -19,13 +19,13 @@ jobs:
       fail-fast: false
       matrix:
         include:
-          - { name: linux-python3.10-minimum   , test-tox-env: py310-minimum , python-ver: "3.10", os: ubuntu-latest }
+          - { name: linux-python3.11-minimum   , test-tox-env: py311-minimum , python-ver: "3.11", os: ubuntu-latest }
           - { name: linux-python3.14           , test-tox-env: py314         , python-ver: "3.14", os: ubuntu-latest }
           - { name: linux-python3.14-optional  , test-tox-env: py314-optional, python-ver: "3.14", os: ubuntu-latest }
-          - { name: windows-python3.10-minimum , test-tox-env: py310-minimum , python-ver: "3.10", os: windows-latest }
+          - { name: windows-python3.11-minimum , test-tox-env: py311-minimum , python-ver: "3.11", os: windows-latest }
           - { name: windows-python3.14         , test-tox-env: py314         , python-ver: "3.14", os: windows-latest }
           - { name: windows-python3.14-optional, test-tox-env: py314-optional, python-ver: "3.14", os: windows-latest }
-          - { name: macos-python3.10-minimum   , test-tox-env: py310-minimum , python-ver: "3.10", os: macos-15-intel }
+          - { name: macos-python3.11-minimum   , test-tox-env: py311-minimum , python-ver: "3.11", os: macos-15-intel }
           - { name: macos-python3.13           , test-tox-env: py313         , python-ver: "3.13", os: macos-latest }
           - { name: macos-python3.13-optional  , test-tox-env: py313-optional, python-ver: "3.13", os: macos-latest }
           # No prebuilt wheels for numcodecs==0.15.1 for Python 3.14 on PyPI and cannot build numcodecs == 0.15.1 on macos-latest
@@ -79,9 +79,9 @@ jobs:
       fail-fast: false
       matrix:
         include:
-          - { name: linux-gallery-python3.10-minimum   , test-tox-env: gallery-py310-minimum , python-ver: "3.10", os: ubuntu-latest }
+          - { name: linux-gallery-python3.11-minimum   , test-tox-env: gallery-py311-minimum , python-ver: "3.11", os: ubuntu-latest }
           - { name: linux-gallery-python3.14-optional  , test-tox-env: gallery-py314-optional, python-ver: "3.14", os: ubuntu-latest }
-          - { name: windows-gallery-python3.10-minimum , test-tox-env: gallery-py310-minimum , python-ver: "3.10", os: windows-latest }
+          - { name: windows-gallery-python3.11-minimum , test-tox-env: gallery-py311-minimum , python-ver: "3.11", os: windows-latest }
           - { name: windows-gallery-python3.14-optional, test-tox-env: gallery-py314-optional, python-ver: "3.14", os: windows-latest }
     steps:
       - name: Checkout repo
@@ -117,7 +117,7 @@ jobs:
       fail-fast: false
       matrix:
         include:
-          - { name: conda-linux-python3.10-minimum , test-tox-env: py310-minimum , python-ver: "3.10", os: ubuntu-latest }
+          - { name: conda-linux-python3.11-minimum , test-tox-env: py311-minimum , python-ver: "3.11", os: ubuntu-latest }
           - { name: conda-linux-python3.14         , test-tox-env: py314         , python-ver: "3.14", os: ubuntu-latest }
           - { name: conda-linux-python3.14-optional, test-tox-env: py314-optional, python-ver: "3.14", os: ubuntu-latest }
     steps:

docs/gallery/plot_s3_streaming.py

@@ -143,7 +143,7 @@
 #     Zarr store. This step is critical before uploading modified files to S3.
 #
 #     For more details on consolidated metadata, see the
-#     :zarr-docs:`Zarr documentation <tutorial.html#consolidating-metadata>` and the
+#     `Zarr documentation <https://zarr.readthedocs.io/en/stable/user-guide/consolidated_metadata.html>`_ and the
 #     :ref:`sec-zarr-storage` section of the hdmf-zarr documentation.
 
 ###############################################################################

docs/gallery/plot_zarr_dataset_io.py

@@ -41,13 +41,13 @@
 # To define custom settings for write (e.g., for chunking and compression) we simply
 # wrap our data array using  :py:class:`~hdmf_zarr.utils.ZarrDataIO`.
 
-from numcodecs import Blosc
+from zarr.codecs import BloscCodec
 
 data_with_data_io = ZarrDataIO(
     data=data * 3,
     chunks=(10, 10),
     fillvalue=0,
-    compressor=Blosc(cname='zstd', clevel=1, shuffle=Blosc.SHUFFLE)
+    compressor=BloscCodec(cname='zstd', clevel=1, shuffle='shuffle')
 )
 
 ###############################################################################
@@ -103,10 +103,11 @@
 # Check dataset settings used.
 #
 for c in intable.columns:
-    print("Name=%s, Chunks=% s, Compressor=%s" %
+    compressors = c.data.compressors if hasattr(c.data, 'compressors') else []
+    print("Name=%s, Chunks=%s, Compressors=%s" %
           (c.name,
            str(c.data.chunks),
-           str(c.data.compressor)))
+           str(compressors)))
 
 ###############################################################################
 #

docs/gallery/plot_zarr_io.py

@@ -145,8 +145,7 @@
 # -----------------------------------
 #
 # :py:class:`~hdmf_zarr.backend.ZarrIO` supports a subset of data stores available
-# for Zarr, e.g., :py:class`~zarr.storage.DirectoryStore`, :py:class`~zarr.storage.TempStore`,
-# and :py:class`~zarr.storage.NestedDirectoryStore`. The supported stores are defined
+# for Zarr, e.g., :py:class:`~zarr.storage.LocalStore`. The supported stores are defined
 # in :py:attr:`~hdmf_zarr.backend.SUPPORTED_ZARR_STORES`. A main limitation to supporting
 # all possible Zarr stores in :py:class:`~hdmf_zarr.backend.ZarrIO` is due to the fact that
 # Zarr does not support links and references.
@@ -157,29 +156,29 @@
 #     new stores with :py:class:`~hdmf_zarr.backend.ZarrIO`.
 #
 # To use a store other than the default, we simply need to instantiate the store
-# and set pass it to :py:class:`~hdmf_zarr.backend.ZarrIO` via the ``path`` parameter.
-# Here we use a :py:class`~zarr.storage.NestedDirectoryStore` to write a simple
+# and pass it to :py:class:`~hdmf_zarr.backend.ZarrIO` via the ``path`` parameter.
+# Here we use a :py:class:`~zarr.storage.LocalStore` to write a simple
 # :py:class:`hdmf.common.CSRMatrix` container to disk.
 #
 
-from zarr.storage import NestedDirectoryStore
+from zarr.storage import LocalStore
 from hdmf.common import CSRMatrix
 
-zarr_nsd_dir = "example_nested_store.zarr"
-store = NestedDirectoryStore(zarr_dir)
+zarr_ls_dir = "example_local_store.zarr"
+store = LocalStore(zarr_ls_dir)
 csr_container = CSRMatrix(
     name=ROOT_NAME,
     data=[1, 2, 3, 4, 5, 6],
     indices=[0, 2, 2, 0, 1, 2],
     indptr=[0, 2, 3, 6],
     shape=(3, 3))
 
-# Write the csr_container to Zarr using a NestedDirectoryStore
-with ZarrIO(path=zarr_nsd_dir,  manager=get_manager(), mode='w') as zarr_io:
+# Write the csr_container to Zarr using a LocalStore
+with ZarrIO(path=zarr_ls_dir,  manager=get_manager(), mode='w') as zarr_io:
     zarr_io.write(csr_container)
 
 # Read the CSR matrix to confirm the data was written correctly
-with ZarrIO(path=zarr_nsd_dir, manager=get_manager(), mode='r') as zarr_io:
+with ZarrIO(path=zarr_ls_dir, manager=get_manager(), mode='r') as zarr_io:
     csr_read = zarr_io.read()
     print(" data=%s\n indices=%s\n indptr=%s\n shape=%s" %
           (str(csr_read.data), str(csr_read.indices), str(csr_read.indptr), str(csr_read.shape)))

docs/source/conf.py

@@ -83,14 +83,14 @@
     'pandas': ('https://pandas.pydata.org/pandas-docs/stable/', None),
     'hdmf': ('https://hdmf.readthedocs.io/en/stable/', None),
     'pynwb': ('https://pynwb.readthedocs.io/en/stable/', None),
-    'zarr': ('https://zarr.readthedocs.io/en/v2.18.4/', None)
+    'zarr': ('https://zarr.readthedocs.io/en/stable/', None)
 }
 
 # Use this for mapping to external links
 extlinks = {
     'pynwb-docs': ('https://pynwb.readthedocs.io/en/stable/%s', '%s'),
     'hdmf-docs': ('https://hdmf.readthedocs.io/en/stable/%s', '%s'),
-    'zarr-docs': ('https://zarr.readthedocs.io/en/v2.18.4/%s', '%s')
+    'zarr-docs': ('https://zarr.readthedocs.io/en/stable/%s', '%s')
 }
 
 # Add any paths that contain templates here, relative to this directory.

docs/source/index.rst

@@ -6,15 +6,14 @@
 Welcome to hdmf-zarr's documentation!
 =====================================
 
-**hdmf_zarr** implements a Zarr v2 backend for `HDMF <https://hdmf.readthedocs.io/en/stable/>`_ as well as
+**hdmf_zarr** implements a `Zarr <https://zarr.readthedocs.io/en/stable/>`_ backend for
+`HDMF <https://hdmf.readthedocs.io/en/stable/>`_ as well as
 convenience classes for integration of Zarr with `PyNWB <https://pynwb.readthedocs.io/en/stable/>`_ to
-support writing of NWB files to `Zarr v2 <https://zarr.readthedocs.io/en/v2.18.4/>`_.
+support writing of NWB files to Zarr.
 
 **Status:** The Zarr backend is **under development** and may still change. See the
 :ref:`sec-overview` section for a description of available features and known limitations of hdmf-zarr.
 
-Support for Zarr v3 is planned. You can track progress of the support in https://github.com/hdmf-dev/hdmf-zarr/issues/202.
-
 
 Citing hdmf-zarr
 ^^^^^^^^^^^^^^^^

docs/source/integrating_data_stores.rst

@@ -5,7 +5,7 @@ Integrating New Zarr Data Stores
 ================================
 
 :py:class:`~hdmf_zarr.backend.ZarrIO` by default uses the Zarr
-:zarr-docs:`DirectoryStore <api/storage.html#zarr.storage.DirectoryStore>` via
+:zarr-docs:`LocalStore <api/zarr/storage/index.html#zarr.storage.LocalStore>` via
 the :py:meth:`zarr.convenience.open` method. :py:class:`~hdmf_zarr.backend.ZarrIO` further
 supports all stores listed in :py:class:`~hdmf_zarr.backend.SUPPORTED_ZARR_STORES`.
 Users can specify a particular store using the ``path`` parameter when creating a new

docs/source/overview.rst

@@ -30,7 +30,9 @@ Known Limitations
 -----------------
 
 - The Zarr backend is currently experimental and may still change.
-- Attributes are stored as JSON documents in Zarr (using the DirectoryStore). As such, all attributes must be JSON serializable. The :py:class:`~hdmf_zarr.backend.ZarrIO` backend attempts to cast types to JSON serializable types as much as possible.
-- Currently the :py:class:`~hdmf_zarr.backend.ZarrIO` backend supports Zarr's directory-based stores :py:class:`~zarr.storage.DirectoryStore`, :py:class:`~zarr.storage.NestedDirectoryStore`, and :py:class:`~zarr.storage.TempStore`. Other `Zarr stores <https://zarr.readthedocs.io/en/v2.18.4/api/storage.html>`_ could be added but will require proper treatment of links and references for those backends as links are not supported in Zarr (see `zarr-python issues #389 <https://github.com/zarr-developers/zarr-python/issues/389>`_.
-- Exporting of HDF5 files with external links is not yet fully implemented/tested. (see `hdmf-zarr issue #49 <https://github.com/hdmf-dev/hdmf-zarr/issues/49>`_.
+- **Zarr v3 only:** This version of hdmf-zarr writes zarr v3 format only and cannot read files written in zarr v2 format. To read zarr v2 files, use hdmf-zarr 0.x with zarr-python v2. The `zarr CLI migration tool <https://zarr.readthedocs.io/en/stable/user-guide/cli/>`_ may also be used to convert v2 files to v3 format.
+- Attributes are stored as JSON documents in Zarr (using the :py:class:`~zarr.storage.LocalStore`). As such, all attributes must be JSON serializable. The :py:class:`~hdmf_zarr.backend.ZarrIO` backend attempts to cast types to JSON serializable types as much as possible.
+- Currently the :py:class:`~hdmf_zarr.backend.ZarrIO` backend supports Zarr's :py:class:`~zarr.storage.LocalStore` for local storage and :py:class:`~zarr.storage.FsspecStore` for remote read-only access. Other `Zarr stores <https://zarr.readthedocs.io/en/stable/user-guide/storage/>`_ could be added but will require proper treatment of links and references for those backends as links are not supported in Zarr (see `zarr-python issues #389 <https://github.com/zarr-developers/zarr-python/issues/389>`_).
+- **Compound dtypes with strings:** String and reference fields in compound data types are stored as fixed-length Unicode strings (``FixedLengthUTF32``) in zarr v3. The ``FixedLengthUTF32`` data type is currently not part of the Zarr V3 specification and may not be supported by other zarr implementations.
+- Exporting of HDF5 files with external links is not yet fully implemented/tested (see `hdmf-zarr issue #49 <https://github.com/hdmf-dev/hdmf-zarr/issues/49>`_).
 - Special characters (e.g., ``:``, ``<``, ``>``, ``"``, ``/``, ``\``, ``|``, ``?``, or ``*``) may not be supported by all file systems (e.g., on Windows) and as such should not be used as part of the names of Datasets or Groups as Zarr needs to create folders on the filesystem for these objects.

docs/source/storage.rst

@@ -4,7 +4,7 @@
 Storage Specification
 =====================
 
-hdmf-zarr currently uses the Zarr :zarr-docs:`DirectoryStore <api/storage.html#zarr.storage.DirectoryStore>`,
+hdmf-zarr currently uses the Zarr :zarr-docs:`LocalStore <api/zarr/storage/index.html#zarr.storage.LocalStore>`,
 which uses directories and files on a standard file system to serialize data.
 
 Format Mapping
@@ -247,12 +247,9 @@ Zarr file. The individual object references are defined in the
 :py:class:`~hdmf_zarr.backend.ZarrIO` as py:class:`~hdmf_zarr.utils.ZarrReference` object created via
 the :py:meth:`~hdmf_zarr.backend.ZarrIO._create_ref` helper function.
 
-By default, :py:class:`~hdmf_zarr.backend.ZarrIO` uses the ``numcodecs.pickles.Pickle`` codec to
-encode object references defined as py:class:`~hdmf_zarr.utils.ZarrReference` dicts in datasets.
-Users may set the codec used to encode objects in Zarr datasets via the ``object_codec_class``
-parameter of the :py:func:`~hdmf_zarr.backend.ZarrIO.__init__` constructor of
-:py:class:`~hdmf_zarr.backend.ZarrIO`. E.g.,  we could use
-``ZarrIO( ... , object_codec_class=numcodecs.JSON)`` to serialize objects using JSON.
+In zarr v3, :py:class:`~hdmf_zarr.backend.ZarrIO` stores object references as JSON-serialized strings
+in variable-length string (``StringDType``) datasets. Each element is a JSON string encoding a
+py:class:`~hdmf_zarr.utils.ZarrReference` dict.
 
 Storing object references in Attributes
 ---------------------------------------
@@ -337,6 +334,14 @@ The mappings of data types is as follows
     |                          | ``2018-09-28T14:43:54.123+02:00``  |                |
     +--------------------------+------------------------------------+----------------+
 
+.. note::
+
+    In zarr v3, string and reference fields within compound dtypes are stored as fixed-length
+    Unicode strings (``FixedLengthUTF32``). The string length is dynamically sized to fit the
+    actual data, with a minimum of :py:attr:`~hdmf_zarr.backend.COMPOUND_DTYPE_MIN_STRING_LENGTH`
+    characters to allow for appending rows with longer values. Reference fields are stored as
+    JSON-serialized strings within these fixed-length fields.
+
 .. note::
 
     For compound dtypes, the ``zarr_dtype`` attribute is stored as a list of dictionaries,