diff --git a/_request.json b/_request.json new file mode 100644 index 000000000..8d91a1aaa --- /dev/null +++ b/_request.json @@ -0,0 +1 @@ +{"input_config": {"store_id": "memory", "data_id": "S2L2A.zarr"}, "cube_config": {"variable_names": ["B01", "B02", "B03"], "crs": "WGS84", "bbox": [12.2, 52.1, 13.9, 54.8], "spatial_res": 0.05, "time_range": ["2010-01-01", null], "time_period": "4D", "chunks": {"time": null, "lat": 90, "lon": 90}, "metadata": {"title": "A S2L2A subset"}, "variable_metadata": {"B01": {"long_name": "Band 1"}, "B02": {"long_name": "Band 2"}, "B03": {"long_name": "Band 3"}}}, "output_config": {"store_id": "memory", "data_id": "CHL.zarr", "replace": true}, "callback_config": {"api_uri": "https://xcube-gen.test/api/v1/jobs/tomtom/iamajob/callback", "access_token": "dfsvdfsv"}} \ No newline at end of file diff --git a/_request.yaml b/_request.yaml new file mode 100644 index 000000000..76ec276ce --- /dev/null +++ b/_request.yaml @@ -0,0 +1,39 @@ +callback_config: + access_token: dfsvdfsv + api_uri: https://xcube-gen.test/api/v1/jobs/tomtom/iamajob/callback +cube_config: + bbox: + - 12.2 + - 52.1 + - 13.9 + - 54.8 + chunks: + lat: 90 + lon: 90 + time: null + crs: WGS84 + metadata: + title: A S2L2A subset + spatial_res: 0.05 + time_period: 4D + time_range: + - '2010-01-01' + - null + variable_metadata: + B01: + long_name: Band 1 + B02: + long_name: Band 2 + B03: + long_name: Band 3 + variable_names: + - B01 + - B02 + - B03 +input_config: + data_id: S2L2A.zarr + store_id: memory +output_config: + data_id: CHL.zarr + replace: true + store_id: memory diff --git a/eurodatacube-test/xcube-eea/test.zarr/A/c/0/0/0 b/eurodatacube-test/xcube-eea/test.zarr/A/c/0/0/0 new file mode 100644 index 000000000..3037fe80e Binary files /dev/null and b/eurodatacube-test/xcube-eea/test.zarr/A/c/0/0/0 differ diff --git a/eurodatacube-test/xcube-eea/test.zarr/A/c/1/0/0 b/eurodatacube-test/xcube-eea/test.zarr/A/c/1/0/0 new file mode 100644 index 000000000..6833c4a94 Binary files /dev/null and b/eurodatacube-test/xcube-eea/test.zarr/A/c/1/0/0 differ diff --git a/eurodatacube-test/xcube-eea/test.zarr/A/zarr.json b/eurodatacube-test/xcube-eea/test.zarr/A/zarr.json new file mode 100644 index 000000000..64c8b32f5 --- /dev/null +++ b/eurodatacube-test/xcube-eea/test.zarr/A/zarr.json @@ -0,0 +1,51 @@ +{ + "shape": [ + 5, + 100, + 100 + ], + "data_type": "float64", + "chunk_grid": { + "name": "regular", + "configuration": { + "chunk_shape": [ + 3, + 100, + 100 + ] + } + }, + "chunk_key_encoding": { + "name": "default", + "configuration": { + "separator": "/" + } + }, + "fill_value": "NaN", + "codecs": [ + { + "name": "bytes", + "configuration": { + "endian": "little" + } + }, + { + "name": "zstd", + "configuration": { + "level": 0, + "checksum": false + } + } + ], + "attributes": { + "_FillValue": "AAAAAAAA+H8=" + }, + "dimension_names": [ + "time", + "y", + "x" + ], + "zarr_format": 3, + "node_type": "array", + "storage_transformers": [] +} \ No newline at end of file diff --git a/eurodatacube-test/xcube-eea/test.zarr/B/c/0/0/0 b/eurodatacube-test/xcube-eea/test.zarr/B/c/0/0/0 new file mode 100644 index 000000000..f0c415f84 Binary files /dev/null and b/eurodatacube-test/xcube-eea/test.zarr/B/c/0/0/0 differ diff --git a/eurodatacube-test/xcube-eea/test.zarr/B/c/1/0/0 b/eurodatacube-test/xcube-eea/test.zarr/B/c/1/0/0 new file mode 100644 index 000000000..675cce54c Binary files /dev/null and b/eurodatacube-test/xcube-eea/test.zarr/B/c/1/0/0 differ diff --git a/eurodatacube-test/xcube-eea/test.zarr/B/zarr.json b/eurodatacube-test/xcube-eea/test.zarr/B/zarr.json new file mode 100644 index 000000000..64c8b32f5 --- /dev/null +++ b/eurodatacube-test/xcube-eea/test.zarr/B/zarr.json @@ -0,0 +1,51 @@ +{ + "shape": [ + 5, + 100, + 100 + ], + "data_type": "float64", + "chunk_grid": { + "name": "regular", + "configuration": { + "chunk_shape": [ + 3, + 100, + 100 + ] + } + }, + "chunk_key_encoding": { + "name": "default", + "configuration": { + "separator": "/" + } + }, + "fill_value": "NaN", + "codecs": [ + { + "name": "bytes", + "configuration": { + "endian": "little" + } + }, + { + "name": "zstd", + "configuration": { + "level": 0, + "checksum": false + } + } + ], + "attributes": { + "_FillValue": "AAAAAAAA+H8=" + }, + "dimension_names": [ + "time", + "y", + "x" + ], + "zarr_format": 3, + "node_type": "array", + "storage_transformers": [] +} \ No newline at end of file diff --git a/eurodatacube-test/xcube-eea/test.zarr/time/c/0 b/eurodatacube-test/xcube-eea/test.zarr/time/c/0 new file mode 100644 index 000000000..17d4981e7 Binary files /dev/null and b/eurodatacube-test/xcube-eea/test.zarr/time/c/0 differ diff --git a/eurodatacube-test/xcube-eea/test.zarr/time/zarr.json b/eurodatacube-test/xcube-eea/test.zarr/time/zarr.json new file mode 100644 index 000000000..8b967f603 --- /dev/null +++ b/eurodatacube-test/xcube-eea/test.zarr/time/zarr.json @@ -0,0 +1,46 @@ +{ + "shape": [ + 5 + ], + "data_type": "int64", + "chunk_grid": { + "name": "regular", + "configuration": { + "chunk_shape": [ + 5 + ] + } + }, + "chunk_key_encoding": { + "name": "default", + "configuration": { + "separator": "/" + } + }, + "fill_value": 0, + "codecs": [ + { + "name": "bytes", + "configuration": { + "endian": "little" + } + }, + { + "name": "zstd", + "configuration": { + "level": 0, + "checksum": false + } + } + ], + "attributes": { + "units": "seconds since 1970-01-01", + "calendar": "proleptic_gregorian" + }, + "dimension_names": [ + "time" + ], + "zarr_format": 3, + "node_type": "array", + "storage_transformers": [] +} \ No newline at end of file diff --git a/eurodatacube-test/xcube-eea/test.zarr/x/c/0 b/eurodatacube-test/xcube-eea/test.zarr/x/c/0 new file mode 100644 index 000000000..2743c0c68 Binary files /dev/null and b/eurodatacube-test/xcube-eea/test.zarr/x/c/0 differ diff --git a/eurodatacube-test/xcube-eea/test.zarr/x/zarr.json b/eurodatacube-test/xcube-eea/test.zarr/x/zarr.json new file mode 100644 index 000000000..e4d127b5b --- /dev/null +++ b/eurodatacube-test/xcube-eea/test.zarr/x/zarr.json @@ -0,0 +1,48 @@ +{ + "shape": [ + 100 + ], + "data_type": "float64", + "chunk_grid": { + "name": "regular", + "configuration": { + "chunk_shape": [ + 100 + ] + } + }, + "chunk_key_encoding": { + "name": "default", + "configuration": { + "separator": "/" + } + }, + "fill_value": "NaN", + "codecs": [ + { + "name": "bytes", + "configuration": { + "endian": "little" + } + }, + { + "name": "zstd", + "configuration": { + "level": 0, + "checksum": false + } + } + ], + "attributes": { + "units": "metres", + "long_name": "x coordinate of projection", + "standard_name": "projection_x_coordinate", + "_FillValue": "AAAAAAAA+H8=" + }, + "dimension_names": [ + "x" + ], + "zarr_format": 3, + "node_type": "array", + "storage_transformers": [] +} \ No newline at end of file diff --git a/eurodatacube-test/xcube-eea/test.zarr/y/c/0 b/eurodatacube-test/xcube-eea/test.zarr/y/c/0 new file mode 100644 index 000000000..2743c0c68 Binary files /dev/null and b/eurodatacube-test/xcube-eea/test.zarr/y/c/0 differ diff --git a/eurodatacube-test/xcube-eea/test.zarr/y/zarr.json b/eurodatacube-test/xcube-eea/test.zarr/y/zarr.json new file mode 100644 index 000000000..f735fb824 --- /dev/null +++ b/eurodatacube-test/xcube-eea/test.zarr/y/zarr.json @@ -0,0 +1,48 @@ +{ + "shape": [ + 100 + ], + "data_type": "float64", + "chunk_grid": { + "name": "regular", + "configuration": { + "chunk_shape": [ + 100 + ] + } + }, + "chunk_key_encoding": { + "name": "default", + "configuration": { + "separator": "/" + } + }, + "fill_value": "NaN", + "codecs": [ + { + "name": "bytes", + "configuration": { + "endian": "little" + } + }, + { + "name": "zstd", + "configuration": { + "level": 0, + "checksum": false + } + } + ], + "attributes": { + "units": "metres", + "long_name": "y coordinate of projection", + "standard_name": "projection_y_coordinate", + "_FillValue": "AAAAAAAA+H8=" + }, + "dimension_names": [ + "y" + ], + "zarr_format": 3, + "node_type": "array", + "storage_transformers": [] +} \ No newline at end of file diff --git a/eurodatacube-test/xcube-eea/test.zarr/zarr.json b/eurodatacube-test/xcube-eea/test.zarr/zarr.json new file mode 100644 index 000000000..91c7745ed --- /dev/null +++ b/eurodatacube-test/xcube-eea/test.zarr/zarr.json @@ -0,0 +1,260 @@ +{ + "attributes": { + "Conventions": "CF-1.7", + "title": "Test Cube", + "time_coverage_start": "2010-01-01T00:00:00.000000000", + "time_coverage_end": "2010-01-06T00:00:00.000000000" + }, + "zarr_format": 3, + "consolidated_metadata": { + "kind": "inline", + "must_understand": false, + "metadata": { + "A": { + "shape": [ + 5, + 100, + 100 + ], + "data_type": "float64", + "chunk_grid": { + "name": "regular", + "configuration": { + "chunk_shape": [ + 3, + 100, + 100 + ] + } + }, + "chunk_key_encoding": { + "name": "default", + "configuration": { + "separator": "/" + } + }, + "fill_value": "NaN", + "codecs": [ + { + "name": "bytes", + "configuration": { + "endian": "little" + } + }, + { + "name": "zstd", + "configuration": { + "level": 0, + "checksum": false + } + } + ], + "attributes": { + "_FillValue": "AAAAAAAA+H8=" + }, + "dimension_names": [ + "time", + "y", + "x" + ], + "zarr_format": 3, + "node_type": "array", + "storage_transformers": [] + }, + "B": { + "shape": [ + 5, + 100, + 100 + ], + "data_type": "float64", + "chunk_grid": { + "name": "regular", + "configuration": { + "chunk_shape": [ + 3, + 100, + 100 + ] + } + }, + "chunk_key_encoding": { + "name": "default", + "configuration": { + "separator": "/" + } + }, + "fill_value": "NaN", + "codecs": [ + { + "name": "bytes", + "configuration": { + "endian": "little" + } + }, + { + "name": "zstd", + "configuration": { + "level": 0, + "checksum": false + } + } + ], + "attributes": { + "_FillValue": "AAAAAAAA+H8=" + }, + "dimension_names": [ + "time", + "y", + "x" + ], + "zarr_format": 3, + "node_type": "array", + "storage_transformers": [] + }, + "time": { + "shape": [ + 5 + ], + "data_type": "int64", + "chunk_grid": { + "name": "regular", + "configuration": { + "chunk_shape": [ + 5 + ] + } + }, + "chunk_key_encoding": { + "name": "default", + "configuration": { + "separator": "/" + } + }, + "fill_value": 0, + "codecs": [ + { + "name": "bytes", + "configuration": { + "endian": "little" + } + }, + { + "name": "zstd", + "configuration": { + "level": 0, + "checksum": false + } + } + ], + "attributes": { + "units": "seconds since 1970-01-01", + "calendar": "proleptic_gregorian" + }, + "dimension_names": [ + "time" + ], + "zarr_format": 3, + "node_type": "array", + "storage_transformers": [] + }, + "x": { + "shape": [ + 100 + ], + "data_type": "float64", + "chunk_grid": { + "name": "regular", + "configuration": { + "chunk_shape": [ + 100 + ] + } + }, + "chunk_key_encoding": { + "name": "default", + "configuration": { + "separator": "/" + } + }, + "fill_value": "NaN", + "codecs": [ + { + "name": "bytes", + "configuration": { + "endian": "little" + } + }, + { + "name": "zstd", + "configuration": { + "level": 0, + "checksum": false + } + } + ], + "attributes": { + "units": "metres", + "long_name": "x coordinate of projection", + "standard_name": "projection_x_coordinate", + "_FillValue": "AAAAAAAA+H8=" + }, + "dimension_names": [ + "x" + ], + "zarr_format": 3, + "node_type": "array", + "storage_transformers": [] + }, + "y": { + "shape": [ + 100 + ], + "data_type": "float64", + "chunk_grid": { + "name": "regular", + "configuration": { + "chunk_shape": [ + 100 + ] + } + }, + "chunk_key_encoding": { + "name": "default", + "configuration": { + "separator": "/" + } + }, + "fill_value": "NaN", + "codecs": [ + { + "name": "bytes", + "configuration": { + "endian": "little" + } + }, + { + "name": "zstd", + "configuration": { + "level": 0, + "checksum": false + } + } + ], + "attributes": { + "units": "metres", + "long_name": "y coordinate of projection", + "standard_name": "projection_y_coordinate", + "_FillValue": "AAAAAAAA+H8=" + }, + "dimension_names": [ + "y" + ], + "zarr_format": 3, + "node_type": "array", + "storage_transformers": [] + } + } + }, + "node_type": "group" +} \ No newline at end of file diff --git a/examples/notebooks/inputdata/S3-OLCI-L2A.zarr.zip b/examples/notebooks/inputdata/S3-OLCI-L2A.zarr.zip deleted file mode 100644 index 4521b00cb..000000000 Binary files a/examples/notebooks/inputdata/S3-OLCI-L2A.zarr.zip and /dev/null differ diff --git a/examples/notebooks/inputdata/S3-OLCI-L2A.zarr/.zattrs b/examples/notebooks/inputdata/S3-OLCI-L2A.zarr/.zattrs new file mode 100644 index 000000000..666d82630 --- /dev/null +++ b/examples/notebooks/inputdata/S3-OLCI-L2A.zarr/.zattrs @@ -0,0 +1,6 @@ +{ + "Conventions": "CF-1.4", + "product_type": "C2RCC_OLCI", + "start_date": "04-JUL-2018 09:21:55.677316", + "stop_date": "04-JUL-2018 09:23:18.811790" +} \ No newline at end of file diff --git a/examples/serve/demo/cube-1-250-250.levels/0.zarr/.zgroup b/examples/notebooks/inputdata/S3-OLCI-L2A.zarr/.zgroup similarity index 100% rename from examples/serve/demo/cube-1-250-250.levels/0.zarr/.zgroup rename to examples/notebooks/inputdata/S3-OLCI-L2A.zarr/.zgroup diff --git a/examples/notebooks/inputdata/S3-OLCI-L2A.zarr/lat/.zarray b/examples/notebooks/inputdata/S3-OLCI-L2A.zarr/lat/.zarray new file mode 100644 index 000000000..e2384f1ab --- /dev/null +++ b/examples/notebooks/inputdata/S3-OLCI-L2A.zarr/lat/.zarray @@ -0,0 +1,22 @@ +{ + "chunks": [ + 512, + 512 + ], + "compressor": { + "blocksize": 0, + "clevel": 5, + "cname": "lz4", + "id": "blosc", + "shuffle": 1 + }, + "dtype": " Size: 208B\n", + "Dimensions: (time: 2, lat: 3, lon: 3)\n", + "Coordinates:\n", + " * time (time) int64 16B 0 1\n", + " * lat (lat) int64 24B 0 1 2\n", + " * lon (lon) int64 24B 0 1 2\n", + "Data variables:\n", + " B01 (time, lat, lon) float64 144B 0.9488 0.001598 ... 0.931 0.728" + ], + "text/html": [ + "
\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "
<xarray.Dataset> Size: 208B\n",
+       "Dimensions:  (time: 2, lat: 3, lon: 3)\n",
+       "Coordinates:\n",
+       "  * time     (time) int64 16B 0 1\n",
+       "  * lat      (lat) int64 24B 0 1 2\n",
+       "  * lon      (lon) int64 24B 0 1 2\n",
+       "Data variables:\n",
+       "    B01      (time, lat, lon) float64 144B 0.9488 0.001598 ... 0.931 0.728
" + ] + }, + "execution_count": 61, + "metadata": {}, + "output_type": "execute_result" + } + ], + "execution_count": 61 + }, + { + "metadata": { + "ExecuteTime": { + "end_time": "2026-05-12T15:38:31.196360369Z", + "start_time": "2026-05-12T15:38:31.135572454Z" + } + }, + "cell_type": "code", + "source": [ + "fs = fsspec.filesystem(\"memory\")\n", + "fs = AsyncFileSystemWrapper(fs)\n", + "store = fs.get_mapper(\"dataset_1.zarr\")\n", + "data.to_zarr(store, mode=\"w\")" + ], + "id": "dc390501912824de", + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/home/konstantin/micromamba/envs/xcube/lib/python3.13/site-packages/zarr/core/group.py:3289: ZarrUserWarning: Object at dataset_1.zarr/zarr.json is not recognized as a component of a Zarr hierarchy.\n", + " warnings.warn(\n", + "/home/konstantin/micromamba/envs/xcube/lib/python3.13/site-packages/zarr/core/group.py:3289: ZarrUserWarning: Object at dataset_1.zarr/lon is not recognized as a component of a Zarr hierarchy.\n", + " warnings.warn(\n", + "/home/konstantin/micromamba/envs/xcube/lib/python3.13/site-packages/zarr/core/group.py:3289: ZarrUserWarning: Object at dataset_1.zarr/B01 is not recognized as a component of a Zarr hierarchy.\n", + " warnings.warn(\n", + "/home/konstantin/micromamba/envs/xcube/lib/python3.13/site-packages/zarr/core/group.py:3289: ZarrUserWarning: Object at dataset_1.zarr/lat is not recognized as a component of a Zarr hierarchy.\n", + " warnings.warn(\n", + "/home/konstantin/micromamba/envs/xcube/lib/python3.13/site-packages/zarr/core/group.py:3289: ZarrUserWarning: Object at dataset_1.zarr/time is not recognized as a component of a Zarr hierarchy.\n", + " warnings.warn(\n" + ] + }, + { + "data": { + "text/plain": [ + "" + ] + }, + "execution_count": 64, + "metadata": {}, + "output_type": "execute_result" + } + ], + "execution_count": 64 + }, + { + "metadata": { + "ExecuteTime": { + "end_time": "2026-05-12T15:38:46.147359161Z", + "start_time": "2026-05-12T15:38:46.122093458Z" + } + }, + "cell_type": "code", + "source": [ + "# read back\n", + "ds = xr.open_zarr(store, consolidated=False)\n", + "ds" + ], + "id": "2463206ecd9d9959", + "outputs": [ + { + "data": { + "text/plain": [ + " Size: 0B\n", + "Dimensions: ()\n", + "Data variables:\n", + " *empty*" + ], + "text/html": [ + "
\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "
<xarray.Dataset> Size: 0B\n",
+       "Dimensions:  ()\n",
+       "Data variables:\n",
+       "    *empty*
" + ] + }, + "execution_count": 65, + "metadata": {}, + "output_type": "execute_result" + } + ], + "execution_count": 65 + }, + { + "metadata": {}, + "cell_type": "code", + "outputs": [], + "execution_count": null, + "source": "", + "id": "e37f7d93623aa95c" + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 2 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython2", + "version": "2.7.6" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/examples/serve/demo/cube-1-250-250.levels/.zlevels b/examples/serve/demo/cube-1-250-250.levels/.zlevels new file mode 100644 index 000000000..80f73e63a --- /dev/null +++ b/examples/serve/demo/cube-1-250-250.levels/.zlevels @@ -0,0 +1,12 @@ +{ + "version": "1.0", + "num_levels": 3, + "use_saved_levels": false, + "agg_methods": { + "c2rcc_flags": "mean", + "conc_chl": "mean", + "conc_tsm": "mean", + "kd489": "mean", + "quality_flags": "mean" + } +} \ No newline at end of file diff --git a/examples/serve/demo/cube-1-250-250.levels/0.zarr/.zattrs b/examples/serve/demo/cube-1-250-250.levels/0.zarr/.zattrs deleted file mode 100644 index 75cc6f499..000000000 --- a/examples/serve/demo/cube-1-250-250.levels/0.zarr/.zattrs +++ /dev/null @@ -1,4 +0,0 @@ -{ - "Conventions": "CF-1.7", - "coordinates": "lat_bnds time_bnds lon_bnds" -} \ No newline at end of file diff --git a/examples/serve/demo/cube-1-250-250.levels/0.zarr/.zmetadata b/examples/serve/demo/cube-1-250-250.levels/0.zarr/.zmetadata deleted file mode 100644 index ef794d2e0..000000000 --- a/examples/serve/demo/cube-1-250-250.levels/0.zarr/.zmetadata +++ /dev/null @@ -1,471 +0,0 @@ -{ - "metadata": { - ".zattrs": { - "Conventions": "CF-1.7", - "coordinates": "lat_bnds time_bnds lon_bnds" - }, - ".zgroup": { - "zarr_format": 2 - }, - "c2rcc_flags/.zarray": { - "chunks": [ - 1, - 250, - 250 - ], - "compressor": { - "blocksize": 0, - "clevel": 5, - "cname": "lz4", - "id": "blosc", - "shuffle": 1 - }, - "dtype": " None: rimraf(self.TEST_CUBE) + rimraf(self.TEST_CUBE_V3) cube = new_cube( time_periods=3, variables=dict(precipitation=np.nan, temperature=np.nan) ).chunk(dict(time=1, lat=90, lon=90)) fv_encoding = dict(_FillValue=None) encoding = dict(precipitation=fv_encoding, temperature=fv_encoding) - cube.to_zarr(self.TEST_CUBE, encoding=encoding) + + # Zarr v2 cube + cube.to_zarr( + self.TEST_CUBE, + encoding=encoding, + zarr_version=2, + ) + + # Zarr v3 cube + cube = cube.drop_encoding() + cube.to_zarr( + self.TEST_CUBE_V3, + encoding=encoding, + zarr_version=3, + consolidated=False, + ) def tearDown(self) -> None: rimraf(self.TEST_CUBE) + rimraf(self.TEST_CUBE_V3) def test_dry_run(self): result = self.invoke_cli(["prune", self.TEST_CUBE, "-vv", "--dry-run"]) @@ -184,3 +202,21 @@ def monitor(message, level): f"Failed to delete block file {block_file}: " ) ) + + def test_zarr3_noop(self): + result = self.invoke_cli(["prune", self.TEST_CUBE_V3, "-vv"]) + + self.assertEqual(0, result.exit_code) + + self.assertEqual( + ( + "Dataset uses Zarr format 3. " + "Empty chunks are not stored, pruning is unnecessary. " + "Nothing to do.\n" + ), + result.stderr, + ) + ds = xr.open_zarr(self.TEST_CUBE_V3) + assert_cube(ds) + self.assertIn("precipitation", ds) + self.assertIn("temperature", ds) diff --git a/test/cli/test_vars2dim.py b/test/cli/test_vars2dim.py index 52766b758..511930434 100644 --- a/test/cli/test_vars2dim.py +++ b/test/cli/test_vars2dim.py @@ -30,6 +30,6 @@ def test_vars2dim(self): self.assertEqual(("var",), var_names.dims) self.assertTrue(hasattr(var_names, "encoding")) self.assertEqual(3, len(var_names)) - self.assertIn("precipitation", str(var_names[0])) - self.assertIn("soil_moisture", str(var_names[1])) - self.assertIn("temperature", str(var_names[2])) + self.assertIn("precipitation", var_names) + self.assertIn("soil_moisture", var_names) + self.assertIn("temperature", var_names) diff --git a/test/cli/test_verify.py b/test/cli/test_verify.py index a43ee331c..0d2150496 100644 --- a/test/cli/test_verify.py +++ b/test/cli/test_verify.py @@ -44,13 +44,8 @@ def test_verify_failure(self): result = self.invoke_cli(["verify", self.TEST_CUBE]) self.assertEqual(3, result.exit_code) - self.assertEqual( + self.assertIn( "INPUT is not a valid cube due to the following reasons:\n" - "- dimensions of data variable 'chl' must be ('time', ..., 'lat', 'lon')," - " but were ('lat', 'lon') for 'chl'\n" - "- dimensions of all data variables must be same, but found ('lat', 'lon')" - " for 'chl' and ('time', 'lat', 'lon') for 'precipitation'\n" - "- all data variables must have same chunk sizes, but found ((90, 90), (360,))" - " for 'chl' and ((3, 2), (90, 90), (180, 180)) for 'precipitation'\n", + "- dimensions of data variable ", result.stdout, ) diff --git a/test/core/gen/test_gen.py b/test/core/gen/test_gen.py index 330667b04..cda28128d 100644 --- a/test/core/gen/test_gen.py +++ b/test/core/gen/test_gen.py @@ -289,7 +289,7 @@ def test_process_inputs_replace_multiple_zarr(self): time_coverage_end="2017-01-03T12:00:00.000000000", ), ) - self.assertTrue(os.path.exists(os.path.join("l2c.zarr", ".zmetadata"))) + self.assertTrue(os.path.exists(os.path.join("l2c.zarr", "zarr.json"))) def test_input_txt(self): f = open( @@ -312,7 +312,7 @@ def test_input_txt(self): time_coverage_end="2017-01-03T12:00:00.000000000", ), ) - self.assertTrue(os.path.exists(os.path.join("l2c.zarr", ".zmetadata"))) + self.assertTrue(os.path.exists(os.path.join("l2c.zarr", "zarr.json"))) def test_process_chunked_zarr(self): status, output = gen_cube_wrapper( @@ -720,12 +720,16 @@ def test_process_compressed_zarr(self): ds_default_compressor.analysed_sst.values, ds_compressed.analysed_sst.values ) self.assertEqual( - "Blosc(cname='lz4', clevel=5, shuffle=SHUFFLE, blocksize=0)", - str(ds_default_compressor.analysed_sst.encoding["compressor"]), + "(ZstdCodec(level=0, checksum=False),)", + str(ds_default_compressor.analysed_sst.encoding["compressors"]), ) self.assertEqual( - "Blosc(cname='zstd', clevel=1, shuffle=BITSHUFFLE, blocksize=0)", - str(ds_compressed.analysed_sst.encoding["compressor"]), + ( + "(BloscCodec(_tunable_attrs=set(), typesize=8, cname=, clevel=1, shuffle=, " + "blocksize=0),)" + ), + str(ds_compressed.analysed_sst.encoding["compressors"]), ) def assert_cube_ok( diff --git a/test/core/gridmapping/test_dataset.py b/test/core/gridmapping/test_dataset.py index f7678fa56..918435101 100644 --- a/test/core/gridmapping/test_dataset.py +++ b/test/core/gridmapping/test_dataset.py @@ -99,7 +99,7 @@ def test_from_real_olci(self): "examples", "notebooks", "inputdata", - "S3-OLCI-L2A.zarr.zip", + "S3-OLCI-L2A.zarr", ) dataset = xr.open_zarr(olci_l2_path, consolidated=False) diff --git a/test/core/store/fs/test_registry.py b/test/core/store/fs/test_registry.py index 5c4602049..0d2e38a67 100644 --- a/test/core/store/fs/test_registry.py +++ b/test/core/store/fs/test_registry.py @@ -31,7 +31,8 @@ ) from xcube.core.store.fs.registry import get_filename_extensions, new_fs_data_store from xcube.core.store.fs.store import FsDataStore -from xcube.core.zarrstore import GenericZarrStore + +# from xcube.core.zarrstore import GenericZarrStore from xcube.util.temp import new_temp_dir ROOT_DIR = "xcube" @@ -324,12 +325,12 @@ def _assert_zarr_store_direct_ok(self, dataset): self.assertIsInstance(dataset, xr.Dataset) self.assertTrue(hasattr(dataset, "zarr_store")) self.assertIsInstance(dataset.zarr_store.get(), collections.abc.MutableMapping) - self.assertNotIsInstance(dataset.zarr_store.get(), GenericZarrStore) + # self.assertNotIsInstance(dataset.zarr_store.get(), GenericZarrStore) def _assert_zarr_store_generic_ok(self, dataset): self.assertIsInstance(dataset, xr.Dataset) self.assertTrue(hasattr(dataset, "zarr_store")) - self.assertIsInstance(dataset.zarr_store.get(), GenericZarrStore) + # self.assertIsInstance(dataset.zarr_store.get(), GenericZarrStore) def _assert_multi_level_dataset_data_ok(self, ml_dataset): self.assertIsInstance(ml_dataset, xcube.core.mldataset.MultiLevelDataset) @@ -360,7 +361,7 @@ def _assert_multi_level_dataset_data_ok(self, ml_dataset): self.assertIsInstance( dataset.zarr_store.get(), collections.abc.MutableMapping ) - self.assertNotIsInstance(dataset.zarr_store.get(), GenericZarrStore) + # self.assertNotIsInstance(dataset.zarr_store.get(), GenericZarrStore) def _assert_multi_level_dataset_format_with_tile_size( self, data_store: FsDataStore diff --git a/test/core/test_chunkstore.py b/test/core/test_chunkstore.py deleted file mode 100644 index 50e8365c8..000000000 --- a/test/core/test_chunkstore.py +++ /dev/null @@ -1,140 +0,0 @@ -# Copyright (c) 2018-2026 by xcube team and contributors -# Permissions are hereby granted under the terms of the MIT License: -# https://opensource.org/licenses/MIT. - -import unittest -import warnings -from typing import Tuple - -import numpy as np -import xarray as xr -from zarr.storage import MemoryStore - -from xcube.core.chunkstore import ChunkStore, LoggingStore, MutableLoggingStore - - -class ChunkStoreTest(unittest.TestCase): - def test_chunk_store(self): - self._test_chunk_store(trace_store_calls=False) - - def test_chunk_store_with_tracing(self): - self._test_chunk_store(trace_store_calls=True) - - def _test_chunk_store(self, trace_store_calls: bool): - index_var = gen_index_var( - dims=("time", "lat", "lon"), - shape=(4, 8, 16), - chunks=(2, 4, 8), - trace_store_calls=trace_store_calls, - ) - self.assertIsNotNone(index_var) - self.assertEqual((4, 8, 16), index_var.shape) - self.assertEqual(((2, 2), (4, 4), (8, 8)), index_var.chunks) - self.assertEqual(("time", "lat", "lon"), index_var.dims) - - visited_indexes = set() - - def index_var_ufunc(index_var_): - if index_var_.size < 6: - warnings.warn(f"weird variable of size {index_var_.size} received!") - return - nonlocal visited_indexes - index = tuple(map(int, index_var_.ravel()[0:6])) - visited_indexes.add(index) - return index_var_ - - result = xr.apply_ufunc( - index_var_ufunc, - index_var, - dask="parallelized", - output_dtypes=[index_var.dtype], - ) - self.assertIsNotNone(result) - self.assertEqual((4, 8, 16), result.shape) - self.assertEqual(((2, 2), (4, 4), (8, 8)), result.chunks) - self.assertEqual(("time", "lat", "lon"), result.dims) - - values = result.values - self.assertEqual((4, 8, 16), values.shape) - np.testing.assert_array_equal(index_var.values, values) - - print(visited_indexes) - - self.assertEqual(8, len(visited_indexes)) - self.assertEqual( - { - (0, 2, 0, 4, 0, 8), - (2, 4, 0, 4, 0, 8), - (0, 2, 4, 8, 0, 8), - (2, 4, 4, 8, 0, 8), - (0, 2, 0, 4, 8, 16), - (2, 4, 0, 4, 8, 16), - (0, 2, 4, 8, 8, 16), - (2, 4, 4, 8, 8, 16), - }, - visited_indexes, - ) - - -def gen_index_var(dims, shape, chunks, trace_store_calls: bool = False): - # noinspection PyUnusedLocal - def get_chunk(cube_store: ChunkStore, name: str, index: tuple[int, ...]) -> bytes: - data = np.zeros(cube_store.chunks, dtype=np.uint64) - data_view = data.ravel() - if data_view.base is not data: - raise ValueError("view expected") - if data_view.size < cube_store.ndim * 2: - raise ValueError("size too small") - for i in range(cube_store.ndim): - j1 = cube_store.chunks[i] * index[i] - j2 = j1 + cube_store.chunks[i] - data_view[2 * i] = j1 - data_view[2 * i + 1] = j2 - return data.tobytes() - - store = ChunkStore(dims, shape, chunks, trace_store_calls=trace_store_calls) - store.add_lazy_array("__index_var__", " None: - self.zattrs_value = b"" - self.original_store = MemoryStore() - self.original_store.update({"chl/.zattrs": self.zattrs_value}) - - def assertReadOk(self, logging_store: LoggingStore): - # noinspection PyUnresolvedReferences - self.assertEqual([".zattrs"], logging_store.listdir("chl")) - # noinspection PyUnresolvedReferences - self.assertEqual(0, logging_store.getsize("chl")) - self.assertEqual({"chl/.zattrs"}, set(logging_store.keys())) - self.assertEqual(["chl/.zattrs"], list(iter(logging_store))) - self.assertTrue("chl/.zattrs" in logging_store) - self.assertEqual(1, len(logging_store)) - self.assertEqual(self.zattrs_value, logging_store.get("chl/.zattrs")) - # assert original_store not changed - self.assertEqual({"chl/.zattrs"}, set(self.original_store.keys())) - - def assertWriteOk(self, logging_store: MutableLoggingStore): - zarray_value = b"" - logging_store["chl/.zarray"] = zarray_value - self.assertEqual( - {"chl/.zattrs", "chl/.zarray"}, set(self.original_store.keys()) - ) - del logging_store["chl/.zarray"] - self.assertEqual({"chl/.zattrs"}, set(self.original_store.keys())) diff --git a/test/core/test_optimize.py b/test/core/test_optimize.py index c6c8687d8..01391a534 100644 --- a/test/core/test_optimize.py +++ b/test/core/test_optimize.py @@ -25,41 +25,31 @@ OUTPUT_CUBE_PATTERN = "{input}_opt.zarr" INPUT_CUBE_FILE_SET = { - ".zattrs", - ".zgroup", - ".zmetadata", - "A/.zarray", - "A/.zattrs", - "A/0.0.0", - "A/1.0.0", - "A/2.0.0", - "B/.zarray", - "B/.zattrs", - "B/0.0.0", - "B/1.0.0", - "B/2.0.0", - "lat/.zarray", - "lat/.zattrs", - "lat/0", - "lat_bnds/.zarray", - "lat_bnds/.zattrs", - "lat_bnds/0.0", - "lon/.zarray", - "lon/.zattrs", - "lon/0", - "lon_bnds/.zarray", - "lon_bnds/.zattrs", - "lon_bnds/0.0", - "time/.zarray", - "time/.zattrs", - "time/0", - "time/1", - "time/2", - "time_bnds/.zarray", - "time_bnds/.zattrs", - "time_bnds/0.0", - "time_bnds/1.0", - "time_bnds/2.0", + "A/c/0/0/0", + "A/c/1/0/0", + "A/c/2/0/0", + "A/zarr.json", + "B/c/0/0/0", + "B/c/1/0/0", + "B/c/2/0/0", + "B/zarr.json", + "lat/c/0", + "lat/zarr.json", + "lat_bnds/c/0/0", + "lat_bnds/zarr.json", + "lon/c/0", + "lon/zarr.json", + "lon_bnds/c/0/0", + "lon_bnds/zarr.json", + "time/c/0", + "time/c/1", + "time/c/2", + "time/zarr.json", + "time_bnds/c/0/0", + "time_bnds/c/1/0", + "time_bnds/c/2/0", + "time_bnds/zarr.json", + "zarr.json", } @@ -194,21 +184,18 @@ def _assert_consolidated( ): self.assertTrue(os.path.isdir(cube_path)) expected_files = set(INPUT_CUBE_FILE_SET) - expected_files.add(".zmetadata") if cons_time: - expected_files.remove("time/1") - expected_files.remove("time/2") + expected_files.remove("time/c/1") + expected_files.remove("time/c/2") if cons_time_bnds: - expected_files.remove("time_bnds/1.0") - expected_files.remove("time_bnds/2.0") + expected_files.remove("time_bnds/c/1/0") + expected_files.remove("time_bnds/c/2/0") self.assertEqual(expected_files, list_file_set(cube_path)) def test_failures(self): - with self.assertRaises(RuntimeError) as cm: - optimize_dataset("pippo", in_place=True, exception_type=RuntimeError) - self.assertEqual( - "Input path must point to ZARR dataset directory.", f"{cm.exception}" - ) + with self.assertRaises(ValueError) as cm: + optimize_dataset("pippo", in_place=True) + self.assertIn("'pippo' is not a valid Zarr directory", str(cm.exception)) with self.assertRaises(RuntimeError) as cm: optimize_dataset(INPUT_CUBE_PATH, exception_type=RuntimeError) diff --git a/test/core/test_timeslice.py b/test/core/test_timeslice.py index 3ce7d8eb2..aa64ae4ae 100644 --- a/test/core/test_timeslice.py +++ b/test/core/test_timeslice.py @@ -123,7 +123,7 @@ def test_insert_time_slice(self): ) np.testing.assert_equal(cube.time.values, expected) - insert_time_slice(self.CUBE_PATH, 5, self.make_slice("2019-01-06T02:00:00")) + insert_time_slice(self.CUBE_PATH, 10, self.make_slice("2019-01-06T02:00:00")) cube = xr.open_zarr(self.CUBE_PATH) expected = np.array( diff --git a/test/core/zarrstore/__init__.py b/test/core/zarrstore/__init__.py deleted file mode 100644 index a339910ae..000000000 --- a/test/core/zarrstore/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -# Copyright (c) 2018-2026 by xcube team and contributors -# Permissions are hereby granted under the terms of the MIT License: -# https://opensource.org/licenses/MIT. diff --git a/test/core/zarrstore/test_cached.py b/test/core/zarrstore/test_cached.py deleted file mode 100644 index 70ed6c872..000000000 --- a/test/core/zarrstore/test_cached.py +++ /dev/null @@ -1,77 +0,0 @@ -# Copyright (c) 2018-2026 by xcube team and contributors -# Permissions are hereby granted under the terms of the MIT License: -# https://opensource.org/licenses/MIT. - -import unittest - -import pytest -import zarr.storage - -from xcube.core.zarrstore import CachedZarrStore, DiagnosticZarrStore - - -class CachedZarrStoreTest(unittest.TestCase): - def get_store(self) -> CachedZarrStore: - self.store = { - "chl/.zarray": b"", - "chl/.zattrs": b"", - "chl/0.0.0": b"", - "chl/0.0.1": b"", - "chl/0.1.0": b"", - "chl/0.1.1": b"", - } - self.cache = DiagnosticZarrStore({}) - return CachedZarrStore(self.store, self.cache) - - def test_props(self): - store = self.get_store() - self.assertIsInstance(store.store, zarr.storage.BaseStore) - self.assertIsInstance(store.cache, zarr.storage.BaseStore) - - def test_getitem(self): - store = self.get_store() - - self.assertEqual(b"", store["chl/0.1.1"]) - self.assertEqual( - ["__getitem__('chl/0.1.1')", "__setitem__('chl/0.1.1', bytes)"], - self.cache.records, - ) - self.assertIn("chl/0.1.1", self.store) - self.assertIn("chl/0.1.1", self.cache) - - self.cache.records = [] - self.assertEqual(b"", store["chl/0.1.1"]) - self.assertEqual(["__getitem__('chl/0.1.1')"], self.cache.records) - - def test_len(self): - store = self.get_store() - self.assertEqual(6, len(store)) - - def test_iter(self): - store = self.get_store() - self.assertEqual( - [ - "chl/.zarray", - "chl/.zattrs", - "chl/0.0.0", - "chl/0.0.1", - "chl/0.1.0", - "chl/0.1.1", - ], - list(iter(store)), - ) - - def test_contains(self): - store = self.get_store() - self.assertIn("chl/.zarray", store) - self.assertNotIn("chl", store) - - def test_setitem(self): - store = self.get_store() - with pytest.raises(NotImplementedError): - store["chl/0.0.1"] = b"" - - def test_delitem(self): - store = self.get_store() - with pytest.raises(NotImplementedError): - del store["chl/0.0.1"] diff --git a/test/core/zarrstore/test_generic.py b/test/core/zarrstore/test_generic.py deleted file mode 100644 index 813314c7d..000000000 --- a/test/core/zarrstore/test_generic.py +++ /dev/null @@ -1,1019 +0,0 @@ -# Copyright (c) 2018-2026 by xcube team and contributors -# Permissions are hereby granted under the terms of the MIT License: -# https://opensource.org/licenses/MIT. - -import unittest -from test.s3test import MOTO_SERVER_ENDPOINT_URL, S3Test -from typing import Any, Dict - -import numpy as np -import pytest -import s3fs -import xarray as xr - -from xcube.core.new import new_cube -from xcube.core.zarrstore.diagnostic import DiagnosticZarrStore -from xcube.core.zarrstore.generic import ( - GenericArray, - GenericZarrStore, - dict_to_bytes, - get_array_slices, - get_chunk_indexes, - get_chunk_padding, - get_chunk_shape, - ndarray_to_bytes, -) - - -# noinspection PyMethodMayBeStatic -class GenericArrayTest(unittest.TestCase): - data = np.linspace(1, 4, 4) - - def get_data(self): - return self.data - - def test_defaults(self): - self.assertEqual({}, GenericArray()) - - def test_finalize_converts_fill_value(self): - data = np.linspace(1, 4, 4, dtype=np.uint16) - - # noinspection PyTypeChecker - self.assertEqual( - { - "name": "x", - "dtype": " GenericZarrStore: - store = GenericZarrStore( - array_defaults=GenericArray( - dims=("time", "y", "x"), shape=shape, chunks=chunks - ) - ) - - t_size, y_size, x_size = shape - store.add_array(name="x", dims="x", data=np.linspace(0.0, 1.0, x_size)) - store.add_array(name="y", dims="y", data=np.linspace(0.0, 1.0, y_size)) - store.add_array(name="time", dims="time", data=np.linspace(1, 365, t_size)) - - store.add_array(name="chl", dtype=np.dtype(np.float32).str, get_data=get_data) - - store.add_array(name="spatial_ref", dims=(), data=np.array(0)) - - return store - - def setUp(self) -> None: - self.chunk_shapes = set() - self.chunk_indexes = set() - - def get_data(self, chunk_info=None, array_info=None): - chunk_index = chunk_info["index"] - it, iy, ix = chunk_index - st, sy, sx = array_info["shape"] - nt, ny, nx = array_info["chunks"] - pt = it * nt - py = iy * ny - px = ix * nx - value = (pt * sy + py) * sx + px - self.chunk_shapes.add(chunk_info["shape"]) - self.chunk_indexes.add(chunk_index) - return np.full((nt, ny, nx), value, dtype=np.float32) - - def test_add_array_validates_name(self): - store = self.new_zarr_store((3, 6, 8), (1, 2, 4), self.get_data) - - tsm = np.zeros((3, 6, 8)) - with pytest.raises(ValueError, match="array 'chl' is already defined"): - store.add_array(name="chl", dims=["time", "y", "x"], data=tsm) - - def test_add_array_validates_dim_sizes(self): - store = self.new_zarr_store((3, 6, 8), (1, 2, 4), self.get_data) - - tsm = np.zeros((3, 10, 8)) - with pytest.raises( - ValueError, - match="array 'tsm' defines" - " dimension 'y' with size 10," - " but existing size is 6", - ): - store.add_array(name="tsm", dims=["time", "y", "x"], data=tsm) - - def test_store_override_flags(self): - store = self.new_zarr_store((3, 6, 8), (1, 2, 4), self.get_data) - self.assertEqual(True, store.is_listable()) - self.assertEqual(True, store.is_readable()) - self.assertEqual(True, store.is_erasable()) - self.assertEqual(False, store.is_writeable()) - - def test_store_override_keys(self): - store = self.new_zarr_store((3, 6, 8), (1, 2, 4), self.get_data) - self.assertEqual( - { - ".zmetadata", - ".zgroup", - ".zattrs", - "x/.zarray", - "x/.zattrs", - "x/0", - "y/.zarray", - "y/.zattrs", - "y/0", - "time/.zarray", - "time/.zattrs", - "time/0", - "chl/.zarray", - "chl/.zattrs", - "chl/0.0.0", - "chl/0.0.1", - "chl/0.1.0", - "chl/0.1.1", - "chl/0.2.0", - "chl/0.2.1", - "chl/1.0.0", - "chl/1.0.1", - "chl/1.1.0", - "chl/1.1.1", - "chl/1.2.0", - "chl/1.2.1", - "chl/2.0.0", - "chl/2.0.1", - "chl/2.1.0", - "chl/2.1.1", - "chl/2.2.0", - "chl/2.2.1", - "spatial_ref/0", - "spatial_ref/.zarray", - "spatial_ref/.zattrs", - }, - set(store.keys()), - ) - - def test_store_override_listdir(self): - store = self.new_zarr_store((3, 6, 8), (1, 2, 4), self.get_data) - self.assertEqual( - { - ".zmetadata", - ".zgroup", - ".zattrs", - "x", - "y", - "time", - "chl", - "spatial_ref", - }, - set(store.listdir("")), - ) - - self.assertEqual( - [ - "time/.zarray", - "time/.zattrs", - "time/0", - ], - store.listdir("time"), - ) - - self.assertEqual( - [ - "chl/.zarray", - "chl/.zattrs", - "chl/0.0.0", - "chl/0.0.1", - "chl/0.1.0", - "chl/0.1.1", - "chl/0.2.0", - "chl/0.2.1", - "chl/1.0.0", - "chl/1.0.1", - "chl/1.1.0", - "chl/1.1.1", - "chl/1.2.0", - "chl/1.2.1", - "chl/2.0.0", - "chl/2.0.1", - "chl/2.1.0", - "chl/2.1.1", - "chl/2.2.0", - "chl/2.2.1", - ], - store.listdir("chl"), - ) - - def test_store_override_rmdir(self): - store = self.new_zarr_store((3, 6, 8), (1, 2, 4), self.get_data) - store.rmdir("chl") - self.assertEqual( - [".zattrs", ".zgroup", ".zmetadata", "spatial_ref", "time", "x", "y"], - store.listdir(""), - ) - - # Also remove dimension sizes from object - store.rmdir("x") - store.rmdir("y") - store.rmdir("time") - - with pytest.raises(ValueError, match="chl: can only remove existing arrays"): - store.rmdir("chl") - - def test_store_override_rename(self): - store = self.new_zarr_store((3, 6, 8), (1, 2, 4), self.get_data) - store.rename("chl", "chl_old") - self.assertEqual( - [ - ".zattrs", - ".zgroup", - ".zmetadata", - "chl_old", - "spatial_ref", - "time", - "x", - "y", - ], - store.listdir(""), - ) - - with pytest.raises( - ValueError, match="can only rename arrays, but 'tsm' is not an array" - ): - store.rename("tsm", "tsm_new") - - with pytest.raises( - ValueError, - match="cannot rename array 'chl_old' into 'x' because it already exists", - ): - store.rename("chl_old", "x") - - with pytest.raises( - ValueError, match="cannot rename array 'chl_old' into 'chl/0'" - ): - store.rename("chl_old", "chl/0") - - def test_store_override_close(self): - store = self.new_zarr_store((3, 6, 8), (1, 2, 4), self.get_data) - - _array_info = None - - def handle_close(array_info): - nonlocal _array_info - _array_info = array_info - - tsm = np.zeros((3, 6, 8)) - store.add_array( - name="tsm", dims=["time", "y", "x"], data=tsm, on_close=handle_close - ) - - store.close() - self.assertIsInstance(_array_info, dict) - self.assertEqual("tsm", _array_info.get("name")) - - def test_store_override_iter(self): - store = self.new_zarr_store((3, 6, 8), (1, 2, 4), self.get_data) - self.assertEqual(set(iter(store.keys())), set(iter(store))) - - def test_store_override_len(self): - store = self.new_zarr_store((3, 6, 8), (1, 2, 4), self.get_data) - self.assertEqual(len(list(store.keys())), len(store)) - - def test_store_override_contains(self): - store = self.new_zarr_store((3, 6, 8), (1, 2, 4), self.get_data) - self.assertTrue(".zmetadata" in store) - self.assertTrue(".zattrs" in store) - self.assertTrue(".zgroup" in store) - self.assertFalse("x" in store) - self.assertTrue("x/.zarray" in store) - self.assertTrue("x/.zattrs" in store) - self.assertTrue("x/0" in store) - self.assertFalse("a" in store) - self.assertFalse("a/0" in store) - self.assertFalse("x/a" in store) - - def test_store_override_getitem(self): - store = self.new_zarr_store((3, 6, 8), (1, 2, 4), self.get_data) - self.assertIsInstance(store[".zattrs"], bytes) - self.assertIsInstance(store["x/.zarray"], bytes) - self.assertIsInstance(store["x/0"], bytes) - - with pytest.raises(KeyError, match="x"): - # noinspection PyUnusedLocal - a = store["x"] - - def test_store_override_setitem(self): - store = self.new_zarr_store((3, 6, 8), (1, 2, 4), self.get_data) - with pytest.raises( - TypeError, - match="xcube.core.zarrstore.generic.GenericZarrStore is read-only", - ): - store["tsm/0.0.0"] = np.zeros((1, 2, 4)).tobytes() - - def test_store_override_delitem(self): - store = self.new_zarr_store((3, 6, 8), (1, 2, 4), self.get_data) - del store["x"] - self.assertFalse("x" in store) - - def test_zarr_store_shape_not_multiple_of_chunks(self): - shape = 3, 6, 8 - chunks = 1, 2, 5 - store = self.new_zarr_store(shape, chunks, get_data=self.get_data) - - ds = xr.open_zarr(store) - - self.assertEqual({"x", "y", "time"}, set(ds.coords)) - self.assertEqual({"spatial_ref", "chl"}, set(ds.data_vars)) - - self.assertEqual(np.float32, ds.chl.dtype) - self.assertEqual(shape, ds.chl.shape) - ds.chl.load() - self.assertEqual({(1, 2, 3), (1, 2, 5)}, self.chunk_shapes) - self.assertEqual( - { - (0, 0, 0), - (0, 0, 1), - (0, 1, 0), - (0, 1, 1), - (0, 2, 0), - (0, 2, 1), - (1, 0, 0), - (1, 0, 1), - (1, 1, 0), - (1, 1, 1), - (1, 2, 0), - (1, 2, 1), - (2, 0, 0), - (2, 0, 1), - (2, 1, 0), - (2, 1, 1), - (2, 2, 0), - (2, 2, 1), - }, - self.chunk_indexes, - ) - - np.testing.assert_array_equal( - ds.chl.data[0], - np.array( - [ - [0.0, 0.0, 0.0, 0.0, 0.0, 5.0, 5.0, 5.0], - [0.0, 0.0, 0.0, 0.0, 0.0, 5.0, 5.0, 5.0], - [16.0, 16.0, 16.0, 16.0, 16.0, 21.0, 21.0, 21.0], - [16.0, 16.0, 16.0, 16.0, 16.0, 21.0, 21.0, 21.0], - [32.0, 32.0, 32.0, 32.0, 32.0, 37.0, 37.0, 37.0], - [32.0, 32.0, 32.0, 32.0, 32.0, 37.0, 37.0, 37.0], - ], - dtype=np.float32, - ), - ) - - np.testing.assert_array_equal( - ds.chl.data[1], - np.array( - [ - [48.0, 48.0, 48.0, 48.0, 48.0, 53.0, 53.0, 53.0], - [48.0, 48.0, 48.0, 48.0, 48.0, 53.0, 53.0, 53.0], - [64.0, 64.0, 64.0, 64.0, 64.0, 69.0, 69.0, 69.0], - [64.0, 64.0, 64.0, 64.0, 64.0, 69.0, 69.0, 69.0], - [80.0, 80.0, 80.0, 80.0, 80.0, 85.0, 85.0, 85.0], - [80.0, 80.0, 80.0, 80.0, 80.0, 85.0, 85.0, 85.0], - ], - dtype=np.float32, - ), - ) - - np.testing.assert_array_equal( - ds.chl.data[2], - np.array( - [ - [96.0, 96.0, 96.0, 96.0, 96.0, 101.0, 101.0, 101.0], - [96.0, 96.0, 96.0, 96.0, 96.0, 101.0, 101.0, 101.0], - [112.0, 112.0, 112.0, 112.0, 112.0, 117.0, 117.0, 117.0], - [112.0, 112.0, 112.0, 112.0, 112.0, 117.0, 117.0, 117.0], - [128.0, 128.0, 128.0, 128.0, 128.0, 133.0, 133.0, 133.0], - [128.0, 128.0, 128.0, 128.0, 128.0, 133.0, 133.0, 133.0], - ], - dtype=np.float32, - ), - ) - - def test_zarr_store_shape_multiple_of_chunks(self): - shape = 3, 6, 8 - chunks = 1, 2, 4 - store = self.new_zarr_store(shape, chunks, get_data=self.get_data) - - ds = xr.open_zarr(store) - - self.assertEqual({"x", "y", "time"}, set(ds.coords)) - self.assertEqual({"spatial_ref", "chl"}, set(ds.data_vars)) - - self.assertEqual(np.float32, ds.chl.dtype) - self.assertEqual(shape, ds.chl.shape) - ds.chl.load() - self.assertEqual({(1, 2, 4)}, self.chunk_shapes) - self.assertEqual( - { - (0, 0, 0), - (0, 0, 1), - (0, 1, 0), - (0, 1, 1), - (0, 2, 0), - (0, 2, 1), - (1, 0, 0), - (1, 0, 1), - (1, 1, 0), - (1, 1, 1), - (1, 2, 0), - (1, 2, 1), - (2, 0, 0), - (2, 0, 1), - (2, 1, 0), - (2, 1, 1), - (2, 2, 0), - (2, 2, 1), - }, - self.chunk_indexes, - ) - - np.testing.assert_array_equal( - ds.chl.data[0], - np.array( - [ - [0.0, 0.0, 0.0, 0.0, 4.0, 4.0, 4.0, 4.0], - [0.0, 0.0, 0.0, 0.0, 4.0, 4.0, 4.0, 4.0], - [16.0, 16.0, 16.0, 16.0, 20.0, 20.0, 20.0, 20.0], - [16.0, 16.0, 16.0, 16.0, 20.0, 20.0, 20.0, 20.0], - [32.0, 32.0, 32.0, 32.0, 36.0, 36.0, 36.0, 36.0], - [32.0, 32.0, 32.0, 32.0, 36.0, 36.0, 36.0, 36.0], - ], - dtype=np.float32, - ), - ) - - np.testing.assert_array_equal( - ds.chl.data[1], - np.array( - [ - [48.0, 48.0, 48.0, 48.0, 52.0, 52.0, 52.0, 52.0], - [48.0, 48.0, 48.0, 48.0, 52.0, 52.0, 52.0, 52.0], - [64.0, 64.0, 64.0, 64.0, 68.0, 68.0, 68.0, 68.0], - [64.0, 64.0, 64.0, 64.0, 68.0, 68.0, 68.0, 68.0], - [80.0, 80.0, 80.0, 80.0, 84.0, 84.0, 84.0, 84.0], - [80.0, 80.0, 80.0, 80.0, 84.0, 84.0, 84.0, 84.0], - ], - dtype=np.float32, - ), - ) - - np.testing.assert_array_equal( - ds.chl.data[2], - np.array( - [ - [96.0, 96.0, 96.0, 96.0, 100.0, 100.0, 100.0, 100.0], - [96.0, 96.0, 96.0, 96.0, 100.0, 100.0, 100.0, 100.0], - [112.0, 112.0, 112.0, 112.0, 116.0, 116.0, 116.0, 116.0], - [112.0, 112.0, 112.0, 112.0, 116.0, 116.0, 116.0, 116.0], - [128.0, 128.0, 128.0, 128.0, 132.0, 132.0, 132.0, 132.0], - [128.0, 128.0, 128.0, 128.0, 132.0, 132.0, 132.0, 132.0], - ], - dtype=np.float32, - ), - ) - - def test_from_dataset(self): - store1 = self.new_zarr_store((3, 6, 8), (1, 2, 4), self.get_data) - dataset1: xr.Dataset = xr.open_zarr(store1) - - store2 = GenericZarrStore.from_dataset(dataset1) - self.assertIsInstance(store2, GenericZarrStore) - self.assertIsNot(store2, store1) - - dataset2: xr.Dataset = xr.open_zarr(store2) - - xr.testing.assert_equal(dataset2, dataset1) - - dataset1.load() - dataset2.load() - xr.testing.assert_equal(dataset2, dataset1) - - -class GenericZarrStoreHelpersTest(unittest.TestCase): - def test_get_chunk_indexes(self): - self.assertEqual([(0,)], list(get_chunk_indexes(()))) - self.assertEqual([(0,), (1,), (2,), (3,)], list(get_chunk_indexes((4,)))) - self.assertEqual( - [ - (0, 0, 0), - (0, 0, 1), - (0, 0, 2), - (0, 0, 3), - (0, 1, 0), - (0, 1, 1), - (0, 1, 2), - (0, 1, 3), - (0, 2, 0), - (0, 2, 1), - (0, 2, 2), - (0, 2, 3), - (1, 0, 0), - (1, 0, 1), - (1, 0, 2), - (1, 0, 3), - (1, 1, 0), - (1, 1, 1), - (1, 1, 2), - (1, 1, 3), - (1, 2, 0), - (1, 2, 1), - (1, 2, 2), - (1, 2, 3), - ], - list(get_chunk_indexes((2, 3, 4))), - ) - - def test_get_chunk_shape(self): - shape = (3, 6, 12) - chunks = (1, 3, 4) - self.assertEqual(chunks, get_chunk_shape(shape, chunks, (0, 0, 0))) - self.assertEqual(chunks, get_chunk_shape(shape, chunks, (2, 1, 2))) - - chunks = (1, 4, 8) - self.assertEqual(chunks, get_chunk_shape(shape, chunks, (0, 0, 0))) - self.assertEqual((1, 2, 4), get_chunk_shape(shape, chunks, (2, 1, 2))) - - def test_get_array_slices(self): - shape = (3, 6, 12) - chunks = (1, 3, 4) - self.assertEqual( - (slice(0, 1), slice(0, 3), slice(0, 4)), - get_array_slices(shape, chunks, (0, 0, 0)), - ) - self.assertEqual( - (slice(2, 3), slice(3, 6), slice(8, 12)), - get_array_slices(shape, chunks, (2, 1, 2)), - ) - - chunks = (1, 4, 8) - self.assertEqual( - (slice(0, 1), slice(0, 4), slice(0, 8)), - get_array_slices(shape, chunks, (0, 0, 0)), - ) - self.assertEqual( - (slice(2, 3), slice(4, 6), slice(16, 20)), - get_array_slices(shape, chunks, (2, 1, 2)), - ) - - def test_get_chunk_padding(self): - shape = (3, 6, 12) - chunks = (1, 3, 4) - self.assertEqual( - ((0, 0), (0, 0), (0, 0)), get_chunk_padding(shape, chunks, (0, 0, 0)) - ) - self.assertEqual( - ((0, 0), (0, 0), (0, 0)), get_chunk_padding(shape, chunks, (2, 1, 2)) - ) - - chunks = (1, 4, 8) - self.assertEqual( - ((0, 0), (0, 0), (0, 0)), get_chunk_padding(shape, chunks, (0, 0, 0)) - ) - - chunks = (1, 4, 8) - self.assertEqual( - ((0, 0), (0, 2), (0, 4)), get_chunk_padding(shape, chunks, (0, 1, 1)) - ) - - -class CommonZarrStoreTest(unittest.TestCase): - """This test is used to assert that Zarr stores - behave as expected with xarray, because GenericArrayStore - expects the behavior tested here. - """ - - def setUp(self) -> None: - self.dtype = np.dtype(np.int16) - self.store: dict[str, Any] = { - ".zgroup": dict_to_bytes( - { - "zarr_format": 2, - } - ), - ".zattrs": dict_to_bytes({}), - "x/.zarray": dict_to_bytes( - { - "zarr_format": 2, - "dtype": self.dtype.str, - "shape": [8], - "chunks": [4], - "order": "C", - "compressor": None, - "filters": None, - "fill_value": 7, - } - ), - "x/.zattrs": dict_to_bytes( - { - "_ARRAY_DIMENSIONS": ["x"], - } - ), - "x/0": ndarray_to_bytes(np.linspace(1, 4, 4, dtype=self.dtype)), - "x/1": ndarray_to_bytes(np.linspace(5, 8, 4, dtype=self.dtype)), - } - - def test_works_with_bytes_chunks(self): - ds = xr.open_zarr(self.store, consolidated=False, decode_cf=False) - self.assertEqual([1, 2, 3, 4, 5, 6, 7, 8], list(ds.x.values)) - - ds = xr.open_zarr(self.store, consolidated=False, decode_cf=True) - np.testing.assert_array_equal( - np.array([1.0, 2.0, 3.0, 4.0, 5.0, 6.0, float("nan"), 8.0]), ds.x.values - ) - - def test_works_with_ndarray_chunks(self): - # Here, x's chunks are numpy arrays rather than bytes! - self.store.update( - { - "x/0": np.linspace(1, 4, 4, dtype=self.dtype), - "x/1": np.linspace(5, 8, 4, dtype=self.dtype), - } - ) - - ds = xr.open_zarr(self.store, consolidated=False, decode_cf=False) - self.assertEqual([1, 2, 3, 4, 5, 6, 7, 8], list(ds.x.values)) - - -class CommonS3ZarrStoreTest(S3Test): - """This test is used to assert that the s3fs Zarr store - behaves as expected with xarray. - """ - - def test_it(self): - cube = new_cube(variables=dict(conc_chl=0.5)).chunk( - dict(time=1, lat=90, lon=90) - ) - - s3 = s3fs.S3FileSystem( - anon=False, - client_kwargs=dict( - endpoint_url=MOTO_SERVER_ENDPOINT_URL, - ), - ) - - s3.mkdir("xcube-test") - s3.mkdir("xcube-test/cube.zarr") - zarr_store = s3.get_mapper("xcube-test/cube.zarr") - cube.to_zarr(zarr_store) - - zarr_store = s3.get_mapper("xcube-test/cube.zarr") - zarr_store = DiagnosticZarrStore(zarr_store) - - dataset = xr.open_zarr(zarr_store) - self.assertIsInstance(dataset, xr.Dataset) - - # print(zarr_store.records) - - self.assertIn("__getitem__('.zmetadata')", zarr_store.records) - self.assertIn("__getitem__('lon/0')", zarr_store.records) - self.assertIn("__getitem__('lat/0')", zarr_store.records) - self.assertIn("__getitem__('time/0')", zarr_store.records) - - # Assert that Zarr used __getitem__ only - for r in zarr_store.records: - if not r.startswith("__getitem__"): - self.fail(f"Unexpected store call: {r}") - - zarr_store.records = [] - # noinspection PyUnusedLocal - values = dataset.conc_chl.isel(time=0).values - - # Assert that Zarr used __getitem__ only - for r in zarr_store.records: - if not r.startswith("__getitem__"): - self.fail(f"Unexpected store call: {r}") diff --git a/test/core/zarrstore/test_holder.py b/test/core/zarrstore/test_holder.py deleted file mode 100644 index fed658ca3..000000000 --- a/test/core/zarrstore/test_holder.py +++ /dev/null @@ -1,49 +0,0 @@ -# Copyright (c) 2018-2026 by xcube team and contributors -# Permissions are hereby granted under the terms of the MIT License: -# https://opensource.org/licenses/MIT. - -import unittest - -import pytest -import xarray as xr - -from xcube.core.zarrstore.generic import GenericZarrStore -from xcube.core.zarrstore.holder import ZarrStoreHolder - - -class ZarrStoreHolderTest(unittest.TestCase): - def test_zarr_store_holder_present(self): - dataset = xr.Dataset() - self.assertIsNotNone(dataset.zarr_store) - self.assertIsInstance(dataset.zarr_store, ZarrStoreHolder) - - def test_zarr_store_holder_default(self): - dataset = xr.Dataset() - self.assertIsInstance(dataset.zarr_store.get(), GenericZarrStore) - self.assertIs(dataset.zarr_store.get(), dataset.zarr_store.get()) - - def test_zarr_store_holder_set(self): - dataset = xr.Dataset() - zarr_store = dict() - dataset.zarr_store.set(zarr_store) - self.assertIs(zarr_store, dataset.zarr_store.get()) - self.assertIs(dataset.zarr_store.get(), dataset.zarr_store.get()) - - def test_zarr_store_holder_reset(self): - dataset = xr.Dataset() - zarr_store = dict() - dataset.zarr_store.set(zarr_store) - dataset.zarr_store.reset() - self.assertIsInstance(dataset.zarr_store.get(), GenericZarrStore) - self.assertIs(dataset.zarr_store.get(), dataset.zarr_store.get()) - - # noinspection PyMethodMayBeStatic - def test_zarr_store_type_check(self): - dataset = xr.Dataset() - with pytest.raises( - TypeError, - match="zarr_store must be an instance of" - " ," - " was ", - ): - dataset.zarr_store.set(42) diff --git a/test/core/zarrstore/test_logging.py b/test/core/zarrstore/test_logging.py deleted file mode 100644 index 075f30260..000000000 --- a/test/core/zarrstore/test_logging.py +++ /dev/null @@ -1,43 +0,0 @@ -# Copyright (c) 2018-2026 by xcube team and contributors -# Permissions are hereby granted under the terms of the MIT License: -# https://opensource.org/licenses/MIT. - - -import unittest - -from zarr.storage import MemoryStore - -from xcube.core.zarrstore import LoggingZarrStore - - -class LoggingZarrStoreTest(unittest.TestCase): - def setUp(self) -> None: - self.zattrs_value = b"" - self.original_store = MemoryStore() - self.original_store.update({"chl/.zattrs": self.zattrs_value}) - - def test_read(self): - logging_store = LoggingZarrStore(self.original_store) - - # noinspection PyUnresolvedReferences - self.assertEqual([".zattrs"], logging_store.listdir("chl")) - # noinspection PyUnresolvedReferences - self.assertEqual(0, logging_store.getsize("chl")) - self.assertEqual({"chl/.zattrs"}, set(logging_store.keys())) - self.assertEqual(["chl/.zattrs"], list(iter(logging_store))) - self.assertTrue("chl/.zattrs" in logging_store) - self.assertEqual(1, len(logging_store)) - self.assertEqual(self.zattrs_value, logging_store.get("chl/.zattrs")) - # assert original_store not changed - self.assertEqual({"chl/.zattrs"}, set(self.original_store.keys())) - - def test_write(self): - logging_store = LoggingZarrStore(self.original_store) - - zarray_value = b"" - logging_store["chl/.zarray"] = zarray_value - self.assertEqual( - {"chl/.zattrs", "chl/.zarray"}, set(self.original_store.keys()) - ) - del logging_store["chl/.zarray"] - self.assertEqual({"chl/.zattrs"}, set(self.original_store.keys())) diff --git a/test/webapi/s3/test_objectstorage.py b/test/webapi/s3/test_objectstorage.py index c167d4d4d..384ffdf65 100644 --- a/test/webapi/s3/test_objectstorage.py +++ b/test/webapi/s3/test_objectstorage.py @@ -8,7 +8,7 @@ from xcube.core.new import new_cube # noinspection PyUnresolvedReferences -from xcube.core.zarrstore import ZarrStoreHolder +# from xcube.core.zarrstore import ZarrStoreHolder from xcube.webapi.s3.objectstorage import ObjectStorage diff --git a/xcube/cli/patch.py b/xcube/cli/patch.py index efddf3ca8..e16da6467 100644 --- a/xcube/cli/patch.py +++ b/xcube/cli/patch.py @@ -168,7 +168,7 @@ def _patch_dataset( LOG.info(f"{k}: {del_count} attribute(s) deleted") if not dry_run: - zarr.convenience.consolidate_metadata(zarr_store) + zarr.consolidate_metadata(zarr_store) LOG.info(f"Consolidated {dataset_path}") diff --git a/xcube/cli/prune.py b/xcube/cli/prune.py index ad74a1ebc..70154b5ad 100644 --- a/xcube/cli/prune.py +++ b/xcube/cli/prune.py @@ -7,6 +7,7 @@ from typing import Callable import click +import zarr from xcube.cli.common import ( cli_option_quiet, @@ -59,6 +60,22 @@ def _prune(input_path: str, dry_run: bool, monitor: Monitor): if input_format != FORMAT_NAME_ZARR: raise click.ClickException("input must be a dataset in Zarr format") + # Detect Zarr format version + try: + zarr_group = zarr.open_group(input_path, mode="r") + zarr_format = getattr(zarr_group.metadata, "zarr_format", 2) + except Exception: + zarr_format = 2 + + if zarr_format >= 3: + monitor( + "Dataset uses Zarr format 3. " + "Empty chunks are not stored, pruning is unnecessary. " + "Nothing to do.", + 1, + ) + return + num_deleted_total = 0 monitor(f"Opening dataset from {input_path!r}...", 1) diff --git a/xcube/core/chunkstore.py b/xcube/core/chunkstore.py deleted file mode 100644 index f18607b48..000000000 --- a/xcube/core/chunkstore.py +++ /dev/null @@ -1,266 +0,0 @@ -# Copyright (c) 2018-2026 by xcube team and contributors -# Permissions are hereby granted under the terms of the MIT License: -# https://opensource.org/licenses/MIT. - -import itertools -import json -from collections.abc import MutableMapping, Sequence -from typing import Any, Callable, Dict, Tuple, Union - -import numpy as np - -__author__ = "Norman Fomferra (Brockmann Consult GmbH)" - -import collections.abc -from collections.abc import Iterable, Iterator, KeysView -from logging import Logger -from typing import Optional - -from deprecated import deprecated - -from xcube.constants import LOG, LOG_LEVEL_TRACE -from xcube.core.zarrstore import LoggingZarrStore -from xcube.util.assertions import assert_instance - -GetChunk = Callable[["ChunkStore", str, tuple[int, ...]], bytes] - - -# Note, we cannot remove this deprecated code as long as -# xcube.core.compute.compute_dataset() is using it. -@deprecated( - reason="This class shall no longer be used." - " If similar functionality is needed," - " use xcube.core.zarrstore.GenericZarrStore", - version="0.12.1", -) -class ChunkStore(MutableMapping): - """A Zarr Store that generates datasets by allowing data variables to - fetch or compute their chunks by a user-defined function *get_chunk*. - Implements the standard Python ``MutableMapping`` interface. - - This is how the *get_chunk* function is called::: - - data = get_chunk(chunk_store, var_name, chunk_indexes) - - where ``chunk_store`` is this store, ``var_name`` is the name of - the variable for which data is fetched, and ``chunk_indexes`` is - a tuple of zero-based, integer chunk indexes. The result must be - a Python *bytes* object. - - Args: - dims: Dimension names of all data variables, - e.g. ('time', 'lat', 'lon'). - shape: Shape of all data variables according to *dims*, - e.g. (512, 720, 1480). - chunks: Chunk sizes of all data variables according to *dims*, - e.g. (128, 180, 180). - attrs: Global dataset attributes. - get_chunk: Default chunk fetching/computing function. - trace_store_calls: Whether to log calls - into the ``MutableMapping`` interface. - """ - - def __init__( - self, - dims: Sequence[str], - shape: Sequence[int], - chunks: Sequence[int], - attrs: dict[str, Any] = None, - get_chunk: GetChunk = None, - trace_store_calls: bool = False, - ): - self._ndim = len(dims) - self._dims = tuple(dims) - self._shape = tuple(shape) - self._chunks = tuple(chunks) - self._get_chunk = get_chunk - self._trace_store_calls = trace_store_calls - - # setup Virtual File System (vfs) - self._vfs = { - ".zgroup": _dict_to_bytes(dict(zarr_format=2)), - ".zattrs": _dict_to_bytes(attrs or dict()), - } - - @property - def ndim(self) -> int: - return self._ndim - - @property - def dims(self) -> tuple[str, ...]: - return self._dims - - @property - def shape(self) -> tuple[int, ...]: - return self._shape - - @property - def chunks(self) -> tuple[int, ...]: - return self._chunks - - def add_array(self, name: str, array: np.ndarray, attrs: dict): - shape = list(map(int, array.shape)) - dtype = str(array.dtype.str) - array_metadata = { - "zarr_format": 2, - "chunks": shape, - "shape": shape, - "dtype": dtype, - "fill_value": None, - "compressor": None, - "filters": None, - "order": "C", - } - self._vfs[name] = _str_to_bytes("") - self._vfs[name + "/.zarray"] = _dict_to_bytes(array_metadata) - self._vfs[name + "/.zattrs"] = _dict_to_bytes(attrs) - self._vfs[name + "/" + (".".join(["0"] * array.ndim))] = bytes(array) - - def add_lazy_array( - self, - name: str, - dtype: str, - fill_value: Union[int, float] = None, - compressor: dict[str, Any] = None, - filters=None, - order: str = "C", - attrs: dict[str, Any] = None, - get_chunk: GetChunk = None, - ): - get_chunk = get_chunk or self._get_chunk - if get_chunk is None: - raise ValueError("get_chunk must be given as there is no default") - - array_metadata = dict( - zarr_format=2, - shape=self._shape, - chunks=self._chunks, - compressor=compressor, - dtype=dtype, - fill_value=fill_value, - filters=filters, - order=order, - ) - - self._vfs[name] = _str_to_bytes("") - self._vfs[name + "/.zarray"] = _dict_to_bytes(array_metadata) - self._vfs[name + "/.zattrs"] = _dict_to_bytes( - dict(_ARRAY_DIMENSIONS=self._dims, **(attrs or dict())) - ) - - nums = np.array(self._shape) // np.array(self._chunks) - indexes = itertools.product(*tuple(map(range, map(int, nums)))) - for index in indexes: - filename = ".".join(map(str, index)) - # noinspection PyTypeChecker - self._vfs[name + "/" + filename] = name, index, get_chunk - - @property - def _class_name(self): - return self.__module__ + "." + self.__class__.__name__ - - ########################################################################## - # Zarr Store (MutableMapping) implementation - ########################################################################## - - def keys(self) -> KeysView[str]: - if self._trace_store_calls: - _trace(f"{self._class_name}.keys()") - return self._vfs.keys() - - def listdir(self, key: str) -> Iterable[str]: - if self._trace_store_calls: - _trace(f"{self._class_name}.listdir(key={key!r})") - if key == "": - return (k for k in self._vfs.keys() if "/" not in k) - else: - prefix = key + "/" - start = len(prefix) - return ( - k - for k in self._vfs.keys() - if k.startswith(prefix) and k.find("/", start) == -1 - ) - - def getsize(self, key: str) -> int: - if self._trace_store_calls: - _trace(f"{self._class_name}.getsize(key={key!r})") - return len(self._vfs[key]) - - def __iter__(self) -> Iterator[str]: - if self._trace_store_calls: - _trace(f"{self._class_name}.__iter__()") - return iter(self._vfs.keys()) - - def __len__(self) -> int: - if self._trace_store_calls: - _trace(f"{self._class_name}.__len__()") - return len(self._vfs.keys()) - - def __contains__(self, key) -> bool: - if self._trace_store_calls: - _trace(f"{self._class_name}.__contains__(key={key!r})") - return key in self._vfs - - def __getitem__(self, key: str) -> bytes: - if self._trace_store_calls: - _trace(f"{self._class_name}.__getitem__(key={key!r})") - value = self._vfs[key] - if isinstance(value, tuple): - name, index, get_chunk = value - return get_chunk(self, name, index) - return value - - def __setitem__(self, key: str, value: bytes) -> None: - if self._trace_store_calls: - _trace(f"{self._class_name}.__setitem__(key={key!r}, value={value!r})") - raise TypeError(f"{self._class_name} is read-only") - - def __delitem__(self, key: str) -> None: - if self._trace_store_calls: - _trace(f"{self._class_name}.__delitem__(key={key!r})") - raise TypeError(f"{self._class_name} is read-only") - - -def _trace(msg: str): - LOG.log(LOG_LEVEL_TRACE, msg) - - -def _dict_to_bytes(d: dict): - return _str_to_bytes(json.dumps(d, indent=2)) - - -def _str_to_bytes(s: str): - return bytes(s, encoding="utf-8") - - -@deprecated( - reason="This class has been moved," - " use xcube.core.zarrstore.LoggingZarrStore" - " instead.", - version="0.12.1", -) -class LoggingStore(LoggingZarrStore): - """A Zarr Store that logs all method calls on another store *other* - including execution time. - """ - - @classmethod - def new( - cls, - other: collections.abc.MutableMapping, - logger: Logger = LOG, - name: Optional[str] = None, - ): - assert_instance(other, collections.abc.MutableMapping) - return cls(other, logger=logger, name=name) - - -@deprecated( - reason="This class has been moved," - " use xcube.core.zarrstore.LoggingZarrStore" - " instead.", - version="0.12.1", -) -class MutableLoggingStore(LoggingStore): - """Mutable version of :class:`LoggingStore`.""" diff --git a/xcube/core/compute.py b/xcube/core/compute.py index e194aa963..7a89ee9a8 100644 --- a/xcube/core/compute.py +++ b/xcube/core/compute.py @@ -7,10 +7,10 @@ from collections.abc import Sequence from typing import AbstractSet, Any, Callable, Dict, Tuple, Union +import dask.array as da import numpy as np import xarray as xr -from xcube.core.chunkstore import ChunkStore from xcube.core.schema import CubeSchema from xcube.core.verify import assert_cube @@ -336,30 +336,43 @@ def _inspect_cube_func(cube_func: CubeFunc, input_var_names: Sequence[str] = Non return has_input_params, has_dim_coords, has_dim_ranges + def _gen_index_var(cube_schema: CubeSchema): dims = cube_schema.dims shape = cube_schema.shape chunks = cube_schema.chunks + ndim = len(dims) + + def make_chunk(block, block_info=None): + """Create one chunk filled with chunk index metadata.""" + info = block_info[None] + chunk_shape = info["chunk-shape"] + chunk_location = info["chunk-location"] + data = np.zeros(chunk_shape, dtype=np.uint64) + flat = data.ravel() + + if flat.size < ndim * 2: + raise ValueError("chunk size too small") + + for i in range(ndim): + start = chunks[i] * chunk_location[i] + end = start + chunk_shape[i] + flat[2 * i] = start + flat[2 * i + 1] = end + + return data + + template = da.empty(shape, chunks=chunks, dtype=np.uint64) + index_data = da.map_blocks( + make_chunk, + template, + dtype=np.uint64, + ) + index_var = xr.DataArray( + index_data, + dims=dims, + coords=cube_schema.coords, + name="__index_var__", + ) - # noinspection PyUnusedLocal - def get_chunk(cube_store: ChunkStore, name: str, index: tuple[int, ...]) -> bytes: - data = np.zeros(cube_store.chunks, dtype=np.uint64) - data_view = data.ravel() - if data_view.base is not data: - raise ValueError("view expected") - if data_view.size < cube_store.ndim * 2: - raise ValueError("size too small") - for i in range(cube_store.ndim): - j1 = cube_store.chunks[i] * index[i] - j2 = j1 + cube_store.chunks[i] - data_view[2 * i] = j1 - data_view[2 * i + 1] = j2 - return data.tobytes() - - store = ChunkStore(dims, shape, chunks) - store.add_lazy_array("__index_var__", " 0: - path_or_store = zarr.LRUStoreCache( - path_or_store, max_size=max_cache_size - ) + # caching is no longer a store concern; it is handled by async IO layers + # if max_cache_size is not None and max_cache_size > 0: + # path_or_store = zarr.LRUStoreCache( + # path_or_store, max_size=max_cache_size + # ) return xr.open_zarr(path_or_store, consolidated=consolidated, **kwargs) def write( @@ -542,14 +543,24 @@ def _get_write_encodings(cls, dataset, compressor, chunksizes, packing): encoding[var_name] = dict(packing[var_name]) if compressor: - compressor = zarr.Blosc(**compressor) + # fix shuffle int -> enum string + shuffle_map = { + 0: "none", + 1: "shuffle", + 2: "bitshuffle", + } + if "shuffle" in compressor and isinstance(compressor["shuffle"], int): + compressor["shuffle"] = shuffle_map.get(compressor["shuffle"], "shuffle") + compressor = zarr.codecs.BloscCodec(**compressor) if encoding: - for var_name in encoding.keys(): - encoding[var_name].update(compressor=compressor) + for var_name in encoding: + encoding[var_name].update( + compressors=(compressor,) + ) else: encoding = { - var_name: dict(compressor=compressor) + var_name: {"compressors": (compressor,)} for var_name in dataset.data_vars } return encoding diff --git a/xcube/core/gridmapping/cfconv.py b/xcube/core/gridmapping/cfconv.py index 7d95b9187..1a72394a3 100644 --- a/xcube/core/gridmapping/cfconv.py +++ b/xcube/core/gridmapping/cfconv.py @@ -10,7 +10,7 @@ import pyproj import xarray as xr import zarr -import zarr.convenience +import zarr.storage from xcube.core.schema import get_dataset_chunks from xcube.util.assertions import assert_instance @@ -313,7 +313,7 @@ def _find_dataset_tile_size( def add_spatial_ref( - dataset_store: zarr.convenience.StoreLike, + dataset_store: zarr.storage.StoreLike, crs: pyproj.CRS, crs_var_name: Optional[str] = "spatial_ref", xy_dim_names: Optional[tuple[str, str]] = None, @@ -350,4 +350,4 @@ def add_spatial_ref( ): item.attrs["grid_mapping"] = crs_var_name - zarr.convenience.consolidate_metadata(dataset_store) + zarr.consolidate_metadata(dataset_store) diff --git a/xcube/core/mldataset/fs.py b/xcube/core/mldataset/fs.py index f0082e21a..4b3989e91 100644 --- a/xcube/core/mldataset/fs.py +++ b/xcube/core/mldataset/fs.py @@ -16,11 +16,15 @@ import numpy as np import xarray as xr import zarr +import posixpath +from fsspec import AbstractFileSystem +from fsspec.asyn import AsyncFileSystem +from fsspec.implementations.asyn_wrapper import AsyncFileSystemWrapper +import zarr.storage -# noinspection PyUnresolvedReferences -import xcube.core.zarrstore from xcube.core.gridmapping import GridMapping from xcube.core.subsampling import AggMethod, AggMethods + from xcube.util.assertions import assert_instance from xcube.util.fspath import get_fs_path_class, resolve_path from xcube.util.types import ScalarOrPair, normalize_scalar_or_pair @@ -148,22 +152,28 @@ def _get_dataset_lazily(self, index: int, parameters) -> xr.Dataset: # TODO: complete logic here engine = base_dataset_open_params.pop("engine", "zarr") - level_zarr_store = fs.get_mapper(str(level_path)) - - consolidated = ( - self._consolidate - if self._consolidate is not None - else (".zmetadata" in level_zarr_store) - ) - - if isinstance(cache_size, int) and cache_size >= self._MIN_CACHE_SIZE: - # compute cache size for level weighted by - # size in pixels for each level - cache_size = math.ceil(self.size_weights[index] * cache_size) - if cache_size >= self._MIN_CACHE_SIZE: - level_zarr_store = zarr.LRUStoreCache( - level_zarr_store, max_size=cache_size - ) + level_zarr_store = self.get_zarr_store(str(level_path), fs, None) + + async def compute_consolidated( + level_zarr_store: zarr.storage.StoreLike, + ) -> bool: + if self._consolidate is not None: + return self._consolidate + try: + return await level_zarr_store.get(".zmetadata") is not None + except Exception: + return False + + consolidated = compute_consolidated(level_zarr_store) + + # if isinstance(cache_size, int) and cache_size >= self._MIN_CACHE_SIZE: + # # compute cache size for level weighted by + # # size in pixels for each level + # cache_size = math.ceil(self.size_weights[index] * cache_size) + # if cache_size >= self._MIN_CACHE_SIZE: + # level_zarr_store = zarr.LRUStoreCache( + # level_zarr_store, max_size=cache_size + # ) try: level_dataset = xr.open_zarr( @@ -174,7 +184,7 @@ def _get_dataset_lazily(self, index: int, parameters) -> xr.Dataset: f"Failed to open dataset {level_path!r}: {e}" ) from e - level_dataset.zarr_store.set(level_zarr_store) + # level_dataset.zarr_store.set(level_zarr_store) return level_dataset @staticmethod @@ -347,7 +357,8 @@ def write_dataset( else: # Write level "{index}.zarr" level_path = data_path / f"{index}.zarr" - level_zarr_store = fs.get_mapper(str(level_path), create=True) + level_zarr_store = cls.get_zarr_store(str(level_path), fs, None) + try: level_dataset.to_zarr( level_zarr_store, @@ -361,14 +372,32 @@ def write_dataset( f"Failed to write dataset {path}: {e}" ) from e if use_saved_levels: - level_dataset = xr.open_zarr( - level_zarr_store, consolidated=consolidated - ) - level_dataset.zarr_store.set(level_zarr_store) + level_dataset = xr.open_zarr(level_path, consolidated=consolidated) + level_dataset.zarr_store.set(str(level_path)) ml_dataset.set_dataset(index, level_dataset) return path + @staticmethod + def get_zarr_store( + data_id: str, + fs: AbstractFileSystem | AsyncFileSystem, + root: str | None = None, + ) -> zarr.storage.StoreLike: + path = data_id + + if root: + path = posixpath.join(root, data_id) + + if "local" in fs.protocol: + return zarr.storage.LocalStore(path) + if fs.protocol == "memory": + return zarr.storage.MemoryStore() + if fs.protocol == "ftp": + fs_async = AsyncFileSystemWrapper(fs) + return zarr.storage.FsspecStore(fs_async, path=data_id) + return zarr.storage.FsspecStore(fs, path=data_id) + class FsMultiLevelDatasetError(ValueError): def __init__(self, message: str): diff --git a/xcube/core/optimize.py b/xcube/core/optimize.py index 0fa9ae46d..8561ea078 100644 --- a/xcube/core/optimize.py +++ b/xcube/core/optimize.py @@ -46,9 +46,11 @@ def optimize_dataset( variables. exception_type: Type of exception to be used on value errors. """ - - if not os.path.isfile(os.path.join(input_path, ".zgroup")): - raise exception_type("Input path must point to ZARR dataset directory.") + if not ( + os.path.isfile(os.path.join(input_path, ".zgroup")) + or os.path.isfile(os.path.join(input_path, "zarr.json")) + ): + raise ValueError(f"{input_path!r} is not a valid Zarr directory") input_path = os.path.abspath(os.path.normpath(input_path)) @@ -76,4 +78,4 @@ def optimize_dataset( var_names = tuple(unchunk_coords) unchunk_dataset(output_path, var_names=var_names, coords_only=True) - zarr.convenience.consolidate_metadata(output_path) + zarr.consolidate_metadata(output_path) diff --git a/xcube/core/store/fs/accessor.py b/xcube/core/store/fs/accessor.py index f257bd4b0..15eace4ec 100644 --- a/xcube/core/store/fs/accessor.py +++ b/xcube/core/store/fs/accessor.py @@ -118,6 +118,7 @@ def load_fs( fsspec.filesystem( protocol, use_listings_cache=use_listings_cache, + asynchronous=True, **(storage_options or {}), ), root, diff --git a/xcube/core/store/fs/impl/dataset.py b/xcube/core/store/fs/impl/dataset.py index 8344bae52..f0d3b43d5 100644 --- a/xcube/core/store/fs/impl/dataset.py +++ b/xcube/core/store/fs/impl/dataset.py @@ -3,15 +3,17 @@ # https://opensource.org/licenses/MIT. from abc import ABC -from typing import Optional + import xarray as xr -import zarr +import posixpath +from fsspec import AbstractFileSystem +from fsspec.asyn import AsyncFileSystem +from fsspec.implementations.asyn_wrapper import AsyncFileSystemWrapper +import zarr.storage # Note, we need the following reference to register the # xarray property accessor -# noinspection PyUnresolvedReferences -from xcube.core.zarrstore import LoggingZarrStore from xcube.util.assertions import assert_instance from xcube.util.fspath import is_https_fs, is_local_fs from xcube.util.jsonschema import ( @@ -138,6 +140,8 @@ def get_data_type(cls) -> DataType: class DatasetZarrFsDataAccessor(DatasetFsDataAccessor): """Opener/writer extension name: 'dataset:zarr:'.""" + zarr_store = None + @classmethod def get_format_id(cls) -> str: return "zarr" @@ -155,11 +159,12 @@ def open_data(self, data_id: str, **open_params) -> xr.Dataset: consolidated = open_params.pop( "consolidated", fs.exists(f"{data_id}/.zmetadata") ) - zarr_store = fs.get_mapper(data_id) - if isinstance(cache_size, int) and cache_size > 0: - zarr_store = zarr.LRUStoreCache(zarr_store, max_size=cache_size) - if log_access: - zarr_store = LoggingZarrStore(zarr_store, name=f"zarr_store({data_id!r})") + + zarr_store = self.get_zarr_store(data_id, fs, root) + # if isinstance(cache_size, int) and cache_size > 0: + # zarr_store = zarr.LRUStoreCache(zarr_store, max_size=cache_size) + # if log_access: + # zarr_store = LoggingZarrStore(zarr_store, name=f"zarr_store({data_id!r})") # TODO: test whether we really need to distinguish here as we know # we are opening a zarr dataset, even without another backend. if engine == "zarr": @@ -177,7 +182,7 @@ def open_data(self, data_id: str, **open_params) -> xr.Dataset: f"Failed to open dataset {data_id!r} using engine {engine!r}: {e}" ) from e - dataset.zarr_store.set(zarr_store) + # dataset.zarr_store.set(zarr_store) return dataset @@ -191,10 +196,10 @@ def write_data( assert_instance(data, xr.Dataset, name="data") assert_instance(data_id, str, name="data_id") fs, root, write_params = self.load_fs(write_params) - zarr_store = fs.get_mapper(data_id, create=True) - log_access = write_params.pop("log_access", None) - if log_access: - zarr_store = LoggingZarrStore(zarr_store, name=f"zarr_store({data_id!r})") + zarr_store = self.get_zarr_store(data_id, fs, root) + # log_access = write_params.pop("log_access", None) + # if log_access: + # zarr_store = LoggingZarrStore(zarr_store, name=f"zarr_store({data_id!r})") consolidated = write_params.pop("consolidated", True) try: data.to_zarr( @@ -212,6 +217,30 @@ def delete_data(self, data_id: str, **delete_params): delete_params.pop("recursive", None) fs.delete(data_id, recursive=True, **delete_params) + def get_zarr_store( + self, + data_id: str, + fs: AbstractFileSystem | AsyncFileSystem, + root: str | None = None, + ) -> zarr.storage.StoreLike: + + if self.zarr_store is not None: + return self.zarr_store + + if "local" in fs.protocol: + if root: + data_id = posixpath.join(root, data_id) + self.zarr_store = zarr.storage.LocalStore(data_id) + elif fs.protocol == "memory": + self.zarr_store = zarr.storage.MemoryStore() + elif fs.protocol == "ftp": + fs_async = AsyncFileSystemWrapper(fs) + self.zarr_store = zarr.storage.FsspecStore(fs_async, path=data_id) + else: + self.zarr_store = zarr.storage.FsspecStore(fs, path=data_id) + + return self.zarr_store + NETCDF_OPEN_DATA_PARAMS_SCHEMA = JsonObjectSchema( properties=dict( diff --git a/xcube/core/store/fs/store.py b/xcube/core/store/fs/store.py index 30ef5bedd..f4fc734c5 100644 --- a/xcube/core/store/fs/store.py +++ b/xcube/core/store/fs/store.py @@ -212,11 +212,13 @@ def get_data_types(cls) -> tuple[str, ...]: find_data_opener_extensions ) return tuple( - sorted({ - data_type - for types_tuple in format_to_data_type_aliases.values() - for data_type in types_tuple - }) + sorted( + { + data_type + for types_tuple in format_to_data_type_aliases.values() + for data_type in types_tuple + } + ) ) def get_data_types_for_data(self, data_id: str) -> tuple[str, ...]: @@ -320,8 +322,8 @@ def open_data( ) open_params_schema = self._get_open_data_params_schema(opener, data_id) assert_valid_params(open_params, name="open_params", schema=open_params_schema) - fs_path = self._convert_data_id_into_fs_path(data_id) - return opener.open_data(fs_path, fs=self.fs, **open_params) + # fs_path = self._convert_data_id_into_fs_path(data_id) + return opener.open_data(data_id, fs=self.fs, **open_params) def get_data_writer_ids(self, data_type: str = None) -> tuple[str, ...]: data_type = DataType.normalize(data_type) @@ -356,14 +358,14 @@ def write_data( write_params, name="write_params", schema=write_params_schema ) data_id = self._ensure_valid_data_id(writer_id, data_id=data_id) - fs_path = self._convert_data_id_into_fs_path(data_id) + # fs_path = self._convert_data_id_into_fs_path(data_id) self.fs.makedirs(self.root, exist_ok=True) - written_fs_path = writer.write_data( - data, fs_path, replace=replace, fs=self.fs, root=self.root, **write_params + written_data_id = writer.write_data( + data, data_id, replace=replace, fs=self.fs, root=self.root, **write_params ) # Verify, accessors fulfill their write_data() contract assert_true( - fs_path == written_fs_path, + data_id == written_data_id, message="FsDataAccessor implementations must return the data_id passed in.", ) # Return original data_id (which is a relative path). diff --git a/xcube/core/timeslice.py b/xcube/core/timeslice.py index 9f847046a..62753b9c5 100644 --- a/xcube/core/timeslice.py +++ b/xcube/core/timeslice.py @@ -2,6 +2,8 @@ # Permissions are hereby granted under the terms of the MIT License: # https://opensource.org/licenses/MIT. +import shutil +import os import tempfile from collections.abc import MutableMapping from typing import Dict, Tuple, Union @@ -11,7 +13,6 @@ import zarr from xcube.core.chunk import chunk_dataset -from xcube.core.unchunk import unchunk_dataset DEFAULT_TIME_EPS = np.array(1000 * 1000, dtype="timedelta64[ns]") @@ -74,21 +75,20 @@ def append_time_slice( if chunk_sizes: time_slice = chunk_dataset(time_slice, chunk_sizes, format_name="zarr") - # Unfortunately time_slice.to_zarr(store, mode='a', append_dim='time') will replace global attributes of store - # with attributes of time_slice (xarray bug?), which are usually empty in our case. - # Hence, we must save our old attributes in a copy of time_slice. ds = zarr.open_group(store, mode="r") + time_slice = time_slice.copy() time_slice.attrs.update(ds.attrs) - if "coordinates" in time_slice.attrs: - # Remove 'coordinates', otherwise we get - # ValueError: cannot serialize coordinates because the global attribute 'coordinates' already exists - # from next time_slice.to_zarr(...) call. - time_slice.attrs.pop("coordinates") - time_slice.to_zarr(store, mode="a", append_dim="time", consolidated=True) + # remove legacy attribute conflict + time_slice.attrs.pop("coordinates", None) - unchunk_dataset(store, coords_only=True) + time_slice.to_zarr( + store, + mode="a", + append_dim="time", + consolidated=True, + ) def insert_time_slice( @@ -145,49 +145,32 @@ def update_time_slice( mode: Update mode, 'insert' or 'replace' chunk_sizes: desired chunk sizes """ - if mode not in ("insert", "replace"): raise ValueError(f"illegal mode value: {mode!r}") insert_mode = mode == "insert" - time_var_names = [] - encoding = {} - with xr.open_zarr(store) as cube: - for var_name in cube.variables: - var = cube[var_name] - if var.ndim >= 1 and "time" in var.dims: - if var.dims[0] != "time": - raise ValueError( - f"dimension 'time' of variable {var_name!r} must be first dimension" - ) - time_var_names.append(var_name) - enc = dict(cube[var_name].encoding) - # xarray 0.17+ supports engine preferred chunks if exposed by the backend - # zarr does that, but when we use the new 'preferred_chunks' when writing to zarr - # it raises and says, 'preferred_chunks' is an unsupported encoding - if "preferred_chunks" in enc: - del enc["preferred_chunks"] - encoding[var_name] = enc - if chunk_sizes: time_slice = chunk_dataset(time_slice, chunk_sizes, format_name="zarr") - temp_dir = tempfile.TemporaryDirectory(prefix="xcube-time-slice-", suffix=".zarr") - time_slice.to_zarr(temp_dir.name, encoding=encoding) - slice_root_group = zarr.open(temp_dir.name, mode="r") - slice_arrays = dict(slice_root_group.arrays()) - - cube_root_group = zarr.open(store, mode="r+") - for var_name, var_array in cube_root_group.arrays(): - if var_name in time_var_names: - slice_array = slice_arrays[var_name] - if insert_mode: - # Add one empty time step - empty = zarr.creation.empty(slice_array.shape, dtype=var_array.dtype) - var_array.append(empty, axis=0) - # Shift contents - var_array[insert_index + 1 :, ...] = var_array[insert_index:-1, ...] - # Replace slice - var_array[insert_index, ...] = slice_array[0] - - unchunk_dataset(store, coords_only=True) + + cube = xr.open_zarr(store) + + # --- split dataset into before / after --- + if insert_mode: + before = cube.isel(time=slice(0, insert_index)) + after = cube.isel(time=slice(insert_index, None)) + new_ds = xr.concat([before, time_slice, after], dim="time") + else: + before = cube.isel(time=slice(0, insert_index)) + after = cube.isel(time=slice(insert_index + 1, None)) + new_ds = xr.concat([before, time_slice, after], dim="time") + new_ds = xr.unify_chunks(new_ds)[0] + + # preserve global attrs + new_ds.attrs.update(cube.attrs) + + tmp = tempfile.mkdtemp(prefix="xcube-timeslice-") + new_ds.to_zarr(tmp, mode="w", consolidated=True) + if os.path.isdir(store): + shutil.rmtree(store) + shutil.move(tmp, store) diff --git a/xcube/core/unchunk.py b/xcube/core/unchunk.py index 6d6124209..b43698769 100644 --- a/xcube/core/unchunk.py +++ b/xcube/core/unchunk.py @@ -5,7 +5,6 @@ import json import os.path from collections.abc import Sequence -from typing import List import numpy as np import xarray as xr @@ -23,8 +22,10 @@ def unchunk_dataset( coords_only: Un-chunk coordinate variables only. """ - is_zarr = os.path.isfile(os.path.join(dataset_path, ".zgroup")) - if not is_zarr: + if not ( + os.path.isfile(os.path.join(dataset_path, ".zgroup")) + or os.path.isfile(os.path.join(dataset_path, "zarr.json")) + ): raise ValueError(f"{dataset_path!r} is not a valid Zarr directory") with xr.open_zarr(dataset_path) as dataset: @@ -50,31 +51,42 @@ def unchunk_dataset( def _unchunk_vars(dataset_path: str, var_names: list[str]): + root = zarr.open_group(dataset_path, mode="a") + for var_name in var_names: - var_path = os.path.join(dataset_path, var_name) - - # Optimization: if "shape" and "chunks" are equal in ${var}/.zarray, we are done - var_array_info_path = os.path.join(var_path, ".zarray") - with open(var_array_info_path) as fp: - var_array_info = json.load(fp) - if var_array_info.get("shape") == var_array_info.get("chunks"): - continue - - # Open array and remove chunks from the data - var_array = zarr.convenience.open_array(var_path, "r+") - if var_array.shape != var_array.chunks: - # TODO (forman): Fully loading data is inefficient and dangerous for large arrays. - # Instead save unchunked to temp and replace existing chunked array dir with temp. - # Fully load data and attrs so we no longer depend on files - data = np.array(var_array) - attributes = var_array.attrs.asdict() - # Save array data - zarr.convenience.save_array( - var_path, data, chunks=False, fill_value=var_array.fill_value - ) - # zarr.convenience.save_array() does not seem save user attributes (file ".zattrs" not written), - # therefore we must modify attrs explicitly: - var_array = zarr.convenience.open_array(var_path, "r+") - var_array.attrs.update(attributes) + arr = root[var_name] + + # already unchunked + if tuple(arr.shape) == tuple(arr.chunks): + continue + + # fully load data + data = np.asarray(arr) + + # preserve metadata + attrs = dict(arr.attrs) + fill_value = arr.fill_value + + # Zarr v3 metadata + dimension_names = getattr(arr.metadata, "dimension_names", None) + compressors = getattr(arr.metadata, "compressors", None) + filters = getattr(arr.metadata, "filters", None) + + # remove old array completely + del root[var_name] + + # recreate as single chunk + new_arr = root.create_array( + name=var_name, + data=data, + chunks=data.shape, + fill_value=fill_value, + dimension_names=dimension_names, + compressors=compressors, + filters=filters, + ) + + # restore attrs + new_arr.attrs.update(attrs) zarr.consolidate_metadata(dataset_path) diff --git a/xcube/core/zarrstore/__init__.py b/xcube/core/zarrstore/__init__.py index 88aca7ea4..1cd713b7a 100644 --- a/xcube/core/zarrstore/__init__.py +++ b/xcube/core/zarrstore/__init__.py @@ -2,8 +2,8 @@ # Permissions are hereby granted under the terms of the MIT License: # https://opensource.org/licenses/MIT. -from .cached import CachedZarrStore -from .diagnostic import DiagnosticZarrStore -from .generic import GenericArray, GenericArrayLike, GenericZarrStore +# from .cached import CachedZarrStore +# from .diagnostic import DiagnosticZarrStore +# from .generic import GenericArray, GenericArrayLike, GenericZarrStore from .holder import ZarrStoreHolder -from .logging import LoggingZarrStore +# from .logging import LoggingZarrStore diff --git a/xcube/core/zarrstore/holder.py b/xcube/core/zarrstore/holder.py index 555c2b739..8ebfdc6a0 100644 --- a/xcube/core/zarrstore/holder.py +++ b/xcube/core/zarrstore/holder.py @@ -45,28 +45,12 @@ def __init__(self, dataset: xr.Dataset): self._zarr_store: Optional[collections.abc.MutableMapping] = None self._lock = threading.RLock() - def get(self) -> collections.abc.MutableMapping: + def get(self) -> collections.abc.MutableMapping | None: """Get the Zarr store of a dataset. - If no Zarr store has been set, the method will use - ``GenericZarrStore.from_dataset()`` to create and set - one. Returns: The Zarr store. """ - if self._zarr_store is None: - # Double-checked locking pattern - with self._lock: - if self._zarr_store is None: - from xcube.core.zarrstore import GenericZarrStore - - self._zarr_store = GenericZarrStore.from_dataset(self._dataset) - source = self._dataset.encoding.get("source", "?") - LOG.warning( - f"dataset {source!r} is assigned a" - f" GenericZarrStore which may introduce" - f" performance penalties" - ) return self._zarr_store def set(self, zarr_store: collections.abc.MutableMapping) -> None: diff --git a/xcube/testing/data/ds.levels/0.zarr/spatial_ref/zarr.json b/xcube/testing/data/ds.levels/0.zarr/spatial_ref/zarr.json new file mode 100644 index 000000000..c1c6262cb --- /dev/null +++ b/xcube/testing/data/ds.levels/0.zarr/spatial_ref/zarr.json @@ -0,0 +1,47 @@ +{ + "shape": [], + "data_type": "int64", + "chunk_grid": { + "name": "regular", + "configuration": { + "chunk_shape": [] + } + }, + "chunk_key_encoding": { + "name": "default", + "configuration": { + "separator": "/" + } + }, + "fill_value": 0, + "codecs": [ + { + "name": "bytes", + "configuration": { + "endian": "little" + } + }, + { + "name": "zstd", + "configuration": { + "level": 0, + "checksum": false + } + } + ], + "attributes": { + "crs_wkt": "GEOGCRS[\"WGS 84 (CRS84)\",ENSEMBLE[\"World Geodetic System 1984 ensemble\",MEMBER[\"World Geodetic System 1984 (Transit)\"],MEMBER[\"World Geodetic System 1984 (G730)\"],MEMBER[\"World Geodetic System 1984 (G873)\"],MEMBER[\"World Geodetic System 1984 (G1150)\"],MEMBER[\"World Geodetic System 1984 (G1674)\"],MEMBER[\"World Geodetic System 1984 (G1762)\"],MEMBER[\"World Geodetic System 1984 (G2139)\"],MEMBER[\"World Geodetic System 1984 (G2296)\"],ELLIPSOID[\"WGS 84\",6378137,298.257223563,LENGTHUNIT[\"metre\",1]],ENSEMBLEACCURACY[2.0]],PRIMEM[\"Greenwich\",0,ANGLEUNIT[\"degree\",0.0174532925199433]],CS[ellipsoidal,2],AXIS[\"geodetic longitude (Lon)\",east,ORDER[1],ANGLEUNIT[\"degree\",0.0174532925199433]],AXIS[\"geodetic latitude (Lat)\",north,ORDER[2],ANGLEUNIT[\"degree\",0.0174532925199433]],USAGE[SCOPE[\"Not known.\"],AREA[\"World.\"],BBOX[-90,-180,90,180]],ID[\"OGC\",\"CRS84\"]]", + "semi_major_axis": 6378137.0, + "semi_minor_axis": 6356752.314245179, + "inverse_flattening": 298.257223563, + "reference_ellipsoid_name": "WGS 84", + "longitude_of_prime_meridian": 0.0, + "prime_meridian_name": "Greenwich", + "geographic_crs_name": "WGS 84 (CRS84)", + "horizontal_datum_name": "World Geodetic System 1984 ensemble", + "grid_mapping_name": "latitude_longitude" + }, + "zarr_format": 3, + "node_type": "array", + "storage_transformers": [] +} \ No newline at end of file diff --git a/xcube/testing/data/ds.levels/0.zarr/time/c/0 b/xcube/testing/data/ds.levels/0.zarr/time/c/0 new file mode 100644 index 000000000..17d4981e7 Binary files /dev/null and b/xcube/testing/data/ds.levels/0.zarr/time/c/0 differ diff --git a/xcube/testing/data/ds.levels/0.zarr/time/zarr.json b/xcube/testing/data/ds.levels/0.zarr/time/zarr.json new file mode 100644 index 000000000..1b696e88e --- /dev/null +++ b/xcube/testing/data/ds.levels/0.zarr/time/zarr.json @@ -0,0 +1,47 @@ +{ + "shape": [ + 5 + ], + "data_type": "int64", + "chunk_grid": { + "name": "regular", + "configuration": { + "chunk_shape": [ + 5 + ] + } + }, + "chunk_key_encoding": { + "name": "default", + "configuration": { + "separator": "/" + } + }, + "fill_value": 0, + "codecs": [ + { + "name": "bytes", + "configuration": { + "endian": "little" + } + }, + { + "name": "zstd", + "configuration": { + "level": 0, + "checksum": false + } + } + ], + "attributes": { + "bounds": "time_bnds", + "units": "seconds since 1970-01-01", + "calendar": "proleptic_gregorian" + }, + "dimension_names": [ + "time" + ], + "zarr_format": 3, + "node_type": "array", + "storage_transformers": [] +} \ No newline at end of file diff --git a/xcube/testing/data/ds.levels/0.zarr/time_bnds/c/0/0 b/xcube/testing/data/ds.levels/0.zarr/time_bnds/c/0/0 new file mode 100644 index 000000000..f9c4df9cb Binary files /dev/null and b/xcube/testing/data/ds.levels/0.zarr/time_bnds/c/0/0 differ diff --git a/xcube/testing/data/ds.levels/0.zarr/time_bnds/zarr.json b/xcube/testing/data/ds.levels/0.zarr/time_bnds/zarr.json new file mode 100644 index 000000000..82c977e17 --- /dev/null +++ b/xcube/testing/data/ds.levels/0.zarr/time_bnds/zarr.json @@ -0,0 +1,49 @@ +{ + "shape": [ + 5, + 2 + ], + "data_type": "int64", + "chunk_grid": { + "name": "regular", + "configuration": { + "chunk_shape": [ + 5, + 2 + ] + } + }, + "chunk_key_encoding": { + "name": "default", + "configuration": { + "separator": "/" + } + }, + "fill_value": 0, + "codecs": [ + { + "name": "bytes", + "configuration": { + "endian": "little" + } + }, + { + "name": "zstd", + "configuration": { + "level": 0, + "checksum": false + } + } + ], + "attributes": { + "units": "seconds since 1970-01-01T00:00:00", + "calendar": "proleptic_gregorian" + }, + "dimension_names": [ + "time", + "bnds" + ], + "zarr_format": 3, + "node_type": "array", + "storage_transformers": [] +} \ No newline at end of file diff --git a/xcube/testing/data/ds.levels/0.zarr/var_a/c/0/0/0 b/xcube/testing/data/ds.levels/0.zarr/var_a/c/0/0/0 new file mode 100644 index 000000000..783927e0d Binary files /dev/null and b/xcube/testing/data/ds.levels/0.zarr/var_a/c/0/0/0 differ diff --git a/xcube/testing/data/ds.levels/0.zarr/var_a/c/0/0/1 b/xcube/testing/data/ds.levels/0.zarr/var_a/c/0/0/1 new file mode 100644 index 000000000..8dabb53da Binary files /dev/null and b/xcube/testing/data/ds.levels/0.zarr/var_a/c/0/0/1 differ diff --git a/xcube/testing/data/ds.levels/0.zarr/var_a/c/0/1/0 b/xcube/testing/data/ds.levels/0.zarr/var_a/c/0/1/0 new file mode 100644 index 000000000..8dabb53da Binary files /dev/null and b/xcube/testing/data/ds.levels/0.zarr/var_a/c/0/1/0 differ diff --git a/xcube/testing/data/ds.levels/0.zarr/var_a/c/0/1/1 b/xcube/testing/data/ds.levels/0.zarr/var_a/c/0/1/1 new file mode 100644 index 000000000..8dabb53da Binary files /dev/null and b/xcube/testing/data/ds.levels/0.zarr/var_a/c/0/1/1 differ diff --git a/xcube/testing/data/ds.levels/0.zarr/var_a/c/1/0/0 b/xcube/testing/data/ds.levels/0.zarr/var_a/c/1/0/0 new file mode 100644 index 000000000..8dabb53da Binary files /dev/null and b/xcube/testing/data/ds.levels/0.zarr/var_a/c/1/0/0 differ diff --git a/xcube/testing/data/ds.levels/0.zarr/var_a/c/1/0/1 b/xcube/testing/data/ds.levels/0.zarr/var_a/c/1/0/1 new file mode 100644 index 000000000..8dabb53da Binary files /dev/null and b/xcube/testing/data/ds.levels/0.zarr/var_a/c/1/0/1 differ diff --git a/xcube/testing/data/ds.levels/0.zarr/var_a/c/1/1/0 b/xcube/testing/data/ds.levels/0.zarr/var_a/c/1/1/0 new file mode 100644 index 000000000..8dabb53da Binary files /dev/null and b/xcube/testing/data/ds.levels/0.zarr/var_a/c/1/1/0 differ diff --git a/xcube/testing/data/ds.levels/0.zarr/var_a/c/1/1/1 b/xcube/testing/data/ds.levels/0.zarr/var_a/c/1/1/1 new file mode 100644 index 000000000..8dabb53da Binary files /dev/null and b/xcube/testing/data/ds.levels/0.zarr/var_a/c/1/1/1 differ diff --git a/xcube/testing/data/ds.levels/0.zarr/var_a/c/2/0/0 b/xcube/testing/data/ds.levels/0.zarr/var_a/c/2/0/0 new file mode 100644 index 000000000..8dabb53da Binary files /dev/null and b/xcube/testing/data/ds.levels/0.zarr/var_a/c/2/0/0 differ diff --git a/xcube/testing/data/ds.levels/0.zarr/var_a/c/2/0/1 b/xcube/testing/data/ds.levels/0.zarr/var_a/c/2/0/1 new file mode 100644 index 000000000..8dabb53da Binary files /dev/null and b/xcube/testing/data/ds.levels/0.zarr/var_a/c/2/0/1 differ diff --git a/xcube/testing/data/ds.levels/0.zarr/var_a/c/2/1/0 b/xcube/testing/data/ds.levels/0.zarr/var_a/c/2/1/0 new file mode 100644 index 000000000..8dabb53da Binary files /dev/null and b/xcube/testing/data/ds.levels/0.zarr/var_a/c/2/1/0 differ diff --git a/xcube/testing/data/ds.levels/0.zarr/var_a/c/2/1/1 b/xcube/testing/data/ds.levels/0.zarr/var_a/c/2/1/1 new file mode 100644 index 000000000..8dabb53da Binary files /dev/null and b/xcube/testing/data/ds.levels/0.zarr/var_a/c/2/1/1 differ diff --git a/xcube/testing/data/ds.levels/0.zarr/var_a/c/3/0/0 b/xcube/testing/data/ds.levels/0.zarr/var_a/c/3/0/0 new file mode 100644 index 000000000..8dabb53da Binary files /dev/null and b/xcube/testing/data/ds.levels/0.zarr/var_a/c/3/0/0 differ diff --git a/xcube/testing/data/ds.levels/0.zarr/var_a/c/3/0/1 b/xcube/testing/data/ds.levels/0.zarr/var_a/c/3/0/1 new file mode 100644 index 000000000..8dabb53da Binary files /dev/null and b/xcube/testing/data/ds.levels/0.zarr/var_a/c/3/0/1 differ diff --git a/xcube/testing/data/ds.levels/0.zarr/var_a/c/3/1/0 b/xcube/testing/data/ds.levels/0.zarr/var_a/c/3/1/0 new file mode 100644 index 000000000..8dabb53da Binary files /dev/null and b/xcube/testing/data/ds.levels/0.zarr/var_a/c/3/1/0 differ diff --git a/xcube/testing/data/ds.levels/0.zarr/var_a/c/3/1/1 b/xcube/testing/data/ds.levels/0.zarr/var_a/c/3/1/1 new file mode 100644 index 000000000..8dabb53da Binary files /dev/null and b/xcube/testing/data/ds.levels/0.zarr/var_a/c/3/1/1 differ diff --git a/xcube/testing/data/ds.levels/0.zarr/var_a/c/4/0/0 b/xcube/testing/data/ds.levels/0.zarr/var_a/c/4/0/0 new file mode 100644 index 000000000..8dabb53da Binary files /dev/null and b/xcube/testing/data/ds.levels/0.zarr/var_a/c/4/0/0 differ diff --git a/xcube/testing/data/ds.levels/0.zarr/var_a/c/4/0/1 b/xcube/testing/data/ds.levels/0.zarr/var_a/c/4/0/1 new file mode 100644 index 000000000..8dabb53da Binary files /dev/null and b/xcube/testing/data/ds.levels/0.zarr/var_a/c/4/0/1 differ diff --git a/xcube/testing/data/ds.levels/0.zarr/var_a/c/4/1/0 b/xcube/testing/data/ds.levels/0.zarr/var_a/c/4/1/0 new file mode 100644 index 000000000..8dabb53da Binary files /dev/null and b/xcube/testing/data/ds.levels/0.zarr/var_a/c/4/1/0 differ diff --git a/xcube/testing/data/ds.levels/0.zarr/var_a/c/4/1/1 b/xcube/testing/data/ds.levels/0.zarr/var_a/c/4/1/1 new file mode 100644 index 000000000..8dabb53da Binary files /dev/null and b/xcube/testing/data/ds.levels/0.zarr/var_a/c/4/1/1 differ diff --git a/xcube/testing/data/ds.levels/0.zarr/var_a/zarr.json b/xcube/testing/data/ds.levels/0.zarr/var_a/zarr.json new file mode 100644 index 000000000..5a4a0702f --- /dev/null +++ b/xcube/testing/data/ds.levels/0.zarr/var_a/zarr.json @@ -0,0 +1,52 @@ +{ + "shape": [ + 5, + 180, + 360 + ], + "data_type": "float64", + "chunk_grid": { + "name": "regular", + "configuration": { + "chunk_shape": [ + 1, + 90, + 180 + ] + } + }, + "chunk_key_encoding": { + "name": "default", + "configuration": { + "separator": "/" + } + }, + "fill_value": "NaN", + "codecs": [ + { + "name": "bytes", + "configuration": { + "endian": "little" + } + }, + { + "name": "zstd", + "configuration": { + "level": 0, + "checksum": false + } + } + ], + "attributes": { + "grid_mapping": "spatial_ref", + "_FillValue": "AAAAAAAA+H8=" + }, + "dimension_names": [ + "time", + "y", + "x" + ], + "zarr_format": 3, + "node_type": "array", + "storage_transformers": [] +} \ No newline at end of file diff --git a/xcube/testing/data/ds.levels/0.zarr/var_b/c/0/0/0 b/xcube/testing/data/ds.levels/0.zarr/var_b/c/0/0/0 new file mode 100644 index 000000000..7f2993c19 Binary files /dev/null and b/xcube/testing/data/ds.levels/0.zarr/var_b/c/0/0/0 differ diff --git a/xcube/testing/data/ds.levels/0.zarr/var_b/c/0/0/1 b/xcube/testing/data/ds.levels/0.zarr/var_b/c/0/0/1 new file mode 100644 index 000000000..b56c63cda Binary files /dev/null and b/xcube/testing/data/ds.levels/0.zarr/var_b/c/0/0/1 differ diff --git a/xcube/testing/data/ds.levels/0.zarr/var_b/c/0/1/0 b/xcube/testing/data/ds.levels/0.zarr/var_b/c/0/1/0 new file mode 100644 index 000000000..b56c63cda Binary files /dev/null and b/xcube/testing/data/ds.levels/0.zarr/var_b/c/0/1/0 differ diff --git a/xcube/testing/data/ds.levels/0.zarr/var_b/c/0/1/1 b/xcube/testing/data/ds.levels/0.zarr/var_b/c/0/1/1 new file mode 100644 index 000000000..b56c63cda Binary files /dev/null and b/xcube/testing/data/ds.levels/0.zarr/var_b/c/0/1/1 differ diff --git a/xcube/testing/data/ds.levels/0.zarr/var_b/c/1/0/0 b/xcube/testing/data/ds.levels/0.zarr/var_b/c/1/0/0 new file mode 100644 index 000000000..b56c63cda Binary files /dev/null and b/xcube/testing/data/ds.levels/0.zarr/var_b/c/1/0/0 differ diff --git a/xcube/testing/data/ds.levels/0.zarr/var_b/c/1/0/1 b/xcube/testing/data/ds.levels/0.zarr/var_b/c/1/0/1 new file mode 100644 index 000000000..b56c63cda Binary files /dev/null and b/xcube/testing/data/ds.levels/0.zarr/var_b/c/1/0/1 differ diff --git a/xcube/testing/data/ds.levels/0.zarr/var_b/c/1/1/0 b/xcube/testing/data/ds.levels/0.zarr/var_b/c/1/1/0 new file mode 100644 index 000000000..b56c63cda Binary files /dev/null and b/xcube/testing/data/ds.levels/0.zarr/var_b/c/1/1/0 differ diff --git a/xcube/testing/data/ds.levels/0.zarr/var_b/c/1/1/1 b/xcube/testing/data/ds.levels/0.zarr/var_b/c/1/1/1 new file mode 100644 index 000000000..b56c63cda Binary files /dev/null and b/xcube/testing/data/ds.levels/0.zarr/var_b/c/1/1/1 differ diff --git a/xcube/testing/data/ds.levels/0.zarr/var_b/c/2/0/0 b/xcube/testing/data/ds.levels/0.zarr/var_b/c/2/0/0 new file mode 100644 index 000000000..b56c63cda Binary files /dev/null and b/xcube/testing/data/ds.levels/0.zarr/var_b/c/2/0/0 differ diff --git a/xcube/testing/data/ds.levels/0.zarr/var_b/c/2/0/1 b/xcube/testing/data/ds.levels/0.zarr/var_b/c/2/0/1 new file mode 100644 index 000000000..b56c63cda Binary files /dev/null and b/xcube/testing/data/ds.levels/0.zarr/var_b/c/2/0/1 differ diff --git a/xcube/testing/data/ds.levels/0.zarr/var_b/c/2/1/0 b/xcube/testing/data/ds.levels/0.zarr/var_b/c/2/1/0 new file mode 100644 index 000000000..b56c63cda Binary files /dev/null and b/xcube/testing/data/ds.levels/0.zarr/var_b/c/2/1/0 differ diff --git a/xcube/testing/data/ds.levels/0.zarr/var_b/c/2/1/1 b/xcube/testing/data/ds.levels/0.zarr/var_b/c/2/1/1 new file mode 100644 index 000000000..b56c63cda Binary files /dev/null and b/xcube/testing/data/ds.levels/0.zarr/var_b/c/2/1/1 differ diff --git a/xcube/testing/data/ds.levels/0.zarr/var_b/c/3/0/0 b/xcube/testing/data/ds.levels/0.zarr/var_b/c/3/0/0 new file mode 100644 index 000000000..b56c63cda Binary files /dev/null and b/xcube/testing/data/ds.levels/0.zarr/var_b/c/3/0/0 differ diff --git a/xcube/testing/data/ds.levels/0.zarr/var_b/c/3/0/1 b/xcube/testing/data/ds.levels/0.zarr/var_b/c/3/0/1 new file mode 100644 index 000000000..b56c63cda Binary files /dev/null and b/xcube/testing/data/ds.levels/0.zarr/var_b/c/3/0/1 differ diff --git a/xcube/testing/data/ds.levels/0.zarr/var_b/c/3/1/0 b/xcube/testing/data/ds.levels/0.zarr/var_b/c/3/1/0 new file mode 100644 index 000000000..b56c63cda Binary files /dev/null and b/xcube/testing/data/ds.levels/0.zarr/var_b/c/3/1/0 differ diff --git a/xcube/testing/data/ds.levels/0.zarr/var_b/c/3/1/1 b/xcube/testing/data/ds.levels/0.zarr/var_b/c/3/1/1 new file mode 100644 index 000000000..b56c63cda Binary files /dev/null and b/xcube/testing/data/ds.levels/0.zarr/var_b/c/3/1/1 differ diff --git a/xcube/testing/data/ds.levels/0.zarr/var_b/c/4/0/0 b/xcube/testing/data/ds.levels/0.zarr/var_b/c/4/0/0 new file mode 100644 index 000000000..b56c63cda Binary files /dev/null and b/xcube/testing/data/ds.levels/0.zarr/var_b/c/4/0/0 differ diff --git a/xcube/testing/data/ds.levels/0.zarr/var_b/c/4/0/1 b/xcube/testing/data/ds.levels/0.zarr/var_b/c/4/0/1 new file mode 100644 index 000000000..b56c63cda Binary files /dev/null and b/xcube/testing/data/ds.levels/0.zarr/var_b/c/4/0/1 differ diff --git a/xcube/testing/data/ds.levels/0.zarr/var_b/c/4/1/0 b/xcube/testing/data/ds.levels/0.zarr/var_b/c/4/1/0 new file mode 100644 index 000000000..b56c63cda Binary files /dev/null and b/xcube/testing/data/ds.levels/0.zarr/var_b/c/4/1/0 differ diff --git a/xcube/testing/data/ds.levels/0.zarr/var_b/c/4/1/1 b/xcube/testing/data/ds.levels/0.zarr/var_b/c/4/1/1 new file mode 100644 index 000000000..b56c63cda Binary files /dev/null and b/xcube/testing/data/ds.levels/0.zarr/var_b/c/4/1/1 differ diff --git a/xcube/testing/data/ds.levels/0.zarr/var_b/zarr.json b/xcube/testing/data/ds.levels/0.zarr/var_b/zarr.json new file mode 100644 index 000000000..73ce4e750 --- /dev/null +++ b/xcube/testing/data/ds.levels/0.zarr/var_b/zarr.json @@ -0,0 +1,54 @@ +{ + "shape": [ + 5, + 180, + 360 + ], + "data_type": "int16", + "chunk_grid": { + "name": "regular", + "configuration": { + "chunk_shape": [ + 1, + 90, + 180 + ] + } + }, + "chunk_key_encoding": { + "name": "default", + "configuration": { + "separator": "/" + } + }, + "fill_value": 0, + "codecs": [ + { + "name": "bytes", + "configuration": { + "endian": "little" + } + }, + { + "name": "zstd", + "configuration": { + "level": 0, + "checksum": false + } + } + ], + "attributes": { + "grid_mapping": "spatial_ref", + "add_offset": -10, + "scale_factor": 0.001, + "_FillValue": -9999 + }, + "dimension_names": [ + "time", + "y", + "x" + ], + "zarr_format": 3, + "node_type": "array", + "storage_transformers": [] +} \ No newline at end of file diff --git a/xcube/testing/data/ds.levels/0.zarr/var_c/c/0/0/0 b/xcube/testing/data/ds.levels/0.zarr/var_c/c/0/0/0 new file mode 100644 index 000000000..3bc8df1ad Binary files /dev/null and b/xcube/testing/data/ds.levels/0.zarr/var_c/c/0/0/0 differ diff --git a/xcube/testing/data/ds.levels/0.zarr/var_c/c/0/0/1 b/xcube/testing/data/ds.levels/0.zarr/var_c/c/0/0/1 new file mode 100644 index 000000000..3bc8df1ad Binary files /dev/null and b/xcube/testing/data/ds.levels/0.zarr/var_c/c/0/0/1 differ diff --git a/xcube/testing/data/ds.levels/0.zarr/var_c/c/0/1/0 b/xcube/testing/data/ds.levels/0.zarr/var_c/c/0/1/0 new file mode 100644 index 000000000..3bc8df1ad Binary files /dev/null and b/xcube/testing/data/ds.levels/0.zarr/var_c/c/0/1/0 differ diff --git a/xcube/testing/data/ds.levels/0.zarr/var_c/c/0/1/1 b/xcube/testing/data/ds.levels/0.zarr/var_c/c/0/1/1 new file mode 100644 index 000000000..3bc8df1ad Binary files /dev/null and b/xcube/testing/data/ds.levels/0.zarr/var_c/c/0/1/1 differ diff --git a/xcube/testing/data/ds.levels/0.zarr/var_c/c/1/0/0 b/xcube/testing/data/ds.levels/0.zarr/var_c/c/1/0/0 new file mode 100644 index 000000000..3bc8df1ad Binary files /dev/null and b/xcube/testing/data/ds.levels/0.zarr/var_c/c/1/0/0 differ diff --git a/xcube/testing/data/ds.levels/0.zarr/var_c/c/1/0/1 b/xcube/testing/data/ds.levels/0.zarr/var_c/c/1/0/1 new file mode 100644 index 000000000..3bc8df1ad Binary files /dev/null and b/xcube/testing/data/ds.levels/0.zarr/var_c/c/1/0/1 differ diff --git a/xcube/testing/data/ds.levels/0.zarr/var_c/c/1/1/0 b/xcube/testing/data/ds.levels/0.zarr/var_c/c/1/1/0 new file mode 100644 index 000000000..3bc8df1ad Binary files /dev/null and b/xcube/testing/data/ds.levels/0.zarr/var_c/c/1/1/0 differ diff --git a/xcube/testing/data/ds.levels/0.zarr/var_c/c/1/1/1 b/xcube/testing/data/ds.levels/0.zarr/var_c/c/1/1/1 new file mode 100644 index 000000000..3bc8df1ad Binary files /dev/null and b/xcube/testing/data/ds.levels/0.zarr/var_c/c/1/1/1 differ diff --git a/xcube/testing/data/ds.levels/0.zarr/var_c/c/2/0/0 b/xcube/testing/data/ds.levels/0.zarr/var_c/c/2/0/0 new file mode 100644 index 000000000..3bc8df1ad Binary files /dev/null and b/xcube/testing/data/ds.levels/0.zarr/var_c/c/2/0/0 differ diff --git a/xcube/testing/data/ds.levels/0.zarr/var_c/c/2/0/1 b/xcube/testing/data/ds.levels/0.zarr/var_c/c/2/0/1 new file mode 100644 index 000000000..3bc8df1ad Binary files /dev/null and b/xcube/testing/data/ds.levels/0.zarr/var_c/c/2/0/1 differ diff --git a/xcube/testing/data/ds.levels/0.zarr/var_c/c/2/1/0 b/xcube/testing/data/ds.levels/0.zarr/var_c/c/2/1/0 new file mode 100644 index 000000000..3bc8df1ad Binary files /dev/null and b/xcube/testing/data/ds.levels/0.zarr/var_c/c/2/1/0 differ diff --git a/xcube/testing/data/ds.levels/0.zarr/var_c/c/2/1/1 b/xcube/testing/data/ds.levels/0.zarr/var_c/c/2/1/1 new file mode 100644 index 000000000..3bc8df1ad Binary files /dev/null and b/xcube/testing/data/ds.levels/0.zarr/var_c/c/2/1/1 differ diff --git a/xcube/testing/data/ds.levels/0.zarr/var_c/c/3/0/0 b/xcube/testing/data/ds.levels/0.zarr/var_c/c/3/0/0 new file mode 100644 index 000000000..3bc8df1ad Binary files /dev/null and b/xcube/testing/data/ds.levels/0.zarr/var_c/c/3/0/0 differ diff --git a/xcube/testing/data/ds.levels/0.zarr/var_c/c/3/0/1 b/xcube/testing/data/ds.levels/0.zarr/var_c/c/3/0/1 new file mode 100644 index 000000000..3bc8df1ad Binary files /dev/null and b/xcube/testing/data/ds.levels/0.zarr/var_c/c/3/0/1 differ diff --git a/xcube/testing/data/ds.levels/0.zarr/var_c/c/3/1/0 b/xcube/testing/data/ds.levels/0.zarr/var_c/c/3/1/0 new file mode 100644 index 000000000..3bc8df1ad Binary files /dev/null and b/xcube/testing/data/ds.levels/0.zarr/var_c/c/3/1/0 differ diff --git a/xcube/testing/data/ds.levels/0.zarr/var_c/c/3/1/1 b/xcube/testing/data/ds.levels/0.zarr/var_c/c/3/1/1 new file mode 100644 index 000000000..3bc8df1ad Binary files /dev/null and b/xcube/testing/data/ds.levels/0.zarr/var_c/c/3/1/1 differ diff --git a/xcube/testing/data/ds.levels/0.zarr/var_c/c/4/0/0 b/xcube/testing/data/ds.levels/0.zarr/var_c/c/4/0/0 new file mode 100644 index 000000000..3bc8df1ad Binary files /dev/null and b/xcube/testing/data/ds.levels/0.zarr/var_c/c/4/0/0 differ diff --git a/xcube/testing/data/ds.levels/0.zarr/var_c/c/4/0/1 b/xcube/testing/data/ds.levels/0.zarr/var_c/c/4/0/1 new file mode 100644 index 000000000..3bc8df1ad Binary files /dev/null and b/xcube/testing/data/ds.levels/0.zarr/var_c/c/4/0/1 differ diff --git a/xcube/testing/data/ds.levels/0.zarr/var_c/c/4/1/0 b/xcube/testing/data/ds.levels/0.zarr/var_c/c/4/1/0 new file mode 100644 index 000000000..3bc8df1ad Binary files /dev/null and b/xcube/testing/data/ds.levels/0.zarr/var_c/c/4/1/0 differ diff --git a/xcube/testing/data/ds.levels/0.zarr/var_c/c/4/1/1 b/xcube/testing/data/ds.levels/0.zarr/var_c/c/4/1/1 new file mode 100644 index 000000000..3bc8df1ad Binary files /dev/null and b/xcube/testing/data/ds.levels/0.zarr/var_c/c/4/1/1 differ diff --git a/xcube/testing/data/ds.levels/0.zarr/var_c/zarr.json b/xcube/testing/data/ds.levels/0.zarr/var_c/zarr.json new file mode 100644 index 000000000..b7c264212 --- /dev/null +++ b/xcube/testing/data/ds.levels/0.zarr/var_c/zarr.json @@ -0,0 +1,48 @@ +{ + "shape": [ + 5, + 180, + 360 + ], + "data_type": "uint8", + "chunk_grid": { + "name": "regular", + "configuration": { + "chunk_shape": [ + 1, + 90, + 180 + ] + } + }, + "chunk_key_encoding": { + "name": "default", + "configuration": { + "separator": "/" + } + }, + "fill_value": 0, + "codecs": [ + { + "name": "bytes" + }, + { + "name": "zstd", + "configuration": { + "level": 0, + "checksum": false + } + } + ], + "attributes": { + "grid_mapping": "spatial_ref" + }, + "dimension_names": [ + "time", + "y", + "x" + ], + "zarr_format": 3, + "node_type": "array", + "storage_transformers": [] +} \ No newline at end of file diff --git a/xcube/testing/data/ds.levels/0.zarr/x/c/0 b/xcube/testing/data/ds.levels/0.zarr/x/c/0 new file mode 100644 index 000000000..2e6dffb50 Binary files /dev/null and b/xcube/testing/data/ds.levels/0.zarr/x/c/0 differ diff --git a/xcube/testing/data/ds.levels/0.zarr/x/zarr.json b/xcube/testing/data/ds.levels/0.zarr/x/zarr.json new file mode 100644 index 000000000..6d9598354 --- /dev/null +++ b/xcube/testing/data/ds.levels/0.zarr/x/zarr.json @@ -0,0 +1,49 @@ +{ + "shape": [ + 360 + ], + "data_type": "float64", + "chunk_grid": { + "name": "regular", + "configuration": { + "chunk_shape": [ + 360 + ] + } + }, + "chunk_key_encoding": { + "name": "default", + "configuration": { + "separator": "/" + } + }, + "fill_value": "NaN", + "codecs": [ + { + "name": "bytes", + "configuration": { + "endian": "little" + } + }, + { + "name": "zstd", + "configuration": { + "level": 0, + "checksum": false + } + } + ], + "attributes": { + "units": "degrees_east", + "long_name": "longitude", + "standard_name": "longitude", + "bounds": "x_bnds", + "_FillValue": "AAAAAAAA+H8=" + }, + "dimension_names": [ + "x" + ], + "zarr_format": 3, + "node_type": "array", + "storage_transformers": [] +} \ No newline at end of file diff --git a/xcube/testing/data/ds.levels/0.zarr/x_bnds/c/0/0 b/xcube/testing/data/ds.levels/0.zarr/x_bnds/c/0/0 new file mode 100644 index 000000000..a367ea83e Binary files /dev/null and b/xcube/testing/data/ds.levels/0.zarr/x_bnds/c/0/0 differ diff --git a/xcube/testing/data/ds.levels/0.zarr/x_bnds/zarr.json b/xcube/testing/data/ds.levels/0.zarr/x_bnds/zarr.json new file mode 100644 index 000000000..80401d407 --- /dev/null +++ b/xcube/testing/data/ds.levels/0.zarr/x_bnds/zarr.json @@ -0,0 +1,49 @@ +{ + "shape": [ + 360, + 2 + ], + "data_type": "float64", + "chunk_grid": { + "name": "regular", + "configuration": { + "chunk_shape": [ + 360, + 2 + ] + } + }, + "chunk_key_encoding": { + "name": "default", + "configuration": { + "separator": "/" + } + }, + "fill_value": "NaN", + "codecs": [ + { + "name": "bytes", + "configuration": { + "endian": "little" + } + }, + { + "name": "zstd", + "configuration": { + "level": 0, + "checksum": false + } + } + ], + "attributes": { + "units": "degrees_east", + "_FillValue": "AAAAAAAA+H8=" + }, + "dimension_names": [ + "x", + "bnds" + ], + "zarr_format": 3, + "node_type": "array", + "storage_transformers": [] +} \ No newline at end of file diff --git a/xcube/testing/data/ds.levels/0.zarr/y/c/0 b/xcube/testing/data/ds.levels/0.zarr/y/c/0 new file mode 100644 index 000000000..a731c8c5b Binary files /dev/null and b/xcube/testing/data/ds.levels/0.zarr/y/c/0 differ diff --git a/xcube/testing/data/ds.levels/0.zarr/y/zarr.json b/xcube/testing/data/ds.levels/0.zarr/y/zarr.json new file mode 100644 index 000000000..1ae5b39a9 --- /dev/null +++ b/xcube/testing/data/ds.levels/0.zarr/y/zarr.json @@ -0,0 +1,49 @@ +{ + "shape": [ + 180 + ], + "data_type": "float64", + "chunk_grid": { + "name": "regular", + "configuration": { + "chunk_shape": [ + 180 + ] + } + }, + "chunk_key_encoding": { + "name": "default", + "configuration": { + "separator": "/" + } + }, + "fill_value": "NaN", + "codecs": [ + { + "name": "bytes", + "configuration": { + "endian": "little" + } + }, + { + "name": "zstd", + "configuration": { + "level": 0, + "checksum": false + } + } + ], + "attributes": { + "units": "degrees_north", + "long_name": "latitude", + "standard_name": "latitude", + "bounds": "y_bnds", + "_FillValue": "AAAAAAAA+H8=" + }, + "dimension_names": [ + "y" + ], + "zarr_format": 3, + "node_type": "array", + "storage_transformers": [] +} \ No newline at end of file diff --git a/xcube/testing/data/ds.levels/0.zarr/y_bnds/c/0/0 b/xcube/testing/data/ds.levels/0.zarr/y_bnds/c/0/0 new file mode 100644 index 000000000..78bc7ba5a Binary files /dev/null and b/xcube/testing/data/ds.levels/0.zarr/y_bnds/c/0/0 differ diff --git a/xcube/testing/data/ds.levels/0.zarr/y_bnds/zarr.json b/xcube/testing/data/ds.levels/0.zarr/y_bnds/zarr.json new file mode 100644 index 000000000..d9917f27e --- /dev/null +++ b/xcube/testing/data/ds.levels/0.zarr/y_bnds/zarr.json @@ -0,0 +1,49 @@ +{ + "shape": [ + 180, + 2 + ], + "data_type": "float64", + "chunk_grid": { + "name": "regular", + "configuration": { + "chunk_shape": [ + 180, + 2 + ] + } + }, + "chunk_key_encoding": { + "name": "default", + "configuration": { + "separator": "/" + } + }, + "fill_value": "NaN", + "codecs": [ + { + "name": "bytes", + "configuration": { + "endian": "little" + } + }, + { + "name": "zstd", + "configuration": { + "level": 0, + "checksum": false + } + } + ], + "attributes": { + "units": "degrees_north", + "_FillValue": "AAAAAAAA+H8=" + }, + "dimension_names": [ + "y", + "bnds" + ], + "zarr_format": 3, + "node_type": "array", + "storage_transformers": [] +} \ No newline at end of file diff --git a/xcube/testing/data/ds.levels/0.zarr/zarr.json b/xcube/testing/data/ds.levels/0.zarr/zarr.json new file mode 100644 index 000000000..fc71d0fb5 --- /dev/null +++ b/xcube/testing/data/ds.levels/0.zarr/zarr.json @@ -0,0 +1,516 @@ +{ + "attributes": { + "Conventions": "CF-1.7", + "title": "Test Cube", + "time_coverage_start": "2010-01-01T00:00:00.000000000", + "time_coverage_end": "2010-01-06T00:00:00.000000000", + "geospatial_lon_min": -180.0, + "geospatial_lon_max": 180.0, + "geospatial_lon_units": "degrees_east", + "geospatial_lat_min": -90.0, + "geospatial_lat_max": 90.0, + "geospatial_lat_units": "degrees_north", + "coordinates": "time_bnds x_bnds y_bnds" + }, + "zarr_format": 3, + "consolidated_metadata": { + "kind": "inline", + "must_understand": false, + "metadata": { + "spatial_ref": { + "shape": [], + "data_type": "int64", + "chunk_grid": { + "name": "regular", + "configuration": { + "chunk_shape": [] + } + }, + "chunk_key_encoding": { + "name": "default", + "configuration": { + "separator": "/" + } + }, + "fill_value": 0, + "codecs": [ + { + "name": "bytes", + "configuration": { + "endian": "little" + } + }, + { + "name": "zstd", + "configuration": { + "level": 0, + "checksum": false + } + } + ], + "attributes": { + "crs_wkt": "GEOGCRS[\"WGS 84 (CRS84)\",ENSEMBLE[\"World Geodetic System 1984 ensemble\",MEMBER[\"World Geodetic System 1984 (Transit)\"],MEMBER[\"World Geodetic System 1984 (G730)\"],MEMBER[\"World Geodetic System 1984 (G873)\"],MEMBER[\"World Geodetic System 1984 (G1150)\"],MEMBER[\"World Geodetic System 1984 (G1674)\"],MEMBER[\"World Geodetic System 1984 (G1762)\"],MEMBER[\"World Geodetic System 1984 (G2139)\"],MEMBER[\"World Geodetic System 1984 (G2296)\"],ELLIPSOID[\"WGS 84\",6378137,298.257223563,LENGTHUNIT[\"metre\",1]],ENSEMBLEACCURACY[2.0]],PRIMEM[\"Greenwich\",0,ANGLEUNIT[\"degree\",0.0174532925199433]],CS[ellipsoidal,2],AXIS[\"geodetic longitude (Lon)\",east,ORDER[1],ANGLEUNIT[\"degree\",0.0174532925199433]],AXIS[\"geodetic latitude (Lat)\",north,ORDER[2],ANGLEUNIT[\"degree\",0.0174532925199433]],USAGE[SCOPE[\"Not known.\"],AREA[\"World.\"],BBOX[-90,-180,90,180]],ID[\"OGC\",\"CRS84\"]]", + "semi_major_axis": 6378137.0, + "semi_minor_axis": 6356752.314245179, + "inverse_flattening": 298.257223563, + "reference_ellipsoid_name": "WGS 84", + "longitude_of_prime_meridian": 0.0, + "prime_meridian_name": "Greenwich", + "geographic_crs_name": "WGS 84 (CRS84)", + "horizontal_datum_name": "World Geodetic System 1984 ensemble", + "grid_mapping_name": "latitude_longitude" + }, + "zarr_format": 3, + "node_type": "array", + "storage_transformers": [] + }, + "time": { + "shape": [ + 5 + ], + "data_type": "int64", + "chunk_grid": { + "name": "regular", + "configuration": { + "chunk_shape": [ + 5 + ] + } + }, + "chunk_key_encoding": { + "name": "default", + "configuration": { + "separator": "/" + } + }, + "fill_value": 0, + "codecs": [ + { + "name": "bytes", + "configuration": { + "endian": "little" + } + }, + { + "name": "zstd", + "configuration": { + "level": 0, + "checksum": false + } + } + ], + "attributes": { + "bounds": "time_bnds", + "units": "seconds since 1970-01-01", + "calendar": "proleptic_gregorian" + }, + "dimension_names": [ + "time" + ], + "zarr_format": 3, + "node_type": "array", + "storage_transformers": [] + }, + "time_bnds": { + "shape": [ + 5, + 2 + ], + "data_type": "int64", + "chunk_grid": { + "name": "regular", + "configuration": { + "chunk_shape": [ + 5, + 2 + ] + } + }, + "chunk_key_encoding": { + "name": "default", + "configuration": { + "separator": "/" + } + }, + "fill_value": 0, + "codecs": [ + { + "name": "bytes", + "configuration": { + "endian": "little" + } + }, + { + "name": "zstd", + "configuration": { + "level": 0, + "checksum": false + } + } + ], + "attributes": { + "units": "seconds since 1970-01-01T00:00:00", + "calendar": "proleptic_gregorian" + }, + "dimension_names": [ + "time", + "bnds" + ], + "zarr_format": 3, + "node_type": "array", + "storage_transformers": [] + }, + "var_a": { + "shape": [ + 5, + 180, + 360 + ], + "data_type": "float64", + "chunk_grid": { + "name": "regular", + "configuration": { + "chunk_shape": [ + 1, + 90, + 180 + ] + } + }, + "chunk_key_encoding": { + "name": "default", + "configuration": { + "separator": "/" + } + }, + "fill_value": "NaN", + "codecs": [ + { + "name": "bytes", + "configuration": { + "endian": "little" + } + }, + { + "name": "zstd", + "configuration": { + "level": 0, + "checksum": false + } + } + ], + "attributes": { + "grid_mapping": "spatial_ref", + "_FillValue": "AAAAAAAA+H8=" + }, + "dimension_names": [ + "time", + "y", + "x" + ], + "zarr_format": 3, + "node_type": "array", + "storage_transformers": [] + }, + "var_b": { + "shape": [ + 5, + 180, + 360 + ], + "data_type": "int16", + "chunk_grid": { + "name": "regular", + "configuration": { + "chunk_shape": [ + 1, + 90, + 180 + ] + } + }, + "chunk_key_encoding": { + "name": "default", + "configuration": { + "separator": "/" + } + }, + "fill_value": 0, + "codecs": [ + { + "name": "bytes", + "configuration": { + "endian": "little" + } + }, + { + "name": "zstd", + "configuration": { + "level": 0, + "checksum": false + } + } + ], + "attributes": { + "grid_mapping": "spatial_ref", + "add_offset": -10, + "scale_factor": 0.001, + "_FillValue": -9999 + }, + "dimension_names": [ + "time", + "y", + "x" + ], + "zarr_format": 3, + "node_type": "array", + "storage_transformers": [] + }, + "var_c": { + "shape": [ + 5, + 180, + 360 + ], + "data_type": "uint8", + "chunk_grid": { + "name": "regular", + "configuration": { + "chunk_shape": [ + 1, + 90, + 180 + ] + } + }, + "chunk_key_encoding": { + "name": "default", + "configuration": { + "separator": "/" + } + }, + "fill_value": 0, + "codecs": [ + { + "name": "bytes" + }, + { + "name": "zstd", + "configuration": { + "level": 0, + "checksum": false + } + } + ], + "attributes": { + "grid_mapping": "spatial_ref" + }, + "dimension_names": [ + "time", + "y", + "x" + ], + "zarr_format": 3, + "node_type": "array", + "storage_transformers": [] + }, + "x": { + "shape": [ + 360 + ], + "data_type": "float64", + "chunk_grid": { + "name": "regular", + "configuration": { + "chunk_shape": [ + 360 + ] + } + }, + "chunk_key_encoding": { + "name": "default", + "configuration": { + "separator": "/" + } + }, + "fill_value": "NaN", + "codecs": [ + { + "name": "bytes", + "configuration": { + "endian": "little" + } + }, + { + "name": "zstd", + "configuration": { + "level": 0, + "checksum": false + } + } + ], + "attributes": { + "units": "degrees_east", + "long_name": "longitude", + "standard_name": "longitude", + "bounds": "x_bnds", + "_FillValue": "AAAAAAAA+H8=" + }, + "dimension_names": [ + "x" + ], + "zarr_format": 3, + "node_type": "array", + "storage_transformers": [] + }, + "x_bnds": { + "shape": [ + 360, + 2 + ], + "data_type": "float64", + "chunk_grid": { + "name": "regular", + "configuration": { + "chunk_shape": [ + 360, + 2 + ] + } + }, + "chunk_key_encoding": { + "name": "default", + "configuration": { + "separator": "/" + } + }, + "fill_value": "NaN", + "codecs": [ + { + "name": "bytes", + "configuration": { + "endian": "little" + } + }, + { + "name": "zstd", + "configuration": { + "level": 0, + "checksum": false + } + } + ], + "attributes": { + "units": "degrees_east", + "_FillValue": "AAAAAAAA+H8=" + }, + "dimension_names": [ + "x", + "bnds" + ], + "zarr_format": 3, + "node_type": "array", + "storage_transformers": [] + }, + "y": { + "shape": [ + 180 + ], + "data_type": "float64", + "chunk_grid": { + "name": "regular", + "configuration": { + "chunk_shape": [ + 180 + ] + } + }, + "chunk_key_encoding": { + "name": "default", + "configuration": { + "separator": "/" + } + }, + "fill_value": "NaN", + "codecs": [ + { + "name": "bytes", + "configuration": { + "endian": "little" + } + }, + { + "name": "zstd", + "configuration": { + "level": 0, + "checksum": false + } + } + ], + "attributes": { + "units": "degrees_north", + "long_name": "latitude", + "standard_name": "latitude", + "bounds": "y_bnds", + "_FillValue": "AAAAAAAA+H8=" + }, + "dimension_names": [ + "y" + ], + "zarr_format": 3, + "node_type": "array", + "storage_transformers": [] + }, + "y_bnds": { + "shape": [ + 180, + 2 + ], + "data_type": "float64", + "chunk_grid": { + "name": "regular", + "configuration": { + "chunk_shape": [ + 180, + 2 + ] + } + }, + "chunk_key_encoding": { + "name": "default", + "configuration": { + "separator": "/" + } + }, + "fill_value": "NaN", + "codecs": [ + { + "name": "bytes", + "configuration": { + "endian": "little" + } + }, + { + "name": "zstd", + "configuration": { + "level": 0, + "checksum": false + } + } + ], + "attributes": { + "units": "degrees_north", + "_FillValue": "AAAAAAAA+H8=" + }, + "dimension_names": [ + "y", + "bnds" + ], + "zarr_format": 3, + "node_type": "array", + "storage_transformers": [] + } + } + }, + "node_type": "group" +} \ No newline at end of file diff --git a/xcube/testing/data/ds.levels/1.zarr/spatial_ref/zarr.json b/xcube/testing/data/ds.levels/1.zarr/spatial_ref/zarr.json new file mode 100644 index 000000000..c1c6262cb --- /dev/null +++ b/xcube/testing/data/ds.levels/1.zarr/spatial_ref/zarr.json @@ -0,0 +1,47 @@ +{ + "shape": [], + "data_type": "int64", + "chunk_grid": { + "name": "regular", + "configuration": { + "chunk_shape": [] + } + }, + "chunk_key_encoding": { + "name": "default", + "configuration": { + "separator": "/" + } + }, + "fill_value": 0, + "codecs": [ + { + "name": "bytes", + "configuration": { + "endian": "little" + } + }, + { + "name": "zstd", + "configuration": { + "level": 0, + "checksum": false + } + } + ], + "attributes": { + "crs_wkt": "GEOGCRS[\"WGS 84 (CRS84)\",ENSEMBLE[\"World Geodetic System 1984 ensemble\",MEMBER[\"World Geodetic System 1984 (Transit)\"],MEMBER[\"World Geodetic System 1984 (G730)\"],MEMBER[\"World Geodetic System 1984 (G873)\"],MEMBER[\"World Geodetic System 1984 (G1150)\"],MEMBER[\"World Geodetic System 1984 (G1674)\"],MEMBER[\"World Geodetic System 1984 (G1762)\"],MEMBER[\"World Geodetic System 1984 (G2139)\"],MEMBER[\"World Geodetic System 1984 (G2296)\"],ELLIPSOID[\"WGS 84\",6378137,298.257223563,LENGTHUNIT[\"metre\",1]],ENSEMBLEACCURACY[2.0]],PRIMEM[\"Greenwich\",0,ANGLEUNIT[\"degree\",0.0174532925199433]],CS[ellipsoidal,2],AXIS[\"geodetic longitude (Lon)\",east,ORDER[1],ANGLEUNIT[\"degree\",0.0174532925199433]],AXIS[\"geodetic latitude (Lat)\",north,ORDER[2],ANGLEUNIT[\"degree\",0.0174532925199433]],USAGE[SCOPE[\"Not known.\"],AREA[\"World.\"],BBOX[-90,-180,90,180]],ID[\"OGC\",\"CRS84\"]]", + "semi_major_axis": 6378137.0, + "semi_minor_axis": 6356752.314245179, + "inverse_flattening": 298.257223563, + "reference_ellipsoid_name": "WGS 84", + "longitude_of_prime_meridian": 0.0, + "prime_meridian_name": "Greenwich", + "geographic_crs_name": "WGS 84 (CRS84)", + "horizontal_datum_name": "World Geodetic System 1984 ensemble", + "grid_mapping_name": "latitude_longitude" + }, + "zarr_format": 3, + "node_type": "array", + "storage_transformers": [] +} \ No newline at end of file diff --git a/xcube/testing/data/ds.levels/1.zarr/time/c/0 b/xcube/testing/data/ds.levels/1.zarr/time/c/0 new file mode 100644 index 000000000..17d4981e7 Binary files /dev/null and b/xcube/testing/data/ds.levels/1.zarr/time/c/0 differ diff --git a/xcube/testing/data/ds.levels/1.zarr/time/zarr.json b/xcube/testing/data/ds.levels/1.zarr/time/zarr.json new file mode 100644 index 000000000..1b696e88e --- /dev/null +++ b/xcube/testing/data/ds.levels/1.zarr/time/zarr.json @@ -0,0 +1,47 @@ +{ + "shape": [ + 5 + ], + "data_type": "int64", + "chunk_grid": { + "name": "regular", + "configuration": { + "chunk_shape": [ + 5 + ] + } + }, + "chunk_key_encoding": { + "name": "default", + "configuration": { + "separator": "/" + } + }, + "fill_value": 0, + "codecs": [ + { + "name": "bytes", + "configuration": { + "endian": "little" + } + }, + { + "name": "zstd", + "configuration": { + "level": 0, + "checksum": false + } + } + ], + "attributes": { + "bounds": "time_bnds", + "units": "seconds since 1970-01-01", + "calendar": "proleptic_gregorian" + }, + "dimension_names": [ + "time" + ], + "zarr_format": 3, + "node_type": "array", + "storage_transformers": [] +} \ No newline at end of file diff --git a/xcube/testing/data/ds.levels/1.zarr/var_a/c/0/0/0 b/xcube/testing/data/ds.levels/1.zarr/var_a/c/0/0/0 new file mode 100644 index 000000000..8dabb53da Binary files /dev/null and b/xcube/testing/data/ds.levels/1.zarr/var_a/c/0/0/0 differ diff --git a/xcube/testing/data/ds.levels/1.zarr/var_a/c/1/0/0 b/xcube/testing/data/ds.levels/1.zarr/var_a/c/1/0/0 new file mode 100644 index 000000000..8dabb53da Binary files /dev/null and b/xcube/testing/data/ds.levels/1.zarr/var_a/c/1/0/0 differ diff --git a/xcube/testing/data/ds.levels/1.zarr/var_a/c/2/0/0 b/xcube/testing/data/ds.levels/1.zarr/var_a/c/2/0/0 new file mode 100644 index 000000000..8dabb53da Binary files /dev/null and b/xcube/testing/data/ds.levels/1.zarr/var_a/c/2/0/0 differ diff --git a/xcube/testing/data/ds.levels/1.zarr/var_a/c/3/0/0 b/xcube/testing/data/ds.levels/1.zarr/var_a/c/3/0/0 new file mode 100644 index 000000000..8dabb53da Binary files /dev/null and b/xcube/testing/data/ds.levels/1.zarr/var_a/c/3/0/0 differ diff --git a/xcube/testing/data/ds.levels/1.zarr/var_a/c/4/0/0 b/xcube/testing/data/ds.levels/1.zarr/var_a/c/4/0/0 new file mode 100644 index 000000000..8dabb53da Binary files /dev/null and b/xcube/testing/data/ds.levels/1.zarr/var_a/c/4/0/0 differ diff --git a/xcube/testing/data/ds.levels/1.zarr/var_a/zarr.json b/xcube/testing/data/ds.levels/1.zarr/var_a/zarr.json new file mode 100644 index 000000000..d1104394b --- /dev/null +++ b/xcube/testing/data/ds.levels/1.zarr/var_a/zarr.json @@ -0,0 +1,52 @@ +{ + "shape": [ + 5, + 90, + 180 + ], + "data_type": "float64", + "chunk_grid": { + "name": "regular", + "configuration": { + "chunk_shape": [ + 1, + 90, + 180 + ] + } + }, + "chunk_key_encoding": { + "name": "default", + "configuration": { + "separator": "/" + } + }, + "fill_value": "NaN", + "codecs": [ + { + "name": "bytes", + "configuration": { + "endian": "little" + } + }, + { + "name": "zstd", + "configuration": { + "level": 0, + "checksum": false + } + } + ], + "attributes": { + "grid_mapping": "spatial_ref", + "_FillValue": "AAAAAAAA+H8=" + }, + "dimension_names": [ + "time", + "y", + "x" + ], + "zarr_format": 3, + "node_type": "array", + "storage_transformers": [] +} \ No newline at end of file diff --git a/xcube/testing/data/ds.levels/1.zarr/var_b/c/0/0/0 b/xcube/testing/data/ds.levels/1.zarr/var_b/c/0/0/0 new file mode 100644 index 000000000..b56c63cda Binary files /dev/null and b/xcube/testing/data/ds.levels/1.zarr/var_b/c/0/0/0 differ diff --git a/xcube/testing/data/ds.levels/1.zarr/var_b/c/1/0/0 b/xcube/testing/data/ds.levels/1.zarr/var_b/c/1/0/0 new file mode 100644 index 000000000..b56c63cda Binary files /dev/null and b/xcube/testing/data/ds.levels/1.zarr/var_b/c/1/0/0 differ diff --git a/xcube/testing/data/ds.levels/1.zarr/var_b/c/2/0/0 b/xcube/testing/data/ds.levels/1.zarr/var_b/c/2/0/0 new file mode 100644 index 000000000..b56c63cda Binary files /dev/null and b/xcube/testing/data/ds.levels/1.zarr/var_b/c/2/0/0 differ diff --git a/xcube/testing/data/ds.levels/1.zarr/var_b/c/3/0/0 b/xcube/testing/data/ds.levels/1.zarr/var_b/c/3/0/0 new file mode 100644 index 000000000..b56c63cda Binary files /dev/null and b/xcube/testing/data/ds.levels/1.zarr/var_b/c/3/0/0 differ diff --git a/xcube/testing/data/ds.levels/1.zarr/var_b/c/4/0/0 b/xcube/testing/data/ds.levels/1.zarr/var_b/c/4/0/0 new file mode 100644 index 000000000..b56c63cda Binary files /dev/null and b/xcube/testing/data/ds.levels/1.zarr/var_b/c/4/0/0 differ diff --git a/xcube/testing/data/ds.levels/1.zarr/var_b/zarr.json b/xcube/testing/data/ds.levels/1.zarr/var_b/zarr.json new file mode 100644 index 000000000..07045e1f1 --- /dev/null +++ b/xcube/testing/data/ds.levels/1.zarr/var_b/zarr.json @@ -0,0 +1,54 @@ +{ + "shape": [ + 5, + 90, + 180 + ], + "data_type": "int16", + "chunk_grid": { + "name": "regular", + "configuration": { + "chunk_shape": [ + 1, + 90, + 180 + ] + } + }, + "chunk_key_encoding": { + "name": "default", + "configuration": { + "separator": "/" + } + }, + "fill_value": 0, + "codecs": [ + { + "name": "bytes", + "configuration": { + "endian": "little" + } + }, + { + "name": "zstd", + "configuration": { + "level": 0, + "checksum": false + } + } + ], + "attributes": { + "grid_mapping": "spatial_ref", + "add_offset": -10, + "scale_factor": 0.001, + "_FillValue": -9999 + }, + "dimension_names": [ + "time", + "y", + "x" + ], + "zarr_format": 3, + "node_type": "array", + "storage_transformers": [] +} \ No newline at end of file diff --git a/xcube/testing/data/ds.levels/1.zarr/var_c/c/0/0/0 b/xcube/testing/data/ds.levels/1.zarr/var_c/c/0/0/0 new file mode 100644 index 000000000..3bc8df1ad Binary files /dev/null and b/xcube/testing/data/ds.levels/1.zarr/var_c/c/0/0/0 differ diff --git a/xcube/testing/data/ds.levels/1.zarr/var_c/c/1/0/0 b/xcube/testing/data/ds.levels/1.zarr/var_c/c/1/0/0 new file mode 100644 index 000000000..3bc8df1ad Binary files /dev/null and b/xcube/testing/data/ds.levels/1.zarr/var_c/c/1/0/0 differ diff --git a/xcube/testing/data/ds.levels/1.zarr/var_c/c/2/0/0 b/xcube/testing/data/ds.levels/1.zarr/var_c/c/2/0/0 new file mode 100644 index 000000000..3bc8df1ad Binary files /dev/null and b/xcube/testing/data/ds.levels/1.zarr/var_c/c/2/0/0 differ diff --git a/xcube/testing/data/ds.levels/1.zarr/var_c/c/3/0/0 b/xcube/testing/data/ds.levels/1.zarr/var_c/c/3/0/0 new file mode 100644 index 000000000..3bc8df1ad Binary files /dev/null and b/xcube/testing/data/ds.levels/1.zarr/var_c/c/3/0/0 differ diff --git a/xcube/testing/data/ds.levels/1.zarr/var_c/c/4/0/0 b/xcube/testing/data/ds.levels/1.zarr/var_c/c/4/0/0 new file mode 100644 index 000000000..3bc8df1ad Binary files /dev/null and b/xcube/testing/data/ds.levels/1.zarr/var_c/c/4/0/0 differ diff --git a/xcube/testing/data/ds.levels/1.zarr/var_c/zarr.json b/xcube/testing/data/ds.levels/1.zarr/var_c/zarr.json new file mode 100644 index 000000000..8c12d49f2 --- /dev/null +++ b/xcube/testing/data/ds.levels/1.zarr/var_c/zarr.json @@ -0,0 +1,48 @@ +{ + "shape": [ + 5, + 90, + 180 + ], + "data_type": "uint8", + "chunk_grid": { + "name": "regular", + "configuration": { + "chunk_shape": [ + 1, + 90, + 180 + ] + } + }, + "chunk_key_encoding": { + "name": "default", + "configuration": { + "separator": "/" + } + }, + "fill_value": 0, + "codecs": [ + { + "name": "bytes" + }, + { + "name": "zstd", + "configuration": { + "level": 0, + "checksum": false + } + } + ], + "attributes": { + "grid_mapping": "spatial_ref" + }, + "dimension_names": [ + "time", + "y", + "x" + ], + "zarr_format": 3, + "node_type": "array", + "storage_transformers": [] +} \ No newline at end of file diff --git a/xcube/testing/data/ds.levels/1.zarr/x/c/0 b/xcube/testing/data/ds.levels/1.zarr/x/c/0 new file mode 100644 index 000000000..0007ae611 Binary files /dev/null and b/xcube/testing/data/ds.levels/1.zarr/x/c/0 differ diff --git a/xcube/testing/data/ds.levels/1.zarr/x/zarr.json b/xcube/testing/data/ds.levels/1.zarr/x/zarr.json new file mode 100644 index 000000000..2dbf35930 --- /dev/null +++ b/xcube/testing/data/ds.levels/1.zarr/x/zarr.json @@ -0,0 +1,49 @@ +{ + "shape": [ + 180 + ], + "data_type": "float64", + "chunk_grid": { + "name": "regular", + "configuration": { + "chunk_shape": [ + 180 + ] + } + }, + "chunk_key_encoding": { + "name": "default", + "configuration": { + "separator": "/" + } + }, + "fill_value": "NaN", + "codecs": [ + { + "name": "bytes", + "configuration": { + "endian": "little" + } + }, + { + "name": "zstd", + "configuration": { + "level": 0, + "checksum": false + } + } + ], + "attributes": { + "units": "degrees_east", + "long_name": "longitude", + "standard_name": "longitude", + "bounds": "x_bnds", + "_FillValue": "AAAAAAAA+H8=" + }, + "dimension_names": [ + "x" + ], + "zarr_format": 3, + "node_type": "array", + "storage_transformers": [] +} \ No newline at end of file diff --git a/xcube/testing/data/ds.levels/1.zarr/y/c/0 b/xcube/testing/data/ds.levels/1.zarr/y/c/0 new file mode 100644 index 000000000..1a21df929 Binary files /dev/null and b/xcube/testing/data/ds.levels/1.zarr/y/c/0 differ diff --git a/xcube/testing/data/ds.levels/1.zarr/y/zarr.json b/xcube/testing/data/ds.levels/1.zarr/y/zarr.json new file mode 100644 index 000000000..ba9068e43 --- /dev/null +++ b/xcube/testing/data/ds.levels/1.zarr/y/zarr.json @@ -0,0 +1,49 @@ +{ + "shape": [ + 90 + ], + "data_type": "float64", + "chunk_grid": { + "name": "regular", + "configuration": { + "chunk_shape": [ + 90 + ] + } + }, + "chunk_key_encoding": { + "name": "default", + "configuration": { + "separator": "/" + } + }, + "fill_value": "NaN", + "codecs": [ + { + "name": "bytes", + "configuration": { + "endian": "little" + } + }, + { + "name": "zstd", + "configuration": { + "level": 0, + "checksum": false + } + } + ], + "attributes": { + "units": "degrees_north", + "long_name": "latitude", + "standard_name": "latitude", + "bounds": "y_bnds", + "_FillValue": "AAAAAAAA+H8=" + }, + "dimension_names": [ + "y" + ], + "zarr_format": 3, + "node_type": "array", + "storage_transformers": [] +} \ No newline at end of file diff --git a/xcube/testing/data/ds.levels/1.zarr/zarr.json b/xcube/testing/data/ds.levels/1.zarr/zarr.json new file mode 100644 index 000000000..13fe763f3 --- /dev/null +++ b/xcube/testing/data/ds.levels/1.zarr/zarr.json @@ -0,0 +1,368 @@ +{ + "attributes": { + "Conventions": "CF-1.7", + "title": "Test Cube", + "time_coverage_start": "2010-01-01T00:00:00.000000000", + "time_coverage_end": "2010-01-06T00:00:00.000000000", + "geospatial_lon_min": -180.0, + "geospatial_lon_max": 180.0, + "geospatial_lon_units": "degrees_east", + "geospatial_lat_min": -90.0, + "geospatial_lat_max": 90.0, + "geospatial_lat_units": "degrees_north" + }, + "zarr_format": 3, + "consolidated_metadata": { + "kind": "inline", + "must_understand": false, + "metadata": { + "spatial_ref": { + "shape": [], + "data_type": "int64", + "chunk_grid": { + "name": "regular", + "configuration": { + "chunk_shape": [] + } + }, + "chunk_key_encoding": { + "name": "default", + "configuration": { + "separator": "/" + } + }, + "fill_value": 0, + "codecs": [ + { + "name": "bytes", + "configuration": { + "endian": "little" + } + }, + { + "name": "zstd", + "configuration": { + "level": 0, + "checksum": false + } + } + ], + "attributes": { + "crs_wkt": "GEOGCRS[\"WGS 84 (CRS84)\",ENSEMBLE[\"World Geodetic System 1984 ensemble\",MEMBER[\"World Geodetic System 1984 (Transit)\"],MEMBER[\"World Geodetic System 1984 (G730)\"],MEMBER[\"World Geodetic System 1984 (G873)\"],MEMBER[\"World Geodetic System 1984 (G1150)\"],MEMBER[\"World Geodetic System 1984 (G1674)\"],MEMBER[\"World Geodetic System 1984 (G1762)\"],MEMBER[\"World Geodetic System 1984 (G2139)\"],MEMBER[\"World Geodetic System 1984 (G2296)\"],ELLIPSOID[\"WGS 84\",6378137,298.257223563,LENGTHUNIT[\"metre\",1]],ENSEMBLEACCURACY[2.0]],PRIMEM[\"Greenwich\",0,ANGLEUNIT[\"degree\",0.0174532925199433]],CS[ellipsoidal,2],AXIS[\"geodetic longitude (Lon)\",east,ORDER[1],ANGLEUNIT[\"degree\",0.0174532925199433]],AXIS[\"geodetic latitude (Lat)\",north,ORDER[2],ANGLEUNIT[\"degree\",0.0174532925199433]],USAGE[SCOPE[\"Not known.\"],AREA[\"World.\"],BBOX[-90,-180,90,180]],ID[\"OGC\",\"CRS84\"]]", + "semi_major_axis": 6378137.0, + "semi_minor_axis": 6356752.314245179, + "inverse_flattening": 298.257223563, + "reference_ellipsoid_name": "WGS 84", + "longitude_of_prime_meridian": 0.0, + "prime_meridian_name": "Greenwich", + "geographic_crs_name": "WGS 84 (CRS84)", + "horizontal_datum_name": "World Geodetic System 1984 ensemble", + "grid_mapping_name": "latitude_longitude" + }, + "zarr_format": 3, + "node_type": "array", + "storage_transformers": [] + }, + "time": { + "shape": [ + 5 + ], + "data_type": "int64", + "chunk_grid": { + "name": "regular", + "configuration": { + "chunk_shape": [ + 5 + ] + } + }, + "chunk_key_encoding": { + "name": "default", + "configuration": { + "separator": "/" + } + }, + "fill_value": 0, + "codecs": [ + { + "name": "bytes", + "configuration": { + "endian": "little" + } + }, + { + "name": "zstd", + "configuration": { + "level": 0, + "checksum": false + } + } + ], + "attributes": { + "bounds": "time_bnds", + "units": "seconds since 1970-01-01", + "calendar": "proleptic_gregorian" + }, + "dimension_names": [ + "time" + ], + "zarr_format": 3, + "node_type": "array", + "storage_transformers": [] + }, + "var_a": { + "shape": [ + 5, + 90, + 180 + ], + "data_type": "float64", + "chunk_grid": { + "name": "regular", + "configuration": { + "chunk_shape": [ + 1, + 90, + 180 + ] + } + }, + "chunk_key_encoding": { + "name": "default", + "configuration": { + "separator": "/" + } + }, + "fill_value": "NaN", + "codecs": [ + { + "name": "bytes", + "configuration": { + "endian": "little" + } + }, + { + "name": "zstd", + "configuration": { + "level": 0, + "checksum": false + } + } + ], + "attributes": { + "grid_mapping": "spatial_ref", + "_FillValue": "AAAAAAAA+H8=" + }, + "dimension_names": [ + "time", + "y", + "x" + ], + "zarr_format": 3, + "node_type": "array", + "storage_transformers": [] + }, + "var_b": { + "shape": [ + 5, + 90, + 180 + ], + "data_type": "int16", + "chunk_grid": { + "name": "regular", + "configuration": { + "chunk_shape": [ + 1, + 90, + 180 + ] + } + }, + "chunk_key_encoding": { + "name": "default", + "configuration": { + "separator": "/" + } + }, + "fill_value": 0, + "codecs": [ + { + "name": "bytes", + "configuration": { + "endian": "little" + } + }, + { + "name": "zstd", + "configuration": { + "level": 0, + "checksum": false + } + } + ], + "attributes": { + "grid_mapping": "spatial_ref", + "add_offset": -10, + "scale_factor": 0.001, + "_FillValue": -9999 + }, + "dimension_names": [ + "time", + "y", + "x" + ], + "zarr_format": 3, + "node_type": "array", + "storage_transformers": [] + }, + "var_c": { + "shape": [ + 5, + 90, + 180 + ], + "data_type": "uint8", + "chunk_grid": { + "name": "regular", + "configuration": { + "chunk_shape": [ + 1, + 90, + 180 + ] + } + }, + "chunk_key_encoding": { + "name": "default", + "configuration": { + "separator": "/" + } + }, + "fill_value": 0, + "codecs": [ + { + "name": "bytes" + }, + { + "name": "zstd", + "configuration": { + "level": 0, + "checksum": false + } + } + ], + "attributes": { + "grid_mapping": "spatial_ref" + }, + "dimension_names": [ + "time", + "y", + "x" + ], + "zarr_format": 3, + "node_type": "array", + "storage_transformers": [] + }, + "x": { + "shape": [ + 180 + ], + "data_type": "float64", + "chunk_grid": { + "name": "regular", + "configuration": { + "chunk_shape": [ + 180 + ] + } + }, + "chunk_key_encoding": { + "name": "default", + "configuration": { + "separator": "/" + } + }, + "fill_value": "NaN", + "codecs": [ + { + "name": "bytes", + "configuration": { + "endian": "little" + } + }, + { + "name": "zstd", + "configuration": { + "level": 0, + "checksum": false + } + } + ], + "attributes": { + "units": "degrees_east", + "long_name": "longitude", + "standard_name": "longitude", + "bounds": "x_bnds", + "_FillValue": "AAAAAAAA+H8=" + }, + "dimension_names": [ + "x" + ], + "zarr_format": 3, + "node_type": "array", + "storage_transformers": [] + }, + "y": { + "shape": [ + 90 + ], + "data_type": "float64", + "chunk_grid": { + "name": "regular", + "configuration": { + "chunk_shape": [ + 90 + ] + } + }, + "chunk_key_encoding": { + "name": "default", + "configuration": { + "separator": "/" + } + }, + "fill_value": "NaN", + "codecs": [ + { + "name": "bytes", + "configuration": { + "endian": "little" + } + }, + { + "name": "zstd", + "configuration": { + "level": 0, + "checksum": false + } + } + ], + "attributes": { + "units": "degrees_north", + "long_name": "latitude", + "standard_name": "latitude", + "bounds": "y_bnds", + "_FillValue": "AAAAAAAA+H8=" + }, + "dimension_names": [ + "y" + ], + "zarr_format": 3, + "node_type": "array", + "storage_transformers": [] + } + } + }, + "node_type": "group" +} \ No newline at end of file diff --git a/xcube/testing/data/ds.zarr/spatial_ref/zarr.json b/xcube/testing/data/ds.zarr/spatial_ref/zarr.json new file mode 100644 index 000000000..c1c6262cb --- /dev/null +++ b/xcube/testing/data/ds.zarr/spatial_ref/zarr.json @@ -0,0 +1,47 @@ +{ + "shape": [], + "data_type": "int64", + "chunk_grid": { + "name": "regular", + "configuration": { + "chunk_shape": [] + } + }, + "chunk_key_encoding": { + "name": "default", + "configuration": { + "separator": "/" + } + }, + "fill_value": 0, + "codecs": [ + { + "name": "bytes", + "configuration": { + "endian": "little" + } + }, + { + "name": "zstd", + "configuration": { + "level": 0, + "checksum": false + } + } + ], + "attributes": { + "crs_wkt": "GEOGCRS[\"WGS 84 (CRS84)\",ENSEMBLE[\"World Geodetic System 1984 ensemble\",MEMBER[\"World Geodetic System 1984 (Transit)\"],MEMBER[\"World Geodetic System 1984 (G730)\"],MEMBER[\"World Geodetic System 1984 (G873)\"],MEMBER[\"World Geodetic System 1984 (G1150)\"],MEMBER[\"World Geodetic System 1984 (G1674)\"],MEMBER[\"World Geodetic System 1984 (G1762)\"],MEMBER[\"World Geodetic System 1984 (G2139)\"],MEMBER[\"World Geodetic System 1984 (G2296)\"],ELLIPSOID[\"WGS 84\",6378137,298.257223563,LENGTHUNIT[\"metre\",1]],ENSEMBLEACCURACY[2.0]],PRIMEM[\"Greenwich\",0,ANGLEUNIT[\"degree\",0.0174532925199433]],CS[ellipsoidal,2],AXIS[\"geodetic longitude (Lon)\",east,ORDER[1],ANGLEUNIT[\"degree\",0.0174532925199433]],AXIS[\"geodetic latitude (Lat)\",north,ORDER[2],ANGLEUNIT[\"degree\",0.0174532925199433]],USAGE[SCOPE[\"Not known.\"],AREA[\"World.\"],BBOX[-90,-180,90,180]],ID[\"OGC\",\"CRS84\"]]", + "semi_major_axis": 6378137.0, + "semi_minor_axis": 6356752.314245179, + "inverse_flattening": 298.257223563, + "reference_ellipsoid_name": "WGS 84", + "longitude_of_prime_meridian": 0.0, + "prime_meridian_name": "Greenwich", + "geographic_crs_name": "WGS 84 (CRS84)", + "horizontal_datum_name": "World Geodetic System 1984 ensemble", + "grid_mapping_name": "latitude_longitude" + }, + "zarr_format": 3, + "node_type": "array", + "storage_transformers": [] +} \ No newline at end of file diff --git a/xcube/testing/data/ds.zarr/time/c/0 b/xcube/testing/data/ds.zarr/time/c/0 new file mode 100644 index 000000000..17d4981e7 Binary files /dev/null and b/xcube/testing/data/ds.zarr/time/c/0 differ diff --git a/xcube/testing/data/ds.zarr/time/zarr.json b/xcube/testing/data/ds.zarr/time/zarr.json new file mode 100644 index 000000000..1b696e88e --- /dev/null +++ b/xcube/testing/data/ds.zarr/time/zarr.json @@ -0,0 +1,47 @@ +{ + "shape": [ + 5 + ], + "data_type": "int64", + "chunk_grid": { + "name": "regular", + "configuration": { + "chunk_shape": [ + 5 + ] + } + }, + "chunk_key_encoding": { + "name": "default", + "configuration": { + "separator": "/" + } + }, + "fill_value": 0, + "codecs": [ + { + "name": "bytes", + "configuration": { + "endian": "little" + } + }, + { + "name": "zstd", + "configuration": { + "level": 0, + "checksum": false + } + } + ], + "attributes": { + "bounds": "time_bnds", + "units": "seconds since 1970-01-01", + "calendar": "proleptic_gregorian" + }, + "dimension_names": [ + "time" + ], + "zarr_format": 3, + "node_type": "array", + "storage_transformers": [] +} \ No newline at end of file diff --git a/xcube/testing/data/ds.zarr/time_bnds/c/0/0 b/xcube/testing/data/ds.zarr/time_bnds/c/0/0 new file mode 100644 index 000000000..e90b60475 Binary files /dev/null and b/xcube/testing/data/ds.zarr/time_bnds/c/0/0 differ diff --git a/xcube/testing/data/ds.zarr/time_bnds/c/1/0 b/xcube/testing/data/ds.zarr/time_bnds/c/1/0 new file mode 100644 index 000000000..0d3bae824 Binary files /dev/null and b/xcube/testing/data/ds.zarr/time_bnds/c/1/0 differ diff --git a/xcube/testing/data/ds.zarr/time_bnds/c/2/0 b/xcube/testing/data/ds.zarr/time_bnds/c/2/0 new file mode 100644 index 000000000..c40783f37 Binary files /dev/null and b/xcube/testing/data/ds.zarr/time_bnds/c/2/0 differ diff --git a/xcube/testing/data/ds.zarr/time_bnds/c/3/0 b/xcube/testing/data/ds.zarr/time_bnds/c/3/0 new file mode 100644 index 000000000..da4352d05 Binary files /dev/null and b/xcube/testing/data/ds.zarr/time_bnds/c/3/0 differ diff --git a/xcube/testing/data/ds.zarr/time_bnds/c/4/0 b/xcube/testing/data/ds.zarr/time_bnds/c/4/0 new file mode 100644 index 000000000..a939ee561 Binary files /dev/null and b/xcube/testing/data/ds.zarr/time_bnds/c/4/0 differ diff --git a/xcube/testing/data/ds.zarr/time_bnds/zarr.json b/xcube/testing/data/ds.zarr/time_bnds/zarr.json new file mode 100644 index 000000000..718812cc1 --- /dev/null +++ b/xcube/testing/data/ds.zarr/time_bnds/zarr.json @@ -0,0 +1,49 @@ +{ + "shape": [ + 5, + 2 + ], + "data_type": "int64", + "chunk_grid": { + "name": "regular", + "configuration": { + "chunk_shape": [ + 1, + 2 + ] + } + }, + "chunk_key_encoding": { + "name": "default", + "configuration": { + "separator": "/" + } + }, + "fill_value": 0, + "codecs": [ + { + "name": "bytes", + "configuration": { + "endian": "little" + } + }, + { + "name": "zstd", + "configuration": { + "level": 0, + "checksum": false + } + } + ], + "attributes": { + "units": "seconds since 1970-01-01T00:00:00", + "calendar": "proleptic_gregorian" + }, + "dimension_names": [ + "time", + "bnds" + ], + "zarr_format": 3, + "node_type": "array", + "storage_transformers": [] +} \ No newline at end of file diff --git a/xcube/testing/data/ds.zarr/var_a/c/0/0/0 b/xcube/testing/data/ds.zarr/var_a/c/0/0/0 new file mode 100644 index 000000000..783927e0d Binary files /dev/null and b/xcube/testing/data/ds.zarr/var_a/c/0/0/0 differ diff --git a/xcube/testing/data/ds.zarr/var_a/c/0/0/1 b/xcube/testing/data/ds.zarr/var_a/c/0/0/1 new file mode 100644 index 000000000..8dabb53da Binary files /dev/null and b/xcube/testing/data/ds.zarr/var_a/c/0/0/1 differ diff --git a/xcube/testing/data/ds.zarr/var_a/c/0/1/0 b/xcube/testing/data/ds.zarr/var_a/c/0/1/0 new file mode 100644 index 000000000..8dabb53da Binary files /dev/null and b/xcube/testing/data/ds.zarr/var_a/c/0/1/0 differ diff --git a/xcube/testing/data/ds.zarr/var_a/c/0/1/1 b/xcube/testing/data/ds.zarr/var_a/c/0/1/1 new file mode 100644 index 000000000..8dabb53da Binary files /dev/null and b/xcube/testing/data/ds.zarr/var_a/c/0/1/1 differ diff --git a/xcube/testing/data/ds.zarr/var_a/c/1/0/0 b/xcube/testing/data/ds.zarr/var_a/c/1/0/0 new file mode 100644 index 000000000..8dabb53da Binary files /dev/null and b/xcube/testing/data/ds.zarr/var_a/c/1/0/0 differ diff --git a/xcube/testing/data/ds.zarr/var_a/c/1/0/1 b/xcube/testing/data/ds.zarr/var_a/c/1/0/1 new file mode 100644 index 000000000..8dabb53da Binary files /dev/null and b/xcube/testing/data/ds.zarr/var_a/c/1/0/1 differ diff --git a/xcube/testing/data/ds.zarr/var_a/c/1/1/0 b/xcube/testing/data/ds.zarr/var_a/c/1/1/0 new file mode 100644 index 000000000..8dabb53da Binary files /dev/null and b/xcube/testing/data/ds.zarr/var_a/c/1/1/0 differ diff --git a/xcube/testing/data/ds.zarr/var_a/c/1/1/1 b/xcube/testing/data/ds.zarr/var_a/c/1/1/1 new file mode 100644 index 000000000..8dabb53da Binary files /dev/null and b/xcube/testing/data/ds.zarr/var_a/c/1/1/1 differ diff --git a/xcube/testing/data/ds.zarr/var_a/c/2/0/0 b/xcube/testing/data/ds.zarr/var_a/c/2/0/0 new file mode 100644 index 000000000..8dabb53da Binary files /dev/null and b/xcube/testing/data/ds.zarr/var_a/c/2/0/0 differ diff --git a/xcube/testing/data/ds.zarr/var_a/c/2/0/1 b/xcube/testing/data/ds.zarr/var_a/c/2/0/1 new file mode 100644 index 000000000..8dabb53da Binary files /dev/null and b/xcube/testing/data/ds.zarr/var_a/c/2/0/1 differ diff --git a/xcube/testing/data/ds.zarr/var_a/c/2/1/0 b/xcube/testing/data/ds.zarr/var_a/c/2/1/0 new file mode 100644 index 000000000..8dabb53da Binary files /dev/null and b/xcube/testing/data/ds.zarr/var_a/c/2/1/0 differ diff --git a/xcube/testing/data/ds.zarr/var_a/c/2/1/1 b/xcube/testing/data/ds.zarr/var_a/c/2/1/1 new file mode 100644 index 000000000..8dabb53da Binary files /dev/null and b/xcube/testing/data/ds.zarr/var_a/c/2/1/1 differ diff --git a/xcube/testing/data/ds.zarr/var_a/c/3/0/0 b/xcube/testing/data/ds.zarr/var_a/c/3/0/0 new file mode 100644 index 000000000..8dabb53da Binary files /dev/null and b/xcube/testing/data/ds.zarr/var_a/c/3/0/0 differ diff --git a/xcube/testing/data/ds.zarr/var_a/c/3/0/1 b/xcube/testing/data/ds.zarr/var_a/c/3/0/1 new file mode 100644 index 000000000..8dabb53da Binary files /dev/null and b/xcube/testing/data/ds.zarr/var_a/c/3/0/1 differ diff --git a/xcube/testing/data/ds.zarr/var_a/c/3/1/0 b/xcube/testing/data/ds.zarr/var_a/c/3/1/0 new file mode 100644 index 000000000..8dabb53da Binary files /dev/null and b/xcube/testing/data/ds.zarr/var_a/c/3/1/0 differ diff --git a/xcube/testing/data/ds.zarr/var_a/c/3/1/1 b/xcube/testing/data/ds.zarr/var_a/c/3/1/1 new file mode 100644 index 000000000..8dabb53da Binary files /dev/null and b/xcube/testing/data/ds.zarr/var_a/c/3/1/1 differ diff --git a/xcube/testing/data/ds.zarr/var_a/c/4/0/0 b/xcube/testing/data/ds.zarr/var_a/c/4/0/0 new file mode 100644 index 000000000..8dabb53da Binary files /dev/null and b/xcube/testing/data/ds.zarr/var_a/c/4/0/0 differ diff --git a/xcube/testing/data/ds.zarr/var_a/c/4/0/1 b/xcube/testing/data/ds.zarr/var_a/c/4/0/1 new file mode 100644 index 000000000..8dabb53da Binary files /dev/null and b/xcube/testing/data/ds.zarr/var_a/c/4/0/1 differ diff --git a/xcube/testing/data/ds.zarr/var_a/c/4/1/0 b/xcube/testing/data/ds.zarr/var_a/c/4/1/0 new file mode 100644 index 000000000..8dabb53da Binary files /dev/null and b/xcube/testing/data/ds.zarr/var_a/c/4/1/0 differ diff --git a/xcube/testing/data/ds.zarr/var_a/c/4/1/1 b/xcube/testing/data/ds.zarr/var_a/c/4/1/1 new file mode 100644 index 000000000..8dabb53da Binary files /dev/null and b/xcube/testing/data/ds.zarr/var_a/c/4/1/1 differ diff --git a/xcube/testing/data/ds.zarr/var_a/zarr.json b/xcube/testing/data/ds.zarr/var_a/zarr.json new file mode 100644 index 000000000..5a4a0702f --- /dev/null +++ b/xcube/testing/data/ds.zarr/var_a/zarr.json @@ -0,0 +1,52 @@ +{ + "shape": [ + 5, + 180, + 360 + ], + "data_type": "float64", + "chunk_grid": { + "name": "regular", + "configuration": { + "chunk_shape": [ + 1, + 90, + 180 + ] + } + }, + "chunk_key_encoding": { + "name": "default", + "configuration": { + "separator": "/" + } + }, + "fill_value": "NaN", + "codecs": [ + { + "name": "bytes", + "configuration": { + "endian": "little" + } + }, + { + "name": "zstd", + "configuration": { + "level": 0, + "checksum": false + } + } + ], + "attributes": { + "grid_mapping": "spatial_ref", + "_FillValue": "AAAAAAAA+H8=" + }, + "dimension_names": [ + "time", + "y", + "x" + ], + "zarr_format": 3, + "node_type": "array", + "storage_transformers": [] +} \ No newline at end of file diff --git a/xcube/testing/data/ds.zarr/var_b/c/0/0/0 b/xcube/testing/data/ds.zarr/var_b/c/0/0/0 new file mode 100644 index 000000000..7f2993c19 Binary files /dev/null and b/xcube/testing/data/ds.zarr/var_b/c/0/0/0 differ diff --git a/xcube/testing/data/ds.zarr/var_b/c/0/0/1 b/xcube/testing/data/ds.zarr/var_b/c/0/0/1 new file mode 100644 index 000000000..b56c63cda Binary files /dev/null and b/xcube/testing/data/ds.zarr/var_b/c/0/0/1 differ diff --git a/xcube/testing/data/ds.zarr/var_b/c/0/1/0 b/xcube/testing/data/ds.zarr/var_b/c/0/1/0 new file mode 100644 index 000000000..b56c63cda Binary files /dev/null and b/xcube/testing/data/ds.zarr/var_b/c/0/1/0 differ diff --git a/xcube/testing/data/ds.zarr/var_b/c/0/1/1 b/xcube/testing/data/ds.zarr/var_b/c/0/1/1 new file mode 100644 index 000000000..b56c63cda Binary files /dev/null and b/xcube/testing/data/ds.zarr/var_b/c/0/1/1 differ diff --git a/xcube/testing/data/ds.zarr/var_b/c/1/0/0 b/xcube/testing/data/ds.zarr/var_b/c/1/0/0 new file mode 100644 index 000000000..b56c63cda Binary files /dev/null and b/xcube/testing/data/ds.zarr/var_b/c/1/0/0 differ diff --git a/xcube/testing/data/ds.zarr/var_b/c/1/0/1 b/xcube/testing/data/ds.zarr/var_b/c/1/0/1 new file mode 100644 index 000000000..b56c63cda Binary files /dev/null and b/xcube/testing/data/ds.zarr/var_b/c/1/0/1 differ diff --git a/xcube/testing/data/ds.zarr/var_b/c/1/1/0 b/xcube/testing/data/ds.zarr/var_b/c/1/1/0 new file mode 100644 index 000000000..b56c63cda Binary files /dev/null and b/xcube/testing/data/ds.zarr/var_b/c/1/1/0 differ diff --git a/xcube/testing/data/ds.zarr/var_b/c/1/1/1 b/xcube/testing/data/ds.zarr/var_b/c/1/1/1 new file mode 100644 index 000000000..b56c63cda Binary files /dev/null and b/xcube/testing/data/ds.zarr/var_b/c/1/1/1 differ diff --git a/xcube/testing/data/ds.zarr/var_b/c/2/0/0 b/xcube/testing/data/ds.zarr/var_b/c/2/0/0 new file mode 100644 index 000000000..b56c63cda Binary files /dev/null and b/xcube/testing/data/ds.zarr/var_b/c/2/0/0 differ diff --git a/xcube/testing/data/ds.zarr/var_b/c/2/0/1 b/xcube/testing/data/ds.zarr/var_b/c/2/0/1 new file mode 100644 index 000000000..b56c63cda Binary files /dev/null and b/xcube/testing/data/ds.zarr/var_b/c/2/0/1 differ diff --git a/xcube/testing/data/ds.zarr/var_b/c/2/1/0 b/xcube/testing/data/ds.zarr/var_b/c/2/1/0 new file mode 100644 index 000000000..b56c63cda Binary files /dev/null and b/xcube/testing/data/ds.zarr/var_b/c/2/1/0 differ diff --git a/xcube/testing/data/ds.zarr/var_b/c/2/1/1 b/xcube/testing/data/ds.zarr/var_b/c/2/1/1 new file mode 100644 index 000000000..b56c63cda Binary files /dev/null and b/xcube/testing/data/ds.zarr/var_b/c/2/1/1 differ diff --git a/xcube/testing/data/ds.zarr/var_b/c/3/0/0 b/xcube/testing/data/ds.zarr/var_b/c/3/0/0 new file mode 100644 index 000000000..b56c63cda Binary files /dev/null and b/xcube/testing/data/ds.zarr/var_b/c/3/0/0 differ diff --git a/xcube/testing/data/ds.zarr/var_b/c/3/0/1 b/xcube/testing/data/ds.zarr/var_b/c/3/0/1 new file mode 100644 index 000000000..b56c63cda Binary files /dev/null and b/xcube/testing/data/ds.zarr/var_b/c/3/0/1 differ diff --git a/xcube/testing/data/ds.zarr/var_b/c/3/1/0 b/xcube/testing/data/ds.zarr/var_b/c/3/1/0 new file mode 100644 index 000000000..b56c63cda Binary files /dev/null and b/xcube/testing/data/ds.zarr/var_b/c/3/1/0 differ diff --git a/xcube/testing/data/ds.zarr/var_b/c/3/1/1 b/xcube/testing/data/ds.zarr/var_b/c/3/1/1 new file mode 100644 index 000000000..b56c63cda Binary files /dev/null and b/xcube/testing/data/ds.zarr/var_b/c/3/1/1 differ diff --git a/xcube/testing/data/ds.zarr/var_b/c/4/0/0 b/xcube/testing/data/ds.zarr/var_b/c/4/0/0 new file mode 100644 index 000000000..b56c63cda Binary files /dev/null and b/xcube/testing/data/ds.zarr/var_b/c/4/0/0 differ diff --git a/xcube/testing/data/ds.zarr/var_b/c/4/0/1 b/xcube/testing/data/ds.zarr/var_b/c/4/0/1 new file mode 100644 index 000000000..b56c63cda Binary files /dev/null and b/xcube/testing/data/ds.zarr/var_b/c/4/0/1 differ diff --git a/xcube/testing/data/ds.zarr/var_b/c/4/1/0 b/xcube/testing/data/ds.zarr/var_b/c/4/1/0 new file mode 100644 index 000000000..b56c63cda Binary files /dev/null and b/xcube/testing/data/ds.zarr/var_b/c/4/1/0 differ diff --git a/xcube/testing/data/ds.zarr/var_b/c/4/1/1 b/xcube/testing/data/ds.zarr/var_b/c/4/1/1 new file mode 100644 index 000000000..b56c63cda Binary files /dev/null and b/xcube/testing/data/ds.zarr/var_b/c/4/1/1 differ diff --git a/xcube/testing/data/ds.zarr/var_b/zarr.json b/xcube/testing/data/ds.zarr/var_b/zarr.json new file mode 100644 index 000000000..73ce4e750 --- /dev/null +++ b/xcube/testing/data/ds.zarr/var_b/zarr.json @@ -0,0 +1,54 @@ +{ + "shape": [ + 5, + 180, + 360 + ], + "data_type": "int16", + "chunk_grid": { + "name": "regular", + "configuration": { + "chunk_shape": [ + 1, + 90, + 180 + ] + } + }, + "chunk_key_encoding": { + "name": "default", + "configuration": { + "separator": "/" + } + }, + "fill_value": 0, + "codecs": [ + { + "name": "bytes", + "configuration": { + "endian": "little" + } + }, + { + "name": "zstd", + "configuration": { + "level": 0, + "checksum": false + } + } + ], + "attributes": { + "grid_mapping": "spatial_ref", + "add_offset": -10, + "scale_factor": 0.001, + "_FillValue": -9999 + }, + "dimension_names": [ + "time", + "y", + "x" + ], + "zarr_format": 3, + "node_type": "array", + "storage_transformers": [] +} \ No newline at end of file diff --git a/xcube/testing/data/ds.zarr/var_c/c/0/0/0 b/xcube/testing/data/ds.zarr/var_c/c/0/0/0 new file mode 100644 index 000000000..3bc8df1ad Binary files /dev/null and b/xcube/testing/data/ds.zarr/var_c/c/0/0/0 differ diff --git a/xcube/testing/data/ds.zarr/var_c/c/0/0/1 b/xcube/testing/data/ds.zarr/var_c/c/0/0/1 new file mode 100644 index 000000000..3bc8df1ad Binary files /dev/null and b/xcube/testing/data/ds.zarr/var_c/c/0/0/1 differ diff --git a/xcube/testing/data/ds.zarr/var_c/c/0/1/0 b/xcube/testing/data/ds.zarr/var_c/c/0/1/0 new file mode 100644 index 000000000..3bc8df1ad Binary files /dev/null and b/xcube/testing/data/ds.zarr/var_c/c/0/1/0 differ diff --git a/xcube/testing/data/ds.zarr/var_c/c/0/1/1 b/xcube/testing/data/ds.zarr/var_c/c/0/1/1 new file mode 100644 index 000000000..3bc8df1ad Binary files /dev/null and b/xcube/testing/data/ds.zarr/var_c/c/0/1/1 differ diff --git a/xcube/testing/data/ds.zarr/var_c/c/1/0/0 b/xcube/testing/data/ds.zarr/var_c/c/1/0/0 new file mode 100644 index 000000000..3bc8df1ad Binary files /dev/null and b/xcube/testing/data/ds.zarr/var_c/c/1/0/0 differ diff --git a/xcube/testing/data/ds.zarr/var_c/c/1/0/1 b/xcube/testing/data/ds.zarr/var_c/c/1/0/1 new file mode 100644 index 000000000..3bc8df1ad Binary files /dev/null and b/xcube/testing/data/ds.zarr/var_c/c/1/0/1 differ diff --git a/xcube/testing/data/ds.zarr/var_c/c/1/1/0 b/xcube/testing/data/ds.zarr/var_c/c/1/1/0 new file mode 100644 index 000000000..3bc8df1ad Binary files /dev/null and b/xcube/testing/data/ds.zarr/var_c/c/1/1/0 differ diff --git a/xcube/testing/data/ds.zarr/var_c/c/1/1/1 b/xcube/testing/data/ds.zarr/var_c/c/1/1/1 new file mode 100644 index 000000000..3bc8df1ad Binary files /dev/null and b/xcube/testing/data/ds.zarr/var_c/c/1/1/1 differ diff --git a/xcube/testing/data/ds.zarr/var_c/c/2/0/0 b/xcube/testing/data/ds.zarr/var_c/c/2/0/0 new file mode 100644 index 000000000..3bc8df1ad Binary files /dev/null and b/xcube/testing/data/ds.zarr/var_c/c/2/0/0 differ diff --git a/xcube/testing/data/ds.zarr/var_c/c/2/0/1 b/xcube/testing/data/ds.zarr/var_c/c/2/0/1 new file mode 100644 index 000000000..3bc8df1ad Binary files /dev/null and b/xcube/testing/data/ds.zarr/var_c/c/2/0/1 differ diff --git a/xcube/testing/data/ds.zarr/var_c/c/2/1/0 b/xcube/testing/data/ds.zarr/var_c/c/2/1/0 new file mode 100644 index 000000000..3bc8df1ad Binary files /dev/null and b/xcube/testing/data/ds.zarr/var_c/c/2/1/0 differ diff --git a/xcube/testing/data/ds.zarr/var_c/c/2/1/1 b/xcube/testing/data/ds.zarr/var_c/c/2/1/1 new file mode 100644 index 000000000..3bc8df1ad Binary files /dev/null and b/xcube/testing/data/ds.zarr/var_c/c/2/1/1 differ diff --git a/xcube/testing/data/ds.zarr/var_c/c/3/0/0 b/xcube/testing/data/ds.zarr/var_c/c/3/0/0 new file mode 100644 index 000000000..3bc8df1ad Binary files /dev/null and b/xcube/testing/data/ds.zarr/var_c/c/3/0/0 differ diff --git a/xcube/testing/data/ds.zarr/var_c/c/3/0/1 b/xcube/testing/data/ds.zarr/var_c/c/3/0/1 new file mode 100644 index 000000000..3bc8df1ad Binary files /dev/null and b/xcube/testing/data/ds.zarr/var_c/c/3/0/1 differ diff --git a/xcube/testing/data/ds.zarr/var_c/c/3/1/0 b/xcube/testing/data/ds.zarr/var_c/c/3/1/0 new file mode 100644 index 000000000..3bc8df1ad Binary files /dev/null and b/xcube/testing/data/ds.zarr/var_c/c/3/1/0 differ diff --git a/xcube/testing/data/ds.zarr/var_c/c/3/1/1 b/xcube/testing/data/ds.zarr/var_c/c/3/1/1 new file mode 100644 index 000000000..3bc8df1ad Binary files /dev/null and b/xcube/testing/data/ds.zarr/var_c/c/3/1/1 differ diff --git a/xcube/testing/data/ds.zarr/var_c/c/4/0/0 b/xcube/testing/data/ds.zarr/var_c/c/4/0/0 new file mode 100644 index 000000000..3bc8df1ad Binary files /dev/null and b/xcube/testing/data/ds.zarr/var_c/c/4/0/0 differ diff --git a/xcube/testing/data/ds.zarr/var_c/c/4/0/1 b/xcube/testing/data/ds.zarr/var_c/c/4/0/1 new file mode 100644 index 000000000..3bc8df1ad Binary files /dev/null and b/xcube/testing/data/ds.zarr/var_c/c/4/0/1 differ diff --git a/xcube/testing/data/ds.zarr/var_c/c/4/1/0 b/xcube/testing/data/ds.zarr/var_c/c/4/1/0 new file mode 100644 index 000000000..3bc8df1ad Binary files /dev/null and b/xcube/testing/data/ds.zarr/var_c/c/4/1/0 differ diff --git a/xcube/testing/data/ds.zarr/var_c/c/4/1/1 b/xcube/testing/data/ds.zarr/var_c/c/4/1/1 new file mode 100644 index 000000000..3bc8df1ad Binary files /dev/null and b/xcube/testing/data/ds.zarr/var_c/c/4/1/1 differ diff --git a/xcube/testing/data/ds.zarr/var_c/zarr.json b/xcube/testing/data/ds.zarr/var_c/zarr.json new file mode 100644 index 000000000..b7c264212 --- /dev/null +++ b/xcube/testing/data/ds.zarr/var_c/zarr.json @@ -0,0 +1,48 @@ +{ + "shape": [ + 5, + 180, + 360 + ], + "data_type": "uint8", + "chunk_grid": { + "name": "regular", + "configuration": { + "chunk_shape": [ + 1, + 90, + 180 + ] + } + }, + "chunk_key_encoding": { + "name": "default", + "configuration": { + "separator": "/" + } + }, + "fill_value": 0, + "codecs": [ + { + "name": "bytes" + }, + { + "name": "zstd", + "configuration": { + "level": 0, + "checksum": false + } + } + ], + "attributes": { + "grid_mapping": "spatial_ref" + }, + "dimension_names": [ + "time", + "y", + "x" + ], + "zarr_format": 3, + "node_type": "array", + "storage_transformers": [] +} \ No newline at end of file diff --git a/xcube/testing/data/ds.zarr/x/c/0 b/xcube/testing/data/ds.zarr/x/c/0 new file mode 100644 index 000000000..2e6dffb50 Binary files /dev/null and b/xcube/testing/data/ds.zarr/x/c/0 differ diff --git a/xcube/testing/data/ds.zarr/x/zarr.json b/xcube/testing/data/ds.zarr/x/zarr.json new file mode 100644 index 000000000..6d9598354 --- /dev/null +++ b/xcube/testing/data/ds.zarr/x/zarr.json @@ -0,0 +1,49 @@ +{ + "shape": [ + 360 + ], + "data_type": "float64", + "chunk_grid": { + "name": "regular", + "configuration": { + "chunk_shape": [ + 360 + ] + } + }, + "chunk_key_encoding": { + "name": "default", + "configuration": { + "separator": "/" + } + }, + "fill_value": "NaN", + "codecs": [ + { + "name": "bytes", + "configuration": { + "endian": "little" + } + }, + { + "name": "zstd", + "configuration": { + "level": 0, + "checksum": false + } + } + ], + "attributes": { + "units": "degrees_east", + "long_name": "longitude", + "standard_name": "longitude", + "bounds": "x_bnds", + "_FillValue": "AAAAAAAA+H8=" + }, + "dimension_names": [ + "x" + ], + "zarr_format": 3, + "node_type": "array", + "storage_transformers": [] +} \ No newline at end of file diff --git a/xcube/testing/data/ds.zarr/x_bnds/c/0/0 b/xcube/testing/data/ds.zarr/x_bnds/c/0/0 new file mode 100644 index 000000000..cb36d2b1d Binary files /dev/null and b/xcube/testing/data/ds.zarr/x_bnds/c/0/0 differ diff --git a/xcube/testing/data/ds.zarr/x_bnds/c/1/0 b/xcube/testing/data/ds.zarr/x_bnds/c/1/0 new file mode 100644 index 000000000..86e4f43f8 Binary files /dev/null and b/xcube/testing/data/ds.zarr/x_bnds/c/1/0 differ diff --git a/xcube/testing/data/ds.zarr/x_bnds/zarr.json b/xcube/testing/data/ds.zarr/x_bnds/zarr.json new file mode 100644 index 000000000..58c84414c --- /dev/null +++ b/xcube/testing/data/ds.zarr/x_bnds/zarr.json @@ -0,0 +1,49 @@ +{ + "shape": [ + 360, + 2 + ], + "data_type": "float64", + "chunk_grid": { + "name": "regular", + "configuration": { + "chunk_shape": [ + 180, + 2 + ] + } + }, + "chunk_key_encoding": { + "name": "default", + "configuration": { + "separator": "/" + } + }, + "fill_value": "NaN", + "codecs": [ + { + "name": "bytes", + "configuration": { + "endian": "little" + } + }, + { + "name": "zstd", + "configuration": { + "level": 0, + "checksum": false + } + } + ], + "attributes": { + "units": "degrees_east", + "_FillValue": "AAAAAAAA+H8=" + }, + "dimension_names": [ + "x", + "bnds" + ], + "zarr_format": 3, + "node_type": "array", + "storage_transformers": [] +} \ No newline at end of file diff --git a/xcube/testing/data/ds.zarr/y/c/0 b/xcube/testing/data/ds.zarr/y/c/0 new file mode 100644 index 000000000..a731c8c5b Binary files /dev/null and b/xcube/testing/data/ds.zarr/y/c/0 differ diff --git a/xcube/testing/data/ds.zarr/y/zarr.json b/xcube/testing/data/ds.zarr/y/zarr.json new file mode 100644 index 000000000..1ae5b39a9 --- /dev/null +++ b/xcube/testing/data/ds.zarr/y/zarr.json @@ -0,0 +1,49 @@ +{ + "shape": [ + 180 + ], + "data_type": "float64", + "chunk_grid": { + "name": "regular", + "configuration": { + "chunk_shape": [ + 180 + ] + } + }, + "chunk_key_encoding": { + "name": "default", + "configuration": { + "separator": "/" + } + }, + "fill_value": "NaN", + "codecs": [ + { + "name": "bytes", + "configuration": { + "endian": "little" + } + }, + { + "name": "zstd", + "configuration": { + "level": 0, + "checksum": false + } + } + ], + "attributes": { + "units": "degrees_north", + "long_name": "latitude", + "standard_name": "latitude", + "bounds": "y_bnds", + "_FillValue": "AAAAAAAA+H8=" + }, + "dimension_names": [ + "y" + ], + "zarr_format": 3, + "node_type": "array", + "storage_transformers": [] +} \ No newline at end of file diff --git a/xcube/testing/data/ds.zarr/y_bnds/c/0/0 b/xcube/testing/data/ds.zarr/y_bnds/c/0/0 new file mode 100644 index 000000000..3197a9faa Binary files /dev/null and b/xcube/testing/data/ds.zarr/y_bnds/c/0/0 differ diff --git a/xcube/testing/data/ds.zarr/y_bnds/c/1/0 b/xcube/testing/data/ds.zarr/y_bnds/c/1/0 new file mode 100644 index 000000000..8d094c900 Binary files /dev/null and b/xcube/testing/data/ds.zarr/y_bnds/c/1/0 differ diff --git a/xcube/testing/data/ds.zarr/y_bnds/zarr.json b/xcube/testing/data/ds.zarr/y_bnds/zarr.json new file mode 100644 index 000000000..345ba22ab --- /dev/null +++ b/xcube/testing/data/ds.zarr/y_bnds/zarr.json @@ -0,0 +1,49 @@ +{ + "shape": [ + 180, + 2 + ], + "data_type": "float64", + "chunk_grid": { + "name": "regular", + "configuration": { + "chunk_shape": [ + 90, + 2 + ] + } + }, + "chunk_key_encoding": { + "name": "default", + "configuration": { + "separator": "/" + } + }, + "fill_value": "NaN", + "codecs": [ + { + "name": "bytes", + "configuration": { + "endian": "little" + } + }, + { + "name": "zstd", + "configuration": { + "level": 0, + "checksum": false + } + } + ], + "attributes": { + "units": "degrees_north", + "_FillValue": "AAAAAAAA+H8=" + }, + "dimension_names": [ + "y", + "bnds" + ], + "zarr_format": 3, + "node_type": "array", + "storage_transformers": [] +} \ No newline at end of file diff --git a/xcube/testing/data/ds.zarr/zarr.json b/xcube/testing/data/ds.zarr/zarr.json new file mode 100644 index 000000000..d2818e3ea --- /dev/null +++ b/xcube/testing/data/ds.zarr/zarr.json @@ -0,0 +1,516 @@ +{ + "attributes": { + "Conventions": "CF-1.7", + "title": "Test Cube", + "time_coverage_start": "2010-01-01T00:00:00.000000000", + "time_coverage_end": "2010-01-06T00:00:00.000000000", + "geospatial_lon_min": -180.0, + "geospatial_lon_max": 180.0, + "geospatial_lon_units": "degrees_east", + "geospatial_lat_min": -90.0, + "geospatial_lat_max": 90.0, + "geospatial_lat_units": "degrees_north", + "coordinates": "time_bnds x_bnds y_bnds" + }, + "zarr_format": 3, + "consolidated_metadata": { + "kind": "inline", + "must_understand": false, + "metadata": { + "spatial_ref": { + "shape": [], + "data_type": "int64", + "chunk_grid": { + "name": "regular", + "configuration": { + "chunk_shape": [] + } + }, + "chunk_key_encoding": { + "name": "default", + "configuration": { + "separator": "/" + } + }, + "fill_value": 0, + "codecs": [ + { + "name": "bytes", + "configuration": { + "endian": "little" + } + }, + { + "name": "zstd", + "configuration": { + "level": 0, + "checksum": false + } + } + ], + "attributes": { + "crs_wkt": "GEOGCRS[\"WGS 84 (CRS84)\",ENSEMBLE[\"World Geodetic System 1984 ensemble\",MEMBER[\"World Geodetic System 1984 (Transit)\"],MEMBER[\"World Geodetic System 1984 (G730)\"],MEMBER[\"World Geodetic System 1984 (G873)\"],MEMBER[\"World Geodetic System 1984 (G1150)\"],MEMBER[\"World Geodetic System 1984 (G1674)\"],MEMBER[\"World Geodetic System 1984 (G1762)\"],MEMBER[\"World Geodetic System 1984 (G2139)\"],MEMBER[\"World Geodetic System 1984 (G2296)\"],ELLIPSOID[\"WGS 84\",6378137,298.257223563,LENGTHUNIT[\"metre\",1]],ENSEMBLEACCURACY[2.0]],PRIMEM[\"Greenwich\",0,ANGLEUNIT[\"degree\",0.0174532925199433]],CS[ellipsoidal,2],AXIS[\"geodetic longitude (Lon)\",east,ORDER[1],ANGLEUNIT[\"degree\",0.0174532925199433]],AXIS[\"geodetic latitude (Lat)\",north,ORDER[2],ANGLEUNIT[\"degree\",0.0174532925199433]],USAGE[SCOPE[\"Not known.\"],AREA[\"World.\"],BBOX[-90,-180,90,180]],ID[\"OGC\",\"CRS84\"]]", + "semi_major_axis": 6378137.0, + "semi_minor_axis": 6356752.314245179, + "inverse_flattening": 298.257223563, + "reference_ellipsoid_name": "WGS 84", + "longitude_of_prime_meridian": 0.0, + "prime_meridian_name": "Greenwich", + "geographic_crs_name": "WGS 84 (CRS84)", + "horizontal_datum_name": "World Geodetic System 1984 ensemble", + "grid_mapping_name": "latitude_longitude" + }, + "zarr_format": 3, + "node_type": "array", + "storage_transformers": [] + }, + "time": { + "shape": [ + 5 + ], + "data_type": "int64", + "chunk_grid": { + "name": "regular", + "configuration": { + "chunk_shape": [ + 5 + ] + } + }, + "chunk_key_encoding": { + "name": "default", + "configuration": { + "separator": "/" + } + }, + "fill_value": 0, + "codecs": [ + { + "name": "bytes", + "configuration": { + "endian": "little" + } + }, + { + "name": "zstd", + "configuration": { + "level": 0, + "checksum": false + } + } + ], + "attributes": { + "bounds": "time_bnds", + "units": "seconds since 1970-01-01", + "calendar": "proleptic_gregorian" + }, + "dimension_names": [ + "time" + ], + "zarr_format": 3, + "node_type": "array", + "storage_transformers": [] + }, + "time_bnds": { + "shape": [ + 5, + 2 + ], + "data_type": "int64", + "chunk_grid": { + "name": "regular", + "configuration": { + "chunk_shape": [ + 1, + 2 + ] + } + }, + "chunk_key_encoding": { + "name": "default", + "configuration": { + "separator": "/" + } + }, + "fill_value": 0, + "codecs": [ + { + "name": "bytes", + "configuration": { + "endian": "little" + } + }, + { + "name": "zstd", + "configuration": { + "level": 0, + "checksum": false + } + } + ], + "attributes": { + "units": "seconds since 1970-01-01T00:00:00", + "calendar": "proleptic_gregorian" + }, + "dimension_names": [ + "time", + "bnds" + ], + "zarr_format": 3, + "node_type": "array", + "storage_transformers": [] + }, + "var_a": { + "shape": [ + 5, + 180, + 360 + ], + "data_type": "float64", + "chunk_grid": { + "name": "regular", + "configuration": { + "chunk_shape": [ + 1, + 90, + 180 + ] + } + }, + "chunk_key_encoding": { + "name": "default", + "configuration": { + "separator": "/" + } + }, + "fill_value": "NaN", + "codecs": [ + { + "name": "bytes", + "configuration": { + "endian": "little" + } + }, + { + "name": "zstd", + "configuration": { + "level": 0, + "checksum": false + } + } + ], + "attributes": { + "grid_mapping": "spatial_ref", + "_FillValue": "AAAAAAAA+H8=" + }, + "dimension_names": [ + "time", + "y", + "x" + ], + "zarr_format": 3, + "node_type": "array", + "storage_transformers": [] + }, + "var_b": { + "shape": [ + 5, + 180, + 360 + ], + "data_type": "int16", + "chunk_grid": { + "name": "regular", + "configuration": { + "chunk_shape": [ + 1, + 90, + 180 + ] + } + }, + "chunk_key_encoding": { + "name": "default", + "configuration": { + "separator": "/" + } + }, + "fill_value": 0, + "codecs": [ + { + "name": "bytes", + "configuration": { + "endian": "little" + } + }, + { + "name": "zstd", + "configuration": { + "level": 0, + "checksum": false + } + } + ], + "attributes": { + "grid_mapping": "spatial_ref", + "add_offset": -10, + "scale_factor": 0.001, + "_FillValue": -9999 + }, + "dimension_names": [ + "time", + "y", + "x" + ], + "zarr_format": 3, + "node_type": "array", + "storage_transformers": [] + }, + "var_c": { + "shape": [ + 5, + 180, + 360 + ], + "data_type": "uint8", + "chunk_grid": { + "name": "regular", + "configuration": { + "chunk_shape": [ + 1, + 90, + 180 + ] + } + }, + "chunk_key_encoding": { + "name": "default", + "configuration": { + "separator": "/" + } + }, + "fill_value": 0, + "codecs": [ + { + "name": "bytes" + }, + { + "name": "zstd", + "configuration": { + "level": 0, + "checksum": false + } + } + ], + "attributes": { + "grid_mapping": "spatial_ref" + }, + "dimension_names": [ + "time", + "y", + "x" + ], + "zarr_format": 3, + "node_type": "array", + "storage_transformers": [] + }, + "x": { + "shape": [ + 360 + ], + "data_type": "float64", + "chunk_grid": { + "name": "regular", + "configuration": { + "chunk_shape": [ + 360 + ] + } + }, + "chunk_key_encoding": { + "name": "default", + "configuration": { + "separator": "/" + } + }, + "fill_value": "NaN", + "codecs": [ + { + "name": "bytes", + "configuration": { + "endian": "little" + } + }, + { + "name": "zstd", + "configuration": { + "level": 0, + "checksum": false + } + } + ], + "attributes": { + "units": "degrees_east", + "long_name": "longitude", + "standard_name": "longitude", + "bounds": "x_bnds", + "_FillValue": "AAAAAAAA+H8=" + }, + "dimension_names": [ + "x" + ], + "zarr_format": 3, + "node_type": "array", + "storage_transformers": [] + }, + "x_bnds": { + "shape": [ + 360, + 2 + ], + "data_type": "float64", + "chunk_grid": { + "name": "regular", + "configuration": { + "chunk_shape": [ + 180, + 2 + ] + } + }, + "chunk_key_encoding": { + "name": "default", + "configuration": { + "separator": "/" + } + }, + "fill_value": "NaN", + "codecs": [ + { + "name": "bytes", + "configuration": { + "endian": "little" + } + }, + { + "name": "zstd", + "configuration": { + "level": 0, + "checksum": false + } + } + ], + "attributes": { + "units": "degrees_east", + "_FillValue": "AAAAAAAA+H8=" + }, + "dimension_names": [ + "x", + "bnds" + ], + "zarr_format": 3, + "node_type": "array", + "storage_transformers": [] + }, + "y": { + "shape": [ + 180 + ], + "data_type": "float64", + "chunk_grid": { + "name": "regular", + "configuration": { + "chunk_shape": [ + 180 + ] + } + }, + "chunk_key_encoding": { + "name": "default", + "configuration": { + "separator": "/" + } + }, + "fill_value": "NaN", + "codecs": [ + { + "name": "bytes", + "configuration": { + "endian": "little" + } + }, + { + "name": "zstd", + "configuration": { + "level": 0, + "checksum": false + } + } + ], + "attributes": { + "units": "degrees_north", + "long_name": "latitude", + "standard_name": "latitude", + "bounds": "y_bnds", + "_FillValue": "AAAAAAAA+H8=" + }, + "dimension_names": [ + "y" + ], + "zarr_format": 3, + "node_type": "array", + "storage_transformers": [] + }, + "y_bnds": { + "shape": [ + 180, + 2 + ], + "data_type": "float64", + "chunk_grid": { + "name": "regular", + "configuration": { + "chunk_shape": [ + 90, + 2 + ] + } + }, + "chunk_key_encoding": { + "name": "default", + "configuration": { + "separator": "/" + } + }, + "fill_value": "NaN", + "codecs": [ + { + "name": "bytes", + "configuration": { + "endian": "little" + } + }, + { + "name": "zstd", + "configuration": { + "level": 0, + "checksum": false + } + } + ], + "attributes": { + "units": "degrees_north", + "_FillValue": "AAAAAAAA+H8=" + }, + "dimension_names": [ + "y", + "bnds" + ], + "zarr_format": 3, + "node_type": "array", + "storage_transformers": [] + } + } + }, + "node_type": "group" +} \ No newline at end of file diff --git a/xcube/webapi/s3/objectstorage.py b/xcube/webapi/s3/objectstorage.py index 306d11a1a..bc427cbde 100644 --- a/xcube/webapi/s3/objectstorage.py +++ b/xcube/webapi/s3/objectstorage.py @@ -4,10 +4,9 @@ import collections.abc from collections.abc import Iterator, Mapping -from typing import Tuple, Union +from typing import Tuple, Union, MutableMapping import xarray as xr -import zarr.storage from xcube.core.mldataset import MultiLevelDataset from xcube.server.api import ApiError @@ -71,7 +70,7 @@ def __getitem__(self, key: str) -> bytes: ) return value - def _parse_key(self, key: str) -> tuple[zarr.storage.BaseStore, str]: + def _parse_key(self, key: str) -> tuple[MutableMapping, str]: """Parses a given *key* which is expected to have format "{dataset_id}/{level}.zarr/{*path}" for multi-level datasets and "{dataset_id}/{*path}" for other datasets.