diff --git a/.github/workflows/build-main.yml b/.github/workflows/build-main.yml index 0e3ae80..19a01d9 100644 --- a/.github/workflows/build-main.yml +++ b/.github/workflows/build-main.yml @@ -14,7 +14,7 @@ jobs: steps: - uses: actions/checkout@v4 - name: Set up Java - uses: actions/setup-java@v4 + uses: actions/setup-java@v5 with: java-version: '8' distribution: 'zulu' diff --git a/.github/workflows/build-pr.yml b/.github/workflows/build-pr.yml index 7c51424..3a1fdfb 100644 --- a/.github/workflows/build-pr.yml +++ b/.github/workflows/build-pr.yml @@ -12,7 +12,7 @@ jobs: steps: - uses: actions/checkout@v4 - name: Set up Java - uses: actions/setup-java@v4 + uses: actions/setup-java@v5 with: java-version: '8' distribution: 'zulu' diff --git a/.github/workflows/platform-test.yml b/.github/workflows/platform-test.yml index 9ae99b4..d70f4b5 100644 --- a/.github/workflows/platform-test.yml +++ b/.github/workflows/platform-test.yml @@ -21,7 +21,7 @@ jobs: steps: - uses: actions/checkout@v4 - name: Set up Java - uses: actions/setup-java@v4 + uses: actions/setup-java@v5 with: java-version: '8' distribution: 'zulu' diff --git a/pom.xml b/pom.xml index 1933a63..b456676 100644 --- a/pom.xml +++ b/pom.xml @@ -5,7 +5,7 @@ org.scijava pom-scijava - 40.0.0 + 43.0.0 @@ -139,7 +139,7 @@ sign,deploy-to-scijava - 4.0.0-alpha-9 + 4.0.0-alpha-12 diff --git a/src/main/java/org/janelia/saalfeldlab/n5/hdf5/N5HDF5Reader.java b/src/main/java/org/janelia/saalfeldlab/n5/hdf5/N5HDF5Reader.java index e19a23e..df7377b 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/hdf5/N5HDF5Reader.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/hdf5/N5HDF5Reader.java @@ -689,7 +689,7 @@ public DatasetAttributes getDatasetAttributes(String pathName) { } @Override - public DataBlock readBlock( + public DataBlock readChunk( String pathName, final DatasetAttributes datasetAttributes, final long... gridPosition) throws N5Exception { @@ -713,7 +713,7 @@ public DataBlock readBlock( if (datasetAttributes.getDataType() == DataType.STRING) { final int[] intHdf5CroppedBlockSize = Arrays.stream(hdf5CroppedBlockSize).mapToInt(i -> (int)i).toArray(); MDArray data = reader.string().readMDArrayBlockWithOffset(normalizedPathName, intHdf5CroppedBlockSize, hdf5Offset); - return new StringDataBlock(croppedBlockSize, gridPosition, data.getAsFlatArray()); + return (DataBlock)new StringDataBlock(croppedBlockSize, gridPosition, data.getAsFlatArray()); } final DataType dataType = datasetAttributes.getDataType(); @@ -733,21 +733,21 @@ public DataBlock readBlock( H5Sclose(fileSpaceId); H5Sclose(memorySpaceId); } - return block; + return (DataBlock)block; } @Override - public DataBlock readShard( + public DataBlock readBlock( final String pathName, final DatasetAttributes datasetAttributes, final long... gridPosition) throws N5Exception { - // HDF5 does not support sharding, so readShard is always equivalent to readBlock - return readBlock(pathName, datasetAttributes, gridPosition); + // HDF5 does not support sharding, so readBlock is always equivalent to readChunk + return (DataBlock)readChunk(pathName, datasetAttributes, gridPosition); } @Override - public boolean shardExists(String pathName, DatasetAttributes datasetAttributes, long... gridPosition) throws N5Exception { + public boolean blockExists(String pathName, DatasetAttributes datasetAttributes, long... gridPosition) throws N5Exception { return true; } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/hdf5/N5HDF5Writer.java b/src/main/java/org/janelia/saalfeldlab/n5/hdf5/N5HDF5Writer.java index 9997ccc..84045ce 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/hdf5/N5HDF5Writer.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/hdf5/N5HDF5Writer.java @@ -47,6 +47,9 @@ import org.janelia.saalfeldlab.n5.N5Writer; import org.janelia.saalfeldlab.n5.RawCompression; import org.janelia.saalfeldlab.n5.hdf5.N5HDF5Util.OpenDataSetCache.OpenDataSet; +import org.janelia.saalfeldlab.n5.shard.Nesting.NestedGrid; +import org.janelia.saalfeldlab.n5.shard.Nesting.NestedPosition; +import org.janelia.saalfeldlab.n5.shard.Region; import java.io.File; import java.io.IOException; @@ -55,6 +58,8 @@ import java.util.List; import java.util.Map; import java.util.Map.Entry; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; import static hdf.hdf5lib.H5.H5Dget_space; import static hdf.hdf5lib.H5.H5Dwrite; @@ -527,7 +532,7 @@ public boolean removeAttributes( } @Override - public void writeBlock( + public void writeChunk( String pathName, final DatasetAttributes datasetAttributes, final DataBlock dataBlock) throws N5Exception { @@ -555,29 +560,74 @@ public void writeBlock( } } - public void writeShard( + @Override + public void writeBlock( final String pathName, final DatasetAttributes datasetAttributes, final DataBlock dataBlock) throws N5Exception { - // HDF5 does not support sharding, so readShard is always equivalent to readBlock - writeBlock(pathName, datasetAttributes, dataBlock); + // HDF5 does not support sharding, so writeBlock is always equivalent to writeChunk + writeChunk(pathName, datasetAttributes, dataBlock); } @Override - public boolean deleteBlock(String pathName, final long... gridPosition) throws N5Exception { + public void writeRegion( + String datasetPath, + DatasetAttributes datasetAttributes, + long[] min, + long[] size, + DataBlockSupplier dataBlocks, + boolean writeFully) throws N5Exception { + + final NestedGrid grid = datasetAttributes.getNestedBlockGrid(); + final Region region = new Region(min, size, grid); + for (long[] key : Region.gridPositions(region.minPos().key(), region.maxPos().key())) { + final NestedPosition pos = grid.nestedPosition(key, 0); // HDF5 is never nested, get level 0 + final long[] gridPosition = pos.absolute(0); + final DataBlock existingDataBlock = writeFully || region.fullyContains(pos) + ? null + : readBlock(datasetPath, datasetAttributes, gridPosition); + final DataBlock dataBlock = dataBlocks.get(gridPosition, existingDataBlock); + // null blocks may be provided when they contain only the fill value + // and only non-empty blocks should be written, for example + if (dataBlock == null) { + deleteBlock(datasetPath, datasetAttributes, gridPosition); + } else { + writeBlock(datasetPath, datasetAttributes, dataBlock); + } + } + + } + + public void writeRegion( + String datasetPath, + DatasetAttributes datasetAttributes, + long[] min, + long[] size, + DataBlockSupplier dataBlocks, + boolean writeFully, + ExecutorService exec) throws N5Exception, InterruptedException, ExecutionException { + + // block until the write is complete + exec.submit(() -> { + writeRegion(datasetPath, datasetAttributes, min, size, dataBlocks, writeFully); + }).get(); + } + + @Override + public boolean deleteChunk(String pathName, final long... gridPosition) throws N5Exception { if (pathName.equals("")) pathName = "/"; final DatasetAttributes datasetAttributes = getDatasetAttributes(pathName); - return deleteBlock(pathName, datasetAttributes, gridPosition); + return deleteChunk(pathName, datasetAttributes, gridPosition); } @Override - public boolean deleteBlock(String datasetPath, DatasetAttributes datasetAttributes, long... gridPosition) throws N5Exception { + public boolean deleteChunk(String datasetPath, DatasetAttributes datasetAttributes, long... gridPosition) throws N5Exception { - // deletion is not supported in HDF5, so the block is overwritten with zeros instead + // deletion is not supported in HDF5, so the chunk is overwritten with zeros instead // Consider using defaultValue instead of zero? if (datasetPath.equals("")) @@ -597,13 +647,27 @@ public boolean deleteBlock(String datasetPath, DatasetAttributes datasetAttribut case FLOAT32: case FLOAT64: final DataBlock empty = dataType.createDataBlock(datasetAttributes.getBlockSize(), gridPosition); - writeBlock(datasetPath, datasetAttributes, empty); + writeChunk(datasetPath, datasetAttributes, empty); return true; default: return false; } } + @Override + public boolean deleteBlock(String pathName, final long... gridPosition) throws N5Exception { + + // HDF5 does not support sharding, so deleteBlock is always equivalent to deleteChunk + return deleteChunk(pathName, gridPosition); + } + + @Override + public boolean deleteBlock(String datasetPath, DatasetAttributes datasetAttributes, long... gridPosition) throws N5Exception { + + // HDF5 does not support sharding, so deleteBlock is always equivalent to deleteChunk + return deleteChunk(datasetPath, datasetAttributes, gridPosition); + } + @Override public boolean remove() { diff --git a/src/test/java/org/janelia/saalfeldlab/n5/hdf5/N5HDF5Test.java b/src/test/java/org/janelia/saalfeldlab/n5/hdf5/N5HDF5Test.java index c6ca7da..e0982fc 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/hdf5/N5HDF5Test.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/hdf5/N5HDF5Test.java @@ -190,6 +190,12 @@ public void testMode1WriteReadByteBlock() { public void testWriteReadSerializableBlock() { } + @Override + @Test + @Ignore("HDF5 does not support modifying dataset attributes after creating the dataset.") + public void testDatasetAttributes() { + } + @Override @Test @Ignore("Writing blocks larger than the dimensions of the dataset is invalid for HDF5.") @@ -522,30 +528,30 @@ public void testDelete() { final long[] position1 = {0, 0, 0}; final long[] position2 = {0, 1, 2}; - // non-existant block should be zeros - final DataBlock emptyBlock = n5.readBlock(datasetName, attributes, position1); + // non-existant chunk should be zeros + final DataBlock emptyBlock = n5.readChunk(datasetName, attributes, position1); assertTrue(emptyBlock instanceof ByteArrayDataBlock); final byte[] zeros = new byte[byteBlock.length]; assertArrayEquals(zeros, ((ByteArrayDataBlock)emptyBlock).getData()); final ByteArrayDataBlock dataBlock = new ByteArrayDataBlock(blockSize, position1, byteBlock); - n5.writeBlock(datasetName, attributes, dataBlock); + n5.writeChunk(datasetName, attributes, dataBlock); - // block should exist at position1 but not at position2 - final DataBlock readBlock = n5.readBlock(datasetName, attributes, position1); + // chunk should exist at position1 but not at position2 + final DataBlock readBlock = n5.readChunk(datasetName, attributes, position1); assertNotNull(readBlock); assertTrue(readBlock instanceof ByteArrayDataBlock); assertArrayEquals(byteBlock, ((ByteArrayDataBlock)readBlock).getData()); - assertTrue("deleting existing block should return true", n5.deleteBlock(datasetName, position1)); - assertTrue("hdf5 returns true even on non-existent blocks, since they can only be zeroed out", n5.deleteBlock(datasetName, position1)); - assertTrue("hdf5 returns true even on non-existent blocks, since they can only be zeroed out", n5.deleteBlock(datasetName, position2)); + assertTrue("deleting existing chunk should return true", n5.deleteChunk(datasetName, position1)); + assertTrue("hdf5 returns true even on non-existent chunks, since they can only be zeroed out", n5.deleteChunk(datasetName, position1)); + assertTrue("hdf5 returns true even on non-existent chunks, since they can only be zeroed out", n5.deleteChunk(datasetName, position2)); - // no block should exist anymore - final DataBlock pos1EmptyBlock = n5.readBlock(datasetName, attributes, position1); + // no chunk should exist anymore + final DataBlock pos1EmptyBlock = n5.readChunk(datasetName, attributes, position1); assertArrayEquals(zeros, ((ByteArrayDataBlock)pos1EmptyBlock).getData()); - final DataBlock pos2EmptyBlock = n5.readBlock(datasetName, attributes, position2); + final DataBlock pos2EmptyBlock = n5.readChunk(datasetName, attributes, position2); assertArrayEquals(zeros, ((ByteArrayDataBlock)pos2EmptyBlock).getData()); } }