diff --git a/src/main/java/dev/zarr/zarrjava/core/Array.java b/src/main/java/dev/zarr/zarrjava/core/Array.java index 9f4d5b2..e24f0fb 100644 --- a/src/main/java/dev/zarr/zarrjava/core/Array.java +++ b/src/main/java/dev/zarr/zarrjava/core/Array.java @@ -3,7 +3,6 @@ import dev.zarr.zarrjava.ZarrException; import dev.zarr.zarrjava.core.codec.CodecPipeline; import dev.zarr.zarrjava.store.FilesystemStore; -import dev.zarr.zarrjava.store.Store; import dev.zarr.zarrjava.store.StoreHandle; import dev.zarr.zarrjava.utils.IndexingUtils; import dev.zarr.zarrjava.utils.MultiArrayUtils; @@ -17,9 +16,6 @@ import java.nio.file.Path; import java.nio.file.Paths; import java.util.Arrays; -import java.util.List; -import java.util.Set; -import java.util.stream.Collectors; import java.util.stream.Stream; public abstract class Array extends AbstractNode { @@ -178,6 +174,104 @@ public ucar.ma2.Array readChunk(long[] chunkCoords) throws ZarrException { return codecPipeline.decode(chunkBytes); } + /** + * Deletes chunks that are completely outside the new shape and trims boundary chunks. + * + * @param newShape the new shape of the array + */ + protected void cleanupChunksForResize(long[] newShape) { + ArrayMetadata metadata = metadata(); + final int[] chunkShape = metadata.chunkShape(); + final int ndim = metadata.ndim(); + final dev.zarr.zarrjava.core.chunkkeyencoding.ChunkKeyEncoding chunkKeyEncoding = metadata.chunkKeyEncoding(); + + // Calculate max valid chunk coordinates for the new shape + long[] newMaxChunkCoords = new long[ndim]; + for (int i = 0; i < ndim; i++) { + newMaxChunkCoords[i] = (newShape[i] + chunkShape[i] - 1) / chunkShape[i]; + } + + // Iterate over all possible chunk coordinates in the old shape + long[][] allOldChunkCoords = IndexingUtils.computeChunkCoords(metadata.shape, chunkShape); + + for (long[] chunkCoords : allOldChunkCoords) { + boolean isOutsideBounds = false; + boolean isOnBoundary = false; + + for (int dimIdx = 0; dimIdx < ndim; dimIdx++) { + if (chunkCoords[dimIdx] >= newMaxChunkCoords[dimIdx]) { + isOutsideBounds = true; + break; + } + // Check if this chunk is on the boundary (partially outside new shape) + long chunkEnd = (chunkCoords[dimIdx] + 1) * chunkShape[dimIdx]; + if (chunkEnd > newShape[dimIdx]) { + isOnBoundary = true; + } + } + + String[] chunkKeys = chunkKeyEncoding.encodeChunkKey(chunkCoords); + StoreHandle chunkHandle = storeHandle.resolve(chunkKeys); + + if (isOutsideBounds) { + // Delete chunk that is completely outside + chunkHandle.delete(); + } else if (isOnBoundary) { + // Trim boundary chunk - read, clear out-of-bounds data, write back + try { + trimBoundaryChunk(chunkCoords, newShape, chunkShape); + } catch (ZarrException e) { + throw new RuntimeException(e); + } + } + } + } + + /** + * Trims a boundary chunk by reading it, clearing the out-of-bounds portion, and writing it back. + * + * @param chunkCoords the coordinates of the chunk to trim + * @param newShape the new shape of the array + * @param chunkShape the shape of the chunks + * @throws ZarrException if reading or writing the chunk fails + */ + protected void trimBoundaryChunk(long[] chunkCoords, long[] newShape, int[] chunkShape) throws ZarrException { + ArrayMetadata metadata = metadata(); + final int ndim = metadata.ndim(); + + // Calculate the valid region within this chunk + int[] validShape = new int[ndim]; + boolean needsTrimming = false; + for (int dimIdx = 0; dimIdx < ndim; dimIdx++) { + long chunkStart = chunkCoords[dimIdx] * chunkShape[dimIdx]; + long chunkEnd = chunkStart + chunkShape[dimIdx]; + if (chunkEnd > newShape[dimIdx]) { + validShape[dimIdx] = (int) (newShape[dimIdx] - chunkStart); + needsTrimming = true; + } else { + validShape[dimIdx] = chunkShape[dimIdx]; + } + } + + if (!needsTrimming) { + return; + } + + // Read the existing chunk + ucar.ma2.Array chunkData = readChunk(chunkCoords); + + // Create a new chunk filled with fill value + ucar.ma2.Array newChunkData = metadata.allocateFillValueChunk(); + + // Copy only the valid region + MultiArrayUtils.copyRegion( + chunkData, new int[ndim], newChunkData, new int[ndim], validShape + ); + + // Write the trimmed chunk back + writeChunk(chunkCoords, newChunkData); + } + /** * Writes a ucar.ma2.Array into the Zarr array at the beginning of the Zarr array. The shape of diff --git a/src/main/java/dev/zarr/zarrjava/v2/Array.java b/src/main/java/dev/zarr/zarrjava/v2/Array.java index 87d767a..1d31759 100644 --- a/src/main/java/dev/zarr/zarrjava/v2/Array.java +++ b/src/main/java/dev/zarr/zarrjava/v2/Array.java @@ -198,8 +198,10 @@ private Array writeMetadata(ArrayMetadata newArrayMetadata) throws ZarrException } /** - * Sets a new shape for the Zarr array. It only changes the metadata, no array data is modified or - * deleted. This method returns a new instance of the Zarr array class and the old instance + * Sets a new shape for the Zarr array. Old array data outside the new shape will be deleted. + * If data deletion is not desired, use {@link #resize(long[], boolean)} with + * `resizeMetadataOnly` set to true. + * This method returns a new instance of the Zarr array class and the old instance * becomes invalid. * * @param newShape the new shape of the Zarr array @@ -207,17 +209,36 @@ private Array writeMetadata(ArrayMetadata newArrayMetadata) throws ZarrException * @throws IOException throws IOException if the new metadata cannot be serialized */ public Array resize(long[] newShape) throws ZarrException, IOException { + return resize(newShape, false); + } + + /** + * Sets a new shape for the Zarr array. This method returns a new instance of the Zarr array class + * and the old instance becomes invalid. + * + * @param newShape the new shape of the Zarr array + * @param resizeMetadataOnly if true, only the metadata is updated; if false, chunks outside the new + * bounds are deleted and boundary chunks are trimmed + * @throws ZarrException if the new metadata is invalid + * @throws IOException throws IOException if the new metadata cannot be serialized + */ + public Array resize(long[] newShape, boolean resizeMetadataOnly) throws ZarrException, IOException { if (newShape.length != metadata.ndim()) { throw new IllegalArgumentException( "'newShape' needs to have rank '" + metadata.ndim() + "'."); } + if (!resizeMetadataOnly) { + cleanupChunksForResize(newShape); + } + ArrayMetadata newArrayMetadata = ArrayMetadataBuilder.fromArrayMetadata(metadata) .withShape(newShape) .build(); return writeMetadata(newArrayMetadata); } + /** * Sets the attributes of the Zarr array. It overwrites and removes any existing attributes. This * method returns a new instance of the Zarr array class and the old instance becomes invalid. @@ -245,7 +266,8 @@ public Array setAttributes(Attributes newAttributes) throws ZarrException, IOExc * @throws IOException throws IOException if the new metadata cannot be serialized */ public Array updateAttributes(Function attributeMapper) throws ZarrException, IOException { - return setAttributes(attributeMapper.apply(metadata.attributes)); + Attributes currentAttributes = metadata.attributes != null ? new Attributes(metadata.attributes) : new Attributes(); + return setAttributes(attributeMapper.apply(currentAttributes)); } @Override diff --git a/src/main/java/dev/zarr/zarrjava/v2/Group.java b/src/main/java/dev/zarr/zarrjava/v2/Group.java index d3229e4..73b8cdf 100644 --- a/src/main/java/dev/zarr/zarrjava/v2/Group.java +++ b/src/main/java/dev/zarr/zarrjava/v2/Group.java @@ -283,7 +283,8 @@ public Group setAttributes(Attributes newAttributes) throws ZarrException, IOExc */ public Group updateAttributes(Function attributeMapper) throws ZarrException, IOException { - return setAttributes(attributeMapper.apply(metadata.attributes)); + Attributes currentAttributes = metadata.attributes != null ? new Attributes(metadata.attributes) : new Attributes(); + return setAttributes(attributeMapper.apply(currentAttributes)); } diff --git a/src/main/java/dev/zarr/zarrjava/v3/Array.java b/src/main/java/dev/zarr/zarrjava/v3/Array.java index 72aad1e..be38a9c 100644 --- a/src/main/java/dev/zarr/zarrjava/v3/Array.java +++ b/src/main/java/dev/zarr/zarrjava/v3/Array.java @@ -201,8 +201,10 @@ private Array writeMetadata(ArrayMetadata newArrayMetadata) throws ZarrException } /** - * Sets a new shape for the Zarr array. It only changes the metadata, no array data is modified or - * deleted. This method returns a new instance of the Zarr array class and the old instance + * Sets a new shape for the Zarr array. Old array data outside the new shape will be deleted. + * If data deletion is not desired, use {@link #resize(long[], boolean)} with + * `resizeMetadataOnly` set to true. + * This method returns a new instance of the Zarr array class and the old instance * becomes invalid. * * @param newShape the new shape of the Zarr array @@ -210,17 +212,36 @@ private Array writeMetadata(ArrayMetadata newArrayMetadata) throws ZarrException * @throws IOException throws IOException if the new metadata cannot be serialized */ public Array resize(long[] newShape) throws ZarrException, IOException { + return resize(newShape, false); + } + + /** + * Sets a new shape for the Zarr array. This method returns a new instance of the Zarr array class + * and the old instance becomes invalid. + * + * @param newShape the new shape of the Zarr array + * @param resizeMetadataOnly if true, only the metadata is updated; if false, chunks outside the new + * bounds are deleted and boundary chunks are trimmed + * @throws ZarrException if the new metadata is invalid + * @throws IOException throws IOException if the new metadata cannot be serialized + */ + public Array resize(long[] newShape, boolean resizeMetadataOnly) throws ZarrException, IOException { if (newShape.length != metadata.ndim()) { throw new IllegalArgumentException( "'newShape' needs to have rank '" + metadata.ndim() + "'."); } + if (!resizeMetadataOnly) { + cleanupChunksForResize(newShape); + } + ArrayMetadata newArrayMetadata = ArrayMetadataBuilder.fromArrayMetadata(metadata) .withShape(newShape) .build(); return writeMetadata(newArrayMetadata); } + /** * Sets the attributes of the Zarr array. It overwrites and removes any existing attributes. This * method returns a new instance of the Zarr array class and the old instance becomes invalid. @@ -248,7 +269,8 @@ public Array setAttributes(Attributes newAttributes) throws ZarrException, IOExc * @throws IOException throws IOException if the new metadata cannot be serialized */ public Array updateAttributes(Function attributeMapper) throws ZarrException, IOException { - return setAttributes(attributeMapper.apply(metadata.attributes)); + Attributes currentAttributes = metadata.attributes != null ? new Attributes(metadata.attributes) : new Attributes(); + return setAttributes(attributeMapper.apply(currentAttributes)); } @Override diff --git a/src/main/java/dev/zarr/zarrjava/v3/Group.java b/src/main/java/dev/zarr/zarrjava/v3/Group.java index 5df6d5b..221b7ec 100644 --- a/src/main/java/dev/zarr/zarrjava/v3/Group.java +++ b/src/main/java/dev/zarr/zarrjava/v3/Group.java @@ -285,7 +285,8 @@ private Group writeMetadata(GroupMetadata newGroupMetadata) throws IOException { * @throws IOException if the metadata cannot be serialized */ public Group updateAttributes(Function attributeMapper) throws ZarrException, IOException { - return setAttributes(attributeMapper.apply(metadata.attributes)); + Attributes currentAttributes = metadata.attributes != null ? new Attributes(metadata.attributes) : new Attributes(); + return setAttributes(attributeMapper.apply(currentAttributes)); } /** diff --git a/src/test/java/dev/zarr/zarrjava/ParallelWriteTest.java b/src/test/java/dev/zarr/zarrjava/ParallelWriteTest.java new file mode 100644 index 0000000..cc14959 --- /dev/null +++ b/src/test/java/dev/zarr/zarrjava/ParallelWriteTest.java @@ -0,0 +1,154 @@ +package dev.zarr.zarrjava; + +import dev.zarr.zarrjava.store.FilesystemStore; +import dev.zarr.zarrjava.store.StoreHandle; +import dev.zarr.zarrjava.v3.Array; +import dev.zarr.zarrjava.v3.DataType; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.io.IOException; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.*; + +public class ParallelWriteTest extends ZarrTest { + + @Test + public void testParallelWriteDataSafety() throws IOException, ZarrException { + // Test internal parallelism of write method (using parallel=true) + Path path = TESTOUTPUT.resolve("parallel_write_safety"); + StoreHandle storeHandle = new FilesystemStore(path).resolve(); + + int shape = 1000; + int chunk = 100; + + Array array = Array.create(storeHandle, Array.metadataBuilder() + .withShape(shape, shape) + .withDataType(DataType.INT32) + .withChunkShape(chunk, chunk) + .withFillValue(0) + .build()); + + int[] data = new int[shape * shape]; + // Fill with some deterministic pattern + for (int i = 0; i < shape * shape; i++) { + data[i] = i; + } + + ucar.ma2.Array outputData = ucar.ma2.Array.factory(ucar.ma2.DataType.INT, new int[]{shape, shape}, data); + + // Write in parallel + array.write(outputData, true); + + // Read back + ucar.ma2.Array readData = array.read(); + int[] readArr = (int[]) readData.get1DJavaArray(ucar.ma2.DataType.INT); + + Assertions.assertArrayEquals(data, readArr, "Data read back should match data written in parallel"); + } + + @Test + public void testParallelWriteWithSharding() throws IOException, ZarrException { + // Test internal parallelism with Sharding (nested chunks + shared codec state potential) + Path path = TESTOUTPUT.resolve("parallel_write_sharding"); + StoreHandle storeHandle = new FilesystemStore(path).resolve(); + + int shape = 128; // 128x128 + int shardSize = 64; // Shards are 64x64 + int innerChunk = 32; // Inner chunks 32x32 + + // Metadata with sharding + // With shape 128 and shardSize 64, we have 2x2 = 4 shards. + // Array.write(parallel=true) will likely process these shards concurrently. + dev.zarr.zarrjava.v3.ArrayMetadata metadata = Array.metadataBuilder() + .withShape(shape, shape) + .withDataType(DataType.INT32) + .withChunkShape(shardSize, shardSize) // This sets the shard shape (outer chunks) + .withCodecs(c -> c.withSharding(new int[]{innerChunk, innerChunk}, c2 -> c2.withBytes("LITTLE"))) + .withFillValue(0) + .build(); + + Array array = Array.create(storeHandle, metadata); + + int[] data = new int[shape * shape]; + for (int i = 0; i < shape * shape; i++) { + data[i] = i; + } + + ucar.ma2.Array outputData = ucar.ma2.Array.factory(ucar.ma2.DataType.INT, new int[]{shape, shape}, data); + + // Write in parallel + array.write(outputData, true); + + ucar.ma2.Array readData = array.read(); + int[] readArr = (int[]) readData.get1DJavaArray(ucar.ma2.DataType.INT); + + Assertions.assertArrayEquals(data, readArr, "Sharded data written in parallel should match"); + } + + @Test + public void testConcurrentWritesDifferentChunks() throws IOException, ZarrException, InterruptedException, ExecutionException { + // Test external parallelism (multiple threads calling write on same Array instance) + Path path = TESTOUTPUT.resolve("concurrent_write_safety"); + StoreHandle storeHandle = new FilesystemStore(path).resolve(); + + int chunksX = 10; + int chunksY = 10; + int chunkSize = 50; + int shapeX = chunksX * chunkSize; + int shapeY = chunksY * chunkSize; + + Array array = Array.create(storeHandle, Array.metadataBuilder() + .withShape(shapeX, shapeY) + .withDataType(DataType.INT32) + .withChunkShape(chunkSize, chunkSize) + .withFillValue(-1) + .build()); + + ExecutorService executor = Executors.newFixedThreadPool(8); + List> tasks = new ArrayList<>(); + + for (int i = 0; i < chunksX; i++) { + for (int j = 0; j < chunksY; j++) { + final int cx = i; + final int cy = j; + tasks.add(() -> { + int[] chunkData = new int[chunkSize * chunkSize]; + int val = cx * chunksY + cy; // Unique value per chunk + java.util.Arrays.fill(chunkData, val); + + ucar.ma2.Array ucarArray = ucar.ma2.Array.factory(ucar.ma2.DataType.INT, new int[]{chunkSize, chunkSize}, chunkData); + + // Write to specific chunk offset + long[] offset = new long[]{cx * chunkSize, cy * chunkSize}; + // Use internal parallelism false to isolate external concurrency test mechanism + array.write(offset, ucarArray, false); + return null; + }); + } + } + + List> futures = executor.invokeAll(tasks); + + for (Future f : futures) { + f.get(); // Check for exceptions + } + executor.shutdown(); + + // Verification + ucar.ma2.Array readData = array.read(); + for (int i = 0; i < chunksX; i++) { + for (int j = 0; j < chunksY; j++) { + int expectedVal = i * chunksY + j; + int originX = i * chunkSize; + int originY = j * chunkSize; + + // Verify a pixel in the chunk + int val = readData.getInt(readData.getIndex().set(originX, originY)); + Assertions.assertEquals(expectedVal, val, "Value at chunk " + i + "," + j + " mismatch"); + } + } + } +} diff --git a/src/test/java/dev/zarr/zarrjava/TestUtils.java b/src/test/java/dev/zarr/zarrjava/TestUtils.java index 582dfab..0b21029 100644 --- a/src/test/java/dev/zarr/zarrjava/TestUtils.java +++ b/src/test/java/dev/zarr/zarrjava/TestUtils.java @@ -30,7 +30,7 @@ public void testInversePermutation() { } @Test - public void testComputeChunkCoords(){ + public void testComputeChunkCoords() { long[] arrayShape = new long[]{100, 100}; int[] chunkShape = new int[]{30, 30}; long[] selOffset = new long[]{50, 20}; @@ -56,7 +56,7 @@ public void testComputeChunkCoords(){ } @Test - public void testComputeProjection(){ + public void testComputeProjection() { // chunk (0,2) contains indexes 34-50 along axis 1 // thus the overlap with selection 32-52 is 34-50 // which is offset 2 in the selection and offset 0 in the chunk @@ -71,8 +71,8 @@ public void testComputeProjection(){ chunkCoords, arrayShape, chunkShape, selOffset, selShape ); Assertions.assertArrayEquals(chunkCoords, projection.chunkCoords); - Assertions.assertArrayEquals(new int[]{0,0}, projection.chunkOffset); - Assertions.assertArrayEquals(new int[]{0,2}, projection.outOffset); + Assertions.assertArrayEquals(new int[]{0, 0}, projection.chunkOffset); + Assertions.assertArrayEquals(new int[]{0, 2}, projection.outOffset); Assertions.assertArrayEquals(new int[]{1, 17}, projection.shape); } diff --git a/src/test/java/dev/zarr/zarrjava/ZarrV2Test.java b/src/test/java/dev/zarr/zarrjava/ZarrV2Test.java index 534a351..b5bac3c 100644 --- a/src/test/java/dev/zarr/zarrjava/ZarrV2Test.java +++ b/src/test/java/dev/zarr/zarrjava/ZarrV2Test.java @@ -332,6 +332,31 @@ public void testSetAndUpdateAttributes() throws IOException, ZarrException { assertContainsTestAttributes(array.metadata().attributes()); } + @Test + public void testUpdateAttributesBehavior() throws IOException, ZarrException { + StoreHandle storeHandle = new FilesystemStore(TESTOUTPUT).resolve("testUpdateAttributesBehaviorV2"); + ArrayMetadata arrayMetadata = Array.metadataBuilder() + .withShape(10, 10) + .withDataType(DataType.UINT8) + .withChunks(5, 5) + .withAttributes(new Attributes(b -> b.set("key1", "val1"))) + .build(); + + Array array1 = Array.create(storeHandle, arrayMetadata); + Array array2 = array1.updateAttributes(attrs -> attrs.set("key2", "val2")); + + Assertions.assertNotSame(array1, array2); + Assertions.assertEquals("val1", array1.metadata().attributes().get("key1")); + Assertions.assertNull(array1.metadata().attributes().get("key2")); + + Assertions.assertEquals("val1", array2.metadata().attributes().get("key1")); + Assertions.assertEquals("val2", array2.metadata().attributes().get("key2")); + + // Re-opening should show the updated attributes + Array array3 = Array.open(storeHandle); + Assertions.assertEquals("val2", array3.metadata().attributes().get("key2")); + } + @Test public void testResizeArray() throws IOException, ZarrException { int[] testData = new int[10 * 10]; @@ -358,6 +383,110 @@ public void testResizeArray() throws IOException, ZarrException { int[] expectedData = new int[5 * 5]; Arrays.fill(expectedData, 1); Assertions.assertArrayEquals(expectedData, (int[]) data.get1DJavaArray(ma2DataType)); + + Array reopenedArray = Array.open(storeHandle); + Assertions.assertArrayEquals(new int[]{20, 15}, reopenedArray.read().getShape()); + } + + @Test + public void testResizeArrayShrink() throws IOException, ZarrException { + int[] testData = new int[10 * 10]; + Arrays.setAll(testData, p -> p); + + StoreHandle storeHandle = new FilesystemStore(TESTOUTPUT).resolve("testResizeArrayShrinkV2"); + ArrayMetadata arrayMetadata = Array.metadataBuilder() + .withShape(10, 10) + .withDataType(DataType.UINT32) + .withChunks(5, 5) + .build(); + ucar.ma2.DataType ma2DataType = arrayMetadata.dataType.getMA2DataType(); + Array array = Array.create(storeHandle, arrayMetadata); + array.write(new long[]{0, 0}, ucar.ma2.Array.factory(ma2DataType, new int[]{10, 10}, testData)); + + array = array.resize(new long[]{5, 5}); + Assertions.assertArrayEquals(new int[]{5, 5}, array.read().getShape()); + + ucar.ma2.Array data = array.read(); + int[] expectedData = new int[5 * 5]; + for (int i = 0; i < 5; i++) { + System.arraycopy(testData, i * 10 + 0, expectedData, i * 5 + 0, 5); + } + Assertions.assertArrayEquals(expectedData, (int[]) data.get1DJavaArray(ma2DataType)); + } + + @Test + public void testResizeArrayShrinkWithChunkCleanup() throws IOException, ZarrException { + int[] testData = new int[10 * 10]; + Arrays.setAll(testData, p -> p); + + StoreHandle storeHandle = new FilesystemStore(TESTOUTPUT).resolve("testResizeArrayShrinkWithChunkCleanupV2"); + ArrayMetadata arrayMetadata = Array.metadataBuilder() + .withShape(10, 10) + .withDataType(DataType.UINT32) + .withChunks(5, 5) + .withFillValue(99) + .build(); + ucar.ma2.DataType ma2DataType = arrayMetadata.dataType.getMA2DataType(); + Array array = Array.create(storeHandle, arrayMetadata); + array.write(new long[]{0, 0}, ucar.ma2.Array.factory(ma2DataType, new int[]{10, 10}, testData)); + + // Verify all 4 chunks exist before resize + Assertions.assertTrue(storeHandle.resolve("0.0").exists()); + Assertions.assertTrue(storeHandle.resolve("0.1").exists()); + Assertions.assertTrue(storeHandle.resolve("1.0").exists()); + Assertions.assertTrue(storeHandle.resolve("1.1").exists()); + + // Resize with chunk cleanup (resizeMetadataOnly=false) + array = array.resize(new long[]{5, 5}, false); + Assertions.assertArrayEquals(new int[]{5, 5}, array.read().getShape()); + + // Verify only chunk (0,0) still exists + Assertions.assertTrue(storeHandle.resolve("0.0").exists()); + Assertions.assertFalse(storeHandle.resolve("0.1").exists()); + Assertions.assertFalse(storeHandle.resolve("1.0").exists()); + Assertions.assertFalse(storeHandle.resolve("1.1").exists()); + + ucar.ma2.Array data = array.read(); + int[] expectedData = new int[5 * 5]; + for (int i = 0; i < 5; i++) { + System.arraycopy(testData, i * 10 + 0, expectedData, i * 5 + 0, 5); + } + Assertions.assertArrayEquals(expectedData, (int[]) data.get1DJavaArray(ma2DataType)); + } + + @Test + public void testResizeArrayShrinkWithBoundaryTrimming() throws IOException, ZarrException { + int[] testData = new int[10 * 10]; + Arrays.setAll(testData, p -> p); + + StoreHandle storeHandle = new FilesystemStore(TESTOUTPUT).resolve("testResizeArrayShrinkWithBoundaryTrimmingV2"); + ArrayMetadata arrayMetadata = Array.metadataBuilder() + .withShape(10, 10) + .withDataType(DataType.UINT32) + .withChunks(5, 5) + .withFillValue(99) + .build(); + ucar.ma2.DataType ma2DataType = arrayMetadata.dataType.getMA2DataType(); + Array array = Array.create(storeHandle, arrayMetadata); + array.write(new long[]{0, 0}, ucar.ma2.Array.factory(ma2DataType, new int[]{10, 10}, testData)); + + // Resize to 7x7 (crosses chunk boundary, should trim boundary chunks) + array = array.resize(new long[]{7, 7}, false); + Assertions.assertArrayEquals(new int[]{7, 7}, array.read().getShape()); + + // Verify chunks (0,0), (0,1), (1,0), (1,1) still exist (boundary trimmed, not deleted) + Assertions.assertTrue(storeHandle.resolve("0.0").exists()); + Assertions.assertTrue(storeHandle.resolve("0.1").exists()); + Assertions.assertTrue(storeHandle.resolve("1.0").exists()); + Assertions.assertTrue(storeHandle.resolve("1.1").exists()); + + // Now resize to expand again and check that trimmed area has fill value + array = array.resize(new long[]{10, 10}, true); + ucar.ma2.Array data = array.read(new long[]{7, 0}, new int[]{3, 10}); + // All values in rows 7-9 should be fill value (99) + int[] expectedFillData = new int[3 * 10]; + Arrays.fill(expectedFillData, 99); + Assertions.assertArrayEquals(expectedFillData, (int[]) data.get1DJavaArray(ma2DataType)); } @Test diff --git a/src/test/java/dev/zarr/zarrjava/ZarrV3Test.java b/src/test/java/dev/zarr/zarrjava/ZarrV3Test.java index e8f1e55..3f2079e 100644 --- a/src/test/java/dev/zarr/zarrjava/ZarrV3Test.java +++ b/src/test/java/dev/zarr/zarrjava/ZarrV3Test.java @@ -708,6 +708,31 @@ public void testSetAndUpdateAttributes() throws IOException, ZarrException { assertContainsTestAttributes(array.metadata().attributes()); } + @Test + public void testUpdateAttributesBehavior() throws IOException, ZarrException { + StoreHandle storeHandle = new FilesystemStore(TESTOUTPUT).resolve("testUpdateAttributesBehaviorV3"); + ArrayMetadata arrayMetadata = Array.metadataBuilder() + .withShape(10, 10) + .withDataType(DataType.UINT8) + .withChunkShape(5, 5) + .withAttributes(new Attributes(b -> b.set("key1", "val1"))) + .build(); + + Array array1 = Array.create(storeHandle, arrayMetadata); + Array array2 = array1.updateAttributes(attrs -> attrs.set("key2", "val2")); + + Assertions.assertNotSame(array1, array2); + Assertions.assertEquals("val1", array1.metadata().attributes().get("key1")); + Assertions.assertNull(array1.metadata().attributes().get("key2")); + + Assertions.assertEquals("val1", array2.metadata().attributes().get("key1")); + Assertions.assertEquals("val2", array2.metadata().attributes().get("key2")); + + // Re-opening should show the updated attributes + Array array3 = Array.open(storeHandle); + Assertions.assertEquals("val2", array3.metadata().attributes().get("key2")); + } + @Test public void testResizeArray() throws IOException, ZarrException { int[] testData = new int[10 * 10]; @@ -734,6 +759,110 @@ public void testResizeArray() throws IOException, ZarrException { int[] expectedData = new int[5 * 5]; Arrays.fill(expectedData, 1); Assertions.assertArrayEquals(expectedData, (int[]) data.get1DJavaArray(ma2DataType)); + + Array reopenedArray = Array.open(storeHandle); + Assertions.assertArrayEquals(new int[]{20, 15}, reopenedArray.read().getShape()); + } + + @Test + public void testResizeArrayShrink() throws IOException, ZarrException { + int[] testData = new int[10 * 10]; + Arrays.setAll(testData, p -> p); + + StoreHandle storeHandle = new FilesystemStore(TESTOUTPUT).resolve("testResizeArrayShrinkV3"); + ArrayMetadata arrayMetadata = Array.metadataBuilder() + .withShape(10, 10) + .withDataType(DataType.UINT32) + .withChunkShape(5, 5) + .build(); + ucar.ma2.DataType ma2DataType = arrayMetadata.dataType.getMA2DataType(); + Array array = Array.create(storeHandle, arrayMetadata); + array.write(new long[]{0, 0}, ucar.ma2.Array.factory(ma2DataType, new int[]{10, 10}, testData)); + + array = array.resize(new long[]{5, 5}); + Assertions.assertArrayEquals(new int[]{5, 5}, array.read().getShape()); + + ucar.ma2.Array data = array.read(); + int[] expectedData = new int[5 * 5]; + for (int i = 0; i < 5; i++) { + System.arraycopy(testData, i * 10 + 0, expectedData, i * 5 + 0, 5); + } + Assertions.assertArrayEquals(expectedData, (int[]) data.get1DJavaArray(ma2DataType)); + } + + @Test + public void testResizeArrayShrinkWithChunkCleanup() throws IOException, ZarrException { + int[] testData = new int[10 * 10]; + Arrays.setAll(testData, p -> p); + + StoreHandle storeHandle = new FilesystemStore(TESTOUTPUT).resolve("testResizeArrayShrinkWithChunkCleanupV3"); + ArrayMetadata arrayMetadata = Array.metadataBuilder() + .withShape(10, 10) + .withDataType(DataType.UINT32) + .withChunkShape(5, 5) + .withFillValue(99) + .build(); + ucar.ma2.DataType ma2DataType = arrayMetadata.dataType.getMA2DataType(); + Array array = Array.create(storeHandle, arrayMetadata); + array.write(new long[]{0, 0}, ucar.ma2.Array.factory(ma2DataType, new int[]{10, 10}, testData)); + + // Verify all 4 chunks exist before resize (v3 default encoding has "c" prefix) + Assertions.assertTrue(storeHandle.resolve("c", "0", "0").exists()); + Assertions.assertTrue(storeHandle.resolve("c", "0", "1").exists()); + Assertions.assertTrue(storeHandle.resolve("c", "1", "0").exists()); + Assertions.assertTrue(storeHandle.resolve("c", "1", "1").exists()); + + // Resize with chunk cleanup (resizeMetadataOnly=false) + array = array.resize(new long[]{5, 5}, false); + Assertions.assertArrayEquals(new int[]{5, 5}, array.read().getShape()); + + // Verify only chunk (0,0) still exists + Assertions.assertTrue(storeHandle.resolve("c", "0", "0").exists()); + Assertions.assertFalse(storeHandle.resolve("c", "0", "1").exists()); + Assertions.assertFalse(storeHandle.resolve("c", "1", "0").exists()); + Assertions.assertFalse(storeHandle.resolve("c", "1", "1").exists()); + + ucar.ma2.Array data = array.read(); + int[] expectedData = new int[5 * 5]; + for (int i = 0; i < 5; i++) { + System.arraycopy(testData, i * 10 + 0, expectedData, i * 5 + 0, 5); + } + Assertions.assertArrayEquals(expectedData, (int[]) data.get1DJavaArray(ma2DataType)); + } + + @Test + public void testResizeArrayShrinkWithBoundaryTrimming() throws IOException, ZarrException { + int[] testData = new int[10 * 10]; + Arrays.setAll(testData, p -> p); + + StoreHandle storeHandle = new FilesystemStore(TESTOUTPUT).resolve("testResizeArrayShrinkWithBoundaryTrimmingV3"); + ArrayMetadata arrayMetadata = Array.metadataBuilder() + .withShape(10, 10) + .withDataType(DataType.UINT32) + .withChunkShape(5, 5) + .withFillValue(99) + .build(); + ucar.ma2.DataType ma2DataType = arrayMetadata.dataType.getMA2DataType(); + Array array = Array.create(storeHandle, arrayMetadata); + array.write(new long[]{0, 0}, ucar.ma2.Array.factory(ma2DataType, new int[]{10, 10}, testData)); + + // Resize to 7x7 (crosses chunk boundary, should trim boundary chunks) + array = array.resize(new long[]{7, 7}, false); + Assertions.assertArrayEquals(new int[]{7, 7}, array.read().getShape()); + + // Verify chunks (0,0), (0,1), (1,0), (1,1) still exist (boundary trimmed, not deleted) + Assertions.assertTrue(storeHandle.resolve("c", "0", "0").exists()); + Assertions.assertTrue(storeHandle.resolve("c", "0", "1").exists()); + Assertions.assertTrue(storeHandle.resolve("c", "1", "0").exists()); + Assertions.assertTrue(storeHandle.resolve("c", "1", "1").exists()); + + // Now resize to expand again and check that trimmed area has fill value + array = array.resize(new long[]{10, 10}, true); + ucar.ma2.Array data = array.read(new long[]{7, 0}, new int[]{3, 10}); + // All values in rows 7-9 should be fill value (99) + int[] expectedFillData = new int[3 * 10]; + Arrays.fill(expectedFillData, 99); + Assertions.assertArrayEquals(expectedFillData, (int[]) data.get1DJavaArray(ma2DataType)); } @Test