Skip to content

Commit 8994196

Browse files
konstibobclaude
andauthored
Support suffix reads (negative start) in MemoryStore and ReadOnlyZipStore (#113)
ShardingIndexedCodec reads a shard index located at the end via store.get(keys, -indexSize), i.e. "the last N bytes". FilesystemStore, HttpStore and S3Store handle a negative start, but MemoryStore passed it straight to ByteBuffer.wrap and ReadOnlyZipStore to ZipArchiveInputStream.skip, which throws "Negative skip value". BufferedZipStore delegates to MemoryStore and was affected too. As a result, any partial read of a sharded array (index_location "end") from a zip archive failed, e.g. reading a boundary shard whose valid region is smaller than the shard shape. Convert a negative start into an absolute offset using the entry size, add a StoreTest case for suffix reads for all stores, and an end-to-end test reading a sub-region of a sharded array from a zip. Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
1 parent 4a73a43 commit 8994196

4 files changed

Lines changed: 72 additions & 0 deletions

File tree

‎src/main/java/dev/zarr/zarrjava/store/MemoryStore.java‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -49,6 +49,7 @@ public ByteBuffer get(String[] keys, long start) {
4949
public ByteBuffer get(String[] keys, long start, long end) {
5050
byte[] bytes = map.get(resolveKeys(keys));
5151
if (bytes == null) return null;
52+
if (start < 0) start = Math.max(0, bytes.length + start); // suffix read: the last -start bytes
5253
if (end < 0) end = bytes.length;
5354
if (end > Integer.MAX_VALUE) throw new IllegalArgumentException("End index too large");
5455
return ByteBuffer.wrap(bytes, (int) start, (int) (end - start));
@@ -101,6 +102,7 @@ public String toString() {
101102
public InputStream getInputStream(String[] keys, long start, long end) {
102103
byte[] bytes = map.get(resolveKeys(keys));
103104
if (bytes == null) return null;
105+
if (start < 0) start = Math.max(0, bytes.length + start); // suffix read: the last -start bytes
104106
if (end < 0) end = bytes.length;
105107
if (end > Integer.MAX_VALUE) throw new IllegalArgumentException("End index too large");
106108
return new java.io.ByteArrayInputStream(bytes, (int) start, (int) (end - start));

‎src/main/java/dev/zarr/zarrjava/store/ReadOnlyZipStore.java‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -109,6 +109,10 @@ public ByteBuffer get(String[] keys, long start, long end) {
109109
if (!fileIndex.containsKey(key)) {
110110
return null;
111111
}
112+
if (start < 0) {
113+
// suffix read: the last -start bytes
114+
start = Math.max(0, getSize(keys) + start);
115+
}
112116

113117
InputStream inputStream = underlyingStore.getInputStream();
114118
if (inputStream == null) {
@@ -226,6 +230,10 @@ public InputStream getInputStream(String[] keys, long start, long end) {
226230
if (!fileIndex.containsKey(key)) {
227231
return null;
228232
}
233+
if (start < 0) {
234+
// suffix read: the last -start bytes
235+
start = Math.max(0, getSize(keys) + start);
236+
}
229237

230238
InputStream baseStream = underlyingStore.getInputStream();
231239

‎src/test/java/dev/zarr/zarrjava/store/ReadOnlyZipStoreTest.java‎

Lines changed: 42 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,9 +3,13 @@
33
import dev.zarr.zarrjava.Utils;
44
import dev.zarr.zarrjava.ZarrException;
55
import dev.zarr.zarrjava.core.Group;
6+
import dev.zarr.zarrjava.v3.Array;
7+
import dev.zarr.zarrjava.v3.DataType;
68
import org.junit.jupiter.api.Assertions;
79
import org.junit.jupiter.api.BeforeAll;
810
import org.junit.jupiter.api.Test;
11+
import org.junit.jupiter.params.ParameterizedTest;
12+
import org.junit.jupiter.params.provider.ValueSource;
913

1014
import java.io.IOException;
1115
import java.nio.file.Path;
@@ -123,4 +127,42 @@ public void testReadFromBufferedZipStore() throws ZarrException, IOException {
123127

124128
assertIsTestGroupV3(Group.open(readOnlyZipStore.resolve()), true);
125129
}
130+
131+
@ParameterizedTest
132+
@ValueSource(strings = {"start", "end"})
133+
public void testPartialReadOfShardedArray(String indexLocation) throws ZarrException, IOException {
134+
// partial shard reads fetch the shard index via a suffix read when it is located at the end
135+
Path sourceDir = TESTOUTPUT.resolve("testShardedZipStore_" + indexLocation);
136+
Path targetDir = TESTOUTPUT.resolve("testShardedZipStore_" + indexLocation + ".zip");
137+
Array writeArray = Array.create(new FilesystemStore(sourceDir).resolve(), Array.metadataBuilder()
138+
.withShape(24, 24)
139+
.withDataType(DataType.INT32)
140+
.withChunkShape(16, 16)
141+
.withCodecs(c -> c.withSharding(new int[]{8, 8}, c1 -> c1.withBytes("LITTLE"), indexLocation))
142+
.withFillValue(0)
143+
.build());
144+
int[] data = new int[24 * 24];
145+
for (int i = 0; i < data.length; i++) {
146+
data[i] = i;
147+
}
148+
ucar.ma2.Array expected = ucar.ma2.Array.factory(ucar.ma2.DataType.INT, new int[]{24, 24}, data);
149+
writeArray.write(expected);
150+
151+
Utils.zipFile(sourceDir, targetDir);
152+
153+
Array[] readArrays = {
154+
Array.open(new ReadOnlyZipStore(targetDir).resolve()),
155+
Array.open(new BufferedZipStore(targetDir).resolve())
156+
};
157+
for (Array readArray : readArrays) {
158+
Assertions.assertArrayEquals(data, (int[]) readArray.read().get1DJavaArray(ucar.ma2.DataType.INT));
159+
160+
ucar.ma2.Array partial = readArray.read(new long[]{3, 5}, new long[]{14, 17});
161+
for (int i = 0; i < 14; i++) {
162+
for (int j = 0; j < 17; j++) {
163+
Assertions.assertEquals(data[(i + 3) * 24 + (j + 5)], partial.getInt(i * 17 + j));
164+
}
165+
}
166+
}
167+
}
126168
}

‎src/test/java/dev/zarr/zarrjava/store/StoreTest.java‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -128,6 +128,26 @@ public void testGetWithStartEnd() {
128128
Assertions.assertArrayEquals(expectedBytes, actualBytes);
129129
}
130130

131+
@Test
132+
public void testGetSuffix() {
133+
// a negative start denotes a suffix read (the last -start bytes), as used for sharding indexes
134+
StoreHandle storeHandle = storeHandleWithData();
135+
long size = storeHandle.getSize();
136+
if (size < 20) {
137+
Assertions.fail("Store size is too small to test suffix reads");
138+
}
139+
ByteBuffer buffer = storeHandle.read(-10);
140+
Assertions.assertEquals(10, buffer.remaining());
141+
142+
ByteBuffer fullBuffer = storeHandle.read();
143+
byte[] expectedBytes = new byte[10];
144+
fullBuffer.position((int) (size - 10));
145+
fullBuffer.get(expectedBytes, 0, 10);
146+
byte[] actualBytes = new byte[10];
147+
buffer.get(actualBytes, 0, 10);
148+
Assertions.assertArrayEquals(expectedBytes, actualBytes);
149+
}
150+
131151
@Test
132152
public void testGetRanges() {
133153
StoreHandle storeHandle = storeHandleWithData();

0 commit comments

Comments
 (0)