Merge branch 'main' into docs/from-zero-to-zarr

d-v-b · web-flow · commit 3e1a62d9d760 · 2026-06-18T14:52:01.000+02:00
diff --git a/.github/dependabot.yml b/.github/dependabot.yml
@@ -12,6 +12,19 @@ updates:
           - "*"
     cooldown:
       default-days: 7
+  # Keep the pinned dev tooling in pyproject.toml's [dependency-groups] and the
+  # uv.lock current. Without this the exact pins (e.g. pytest) would never be
+  # bumped automatically and would silently rot.
+  - package-ecosystem: "uv"
+    directory: "/"
+    schedule:
+      interval: "weekly"
+    groups:
+      python-dependencies:
+        patterns:
+          - "*"
+    cooldown:
+      default-days: 7
   - package-ecosystem: "github-actions"
     directory: "/"
     target-branch: "support/v2"
diff --git a/.github/workflows/gpu_test.yml b/.github/workflows/gpu_test.yml
@@ -80,7 +80,7 @@ jobs:
         hatch env run --env "$HATCH_ENV" run-coverage
 
     - name: Upload coverage
-      uses: codecov/codecov-action@e79a6962e0d4c0c17b229090214935d2e33f8354  # v6.0.1
+      uses: codecov/codecov-action@fb8b3582c8e4def4969c97caa2f19720cb33a72f  # v7.0.0
       with:
         token: ${{ secrets.CODECOV_TOKEN }}
         flags: gpu
diff --git a/.github/workflows/hypothesis.yaml b/.github/workflows/hypothesis.yaml
@@ -97,7 +97,7 @@ jobs:
         key: cache-hypothesis-${{ runner.os }}-${{ github.run_id }}
 
     - name: Upload coverage
-      uses: codecov/codecov-action@e79a6962e0d4c0c17b229090214935d2e33f8354 # v6.0.1
+      uses: codecov/codecov-action@fb8b3582c8e4def4969c97caa2f19720cb33a72f # v7.0.0
       with:
         token: ${{ secrets.CODECOV_TOKEN }}
         flags: tests
diff --git a/.github/workflows/prepare_release.yml b/.github/workflows/prepare_release.yml
@@ -42,15 +42,12 @@ jobs:
           fetch-depth: 0
           persist-credentials: false
 
-      - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
-        with:
-          python-version: '3.12'
-
-      - name: Install towncrier
-        run: pip install towncrier
+      - uses: astral-sh/setup-uv@fac544c07dec837d0ccb6301d7b5580bf5edae39 # v8.2.0
 
       - name: Build changelog
-        run: towncrier build --version "$VERSION" --yes
+        # Use the pinned towncrier from the `release` dependency group (single
+        # source of truth) rather than an unpinned standalone install.
+        run: uv run --only-group release towncrier build --version "$VERSION" --yes
         env:
           VERSION: ${{ inputs.version }}
 
diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
@@ -83,7 +83,7 @@ jobs:
         hatch env run --env "$HATCH_ENV" run-coverage
     - name: Upload coverage
       if: ${{ matrix.dependency-set == 'optional' && matrix.os == 'ubuntu-latest' }}
-      uses: codecov/codecov-action@e79a6962e0d4c0c17b229090214935d2e33f8354 # v6.0.1
+      uses: codecov/codecov-action@fb8b3582c8e4def4969c97caa2f19720cb33a72f # v7.0.0
       with:
         token: ${{ secrets.CODECOV_TOKEN }}
         flags: tests
@@ -130,7 +130,7 @@ jobs:
       run: |
         hatch env run --env "$HATCH_ENV" run-coverage
     - name: Upload coverage
-      uses: codecov/codecov-action@e79a6962e0d4c0c17b229090214935d2e33f8354 # v6.0.1
+      uses: codecov/codecov-action@fb8b3582c8e4def4969c97caa2f19720cb33a72f # v7.0.0
       with:
         token: ${{ secrets.CODECOV_TOKEN }}
         flags: tests
diff --git a/changes/4074.bugfix.md b/changes/4074.bugfix.md
@@ -0,0 +1,7 @@
+Fixed several storage and codec bugs:
+
+- Reading a value with a `SuffixByteRequest` larger than the value now correctly returns the whole value (matching HTTP `bytes=-N` suffix-range semantics), instead of silently returning incorrect data for `MemoryStore`.
+- `LoggingStore.get_partial_values` and `FsspecStore.get_partial_values` no longer return empty results when `key_ranges` is passed as a one-shot iterable (e.g. a generator).
+- `Store.getsize_prefix` no longer over-counts sibling keys that merely share a string prefix (e.g. `getsize_prefix("foo")` no longer includes keys under `foobar/`).
+- `ZipStore.close()` no longer raises `AttributeError` when the store was created but never opened (including when used as a context manager without any I/O).
+- `codecs_from_list` now raises a descriptive `TypeError` when a `BytesBytesCodec` immediately follows an `ArrayArrayCodec`, instead of a misleading "Required ArrayBytesCodec was not found" `ValueError`.
diff --git a/pyproject.toml b/pyproject.toml
@@ -83,56 +83,68 @@ Discussions = "https://github.com/zarr-developers/zarr-python/discussions"
 documentation = "https://zarr.readthedocs.io/"
 homepage = "https://github.com/zarr-developers/zarr-python"
 
+# Dev *tooling* is pinned to exact versions for reproducible CI: the hatch envs
+# (see `tool.hatch.envs.*`) and bare `uv run` resolve these groups fresh from
+# PyPI and do NOT consult uv.lock, so an unrelated tooling release can break CI
+# without any change on our side (e.g. the pytest 9.1.0 `duplicate
+# parametrization` regression). Runtime/integration deps (fsspec, obstore, s3fs,
+# botocore, numcodecs, universal-pathlib) are intentionally left floating so the
+# `optional` test matrix keeps exercising their latest releases; their floor and
+# bleeding edge are covered by the `min_deps` and `upstream` hatch envs. Bump the
+# pins deliberately, e.g. via dependabot or `uv lock --upgrade`.
 [dependency-groups]
 test = [
-    "coverage>=7.10",
-    "pytest",
-    "pytest-asyncio",
-    "pytest-cov",
-    "pytest-accept",
-    "numpydoc",
-    "hypothesis",
-    "pytest-xdist",
-    "pytest-benchmark",
-    "pytest-codspeed",
-    "tomlkit",
-    "uv",
+    "coverage==7.14.1",
+    "pytest==9.0.3",
+    "pytest-asyncio==1.4.0",
+    "pytest-cov==7.1.0",
+    "pytest-accept==0.2.3",
+    "numpydoc==1.10.0",
+    "hypothesis==6.155.2",
+    "pytest-xdist==3.8.0",
+    "pytest-benchmark==5.2.3",
+    "pytest-codspeed==5.0.3",
+    "tomlkit==0.15.0",
+    "uv==0.11.19",
 ]
 remote-tests = [
     {include-group = "test"},
     "fsspec>=2023.10.0",
     "obstore>=0.5.1",
     "botocore",
     "s3fs>=2023.10.0",
-    "moto[s3,server]",
-    "requests",
+    "moto[s3,server]==5.2.2",
+    "requests==2.34.2",
+]
+release = [
+    "towncrier==25.8.0",
 ]
 docs = [
     # Doc building
-    "mkdocs-material[imaging]>=9.6.14",
-    "mkdocs>=1.6.1,<2",
-    "mkdocstrings>=0.29.1",
-    "mkdocstrings-python>=1.16.10",
-    "mike>=2.1.3",
-    "mkdocs-jupyter>=0.25.1",
-    "mkdocs-redirects>=1.2.0",
-    "markdown-exec[ansi]",
-    "griffe-inherited-docstrings",
-    "ruff",
+    "mkdocs-material[imaging]==9.7.6",
+    "mkdocs==1.6.1",
+    "mkdocstrings==1.0.4",
+    "mkdocstrings-python==2.0.4",
+    "mike==2.2.0",
+    "mkdocs-jupyter==0.26.3",
+    "mkdocs-redirects==1.2.3",
+    "markdown-exec[ansi]==1.12.1",
+    "griffe-inherited-docstrings==1.1.3",
+    "ruff==0.15.16",
     # Changelog generation
-    "towncrier",
+    {include-group = "release"},
     # Optional dependencies to run examples
     "numcodecs[msgpack]",
     "s3fs>=2023.10.0",
-    "astroid<4",
-    "pytest",
+    "astroid==4.1.2",
+    "pytest==9.0.3",
 ]
 dev = [
     {include-group = "test"},
     {include-group = "remote-tests"},
     {include-group = "docs"},
     "universal-pathlib",
-    "mypy",
+    "mypy==2.1.0",
 ]
 
 [tool.coverage.report]
@@ -381,7 +393,7 @@ show_error_code_links = true
 show_error_context = true
 strict = true
 warn_unreachable = true
-enable_error_code = ["ignore-without-code", "redundant-expr", "truthy-bool"]
+enable_error_code = ["ignore-without-code", "redundant-expr", "truthy-bool", "truthy-iterable"]
 
 [[tool.mypy.overrides]]
 module = [
diff --git a/src/zarr/abc/store.py b/src/zarr/abc/store.py
@@ -536,6 +536,8 @@ async def getsize_prefix(self, prefix: str) -> int:
         from zarr.core.common import concurrent_map
         from zarr.core.config import config
 
+        if prefix != "" and not prefix.endswith("/"):
+            prefix += "/"
         keys = [(x,) async for x in self.list_prefix(prefix)]
         limit = config.get("async.concurrency")
         sizes = await concurrent_map(keys, self.getsize, limit=limit)
diff --git a/src/zarr/core/codec_pipeline.py b/src/zarr/core/codec_pipeline.py
@@ -679,6 +679,7 @@ def codecs_from_list(
                     "must be preceded by either another BytesBytesCodec, or an ArrayBytesCodec. "
                     f"Got {type(prev_codec)} instead."
                 )
+                raise TypeError(msg)
             bytes_bytes += (cur_codec,)
         else:
             raise TypeError
diff --git a/src/zarr/storage/_fsspec.py b/src/zarr/storage/_fsspec.py
@@ -424,30 +424,31 @@ async def get_partial_values(
         key_ranges: Iterable[tuple[str, ByteRequest | None]],
     ) -> list[Buffer | None]:
         # docstring inherited
-        if key_ranges:
-            # _cat_ranges expects a list of paths, start, and end ranges, so we need to reformat each ByteRequest.
-            key_ranges = list(key_ranges)
-            paths: list[str] = []
-            starts: list[int | None] = []
-            stops: list[int | None] = []
-            for key, byte_range in key_ranges:
-                paths.append(_dereference_path(self.path, key))
-                if byte_range is None:
-                    starts.append(None)
-                    stops.append(None)
-                elif isinstance(byte_range, RangeByteRequest):
-                    starts.append(byte_range.start)
-                    stops.append(byte_range.end)
-                elif isinstance(byte_range, OffsetByteRequest):
-                    starts.append(byte_range.offset)
-                    stops.append(None)
-                elif isinstance(byte_range, SuffixByteRequest):
-                    starts.append(-byte_range.suffix)
-                    stops.append(None)
-                else:
-                    raise ValueError(f"Unexpected byte_range, got {byte_range}.")
-        else:
+        # Materialise first: key_ranges may be a one-shot iterable, so a bare
+        # truthiness check (e.g. `if key_ranges`) would be unreliable for an
+        # empty generator. _cat_ranges also expects lists of paths/starts/stops.
+        key_ranges = list(key_ranges)
+        if not key_ranges:
             return []
+        paths: list[str] = []
+        starts: list[int | None] = []
+        stops: list[int | None] = []
+        for key, byte_range in key_ranges:
+            paths.append(_dereference_path(self.path, key))
+            if byte_range is None:
+                starts.append(None)
+                stops.append(None)
+            elif isinstance(byte_range, RangeByteRequest):
+                starts.append(byte_range.start)
+                stops.append(byte_range.end)
+            elif isinstance(byte_range, OffsetByteRequest):
+                starts.append(byte_range.offset)
+                stops.append(None)
+            elif isinstance(byte_range, SuffixByteRequest):
+                starts.append(-byte_range.suffix)
+                stops.append(None)
+            else:
+                raise ValueError(f"Unexpected byte_range, got {byte_range}.")
         # TODO: expectations for exceptions or missing keys?
         res = await self.fs._cat_ranges(paths, starts, stops, on_error="return")
         # the following is an s3-specific condition we probably don't want to leak
diff --git a/src/zarr/storage/_logging.py b/src/zarr/storage/_logging.py
@@ -179,6 +179,7 @@ async def get_partial_values(
         key_ranges: Iterable[tuple[str, ByteRequest | None]],
     ) -> list[Buffer | None]:
         # docstring inherited
+        key_ranges = list(key_ranges)
         keys = ",".join([k[0] for k in key_ranges])
         with self.log(keys):
             return await self._store.get_partial_values(prototype=prototype, key_ranges=key_ranges)
diff --git a/src/zarr/storage/_utils.py b/src/zarr/storage/_utils.py
@@ -153,7 +153,7 @@ def _normalize_byte_range_index(data: Buffer, byte_range: ByteRequest | None) ->
         start = byte_range.offset
         stop = len(data) + 1
     elif isinstance(byte_range, SuffixByteRequest):
-        start = len(data) - byte_range.suffix
+        start = max(0, len(data) - byte_range.suffix)
         stop = len(data) + 1
     else:
         raise ValueError(f"Unexpected byte_range, got {byte_range}.")
diff --git a/src/zarr/storage/_zip.py b/src/zarr/storage/_zip.py
@@ -120,6 +120,8 @@ def __setstate__(self, state: dict[str, Any]) -> None:
 
     def close(self) -> None:
         # docstring inherited
+        if not self._is_open:
+            return
         super().close()
         with self._lock:
             self._zf.close()
diff --git a/src/zarr/testing/stateful.py b/src/zarr/testing/stateful.py
@@ -1,6 +1,6 @@
 import builtins
 import functools
-from collections.abc import Callable
+from collections.abc import Callable, Iterable
 from typing import Any, cast
 
 import hypothesis.extra.numpy as npst
@@ -18,7 +18,12 @@
 
 import zarr
 from zarr import Array
-from zarr.abc.store import Store
+from zarr.abc.store import (
+    OffsetByteRequest,
+    RangeByteRequest,
+    Store,
+    SuffixByteRequest,
+)
 from zarr.codecs.bytes import BytesCodec
 from zarr.core.buffer import Buffer, BufferPrototype, cpu, default_buffer_prototype
 from zarr.core.sync import SyncMixin
@@ -460,7 +465,7 @@ def get(self, key: str, prototype: BufferPrototype) -> Buffer | None:
         return self._sync(self.store.get(key, prototype=prototype))
 
     def get_partial_values(
-        self, key_ranges: builtins.list[Any], prototype: BufferPrototype
+        self, key_ranges: Iterable[Any], prototype: BufferPrototype
     ) -> builtins.list[Buffer | None]:
         return self._sync(self.store.get_partial_values(prototype=prototype, key_ranges=key_ranges))
 
@@ -476,6 +481,9 @@ def clear(self) -> None:
     def exists(self, key: str) -> bool:
         return self._sync(self.store.exists(key))
 
+    def getsize_prefix(self, prefix: str) -> int:
+        return self._sync(self.store.getsize_prefix(prefix))
+
     def list_dir(self, prefix: str) -> None:
         raise NotImplementedError
 
@@ -555,7 +563,9 @@ def get_partial_values(self, data: DataObject) -> None:
             key_ranges(keys=st.sampled_from(sorted(self.model.keys())), max_size=MAX_BINARY_SIZE)
         )
         note(f"(get partial) {key_range=}")
-        obs_maybe = self.store.get_partial_values(key_range, self.prototype)
+        # Pass a one-shot generator rather than a list: stores (and wrappers such
+        # as LoggingStore) must not exhaust the iterable before using it.
+        obs_maybe = self.store.get_partial_values((kr for kr in key_range), self.prototype)
         observed = []
 
         for obs in obs_maybe:
@@ -565,9 +575,23 @@ def get_partial_values(self, data: DataObject) -> None:
         model_vals_ls = []
 
         for key, byte_range in key_range:
-            start = byte_range.start
-            stop = byte_range.end
-            model_vals_ls.append(self.model[key][start:stop])
+            # Independently model each ByteRequest variant (do NOT reuse the
+            # store's _normalize_byte_range_index helper, so this stays an
+            # independent oracle). Bounds may exceed the value length.
+            value = self.model[key]
+            n = len(value)
+            if byte_range is None:
+                expected = value[:]
+            elif isinstance(byte_range, RangeByteRequest):
+                expected = value[byte_range.start : byte_range.end]
+            elif isinstance(byte_range, OffsetByteRequest):
+                expected = value[byte_range.offset :]
+            elif isinstance(byte_range, SuffixByteRequest):
+                # "last suffix bytes"; suffix > n means the whole value.
+                expected = value[max(0, n - byte_range.suffix) :]
+            else:
+                raise AssertionError(f"unexpected byte_range {byte_range!r}")
+            model_vals_ls.append(expected)
 
         assert all(
             obs == exp.to_bytes() for obs, exp in zip(observed, model_vals_ls, strict=True)
@@ -612,6 +636,21 @@ def exists(self, key: str) -> None:
 
         assert self.store.exists(key) == (key in self.model)
 
+    @precondition(lambda self: len(self.model.keys()) > 0)
+    @rule(data=st.data())
+    def getsize_prefix(self, data: DataObject) -> None:
+        # Measure the size under the first path segment of some existing key.
+        # getsize_prefix(node) must count only keys under the directory "node/",
+        # not sibling keys that merely share the string prefix (e.g. measuring
+        # "a" must not include a sibling key "ab/...").
+        key = data.draw(st.sampled_from(sorted(self.model.keys())))
+        node = key.split("/")[0]
+        note(f"(getsize_prefix) {node=}")
+
+        observed = self.store.getsize_prefix(node)
+        expected = sum(len(value) for k, value in self.model.items() if k.startswith(node + "/"))
+        assert observed == expected, (observed, expected, node)
+
     @invariant()
     def check_paths_equal(self) -> None:
         note("Checking that paths are equal")
diff --git a/src/zarr/testing/store.py b/src/zarr/testing/store.py
diff --git a/src/zarr/testing/strategies.py b/src/zarr/testing/strategies.py
diff --git a/tests/test_codec_pipeline.py b/tests/test_codec_pipeline.py
diff --git a/tests/test_store/test_utils.py b/tests/test_store/test_utils.py
diff --git a/tests/test_store/test_zip.py b/tests/test_store/test_zip.py
diff --git a/uv.lock b/uv.lock

Original file line number	Diff line number	Diff line change
`@@ -679,6 +679,7 @@ def codecs_from_list(`
`679`	`679`	`"must be preceded by either another BytesBytesCodec, or an ArrayBytesCodec. "`
`680`	`680`	`f"Got {type(prev_codec)} instead."`
`681`	`681`	`)`
	`682`	`+ raise TypeError(msg)`
`682`	`683`	`bytes_bytes += (cur_codec,)`
`683`	`684`	`else:`
`684`	`685`	`raise TypeError`