From 8da13336dfa60c2b4412097bc5e58488aebbb66c Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sun, 31 May 2026 19:28:07 +0200 Subject: [PATCH 1/2] chore(deps): bump the actions group across 1 directory with 8 updates (#176) Bumps the actions group with 8 updates in the / directory: | Package | From | To | | --- | --- | --- | | [prefix-dev/setup-pixi](https://github.com/prefix-dev/setup-pixi) | `0.9.5` | `0.9.6` | | [codecov/codecov-action](https://github.com/codecov/codecov-action) | `6.0.0` | `6.0.1` | | [github/issue-metrics](https://github.com/github/issue-metrics) | `4.2.2` | `4.2.7` | | [j178/prek-action](https://github.com/j178/prek-action) | `2.0.3` | `2.0.4` | | [actions/upload-artifact](https://github.com/actions/upload-artifact) | `7.0.0` | `7.0.1` | | [actions/download-artifact](https://github.com/actions/download-artifact) | `7.0.0` | `8.0.1` | | [pypa/gh-action-pypi-publish](https://github.com/pypa/gh-action-pypi-publish) | `1.13.0` | `1.14.0` | | [zizmorcore/zizmor-action](https://github.com/zizmorcore/zizmor-action) | `0.5.3` | `0.5.6` | Updates `prefix-dev/setup-pixi` from 0.9.5 to 0.9.6 - [Release notes](https://github.com/prefix-dev/setup-pixi/releases) - [Commits](https://github.com/prefix-dev/setup-pixi/compare/1b2de7f3351f171c8b4dfeb558c639cb58ed4ec0...5185adfbffb4bd703da3010310260805d89ebb11) Updates `codecov/codecov-action` from 6.0.0 to 6.0.1 - [Release notes](https://github.com/codecov/codecov-action/releases) - [Changelog](https://github.com/codecov/codecov-action/blob/main/CHANGELOG.md) - [Commits](https://github.com/codecov/codecov-action/compare/57e3a136b779b570ffcdbf80b3bdc90e7fab3de2...e79a6962e0d4c0c17b229090214935d2e33f8354) Updates `github/issue-metrics` from 4.2.2 to 4.2.7 - [Release notes](https://github.com/github/issue-metrics/releases) - [Commits](https://github.com/github/issue-metrics/compare/c9e9838147fd355dace335ba787f01b6641a400a...1e38d5e62363e14db8019ed7d106b9855bdba6cc) Updates `j178/prek-action` from 2.0.3 to 2.0.4 - [Release notes](https://github.com/j178/prek-action/releases) - [Commits](https://github.com/j178/prek-action/compare/6ad80277337ad479fe43bd70701c3f7f8aa74db3...bdca6f102f98e2b4c7029491a53dfd366469e33d) Updates `actions/upload-artifact` from 7.0.0 to 7.0.1 - [Release notes](https://github.com/actions/upload-artifact/releases) - [Commits](https://github.com/actions/upload-artifact/compare/v7...043fb46d1a93c77aae656e7c1c64a875d1fc6a0a) Updates `actions/download-artifact` from 7.0.0 to 8.0.1 - [Release notes](https://github.com/actions/download-artifact/releases) - [Commits](https://github.com/actions/download-artifact/compare/v7...3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c) Updates `pypa/gh-action-pypi-publish` from 1.13.0 to 1.14.0 - [Release notes](https://github.com/pypa/gh-action-pypi-publish/releases) - [Commits](https://github.com/pypa/gh-action-pypi-publish/compare/v1.13.0...cef221092ed1bacb1cc03d23a2d87d1d172e277b) Updates `zizmorcore/zizmor-action` from 0.5.3 to 0.5.6 - [Release notes](https://github.com/zizmorcore/zizmor-action/releases) - [Commits](https://github.com/zizmorcore/zizmor-action/compare/b1d7e1fb5de872772f31590499237e7cce841e8e...5f14fd08f7cf1cb1609c1e344975f152c7ee938d) --- updated-dependencies: - dependency-name: prefix-dev/setup-pixi dependency-version: 0.9.6 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: actions - dependency-name: codecov/codecov-action dependency-version: 6.0.1 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: actions - dependency-name: github/issue-metrics dependency-version: 4.2.7 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: actions - dependency-name: j178/prek-action dependency-version: 2.0.4 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: actions - dependency-name: actions/upload-artifact dependency-version: 7.0.1 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: actions - dependency-name: actions/download-artifact dependency-version: 8.0.1 dependency-type: direct:production update-type: version-update:semver-major dependency-group: actions - dependency-name: pypa/gh-action-pypi-publish dependency-version: 1.14.0 dependency-type: direct:production update-type: version-update:semver-minor dependency-group: actions - dependency-name: zizmorcore/zizmor-action dependency-version: 0.5.6 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: actions ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/downstream.yml | 2 +- .github/workflows/gpu_test.yml | 2 +- .github/workflows/hypothesis.yaml | 2 +- .github/workflows/issue-metrics.yml | 2 +- .github/workflows/lint.yml | 2 +- .github/workflows/test.yml | 4 ++-- .github/workflows/zarr-metadata-release.yml | 12 ++++++------ .github/workflows/zizmor.yml | 2 +- 8 files changed, 14 insertions(+), 14 deletions(-) diff --git a/.github/workflows/downstream.yml b/.github/workflows/downstream.yml index 74026233c4..3eb6898895 100644 --- a/.github/workflows/downstream.yml +++ b/.github/workflows/downstream.yml @@ -34,7 +34,7 @@ jobs: persist-credentials: false - name: Set up pixi - uses: prefix-dev/setup-pixi@1b2de7f3351f171c8b4dfeb558c639cb58ed4ec0 # v0.9.5 + uses: prefix-dev/setup-pixi@5185adfbffb4bd703da3010310260805d89ebb11 # v0.9.6 with: manifest-path: xarray/pixi.toml diff --git a/.github/workflows/gpu_test.yml b/.github/workflows/gpu_test.yml index 403441b306..333769cb9e 100644 --- a/.github/workflows/gpu_test.yml +++ b/.github/workflows/gpu_test.yml @@ -76,7 +76,7 @@ jobs: hatch env run --env "$HATCH_ENV" run-coverage - name: Upload coverage - uses: codecov/codecov-action@57e3a136b779b570ffcdbf80b3bdc90e7fab3de2 # v6.0.0 + uses: codecov/codecov-action@e79a6962e0d4c0c17b229090214935d2e33f8354 # v6.0.1 with: token: ${{ secrets.CODECOV_TOKEN }} flags: gpu diff --git a/.github/workflows/hypothesis.yaml b/.github/workflows/hypothesis.yaml index 4f9467be7d..a456b2aa0a 100644 --- a/.github/workflows/hypothesis.yaml +++ b/.github/workflows/hypothesis.yaml @@ -93,7 +93,7 @@ jobs: key: cache-hypothesis-${{ runner.os }}-${{ github.run_id }} - name: Upload coverage - uses: codecov/codecov-action@57e3a136b779b570ffcdbf80b3bdc90e7fab3de2 # v6.0.0 + uses: codecov/codecov-action@e79a6962e0d4c0c17b229090214935d2e33f8354 # v6.0.1 with: token: ${{ secrets.CODECOV_TOKEN }} flags: tests diff --git a/.github/workflows/issue-metrics.yml b/.github/workflows/issue-metrics.yml index 14fba5b9ec..510849ef3e 100644 --- a/.github/workflows/issue-metrics.yml +++ b/.github/workflows/issue-metrics.yml @@ -33,7 +33,7 @@ jobs: echo "last_month=$first_day..$last_day" >> "$GITHUB_ENV" - name: Run issue-metrics tool - uses: github/issue-metrics@c9e9838147fd355dace335ba787f01b6641a400a # v4.2.2 + uses: github/issue-metrics@1e38d5e62363e14db8019ed7d106b9855bdba6cc # v4.2.7 env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} SEARCH_QUERY: 'repo:zarr-developers/zarr-python is:issue created:${{ env.last_month }} -reason:"not planned"' diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 768e660ec2..fec211b4dd 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -30,4 +30,4 @@ jobs: uses: astral-sh/setup-uv@08807647e7069bb48b6ef5acd8ec9567f424441b # v8.1.0 with: enable-cache: true - - uses: j178/prek-action@6ad80277337ad479fe43bd70701c3f7f8aa74db3 # v2.0.3 + - uses: j178/prek-action@bdca6f102f98e2b4c7029491a53dfd366469e33d # v2.0.4 diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 03143d3e5b..62e571856b 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -78,7 +78,7 @@ jobs: hatch env run --env "$HATCH_ENV" run-coverage - name: Upload coverage if: ${{ matrix.dependency-set == 'optional' && matrix.os == 'ubuntu-latest' }} - uses: codecov/codecov-action@57e3a136b779b570ffcdbf80b3bdc90e7fab3de2 # v6.0.0 + uses: codecov/codecov-action@e79a6962e0d4c0c17b229090214935d2e33f8354 # v6.0.1 with: token: ${{ secrets.CODECOV_TOKEN }} flags: tests @@ -125,7 +125,7 @@ jobs: run: | hatch env run --env "$HATCH_ENV" run-coverage - name: Upload coverage - uses: codecov/codecov-action@57e3a136b779b570ffcdbf80b3bdc90e7fab3de2 # v6.0.0 + uses: codecov/codecov-action@e79a6962e0d4c0c17b229090214935d2e33f8354 # v6.0.1 with: token: ${{ secrets.CODECOV_TOKEN }} flags: tests diff --git a/.github/workflows/zarr-metadata-release.yml b/.github/workflows/zarr-metadata-release.yml index 809d502f16..9639fcfdd3 100644 --- a/.github/workflows/zarr-metadata-release.yml +++ b/.github/workflows/zarr-metadata-release.yml @@ -35,7 +35,7 @@ jobs: - name: Build run: hatch build - - uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 + - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: zarr-metadata-dist path: packages/zarr-metadata/dist @@ -45,7 +45,7 @@ jobs: needs: [build] runs-on: ubuntu-latest steps: - - uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0 + - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: name: zarr-metadata-dist path: dist @@ -76,7 +76,7 @@ jobs: id-token: write # required for OIDC trusted publishing attestations: write # required for artifact attestations steps: - - uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0 + - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: name: zarr-metadata-dist path: dist @@ -87,7 +87,7 @@ jobs: subject-path: dist/* - name: Publish package to PyPI - uses: pypa/gh-action-pypi-publish@ed0c53931b1dc9bd32cbe73a98c7f6766f8a527e # v1.13.0 + uses: pypa/gh-action-pypi-publish@cef221092ed1bacb1cc03d23a2d87d1d172e277b # v1.14.0 upload_testpypi: name: Upload to TestPyPI @@ -101,7 +101,7 @@ jobs: id-token: write attestations: write steps: - - uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0 + - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: name: zarr-metadata-dist path: dist @@ -112,6 +112,6 @@ jobs: subject-path: dist/* - name: Publish package to TestPyPI - uses: pypa/gh-action-pypi-publish@ed0c53931b1dc9bd32cbe73a98c7f6766f8a527e # v1.13.0 + uses: pypa/gh-action-pypi-publish@cef221092ed1bacb1cc03d23a2d87d1d172e277b # v1.14.0 with: repository-url: https://test.pypi.org/legacy/ diff --git a/.github/workflows/zizmor.yml b/.github/workflows/zizmor.yml index da19f22421..7ac4fe5d0e 100644 --- a/.github/workflows/zizmor.yml +++ b/.github/workflows/zizmor.yml @@ -32,4 +32,4 @@ jobs: persist-credentials: false - name: Run zizmor - uses: zizmorcore/zizmor-action@b1d7e1fb5de872772f31590499237e7cce841e8e # v0.5.3 + uses: zizmorcore/zizmor-action@5f14fd08f7cf1cb1609c1e344975f152c7ee938d # v0.5.6 From 994256f0e793becc41e0c53c2b3da50826c1e4b4 Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Fri, 12 Jun 2026 13:34:51 +0200 Subject: [PATCH 2/2] perf: hoist per-chunk config lookups and parallelize deep group traversal Hoist `async.concurrency` config lookups to once per read/write/encode/ decode operation via an operation-scoped cache (`async_concurrency_scope` / `get_async_concurrency` in `zarr.core.config`), instead of re-reading Donfig config per chunk and per codec stage on the codec pipeline hot path. Config changes between operations are still honored. Drive recursion into sibling subgroups concurrently in `_iter_members_deep` so sibling subtrees overlap their latency, instead of draining each subgroup iterator sequentially. The shared semaphore still bounds total concurrent `getitem` calls; member ordering remains unspecified (as documented). Co-Authored-By: Claude Fable 5 --- src/zarr/abc/codec.py | 8 ++--- src/zarr/core/codec_pipeline.py | 52 +++++++++++++++++--------------- src/zarr/core/config.py | 45 ++++++++++++++++++++++++++++ src/zarr/core/group.py | 53 ++++++++++++++++++++++++++++++--- 4 files changed, 126 insertions(+), 32 deletions(-) diff --git a/src/zarr/abc/codec.py b/src/zarr/abc/codec.py index eed2119aff..099974583d 100644 --- a/src/zarr/abc/codec.py +++ b/src/zarr/abc/codec.py @@ -9,7 +9,7 @@ from zarr.abc.metadata import Metadata from zarr.core.buffer import Buffer, NDBuffer from zarr.core.common import NamedConfig, concurrent_map -from zarr.core.config import config +from zarr.core.config import get_async_concurrency if TYPE_CHECKING: from collections.abc import Awaitable, Callable, Iterable @@ -249,7 +249,7 @@ async def decode_partial( return await concurrent_map( list(batch_info), self._decode_partial_single, - config.get("async.concurrency"), + get_async_concurrency(), ) @@ -286,7 +286,7 @@ async def encode_partial( await concurrent_map( list(batch_info), self._encode_partial_single, - config.get("async.concurrency"), + get_async_concurrency(), ) @@ -493,7 +493,7 @@ async def _batching_helper[CI: CodecInput, CO: CodecOutput]( return await concurrent_map( list(batch_info), _noop_for_none(func), - config.get("async.concurrency"), + get_async_concurrency(), ) diff --git a/src/zarr/core/codec_pipeline.py b/src/zarr/core/codec_pipeline.py index 5c26681d6b..472b564212 100644 --- a/src/zarr/core/codec_pipeline.py +++ b/src/zarr/core/codec_pipeline.py @@ -17,7 +17,7 @@ SupportsSyncCodec, ) from zarr.core.common import concurrent_map -from zarr.core.config import config +from zarr.core.config import async_concurrency_scope, config, get_async_concurrency from zarr.core.indexing import SelectorTuple, is_scalar from zarr.errors import ZarrUserWarning from zarr.registry import register_pipeline @@ -395,7 +395,7 @@ async def read_batch( for byte_getter, array_spec, *_ in batch_info_list ], lambda byte_getter, prototype: byte_getter.get(prototype), - config.get("async.concurrency"), + get_async_concurrency(), ) chunk_array_batch = await self.decode_batch( [ @@ -505,7 +505,7 @@ async def _read_key( for byte_setter, chunk_spec, chunk_selection, _, is_complete_chunk in batch_info ], _read_key, - config.get("async.concurrency"), + get_async_concurrency(), ) chunk_array_decoded = await self.decode_batch( [ @@ -571,7 +571,7 @@ async def _write_key(byte_setter: ByteSetter, chunk_bytes: Buffer | None) -> Non ) ], _write_key, - config.get("async.concurrency"), + get_async_concurrency(), ) async def decode( @@ -579,8 +579,9 @@ async def decode( chunk_bytes_and_specs: Iterable[tuple[Buffer | None, ArraySpec]], ) -> Iterable[NDBuffer | None]: output: list[NDBuffer | None] = [] - for batch_info in batched(chunk_bytes_and_specs, self.batch_size): - output.extend(await self.decode_batch(batch_info)) + with async_concurrency_scope(): + for batch_info in batched(chunk_bytes_and_specs, self.batch_size): + output.extend(await self.decode_batch(batch_info)) return output async def encode( @@ -588,8 +589,9 @@ async def encode( chunk_arrays_and_specs: Iterable[tuple[NDBuffer | None, ArraySpec]], ) -> Iterable[Buffer | None]: output: list[Buffer | None] = [] - for single_batch_info in batched(chunk_arrays_and_specs, self.batch_size): - output.extend(await self.encode_batch(single_batch_info)) + with async_concurrency_scope(): + for single_batch_info in batched(chunk_arrays_and_specs, self.batch_size): + output.extend(await self.encode_batch(single_batch_info)) return output async def read( @@ -598,14 +600,15 @@ async def read( out: NDBuffer, drop_axes: tuple[int, ...] = (), ) -> tuple[GetResult, ...]: - batch_results = await concurrent_map( - [ - (single_batch_info, out, drop_axes) - for single_batch_info in batched(batch_info, self.batch_size) - ], - self.read_batch, - config.get("async.concurrency"), - ) + with async_concurrency_scope(): + batch_results = await concurrent_map( + [ + (single_batch_info, out, drop_axes) + for single_batch_info in batched(batch_info, self.batch_size) + ], + self.read_batch, + get_async_concurrency(), + ) results: list[GetResult] = [] for batch in batch_results: results.extend(batch) @@ -617,14 +620,15 @@ async def write( value: NDBuffer, drop_axes: tuple[int, ...] = (), ) -> None: - await concurrent_map( - [ - (single_batch_info, value, drop_axes) - for single_batch_info in batched(batch_info, self.batch_size) - ], - self.write_batch, - config.get("async.concurrency"), - ) + with async_concurrency_scope(): + await concurrent_map( + [ + (single_batch_info, value, drop_axes) + for single_batch_info in batched(batch_info, self.batch_size) + ], + self.write_batch, + get_async_concurrency(), + ) def codecs_from_list( diff --git a/src/zarr/core/config.py b/src/zarr/core/config.py index 7dcbc78e31..5b3c6585b0 100644 --- a/src/zarr/core/config.py +++ b/src/zarr/core/config.py @@ -29,11 +29,15 @@ from __future__ import annotations +import contextlib +import contextvars from typing import TYPE_CHECKING, Any, Literal, cast from donfig import Config as DConfig if TYPE_CHECKING: + from collections.abc import Iterator + from donfig.config_obj import ConfigSet @@ -148,6 +152,47 @@ def enable_gpu(self) -> ConfigSet: ) +# Operation-scoped cache for ``async.concurrency``. +# +# The per-chunk codec read/write path looks up ``async.concurrency`` many times +# (once per chunk, plus once per codec per chunk via the batching helper). Each +# lookup walks Donfig's nested config. To avoid that, a single read/write/ +# encode/decode operation captures the value once via ``async_concurrency_scope`` +# and the hot paths read it through ``get_async_concurrency``. Outside of a +# scope we fall back to a live ``config.get``, so configuration changes always +# take effect between operations. +_async_concurrency_var: contextvars.ContextVar[int | None] = contextvars.ContextVar( + "zarr_async_concurrency" +) + + +def get_async_concurrency() -> int | None: + """Return the ``async.concurrency`` limit for the current operation. + + Within an :func:`async_concurrency_scope`, returns the value captured once at + the start of the operation. Outside of a scope, falls back to a live + ``config.get`` lookup. + """ + try: + return _async_concurrency_var.get() + except LookupError: + return cast("int | None", config.get("async.concurrency")) + + +@contextlib.contextmanager +def async_concurrency_scope() -> Iterator[None]: + """Capture ``async.concurrency`` once for the duration of an operation. + + Reads of :func:`get_async_concurrency` inside this context return the + captured value rather than performing a fresh ``config.get`` lookup. + """ + token = _async_concurrency_var.set(cast("int | None", config.get("async.concurrency"))) + try: + yield + finally: + _async_concurrency_var.reset(token) + + def parse_indexing_order(data: Any) -> Literal["C", "F"]: if data in ("C", "F"): return cast("Literal['C', 'F']", data) diff --git a/src/zarr/core/group.py b/src/zarr/core/group.py index 52eaa3e144..8aad48a2fb 100644 --- a/src/zarr/core/group.py +++ b/src/zarr/core/group.py @@ -3286,6 +3286,13 @@ async def _iter_members( raise ValueError(f"Unexpected type: {type(fetched_node)}") +class _Drained: + """Sentinel signalling that a subgroup iterator has been fully drained.""" + + +_DRAINED = _Drained() + + async def _iter_members_deep( group: AsyncGroup, *, @@ -3344,10 +3351,48 @@ async def _iter_members_deep( node, max_depth=new_depth, skip_keys=skip_keys, semaphore=semaphore ) - for prefix, subgroup_iter in to_recurse.items(): - async for name, node in subgroup_iter: - key = f"{prefix}/{name}".lstrip("/") - yield key, node + # Recurse into sibling subgroups concurrently rather than draining each + # subgroup's iterator in sequence, so that sibling subtrees overlap their + # latency. The shared ``semaphore`` (threaded into every recursive call) + # still bounds the total number of concurrent ``getitem`` operations. + # Members are not guaranteed to be ordered, so we yield them as they arrive. + queue: asyncio.Queue[tuple[str, AnyAsyncArray | AsyncGroup] | Exception | _Drained] = ( + asyncio.Queue() + ) + + async def _drain( + prefix: str, + subgroup_iter: AsyncGenerator[tuple[str, AnyAsyncArray | AsyncGroup], None], + ) -> None: + try: + async for name, node in subgroup_iter: + key = f"{prefix}/{name}".lstrip("/") + await queue.put((key, node)) + except Exception as exc: + # Forward the error to the consumer so it surfaces from this + # generator just as it would from sequential iteration. + await queue.put(exc) + finally: + await queue.put(_DRAINED) + + tasks = [ + asyncio.create_task(_drain(prefix, subgroup_iter)) + for prefix, subgroup_iter in to_recurse.items() + ] + remaining = len(tasks) + try: + while remaining: + item = await queue.get() + if isinstance(item, _Drained): + remaining -= 1 + elif isinstance(item, Exception): + raise item + else: + yield item + finally: + for task in tasks: + task.cancel() + await asyncio.gather(*tasks, return_exceptions=True) async def _read_metadata_v3(store: Store, path: str) -> ArrayV3Metadata | GroupMetadata: