From 8da13336dfa60c2b4412097bc5e58488aebbb66c Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sun, 31 May 2026 19:28:07 +0200 Subject: [PATCH 1/2] chore(deps): bump the actions group across 1 directory with 8 updates (#176) Bumps the actions group with 8 updates in the / directory: | Package | From | To | | --- | --- | --- | | [prefix-dev/setup-pixi](https://github.com/prefix-dev/setup-pixi) | `0.9.5` | `0.9.6` | | [codecov/codecov-action](https://github.com/codecov/codecov-action) | `6.0.0` | `6.0.1` | | [github/issue-metrics](https://github.com/github/issue-metrics) | `4.2.2` | `4.2.7` | | [j178/prek-action](https://github.com/j178/prek-action) | `2.0.3` | `2.0.4` | | [actions/upload-artifact](https://github.com/actions/upload-artifact) | `7.0.0` | `7.0.1` | | [actions/download-artifact](https://github.com/actions/download-artifact) | `7.0.0` | `8.0.1` | | [pypa/gh-action-pypi-publish](https://github.com/pypa/gh-action-pypi-publish) | `1.13.0` | `1.14.0` | | [zizmorcore/zizmor-action](https://github.com/zizmorcore/zizmor-action) | `0.5.3` | `0.5.6` | Updates `prefix-dev/setup-pixi` from 0.9.5 to 0.9.6 - [Release notes](https://github.com/prefix-dev/setup-pixi/releases) - [Commits](https://github.com/prefix-dev/setup-pixi/compare/1b2de7f3351f171c8b4dfeb558c639cb58ed4ec0...5185adfbffb4bd703da3010310260805d89ebb11) Updates `codecov/codecov-action` from 6.0.0 to 6.0.1 - [Release notes](https://github.com/codecov/codecov-action/releases) - [Changelog](https://github.com/codecov/codecov-action/blob/main/CHANGELOG.md) - [Commits](https://github.com/codecov/codecov-action/compare/57e3a136b779b570ffcdbf80b3bdc90e7fab3de2...e79a6962e0d4c0c17b229090214935d2e33f8354) Updates `github/issue-metrics` from 4.2.2 to 4.2.7 - [Release notes](https://github.com/github/issue-metrics/releases) - [Commits](https://github.com/github/issue-metrics/compare/c9e9838147fd355dace335ba787f01b6641a400a...1e38d5e62363e14db8019ed7d106b9855bdba6cc) Updates `j178/prek-action` from 2.0.3 to 2.0.4 - [Release notes](https://github.com/j178/prek-action/releases) - [Commits](https://github.com/j178/prek-action/compare/6ad80277337ad479fe43bd70701c3f7f8aa74db3...bdca6f102f98e2b4c7029491a53dfd366469e33d) Updates `actions/upload-artifact` from 7.0.0 to 7.0.1 - [Release notes](https://github.com/actions/upload-artifact/releases) - [Commits](https://github.com/actions/upload-artifact/compare/v7...043fb46d1a93c77aae656e7c1c64a875d1fc6a0a) Updates `actions/download-artifact` from 7.0.0 to 8.0.1 - [Release notes](https://github.com/actions/download-artifact/releases) - [Commits](https://github.com/actions/download-artifact/compare/v7...3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c) Updates `pypa/gh-action-pypi-publish` from 1.13.0 to 1.14.0 - [Release notes](https://github.com/pypa/gh-action-pypi-publish/releases) - [Commits](https://github.com/pypa/gh-action-pypi-publish/compare/v1.13.0...cef221092ed1bacb1cc03d23a2d87d1d172e277b) Updates `zizmorcore/zizmor-action` from 0.5.3 to 0.5.6 - [Release notes](https://github.com/zizmorcore/zizmor-action/releases) - [Commits](https://github.com/zizmorcore/zizmor-action/compare/b1d7e1fb5de872772f31590499237e7cce841e8e...5f14fd08f7cf1cb1609c1e344975f152c7ee938d) --- updated-dependencies: - dependency-name: prefix-dev/setup-pixi dependency-version: 0.9.6 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: actions - dependency-name: codecov/codecov-action dependency-version: 6.0.1 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: actions - dependency-name: github/issue-metrics dependency-version: 4.2.7 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: actions - dependency-name: j178/prek-action dependency-version: 2.0.4 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: actions - dependency-name: actions/upload-artifact dependency-version: 7.0.1 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: actions - dependency-name: actions/download-artifact dependency-version: 8.0.1 dependency-type: direct:production update-type: version-update:semver-major dependency-group: actions - dependency-name: pypa/gh-action-pypi-publish dependency-version: 1.14.0 dependency-type: direct:production update-type: version-update:semver-minor dependency-group: actions - dependency-name: zizmorcore/zizmor-action dependency-version: 0.5.6 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: actions ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/downstream.yml | 2 +- .github/workflows/gpu_test.yml | 2 +- .github/workflows/hypothesis.yaml | 2 +- .github/workflows/issue-metrics.yml | 2 +- .github/workflows/lint.yml | 2 +- .github/workflows/test.yml | 4 ++-- .github/workflows/zarr-metadata-release.yml | 12 ++++++------ .github/workflows/zizmor.yml | 2 +- 8 files changed, 14 insertions(+), 14 deletions(-) diff --git a/.github/workflows/downstream.yml b/.github/workflows/downstream.yml index 74026233c4..3eb6898895 100644 --- a/.github/workflows/downstream.yml +++ b/.github/workflows/downstream.yml @@ -34,7 +34,7 @@ jobs: persist-credentials: false - name: Set up pixi - uses: prefix-dev/setup-pixi@1b2de7f3351f171c8b4dfeb558c639cb58ed4ec0 # v0.9.5 + uses: prefix-dev/setup-pixi@5185adfbffb4bd703da3010310260805d89ebb11 # v0.9.6 with: manifest-path: xarray/pixi.toml diff --git a/.github/workflows/gpu_test.yml b/.github/workflows/gpu_test.yml index 403441b306..333769cb9e 100644 --- a/.github/workflows/gpu_test.yml +++ b/.github/workflows/gpu_test.yml @@ -76,7 +76,7 @@ jobs: hatch env run --env "$HATCH_ENV" run-coverage - name: Upload coverage - uses: codecov/codecov-action@57e3a136b779b570ffcdbf80b3bdc90e7fab3de2 # v6.0.0 + uses: codecov/codecov-action@e79a6962e0d4c0c17b229090214935d2e33f8354 # v6.0.1 with: token: ${{ secrets.CODECOV_TOKEN }} flags: gpu diff --git a/.github/workflows/hypothesis.yaml b/.github/workflows/hypothesis.yaml index 4f9467be7d..a456b2aa0a 100644 --- a/.github/workflows/hypothesis.yaml +++ b/.github/workflows/hypothesis.yaml @@ -93,7 +93,7 @@ jobs: key: cache-hypothesis-${{ runner.os }}-${{ github.run_id }} - name: Upload coverage - uses: codecov/codecov-action@57e3a136b779b570ffcdbf80b3bdc90e7fab3de2 # v6.0.0 + uses: codecov/codecov-action@e79a6962e0d4c0c17b229090214935d2e33f8354 # v6.0.1 with: token: ${{ secrets.CODECOV_TOKEN }} flags: tests diff --git a/.github/workflows/issue-metrics.yml b/.github/workflows/issue-metrics.yml index 14fba5b9ec..510849ef3e 100644 --- a/.github/workflows/issue-metrics.yml +++ b/.github/workflows/issue-metrics.yml @@ -33,7 +33,7 @@ jobs: echo "last_month=$first_day..$last_day" >> "$GITHUB_ENV" - name: Run issue-metrics tool - uses: github/issue-metrics@c9e9838147fd355dace335ba787f01b6641a400a # v4.2.2 + uses: github/issue-metrics@1e38d5e62363e14db8019ed7d106b9855bdba6cc # v4.2.7 env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} SEARCH_QUERY: 'repo:zarr-developers/zarr-python is:issue created:${{ env.last_month }} -reason:"not planned"' diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 768e660ec2..fec211b4dd 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -30,4 +30,4 @@ jobs: uses: astral-sh/setup-uv@08807647e7069bb48b6ef5acd8ec9567f424441b # v8.1.0 with: enable-cache: true - - uses: j178/prek-action@6ad80277337ad479fe43bd70701c3f7f8aa74db3 # v2.0.3 + - uses: j178/prek-action@bdca6f102f98e2b4c7029491a53dfd366469e33d # v2.0.4 diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 03143d3e5b..62e571856b 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -78,7 +78,7 @@ jobs: hatch env run --env "$HATCH_ENV" run-coverage - name: Upload coverage if: ${{ matrix.dependency-set == 'optional' && matrix.os == 'ubuntu-latest' }} - uses: codecov/codecov-action@57e3a136b779b570ffcdbf80b3bdc90e7fab3de2 # v6.0.0 + uses: codecov/codecov-action@e79a6962e0d4c0c17b229090214935d2e33f8354 # v6.0.1 with: token: ${{ secrets.CODECOV_TOKEN }} flags: tests @@ -125,7 +125,7 @@ jobs: run: | hatch env run --env "$HATCH_ENV" run-coverage - name: Upload coverage - uses: codecov/codecov-action@57e3a136b779b570ffcdbf80b3bdc90e7fab3de2 # v6.0.0 + uses: codecov/codecov-action@e79a6962e0d4c0c17b229090214935d2e33f8354 # v6.0.1 with: token: ${{ secrets.CODECOV_TOKEN }} flags: tests diff --git a/.github/workflows/zarr-metadata-release.yml b/.github/workflows/zarr-metadata-release.yml index 809d502f16..9639fcfdd3 100644 --- a/.github/workflows/zarr-metadata-release.yml +++ b/.github/workflows/zarr-metadata-release.yml @@ -35,7 +35,7 @@ jobs: - name: Build run: hatch build - - uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 + - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: zarr-metadata-dist path: packages/zarr-metadata/dist @@ -45,7 +45,7 @@ jobs: needs: [build] runs-on: ubuntu-latest steps: - - uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0 + - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: name: zarr-metadata-dist path: dist @@ -76,7 +76,7 @@ jobs: id-token: write # required for OIDC trusted publishing attestations: write # required for artifact attestations steps: - - uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0 + - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: name: zarr-metadata-dist path: dist @@ -87,7 +87,7 @@ jobs: subject-path: dist/* - name: Publish package to PyPI - uses: pypa/gh-action-pypi-publish@ed0c53931b1dc9bd32cbe73a98c7f6766f8a527e # v1.13.0 + uses: pypa/gh-action-pypi-publish@cef221092ed1bacb1cc03d23a2d87d1d172e277b # v1.14.0 upload_testpypi: name: Upload to TestPyPI @@ -101,7 +101,7 @@ jobs: id-token: write attestations: write steps: - - uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0 + - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: name: zarr-metadata-dist path: dist @@ -112,6 +112,6 @@ jobs: subject-path: dist/* - name: Publish package to TestPyPI - uses: pypa/gh-action-pypi-publish@ed0c53931b1dc9bd32cbe73a98c7f6766f8a527e # v1.13.0 + uses: pypa/gh-action-pypi-publish@cef221092ed1bacb1cc03d23a2d87d1d172e277b # v1.14.0 with: repository-url: https://test.pypi.org/legacy/ diff --git a/.github/workflows/zizmor.yml b/.github/workflows/zizmor.yml index da19f22421..7ac4fe5d0e 100644 --- a/.github/workflows/zizmor.yml +++ b/.github/workflows/zizmor.yml @@ -32,4 +32,4 @@ jobs: persist-credentials: false - name: Run zizmor - uses: zizmorcore/zizmor-action@b1d7e1fb5de872772f31590499237e7cce841e8e # v0.5.3 + uses: zizmorcore/zizmor-action@5f14fd08f7cf1cb1609c1e344975f152c7ee938d # v0.5.6 From 5e161efc08672532741b413ba20903d2abb4a5b5 Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Fri, 12 Jun 2026 13:38:15 +0200 Subject: [PATCH 2/2] perf(sharding): cache per-shard invariants in ShardingCodec Several values that are invariant across the shards of an array were recomputed on every shard (and, for the prototype assert, every inner chunk). Cache them on the frozen codec instance following the existing lru_cache-on-self pattern: - codec_pipeline / index codec pipeline: build the CodecPipeline once via cached_property instead of get_pipeline_class().from_codecs(...) per shard (codec_pipeline) and per index encode/decode/size call. - _ShardingByteGetter.get: take the expected BufferPrototype precomputed once per shard instead of calling default_buffer_prototype() (a config + registry lookup) for every inner chunk on the read path. - full-shard BasicIndexer + ChunkGrid + projection list: memoize keyed on shard_shape (which varies for boundary shards) and reuse the immutable projections across same-shaped shards. Behavior is unchanged; the full tests/test_codecs suite passes. Co-Authored-By: Claude Fable 5 --- src/zarr/codecs/sharding.py | 94 +++++++++++++++++++++++-------------- 1 file changed, 58 insertions(+), 36 deletions(-) diff --git a/src/zarr/codecs/sharding.py b/src/zarr/codecs/sharding.py index 657484e9af..19f0452d16 100644 --- a/src/zarr/codecs/sharding.py +++ b/src/zarr/codecs/sharding.py @@ -3,7 +3,7 @@ from collections.abc import Iterable, Mapping, MutableMapping, Sequence from dataclasses import dataclass, replace from enum import Enum -from functools import lru_cache +from functools import cached_property, lru_cache from typing import TYPE_CHECKING, Any, Literal, NamedTuple, cast import numpy as np @@ -98,12 +98,19 @@ def parse_index_location(data: object) -> ShardingCodecIndexLocation: class _ShardingByteGetter(ByteGetter): shard_dict: ShardMapping chunk_coords: tuple[int, ...] + # The prototype required for chunks within this shard. Precomputing it once per shard + # avoids a per-inner-chunk `default_buffer_prototype()` lookup on the hot read path. If + # None (e.g. direct construction in tests), it is resolved lazily in `get`. + expected_prototype: BufferPrototype | None = None async def get( self, prototype: BufferPrototype, byte_range: ByteRequest | None = None ) -> Buffer | None: - assert prototype == default_buffer_prototype(), ( - f"prototype is not supported within shards currently. diff: {prototype} != {default_buffer_prototype()}" + expected_prototype = self.expected_prototype + if expected_prototype is None: + expected_prototype = default_buffer_prototype() + assert prototype == expected_prototype, ( + f"prototype is not supported within shards currently. diff: {prototype} != {expected_prototype}" ) value = self.shard_dict.get(self.chunk_coords) if value is None: @@ -339,6 +346,7 @@ def __init__( # object.__setattr__(self, "_get_chunk_spec", lru_cache()(self._get_chunk_spec)) object.__setattr__(self, "_get_index_chunk_spec", lru_cache()(self._get_index_chunk_spec)) object.__setattr__(self, "_get_chunks_per_shard", lru_cache()(self._get_chunks_per_shard)) + object.__setattr__(self, "_get_full_indexer", lru_cache()(self._get_full_indexer)) # todo: typedict return type def __getstate__(self) -> dict[str, Any]: @@ -358,16 +366,25 @@ def __setstate__(self, state: dict[str, Any]) -> None: # object.__setattr__(self, "_get_chunk_spec", lru_cache()(self._get_chunk_spec)) object.__setattr__(self, "_get_index_chunk_spec", lru_cache()(self._get_index_chunk_spec)) object.__setattr__(self, "_get_chunks_per_shard", lru_cache()(self._get_chunks_per_shard)) + object.__setattr__(self, "_get_full_indexer", lru_cache()(self._get_full_indexer)) @classmethod def from_dict(cls, data: dict[str, JSON]) -> Self: _, configuration_parsed = parse_named_configuration(data, "sharding_indexed") return cls(**configuration_parsed) # type: ignore[arg-type] - @property + @cached_property def codec_pipeline(self) -> CodecPipeline: + # Cached on the (frozen, slot-less) instance: `self.codecs` never changes, so the + # pipeline is built once instead of once per shard in `_decode_single`/`_encode_single`. return get_pipeline_class().from_codecs(self.codecs) + @cached_property + def _index_codec_pipeline(self) -> CodecPipeline: + # Cached for the same reason as `codec_pipeline`; reused by the shard-index + # encode/decode/size helpers below. + return get_pipeline_class().from_codecs(self.index_codecs) + def to_dict(self) -> dict[str, JSON]: return { "name": "sharding_indexed", @@ -422,15 +439,11 @@ async def _decode_single( shard_spec: ArraySpec, ) -> NDBuffer: shard_shape = shard_spec.shape - chunk_shape = self.chunk_shape chunks_per_shard = self._get_chunks_per_shard(shard_spec) chunk_spec = self._get_chunk_spec(shard_spec) + default_prototype = default_buffer_prototype() - indexer = BasicIndexer( - tuple(slice(0, s) for s in shard_shape), - shape=shard_shape, - chunk_grid=ChunkGrid.from_sizes(shard_shape, chunk_shape), - ) + indexer = self._get_full_indexer(shard_shape) # setup output array out = chunk_spec.prototype.nd_buffer.empty( @@ -448,7 +461,9 @@ async def _decode_single( await self.codec_pipeline.read( [ ( - _ShardingByteGetter(shard_dict, chunk_coords), + _ShardingByteGetter( + shard_dict, chunk_coords, expected_prototype=default_prototype + ), chunk_spec, chunk_selection, out_selection, @@ -471,6 +486,7 @@ async def _decode_partial_single( chunk_shape = self.chunk_shape chunks_per_shard = self._get_chunks_per_shard(shard_spec) chunk_spec = self._get_chunk_spec(shard_spec) + default_prototype = default_buffer_prototype() indexer = get_indexer( selection, @@ -514,7 +530,9 @@ async def _decode_partial_single( await self.codec_pipeline.read( [ ( - _ShardingByteGetter(shard_dict, chunk_coords), + _ShardingByteGetter( + shard_dict, chunk_coords, expected_prototype=default_prototype + ), chunk_spec, chunk_selection, out_selection, @@ -554,17 +572,11 @@ async def _encode_single( shard_spec: ArraySpec, ) -> Buffer | None: shard_shape = shard_spec.shape - chunk_shape = self.chunk_shape chunks_per_shard = self._get_chunks_per_shard(shard_spec) chunk_spec = self._get_chunk_spec(shard_spec) + default_prototype = default_buffer_prototype() - indexer = list( - BasicIndexer( - tuple(slice(0, s) for s in shard_shape), - shape=shard_shape, - chunk_grid=ChunkGrid.from_sizes(shard_shape, chunk_shape), - ) - ) + indexer = self._get_full_indexer(shard_shape) # The key order of this intermediate dict is immaterial; the physical layout is # decided later by the `subchunk_write_order` loop in `_encode_shard_dict`. shard_builder = dict.fromkeys(np.ndindex(chunks_per_shard)) @@ -572,7 +584,9 @@ async def _encode_single( await self.codec_pipeline.write( [ ( - _ShardingByteSetter(shard_builder, chunk_coords), + _ShardingByteSetter( + shard_builder, chunk_coords, expected_prototype=default_prototype + ), chunk_spec, chunk_selection, out_selection, @@ -586,7 +600,7 @@ async def _encode_single( return await self._encode_shard_dict( shard_builder, chunks_per_shard=chunks_per_shard, - buffer_prototype=default_buffer_prototype(), + buffer_prototype=default_prototype, ) async def _encode_partial_single( @@ -600,6 +614,7 @@ async def _encode_partial_single( chunk_shape = self.chunk_shape chunks_per_shard = self._get_chunks_per_shard(shard_spec) chunk_spec = self._get_chunk_spec(shard_spec) + default_prototype = default_buffer_prototype() indexer = list( get_indexer( @@ -627,7 +642,9 @@ async def _encode_partial_single( await self.codec_pipeline.write( [ ( - _ShardingByteSetter(shard_dict, chunk_coords), + _ShardingByteSetter( + shard_dict, chunk_coords, expected_prototype=default_prototype + ), chunk_spec, chunk_selection, out_selection, @@ -640,7 +657,7 @@ async def _encode_partial_single( buf = await self._encode_shard_dict( shard_dict, chunks_per_shard=chunks_per_shard, - buffer_prototype=default_buffer_prototype(), + buffer_prototype=default_prototype, ) if buf is None: @@ -711,9 +728,7 @@ async def _decode_shard_index( ) -> _ShardIndex: index_array = next( iter( - await get_pipeline_class() - .from_codecs(self.index_codecs) - .decode( + await self._index_codec_pipeline.decode( [(index_bytes, self._get_index_chunk_spec(chunks_per_shard))], ) ) @@ -725,9 +740,7 @@ async def _decode_shard_index( async def _encode_shard_index(self, index: _ShardIndex) -> Buffer: index_bytes = next( iter( - await get_pipeline_class() - .from_codecs(self.index_codecs) - .encode( + await self._index_codec_pipeline.encode( [ ( get_ndbuffer_class().from_numpy_array(index.offsets_and_lengths), @@ -742,12 +755,8 @@ async def _encode_shard_index(self, index: _ShardIndex) -> Buffer: return index_bytes def _shard_index_size(self, chunks_per_shard: tuple[int, ...]) -> int: - return ( - get_pipeline_class() - .from_codecs(self.index_codecs) - .compute_encoded_size( - 16 * product(chunks_per_shard), self._get_index_chunk_spec(chunks_per_shard) - ) + return self._index_codec_pipeline.compute_encoded_size( + 16 * product(chunks_per_shard), self._get_index_chunk_spec(chunks_per_shard) ) def _get_index_chunk_spec(self, chunks_per_shard: tuple[int, ...]) -> ArraySpec: @@ -761,6 +770,19 @@ def _get_index_chunk_spec(self, chunks_per_shard: tuple[int, ...]) -> ArraySpec: prototype=default_buffer_prototype(), ) + def _get_full_indexer(self, shard_shape: tuple[int, ...]) -> list[ChunkProjection]: + # The full-shard indexer (chunk grid + projections) depends only on `shard_shape` + # and `self.chunk_shape`, both invariant across same-shaped shards. `shard_shape` + # can vary for partial shards at array boundaries, so it keys the cache. The + # returned projections are immutable and safe to reuse across shards. + return list( + BasicIndexer( + tuple(slice(0, s) for s in shard_shape), + shape=shard_shape, + chunk_grid=ChunkGrid.from_sizes(shard_shape, self.chunk_shape), + ) + ) + def _get_chunk_spec(self, shard_spec: ArraySpec) -> ArraySpec: return ArraySpec( shape=self.chunk_shape,