From 9bfe855a6cfd6985073fd5e4e5b5be027628eca8 Mon Sep 17 00:00:00 2001 From: Davis Bennett Date: Tue, 14 Jul 2026 13:25:16 +0200 Subject: [PATCH 1/4] fix: byte-order handling for structured dtypes in the bytes codec (#220) * fix: byte-order handling for structured dtypes in the bytes codec The bytes codec neither byte-swapped structured-dtype fields to its configured endian on encode (numpy reports byteorder '|' for void dtypes, so the top-level byteorder comparison never detected a mismatch) nor honored its endian when decoding, silently corrupting any structured data whose field byte order differed from the stored one (e.g. virtual references to external big-endian data). Encode now detects byte-order mismatches by comparing full dtypes via newbyteorder, and decode reinterprets raw bytes in the stored byte order before converting to the data type's declared byte order, so the stored layout (codec state) and the in-memory layout (array data type) are independent. Closes #4141 Assisted-by: ClaudeCode:claude-fable-5 * test: fold structured byte-order cases into existing bytes codec tests Extend test_endian's parametrization with structured dtypes and test_bytes_codec_sync_roundtrip with endian/dtype parametrization plus stored-layout and decoded-dtype assertions, instead of adding parallel test functions for the same properties. Assisted-by: ClaudeCode:claude-fable-5 * refactor: rename stored_dtype to view_dtype in BytesCodec decode The variable is the dtype used to view the raw chunk bytes (byte order from the codec's endian configuration), not a property of the stored data or of the returned buffer, which always carries the array's declared dtype. Assisted-by: ClaudeCode:claude-fable-5 * docs: note that the decode-side byte-order conversion copies the chunk Assisted-by: ClaudeCode:claude-fable-5 --- changes/4141.bugfix.md | 1 + src/zarr/codecs/bytes.py | 33 +++++++++++----- tests/test_codecs/test_bytes.py | 69 ++++++++++++++++++++++++++------- 3 files changed, 80 insertions(+), 23 deletions(-) create mode 100644 changes/4141.bugfix.md diff --git a/changes/4141.bugfix.md b/changes/4141.bugfix.md new file mode 100644 index 0000000000..6a132da3f5 --- /dev/null +++ b/changes/4141.bugfix.md @@ -0,0 +1 @@ +Fix silent byte-order corruption for structured dtypes with the `bytes` codec: multi-byte fields are now byte-swapped to the codec's configured `endian` on write and decoded honoring it on read, so non-native-endian structured data (e.g. big-endian fields, as produced by virtual references to external data) round-trips correctly. diff --git a/src/zarr/codecs/bytes.py b/src/zarr/codecs/bytes.py index 240c077627..fae762fd08 100644 --- a/src/zarr/codecs/bytes.py +++ b/src/zarr/codecs/bytes.py @@ -100,14 +100,28 @@ def _decode_sync( chunk_spec: ArraySpec, ) -> NDBuffer: endian_str = self.endian + dtype = chunk_spec.dtype.to_native_dtype() + # The byte order of the stored data is set by this codec's `endian` + # configuration; the byte order of the decoded array is set by the array's + # data type. The two are independent: the raw bytes are viewed with a dtype + # in the stored byte order, then converted to the declared dtype if needed. if isinstance(chunk_spec.dtype, HasEndianness): - dtype = replace(chunk_spec.dtype, endianness=endian_str).to_native_dtype() # type: ignore[call-arg] + view_dtype = replace(chunk_spec.dtype, endianness=endian_str).to_native_dtype() # type: ignore[call-arg] + elif isinstance(chunk_spec.dtype, Struct) and endian_str is not None: + # Per the struct data type spec, all multi-byte fields are stored in the + # byte order configured on this codec. + view_dtype = dtype.newbyteorder(endian_str) else: - dtype = chunk_spec.dtype.to_native_dtype() + view_dtype = dtype as_array_like = chunk_bytes.as_array_like() chunk_array = chunk_spec.prototype.nd_buffer.from_ndarray_like( - as_array_like.view(dtype=dtype) # type: ignore[attr-defined] + as_array_like.view(dtype=view_dtype) # type: ignore[attr-defined] ) + if view_dtype != dtype: + # This byte-swapping conversion copies the chunk. The dtype inequality + # guard keeps the common case, where the stored and declared byte orders + # already match, on the zero-copy view path above. + chunk_array = chunk_array.astype(dtype) # ensure correct chunk shape if chunk_array.shape != chunk_spec.shape: @@ -129,13 +143,14 @@ def _encode_sync( chunk_spec: ArraySpec, ) -> Buffer | None: assert isinstance(chunk_array, NDBuffer) - if ( - chunk_array.dtype.itemsize > 1 - and self.endian is not None - and self.endian != chunk_array.byteorder - ): + if chunk_array.dtype.itemsize > 1 and self.endian is not None: + # Compare full dtypes rather than the top-level byteorder: numpy reports + # byteorder '|' for structured dtypes even when their fields are + # byte-order-sensitive, so newbyteorder is the only reliable way to + # detect (and normalize) a byte-order mismatch. new_dtype = chunk_array.dtype.newbyteorder(self.endian) - chunk_array = chunk_array.astype(new_dtype) + if new_dtype != chunk_array.dtype: + chunk_array = chunk_array.astype(new_dtype) nd_array = chunk_array.as_ndarray_like() # Flatten the nd-array (only copy if needed) and reinterpret as bytes diff --git a/tests/test_codecs/test_bytes.py b/tests/test_codecs/test_bytes.py index 03dd0b40c6..ead778f526 100644 --- a/tests/test_codecs/test_bytes.py +++ b/tests/test_codecs/test_bytes.py @@ -33,29 +33,50 @@ @pytest.mark.parametrize("store", ["local", "memory"], indirect=["store"]) -@pytest.mark.parametrize("input_dtype", [">u2", "u2", + "f4"), ("mask", ">i4")], + [("flux", "u2", " None: """ The `bytes` codec stores multi-byte data in the byte order configured on the codec, regardless of the input array's byte order, and reads it back to the - original values. The input-dtype/store-endian cross-product exercises the - encode-side byteswap (input byte order != store byte order) and the no-op - case alike. Compression is disabled so the stored chunk is the codec's raw - output and its byte layout can be asserted directly. + original values. For structured dtypes this applies to every multi-byte + field, per the `struct` data type spec; the struct cases guard against the + endianness bugs from + https://github.com/zarr-developers/zarr-python/issues/4141, where the + encode path never byte-swapped struct fields (numpy reports byteorder '|' + for void dtypes) and the decode path ignored the codec's endian entirely. + The input-dtype/store-endian cross-product exercises the encode-side + byteswap (input byte order != store byte order) and the no-op case alike. + Compression is disabled so the stored chunk is the codec's raw output and + its byte layout can be asserted directly. """ - data = np.arange(0, 256, dtype=input_dtype).reshape((16, 16)) + dtype = np.dtype(input_dtype) + if dtype.fields is None: + data = np.arange(0, 256, dtype=dtype).reshape((16, 16)) + else: + data = np.zeros((16, 16), dtype=dtype) + data["flux"] = np.arange(0, 256).reshape((16, 16)) + data["mask"] = np.arange(256, 512).reshape((16, 16)) path = "endian" spath = StorePath(store, path) a = await zarr.api.asynchronous.create_array( spath, shape=data.shape, chunks=(16, 16), - dtype="uint16", + dtype=dtype, fill_value=0, compressors=None, serializer=BytesCodec(endian=store_endian), @@ -66,8 +87,7 @@ async def test_endian( # The stored chunk is laid out in the byte order configured on the codec. stored = await store.get(f"{path}/c/0/0", prototype=default_buffer_prototype()) assert stored is not None - expected_dtype = ">u2" if store_endian == "big" else " None: assert isinstance(BytesCodec(), SupportsSyncCodec) -def test_bytes_codec_sync_roundtrip() -> None: - codec = BytesCodec() - arr = np.arange(100, dtype="float64") +@pytest.mark.parametrize("endian", ENDIAN) +@pytest.mark.parametrize( + "native_dtype", + [np.dtype("float64"), np.dtype(">u2"), np.dtype([("a", ">f4"), ("b", " None: + """ + The synchronous encode/decode path round-trips data, and the two byte + orders involved are independent: the codec's `endian` configuration governs + only the stored byte layout (every multi-byte value, including struct + fields, is laid out in the codec's byte order regardless of the input + array's byte order), while the decoded buffer's byte order is governed by + the array's data type regardless of the codec's. The mixed-endian struct + case pins that per-field byte order of the in-memory dtype survives a + roundtrip through a single stored byte order. + """ + if native_dtype.fields is None: + arr = np.arange(100, dtype=native_dtype) + else: + arr = np.array([(1.5, 2), (3.5, 4), (5.5, 6), (7.5, 8)], dtype=native_dtype) zdtype = get_data_type_from_native_dtype(arr.dtype) spec = ArraySpec( shape=arr.shape, @@ -91,11 +129,14 @@ def test_bytes_codec_sync_roundtrip() -> None: ) nd_buf: NDBuffer = default_buffer_prototype().nd_buffer.from_numpy_array(arr) - codec = codec.evolve_from_array_spec(spec) + codec = BytesCodec(endian=endian).evolve_from_array_spec(spec) encoded = codec._encode_sync(nd_buf, spec) assert encoded is not None + assert encoded.to_bytes() == arr.astype(native_dtype.newbyteorder(endian)).tobytes() + decoded = codec._decode_sync(encoded, spec) + assert decoded.dtype == zdtype.to_native_dtype() np.testing.assert_array_equal(arr, decoded.as_numpy_array()) From 845a850a4c71f1365d5c89b22efaca4b112b36b6 Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Thu, 16 Jul 2026 22:48:44 +0200 Subject: [PATCH 2/4] fix(store): copy buffers on write in MemoryStore Encoding an uncompressed chunk hands the store a zero-copy view of the caller's array, and MemoryStore keeps whatever it is given alive in a dict rather than serializing it. Mutating the source array after a write therefore rewrote chunks already committed to the store, silently. Stores that serialize on write (LocalStore, ZipStore, remote stores) are unaffected, so they keep the full benefit of #3885. Only MemoryStore pays the copy, and only where it was aliasing to begin with: an uncompressed 34 MB write goes from ~23 ms to ~39 ms, while compressed writes are unchanged. Copying at the store boundary rather than narrowing the fast path in _merge_chunk_array also fixes the single-chunk case, which aliased in v3.2.1 too. Assisted-by: ClaudeCode:claude-opus-4.8 --- src/zarr/core/buffer/core.py | 2 ++ src/zarr/storage/_memory.py | 24 ++++++++++++++-- tests/test_store/test_memory.py | 51 ++++++++++++++++++++++++++++++++- 3 files changed, 73 insertions(+), 4 deletions(-) diff --git a/src/zarr/core/buffer/core.py b/src/zarr/core/buffer/core.py index 497543f88f..b8f7f11cd4 100644 --- a/src/zarr/core/buffer/core.py +++ b/src/zarr/core/buffer/core.py @@ -45,6 +45,8 @@ def __getitem__(self, key: slice) -> Self: ... def __setitem__(self, key: slice, value: Any) -> None: ... + def copy(self) -> Self: ... + @runtime_checkable class NDArrayLike(Protocol): diff --git a/src/zarr/storage/_memory.py b/src/zarr/storage/_memory.py index 5f5b632d4a..97dd355515 100644 --- a/src/zarr/storage/_memory.py +++ b/src/zarr/storage/_memory.py @@ -26,6 +26,18 @@ logger = getLogger(__name__) +def _copy_buffer(value: Buffer) -> Buffer: + """Copy `value` so the store does not retain the caller's memory. + + Encoding a chunk can hand the store a zero-copy view of the user's array + (an uncompressed write is the common case), and unlike stores that + serialize on write, this one keeps whatever it is given alive in a dict. + Without this copy a later mutation of the user's array would rewrite + chunks already committed to the store. + """ + return type(value).from_array_like(value.as_array_like().copy()) + + class MemoryStore(Store): """ Store for local memory. @@ -42,6 +54,12 @@ class MemoryStore(Store): supports_writes supports_deletes supports_listing + + Notes + ----- + Writes copy the buffer they are given, so the store never aliases the + caller's memory. Buffers passed via `store_dict` are the caller's + responsibility and are stored as-is. """ supports_writes: bool = True @@ -117,7 +135,7 @@ def set_sync(self, key: str, value: Buffer) -> None: raise TypeError( f"MemoryStore.set(): `value` must be a Buffer instance. Got an instance of {type(value)} instead." ) - self._store_dict[key] = value + self._store_dict[key] = _copy_buffer(value) def delete_sync(self, key: str) -> None: self._check_writable() @@ -178,13 +196,13 @@ async def set(self, key: str, value: Buffer, byte_range: tuple[int, int] | None buf[byte_range[0] : byte_range[1]] = value self._store_dict[key] = buf else: - self._store_dict[key] = value + self._store_dict[key] = _copy_buffer(value) async def set_if_not_exists(self, key: str, value: Buffer) -> None: # docstring inherited self._check_writable() await self._ensure_open() - self._store_dict.setdefault(key, value) + self._store_dict.setdefault(key, _copy_buffer(value)) async def delete(self, key: str) -> None: # docstring inherited diff --git a/tests/test_store/test_memory.py b/tests/test_store/test_memory.py index 35504718a7..36265423e6 100644 --- a/tests/test_store/test_memory.py +++ b/tests/test_store/test_memory.py @@ -8,7 +8,7 @@ import pytest import zarr -from zarr.core.buffer import Buffer, cpu, gpu +from zarr.core.buffer import Buffer, cpu, default_buffer_prototype, gpu from zarr.errors import ZarrUserWarning from zarr.storage import GpuMemoryStore, ManagedMemoryStore, MemoryStore from zarr.testing.store import StoreTests @@ -76,6 +76,55 @@ async def test_deterministic_size( np.testing.assert_array_equal(a[:3], 1) np.testing.assert_array_equal(a[3:], 0) + @pytest.mark.parametrize("method", ["set", "set_sync", "set_if_not_exists"]) + async def test_set_does_not_retain_caller_buffer(self, store: MemoryStore, method: str) -> None: + """Writing a buffer must not alias the caller's memory. + + MemoryStore keeps whatever it is handed alive in a dict, so retaining + the caller's buffer lets a later mutation of that buffer rewrite data + already committed to the store. + """ + source = np.frombuffer(bytearray(b"\x01\x02\x03\x04"), dtype="B") + value = cpu.Buffer.from_array_like(source) + + if method == "set_sync": + store.set_sync("k", value) + else: + await getattr(store, method)("k", value) + + source[:] = 0xF # mutate the caller's memory after the write + stored = await store.get("k", prototype=default_buffer_prototype()) + assert stored is not None + assert stored.to_bytes() == b"\x01\x02\x03\x04" + + @pytest.mark.parametrize( + "pipeline", + [ + "zarr.core.codec_pipeline.BatchedCodecPipeline", + "zarr.core.codec_pipeline.FusedCodecPipeline", + ], + ) + @pytest.mark.parametrize(("shape", "chunks"), [((30,), (10,)), ((8,), (4,)), ((4,), (4,))]) + def test_write_does_not_alias_source_array( + self, pipeline: str, shape: tuple[int], chunks: tuple[int] + ) -> None: + """Mutating the source array after a write must not corrupt stored chunks. + + Without compression the encoded buffer is a zero-copy view of the + caller's array all the way down to the store, so this covers both the + single-chunk and multi-chunk write paths. + """ + with zarr.config.set({"codec_pipeline.path": pipeline}): + array = zarr.create_array( + store=MemoryStore(), shape=shape, chunks=chunks, dtype="i4", compressors=None + ) + source = np.arange(shape[0], dtype="i4") + expected = source.copy() + array[:] = source + source[:] = -1 + + np.testing.assert_array_equal(array[:], expected) + # --- byte-range-write tests: disabled --- # Byte-range-write support (set_range / set_range_sync / SupportsSetRange) # was removed from this PR pending a decision on the store interface. These From af10bfe051cf73d5e15bce82118f9952c77cfbb9 Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Thu, 16 Jul 2026 22:49:19 +0200 Subject: [PATCH 3/4] docs: add changelog entry for MemoryStore buffer copy Assisted-by: ClaudeCode:claude-opus-4.8 --- changes/224.bugfix.md | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100644 changes/224.bugfix.md diff --git a/changes/224.bugfix.md b/changes/224.bugfix.md new file mode 100644 index 0000000000..6b0d0fcc67 --- /dev/null +++ b/changes/224.bugfix.md @@ -0,0 +1,11 @@ +`MemoryStore` now copies buffers as they are written, so it never retains the +caller's memory. Previously an uncompressed write handed the store a zero-copy +view of the user's array, and mutating that array afterwards would silently +rewrite chunks already committed to the store. + +Only `MemoryStore` is affected: stores that serialize on write, such as +`LocalStore` and `ZipStore`, never aliased the caller's memory. Uncompressed +writes to a `MemoryStore` are correspondingly slower, since the copy that makes +the stored data independent is now actually performed; compressed writes are +unchanged. Buffers supplied through the `store_dict` argument remain the +caller's responsibility and are stored as-is. From cd68f10c832a678848d4939e77d79bc87f4215a2 Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Fri, 17 Jul 2026 13:45:34 +0200 Subject: [PATCH 4/4] docs: correct changelog --- changes/{224.bugfix.md => 4157.bugfix.md} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename changes/{224.bugfix.md => 4157.bugfix.md} (100%) diff --git a/changes/224.bugfix.md b/changes/4157.bugfix.md similarity index 100% rename from changes/224.bugfix.md rename to changes/4157.bugfix.md