diff --git a/src/zarr/abc/store.py b/src/zarr/abc/store.py index 304d0cddb5..5f976944f4 100644 --- a/src/zarr/abc/store.py +++ b/src/zarr/abc/store.py @@ -185,6 +185,20 @@ def _check_writable(self) -> None: if self.read_only: raise ValueError("store was opened in read-only mode and does not support writing") + def _check_value(self, value: object) -> None: + """Raise a TypeError if ``value`` is not a Buffer instance. + + The error message is prefixed with the concrete store class name, e.g. + ``MemoryStore.set(): ...``, so that callers do not have to repeat it. + """ + from zarr.core.buffer import Buffer + + if not isinstance(value, Buffer): + raise TypeError( + f"{type(self).__name__}.set(): `value` must be a Buffer instance. " + f"Got an instance of {type(value)} instead." + ) + @abstractmethod def __eq__(self, value: object) -> bool: """Equality comparison.""" diff --git a/src/zarr/core/dtype/npy/int.py b/src/zarr/core/dtype/npy/int.py index c18fd01dd8..e0c993b801 100644 --- a/src/zarr/core/dtype/npy/int.py +++ b/src/zarr/core/dtype/npy/int.py @@ -61,10 +61,65 @@ class BaseInt[ This class provides methods for serialization and deserialization of integer types in both Zarr v2 and v3 formats, as well as methods for checking and casting scalars. + + Subclasses provide the concrete ``dtype_cls``, ``_zarr_v3_name``, ``_zarr_v2_names``, + and ``item_size`` attributes. Multi-byte integer types additionally inherit from + [`HasEndianness`][zarr.core.dtype.common.HasEndianness]; the byte-order logic in this + base class is gated on that so that the single-byte ``Int8``/``UInt8`` types (which have + no meaningful byte order) reuse the same implementation. """ _zarr_v2_names: ClassVar[tuple[str, ...]] + # Single-byte int types (Int8/UInt8) have no meaningful byte order, so they do not mix in + # HasEndianness. Multi-byte types do; this flag (set True by HasEndianness subclasses) lets the + # shared native-dtype conversion below branch without tripping mypy's issubclass narrowing. + _has_endianness: ClassVar[bool] = False + + @classmethod + def from_native_dtype(cls, dtype: TBaseDType) -> Self: + """ + Create an instance of this data type from a native NumPy dtype. + + Parameters + ---------- + dtype : TBaseDType + The native NumPy dtype. + + Returns + ------- + Self + An instance of this data type. + + Raises + ------ + DataTypeValidationError + If the input dtype is not a valid representation of this data type. + """ + if cls._check_native_dtype(dtype): + kwargs: dict[str, object] = {} + if cls._has_endianness: + kwargs["endianness"] = get_endianness_from_numpy_dtype(dtype) + return cls(**kwargs) + raise DataTypeValidationError( + f"Invalid data type: {dtype}. Expected an instance of {cls.dtype_cls}" + ) + + def to_native_dtype(self) -> DType: + """ + Convert this data type to a native NumPy dtype. + + Returns + ------- + DType + The native NumPy dtype. + """ + if isinstance(self, HasEndianness): + byte_order = endianness_to_numpy_str(self.endianness) + # numpy 2.x stub: newbyteorder widens to base dtype, runtime preserves the subclass + return self.dtype_cls().newbyteorder(byte_order) # type: ignore[no-any-return,call-overload] + return self.dtype_cls() # type: ignore[no-any-return,call-overload] + @classmethod def _check_json_v2(cls, data: object) -> TypeGuard[DTypeConfig_V2[str, None]]: """ @@ -110,6 +165,96 @@ def _check_json_v3(cls, data: object) -> TypeGuard[str]: """ return data == cls._zarr_v3_name + @classmethod + def _from_json_v2(cls, data: DTypeJSON) -> Self: + """ + Create an instance of this data type from Zarr V2-flavored JSON. + + Parameters + ---------- + data : DTypeJSON + The JSON data. + + Returns + ------- + Self + An instance of this data type. + + Raises + ------ + DataTypeValidationError + If the input JSON is not a valid representation of this class. + """ + if cls._check_json_v2(data): + # Going via NumPy ensures that we get the endianness correct without + # annoying string parsing. + name = data["name"] + return cls.from_native_dtype(np.dtype(name)) + msg = ( + f"Invalid JSON representation of {cls.__name__}. Got {data!r}, " + f"expected one of the strings {cls._zarr_v2_names!r}." + ) + raise DataTypeValidationError(msg) + + @classmethod + def _from_json_v3(cls, data: DTypeJSON) -> Self: + """ + Create an instance of this data type from Zarr V3-flavored JSON. + + Parameters + ---------- + data : DTypeJSON + The JSON data. + + Returns + ------- + Self + An instance of this data type. + + Raises + ------ + DataTypeValidationError + If the input JSON is not a valid representation of this class. + """ + if cls._check_json_v3(data): + return cls() + msg = ( + f"Invalid JSON representation of {cls.__name__}. Got {data!r}, " + f"expected the string {cls._zarr_v3_name!r}" + ) + raise DataTypeValidationError(msg) + + @overload + def to_json(self, zarr_format: Literal[2]) -> DTypeConfig_V2[str, None]: ... + + @overload + def to_json(self, zarr_format: Literal[3]) -> str: ... + + def to_json(self, zarr_format: ZarrFormat) -> DTypeConfig_V2[str, None] | str: + """ + Convert the data type to a JSON-serializable form. + + Parameters + ---------- + zarr_format : ZarrFormat + The Zarr format version. + + Returns + ------- + DTypeConfig_V2[str, None] or str + The JSON-serializable representation of the data type. + + Raises + ------ + ValueError + If the zarr_format is not 2 or 3. + """ + if zarr_format == 2: + return {"name": self.to_native_dtype().str, "object_codec_id": None} + elif zarr_format == 3: + return self._zarr_v3_name + raise ValueError(f"zarr_format must be 2 or 3, got {zarr_format}") # pragma: no cover + def _check_scalar(self, data: object) -> TypeGuard[IntLike]: """ Check if the input object is of an IntLike type. @@ -261,125 +406,79 @@ class Int8(BaseInt[np.dtypes.Int8DType, np.int8]): _zarr_v3_name: ClassVar[Literal["int8"]] = "int8" _zarr_v2_names: ClassVar[tuple[Literal["|i1"]]] = ("|i1",) - @classmethod - def from_native_dtype(cls, dtype: TBaseDType) -> Self: + @property + def item_size(self) -> int: """ - Create an Int8 from an np.dtype('int8') instance. - - Parameters - ---------- - dtype : TBaseDType - The np.dtype('int8') instance. + The size of a single scalar in bytes. Returns ------- - Self - An instance of this data type. - - Raises - ------ - DataTypeValidationError - If the input data type is not a valid representation of this class Int8. + int + The size of a single scalar in bytes. """ - if cls._check_native_dtype(dtype): - return cls() - raise DataTypeValidationError( - f"Invalid data type: {dtype}. Expected an instance of {cls.dtype_cls}" - ) + return 1 - def to_native_dtype(self: Self) -> np.dtypes.Int8DType: - """ - Convert the Int8 instance to an np.dtype('int8') instance. - Returns - ------- - np.dtypes.Int8DType - The np.dtype('int8') instance. - """ - return self.dtype_cls() +@dataclass(frozen=True, kw_only=True) +class UInt8(BaseInt[np.dtypes.UInt8DType, np.uint8]): + """ + A Zarr data type for arrays containing 8-bit unsigned integers. - @classmethod - def _from_json_v2(cls, data: DTypeJSON) -> Self: - """ - Create an Int8 from Zarr V2-flavored JSON. + Wraps the [`np.dtypes.UInt8DType`][numpy.dtypes.UInt8DType] data type. Scalars for this data type are instances of [`np.uint8`][numpy.uint8]. - Parameters - ---------- - data : DTypeJSON - The JSON data. + Attributes + ---------- + dtype_cls : np.dtypes.UInt8DType + The class of the underlying NumPy dtype. - Returns - ------- - Self - An instance of this data type. + References + ---------- + This class implements the 8-bit unsigned integer data type defined in Zarr V2 and V3. - Raises - ------ - DataTypeValidationError - If the input JSON is not a valid representation of this class Int8. - """ - if cls._check_json_v2(data): - return cls() - msg = f"Invalid JSON representation of {cls.__name__}. Got {data!r}, expected the string {cls._zarr_v2_names[0]!r}" - raise DataTypeValidationError(msg) + See the [Zarr V2](https://github.com/zarr-developers/zarr-specs/blob/main/docs/v2/v2.0.rst#data-type-encoding) and [Zarr V3](https://github.com/zarr-developers/zarr-specs/blob/main/docs/v3/data-types/index.rst) specification documents for details. + """ - @classmethod - def _from_json_v3(cls, data: DTypeJSON) -> Self: - """ - Create an Int8 from Zarr V3-flavored JSON. + dtype_cls = np.dtypes.UInt8DType + _zarr_v3_name: ClassVar[Literal["uint8"]] = "uint8" + _zarr_v2_names: ClassVar[tuple[Literal["|u1"]]] = ("|u1",) - Parameters - ---------- - data : DTypeJSON - The JSON data. + @property + def item_size(self) -> int: + """ + The size of a single scalar in bytes. Returns ------- - Self - An instance of this data type. - - Raises - ------ - DataTypeValidationError - If the input JSON is not a valid representation of this class Int8. + int + The size of a single scalar in bytes. """ - if cls._check_json_v3(data): - return cls() - msg = f"Invalid JSON representation of {cls.__name__}. Got {data!r}, expected the string {cls._zarr_v3_name!r}" - raise DataTypeValidationError(msg) + return 1 - @overload - def to_json(self, zarr_format: Literal[2]) -> DTypeConfig_V2[Literal["|i1"], None]: ... - @overload - def to_json(self, zarr_format: Literal[3]) -> Literal["int8"]: ... +@dataclass(frozen=True, kw_only=True) +class Int16(BaseInt[np.dtypes.Int16DType, np.int16], HasEndianness): + """ + A Zarr data type for arrays containing 16-bit signed integers. - def to_json( - self, zarr_format: ZarrFormat - ) -> DTypeConfig_V2[Literal["|i1"], None] | Literal["int8"]: - """ - Convert the data type to a JSON-serializable form. + Wraps the [`np.dtypes.Int16DType`][numpy.dtypes.Int16DType] data type. Scalars for this data type are instances of + [`np.int16`][numpy.int16]. - Parameters - ---------- - zarr_format : ZarrFormat - The Zarr format version. + Attributes + ---------- + dtype_cls : np.dtypes.Int16DType + The class of the underlying NumPy dtype. - Returns - ------- - ``DTypeConfig_V2[Literal["|i1"], None] | Literal["int8"]`` - The JSON-serializable representation of the data type. + References + ---------- + This class implements the 16-bit signed integer data type defined in Zarr V2 and V3. - Raises - ------ - ValueError - If the zarr_format is not 2 or 3. - """ - if zarr_format == 2: - return {"name": self._zarr_v2_names[0], "object_codec_id": None} - elif zarr_format == 3: - return self._zarr_v3_name - raise ValueError(f"zarr_format must be 2 or 3, got {zarr_format}") # pragma: no cover + See the [Zarr V2](https://github.com/zarr-developers/zarr-specs/blob/main/docs/v2/v2.0.rst#data-type-encoding) and [Zarr V3](https://github.com/zarr-developers/zarr-specs/blob/main/docs/v3/data-types/index.rst) specification documents for details. + """ + + dtype_cls = np.dtypes.Int16DType + _zarr_v3_name: ClassVar[Literal["int16"]] = "int16" + _zarr_v2_names: ClassVar[tuple[Literal[">i2"], Literal["i2", " int: @@ -391,141 +490,92 @@ def item_size(self) -> int: int The size of a single scalar in bytes. """ - return 1 + return 2 @dataclass(frozen=True, kw_only=True) -class UInt8(BaseInt[np.dtypes.UInt8DType, np.uint8]): +class UInt16(BaseInt[np.dtypes.UInt16DType, np.uint16], HasEndianness): """ - A Zarr data type for arrays containing 8-bit unsigned integers. + A Zarr data type for arrays containing 16-bit unsigned integers. - Wraps the [`np.dtypes.UInt8DType`][numpy.dtypes.UInt8DType] data type. Scalars for this data type are instances of [`np.uint8`][numpy.uint8]. + Wraps the [`np.dtypes.UInt16DType`][numpy.dtypes.UInt16DType] data type. Scalars for this data type are instances of + [`np.uint16`][numpy.uint16]. Attributes ---------- - dtype_cls : np.dtypes.UInt8DType + dtype_cls : np.dtypes.UInt16DType The class of the underlying NumPy dtype. References ---------- - This class implements the 8-bit unsigned integer data type defined in Zarr V2 and V3. + This class implements the unsigned 16-bit unsigned integer data type defined in Zarr V2 and V3. See the [Zarr V2](https://github.com/zarr-developers/zarr-specs/blob/main/docs/v2/v2.0.rst#data-type-encoding) and [Zarr V3](https://github.com/zarr-developers/zarr-specs/blob/main/docs/v3/data-types/index.rst) specification documents for details. """ - dtype_cls = np.dtypes.UInt8DType - _zarr_v3_name: ClassVar[Literal["uint8"]] = "uint8" - _zarr_v2_names: ClassVar[tuple[Literal["|u1"]]] = ("|u1",) - - @classmethod - def from_native_dtype(cls, dtype: TBaseDType) -> Self: - """ - Create a UInt8 from an np.dtype('uint8') instance. - """ - if cls._check_native_dtype(dtype): - return cls() - raise DataTypeValidationError( - f"Invalid data type: {dtype}. Expected an instance of {cls.dtype_cls}" - ) + dtype_cls = np.dtypes.UInt16DType + _zarr_v3_name: ClassVar[Literal["uint16"]] = "uint16" + _zarr_v2_names: ClassVar[tuple[Literal[">u2"], Literal["u2", " np.dtypes.UInt8DType: + @property + def item_size(self) -> int: """ - Create a NumPy unsigned 8-bit integer dtype instance from this UInt8 ZDType. + The size of a single scalar in bytes. Returns ------- - np.dtypes.UInt8DType - The NumPy unsigned 8-bit integer dtype. - """ - - return self.dtype_cls() - - @classmethod - def _from_json_v2(cls, data: DTypeJSON) -> Self: + int + The size of a single scalar in bytes. """ - Create an instance of this data type from Zarr V2-flavored JSON. + return 2 - Parameters - ---------- - data : DTypeJSON - The JSON data. - Returns - ------- - Self - An instance of this data type. +@dataclass(frozen=True, kw_only=True) +class Int32(BaseInt[np.dtypes.Int32DType, np.int32], HasEndianness): + """ + A Zarr data type for arrays containing 32-bit signed integers. - Raises - ------ - DataTypeValidationError - If the input JSON is not a valid representation of this class. - """ + Wraps the [`np.dtypes.Int32DType`][numpy.dtypes.Int32DType] data type. Scalars for this data type are instances of + [`np.int32`][numpy.int32]. - if cls._check_json_v2(data): - return cls() - msg = f"Invalid JSON representation of {cls.__name__}. Got {data!r}, expected the string {cls._zarr_v2_names[0]!r}" - raise DataTypeValidationError(msg) + Attributes + ---------- + dtype_cls : np.dtypes.Int32DType + The class of the underlying NumPy dtype. - @classmethod - def _from_json_v3(cls, data: DTypeJSON) -> Self: - """ - Create an instance of this data type from Zarr V3-flavored JSON. + References + ---------- + This class implements the 32-bit signed integer data type defined in Zarr V2 and V3. - Parameters - ---------- - data : DTypeJSON - The JSON data. + See the [Zarr V2](https://github.com/zarr-developers/zarr-specs/blob/main/docs/v2/v2.0.rst#data-type-encoding) and [Zarr V3](https://github.com/zarr-developers/zarr-specs/blob/main/docs/v3/data-types/index.rst) specification documents for details. + """ - Returns - ------- - Self - An instance of this data type. + dtype_cls = np.dtypes.Int32DType + _zarr_v3_name: ClassVar[Literal["int32"]] = "int32" + _zarr_v2_names: ClassVar[tuple[Literal[">i4"], Literal["i4", " TypeGuard[np.dtypes.Int32DType]: """ - if cls._check_json_v3(data): - return cls() - msg = f"Invalid JSON representation of {cls.__name__}. Got {data!r}, expected the string {cls._zarr_v3_name!r}" - raise DataTypeValidationError(msg) - - @overload - def to_json(self, zarr_format: Literal[2]) -> DTypeConfig_V2[Literal["|u1"], None]: ... - - @overload - def to_json(self, zarr_format: Literal[3]) -> Literal["uint8"]: ... + A type guard that checks if the input is assignable to the type of ``cls.dtype_class`` - def to_json( - self, zarr_format: ZarrFormat - ) -> DTypeConfig_V2[Literal["|u1"], None] | Literal["uint8"]: - """ - Convert the data type to a JSON-serializable form. + This method is overridden for this particular data type because of a Windows-specific issue + where np.dtype('i') creates an instance of ``np.dtypes.IntDType``, rather than an + instance of ``np.dtypes.Int32DType``, even though both represent 32-bit signed integers. Parameters ---------- - zarr_format : ZarrFormat - The Zarr format version. Supported values are 2 and 3. + dtype : TDType + The dtype to check. Returns ------- - ``DTypeConfig_V2[Literal["|u1"], None] | Literal["uint8"]`` - The JSON-serializable representation of the data type. - - Raises - ------ - ValueError - If `zarr_format` is not 2 or 3. + Bool + True if the dtype matches, False otherwise. """ - if zarr_format == 2: - # For Zarr format version 2, return a dictionary with the name and object codec ID. - return {"name": self._zarr_v2_names[0], "object_codec_id": None} - elif zarr_format == 3: - # For Zarr format version 3, return the v3 name as a string. - return self._zarr_v3_name - # Raise an error if the zarr_format is neither 2 nor 3. - raise ValueError(f"zarr_format must be 2 or 3, got {zarr_format}") # pragma: no cover + return super()._check_native_dtype(dtype) or dtype == np.dtypes.Int32DType() @property def item_size(self) -> int: @@ -537,158 +587,93 @@ def item_size(self) -> int: int The size of a single scalar in bytes. """ - return 1 + return 4 @dataclass(frozen=True, kw_only=True) -class Int16(BaseInt[np.dtypes.Int16DType, np.int16], HasEndianness): +class UInt32(BaseInt[np.dtypes.UInt32DType, np.uint32], HasEndianness): """ - A Zarr data type for arrays containing 16-bit signed integers. + A Zarr data type for arrays containing 32-bit unsigned integers. - Wraps the [`np.dtypes.Int16DType`][numpy.dtypes.Int16DType] data type. Scalars for this data type are instances of - [`np.int16`][numpy.int16]. + Wraps the [`np.dtypes.UInt32DType`][numpy.dtypes.UInt32DType] data type. Scalars for this data type are instances of + [`np.uint32`][numpy.uint32]. Attributes ---------- - dtype_cls : np.dtypes.Int16DType + dtype_cls : np.dtypes.UInt32DType The class of the underlying NumPy dtype. References ---------- - This class implements the 16-bit signed integer data type defined in Zarr V2 and V3. + This class implements the 32-bit unsigned integer data type defined in Zarr V2 and V3. See the [Zarr V2](https://github.com/zarr-developers/zarr-specs/blob/main/docs/v2/v2.0.rst#data-type-encoding) and [Zarr V3](https://github.com/zarr-developers/zarr-specs/blob/main/docs/v3/data-types/index.rst) specification documents for details. """ - dtype_cls = np.dtypes.Int16DType - _zarr_v3_name: ClassVar[Literal["int16"]] = "int16" - _zarr_v2_names: ClassVar[tuple[Literal[">i2"], Literal["i2", "u4"], Literal["u4", " Self: - """ - Create an instance of this data type from an np.dtype('int16') instance. - - Parameters - ---------- - dtype : np.dtype - The instance of np.dtype('int16') to create from. - - Returns - ------- - Self - An instance of this data type. - - Raises - ------ - DataTypeValidationError - If the input data type is not an instance of np.dtype('int16'). - """ - if cls._check_native_dtype(dtype): - return cls(endianness=get_endianness_from_numpy_dtype(dtype)) - raise DataTypeValidationError( - f"Invalid data type: {dtype}. Expected an instance of {cls.dtype_cls}" - ) - - def to_native_dtype(self) -> np.dtypes.Int16DType: - """ - Convert the data type to an np.dtype('int16') instance. - - Returns - ------- - np.dtype - The np.dtype('int16') instance. + def _check_native_dtype(cls: type[Self], dtype: TBaseDType) -> TypeGuard[np.dtypes.UInt32DType]: """ - byte_order = endianness_to_numpy_str(self.endianness) - # numpy 2.x stub: newbyteorder widens to base dtype, runtime preserves the concrete subclass - return self.dtype_cls().newbyteorder(byte_order) # type: ignore[return-value] + A type guard that checks if the input is assignable to the type of ``cls.dtype_class`` - @classmethod - def _from_json_v2(cls, data: DTypeJSON) -> Self: - """ - Create an instance of this data type from Zarr V2-flavored JSON. + This method is overridden for this particular data type because of a Windows-specific issue + where ``np.array([1], dtype=np.uint32) & 1`` creates an instance of ``np.dtypes.UIntDType``, + rather than an instance of ``np.dtypes.UInt32DType``, even though both represent 32-bit + unsigned integers. (In contrast to ``np.dtype('i')``, ``np.dtype('u')`` raises an error.) Parameters ---------- - data : DTypeJSON - The JSON data. + dtype : TDType + The dtype to check. Returns ------- - Self - An instance of this data type. - - Raises - ------ - DataTypeValidationError - If the input JSON is not a valid representation of this class. + Bool + True if the dtype matches, False otherwise. """ - if cls._check_json_v2(data): - # Going via NumPy ensures that we get the endianness correct without - # annoying string parsing. - name = data["name"] - return cls.from_native_dtype(np.dtype(name)) - msg = f"Invalid JSON representation of {cls.__name__}. Got {data!r}, expected one of the strings {cls._zarr_v2_names!r}." - raise DataTypeValidationError(msg) + return super()._check_native_dtype(dtype) or dtype == np.dtypes.UInt32DType() - @classmethod - def _from_json_v3(cls, data: DTypeJSON) -> Self: + @property + def item_size(self) -> int: """ - Create an instance of this data type from Zarr V3-flavored JSON. - - Parameters - ---------- - data : DTypeJSON - The JSON data. + The size of a single scalar in bytes. Returns ------- - Self - An instance of this data type. - - Raises - ------ - DataTypeValidationError - If the input JSON is not a valid representation of this class. + int + The size of a single scalar in bytes. """ - if cls._check_json_v3(data): - return cls() - msg = f"Invalid JSON representation of {cls.__name__}. Got {data!r}, expected the string {cls._zarr_v3_name!r}" - raise DataTypeValidationError(msg) + return 4 - @overload - def to_json(self, zarr_format: Literal[2]) -> DTypeConfig_V2[Literal[">i2", " Literal["int16"]: ... +@dataclass(frozen=True, kw_only=True) +class Int64(BaseInt[np.dtypes.Int64DType, np.int64], HasEndianness): + """ + A Zarr data type for arrays containing 64-bit signed integers. - def to_json( - self, zarr_format: ZarrFormat - ) -> DTypeConfig_V2[Literal[">i2", "i2", "i8"], Literal["i8", " int: @@ -700,712 +685,25 @@ def item_size(self) -> int: int The size of a single scalar in bytes. """ - return 2 + return 8 @dataclass(frozen=True, kw_only=True) -class UInt16(BaseInt[np.dtypes.UInt16DType, np.uint16], HasEndianness): +class UInt64(BaseInt[np.dtypes.UInt64DType, np.uint64], HasEndianness): """ - A Zarr data type for arrays containing 16-bit unsigned integers. + A Zarr data type for arrays containing 64-bit unsigned integers. - Wraps the [`np.dtypes.UInt16DType`][numpy.dtypes.UInt16DType] data type. Scalars for this data type are instances of - [`np.uint16`][numpy.uint16]. + Wraps the [`np.dtypes.UInt64DType`][numpy.dtypes.UInt64DType] data type. Scalars for this data type + are instances of [`np.uint64`][numpy.uint64]. Attributes ---------- - dtype_cls : np.dtypes.UInt16DType + dtype_cls: np.dtypes.UInt64DType The class of the underlying NumPy dtype. References ---------- - This class implements the unsigned 16-bit unsigned integer data type defined in Zarr V2 and V3. - - See the [Zarr V2](https://github.com/zarr-developers/zarr-specs/blob/main/docs/v2/v2.0.rst#data-type-encoding) and [Zarr V3](https://github.com/zarr-developers/zarr-specs/blob/main/docs/v3/data-types/index.rst) specification documents for details. - """ - - dtype_cls = np.dtypes.UInt16DType - _zarr_v3_name: ClassVar[Literal["uint16"]] = "uint16" - _zarr_v2_names: ClassVar[tuple[Literal[">u2"], Literal["u2", " Self: - """ - Create an instance of this data type from an np.dtype('uint16') instance. - - Parameters - ---------- - dtype : np.dtype - The NumPy data type. - - Returns - ------- - Self - An instance of this data type. - - Raises - ------ - DataTypeValidationError - If the input data type is not an instance of np.dtype('uint16'). - """ - if cls._check_native_dtype(dtype): - return cls(endianness=get_endianness_from_numpy_dtype(dtype)) - raise DataTypeValidationError( - f"Invalid data type: {dtype}. Expected an instance of {cls.dtype_cls}" - ) - - def to_native_dtype(self) -> np.dtypes.UInt16DType: - """ - Convert the data type to an np.dtype('uint16') instance. - - Returns - ------- - np.dtype - The np.dtype('uint16') instance. - """ - byte_order = endianness_to_numpy_str(self.endianness) - # numpy 2.x stub: newbyteorder widens to base dtype, runtime preserves the concrete subclass - return self.dtype_cls().newbyteorder(byte_order) # type: ignore[return-value] - - @classmethod - def _from_json_v2(cls, data: DTypeJSON) -> Self: - """ - Create an instance of this data type from Zarr V2-flavored JSON. - - Parameters - ---------- - data : DTypeJSON - The JSON data. - - Returns - ------- - Self - An instance of this data type. - - Raises - ------ - DataTypeValidationError - If the input JSON is not a valid representation of this class. - """ - if cls._check_json_v2(data): - # Going via NumPy ensures that we get the endianness correct without - # annoying string parsing. - name = data["name"] - return cls.from_native_dtype(np.dtype(name)) - msg = f"Invalid JSON representation of UInt16. Got {data!r}, expected one of the strings {cls._zarr_v2_names}." - raise DataTypeValidationError(msg) - - @classmethod - def _from_json_v3(cls, data: DTypeJSON) -> Self: - """ - Create an instance of this data type from Zarr V3-flavored JSON. - - Parameters - ---------- - data : DTypeJSON - The JSON data. - - Returns - ------- - Self - An instance of this data type. - - Raises - ------ - DataTypeValidationError - If the input JSON is not a valid representation of this class. - """ - if cls._check_json_v3(data): - return cls() - msg = f"Invalid JSON representation of UInt16. Got {data!r}, expected the string {cls._zarr_v3_name!r}" - raise DataTypeValidationError(msg) - - @overload - def to_json(self, zarr_format: Literal[2]) -> DTypeConfig_V2[Literal[">u2", " Literal["uint16"]: ... - - def to_json( - self, zarr_format: ZarrFormat - ) -> DTypeConfig_V2[Literal[">u2", "u2", " int: - """ - The size of a single scalar in bytes. - - Returns - ------- - int - The size of a single scalar in bytes. - """ - return 2 - - -@dataclass(frozen=True, kw_only=True) -class Int32(BaseInt[np.dtypes.Int32DType, np.int32], HasEndianness): - """ - A Zarr data type for arrays containing 32-bit signed integers. - - Wraps the [`np.dtypes.Int32DType`][numpy.dtypes.Int32DType] data type. Scalars for this data type are instances of - [`np.int32`][numpy.int32]. - - Attributes - ---------- - dtype_cls : np.dtypes.Int32DType - The class of the underlying NumPy dtype. - - References - ---------- - This class implements the 32-bit signed integer data type defined in Zarr V2 and V3. - - See the [Zarr V2](https://github.com/zarr-developers/zarr-specs/blob/main/docs/v2/v2.0.rst#data-type-encoding) and [Zarr V3](https://github.com/zarr-developers/zarr-specs/blob/main/docs/v3/data-types/index.rst) specification documents for details. - """ - - dtype_cls = np.dtypes.Int32DType - _zarr_v3_name: ClassVar[Literal["int32"]] = "int32" - _zarr_v2_names: ClassVar[tuple[Literal[">i4"], Literal["i4", " TypeGuard[np.dtypes.Int32DType]: - """ - A type guard that checks if the input is assignable to the type of ``cls.dtype_class`` - - This method is overridden for this particular data type because of a Windows-specific issue - where np.dtype('i') creates an instance of ``np.dtypes.IntDType``, rather than an - instance of ``np.dtypes.Int32DType``, even though both represent 32-bit signed integers. - - Parameters - ---------- - dtype : TDType - The dtype to check. - - Returns - ------- - Bool - True if the dtype matches, False otherwise. - """ - return super()._check_native_dtype(dtype) or dtype == np.dtypes.Int32DType() - - @classmethod - def from_native_dtype(cls: type[Self], dtype: TBaseDType) -> Self: - """ - Create an Int32 from an np.dtype('int32') instance. - - Parameters - ---------- - dtype : TBaseDType - The np.dtype('int32') instance. - - Returns - ------- - Self - An instance of this data type. - - Raises - ------ - DataTypeValidationError - If the input JSON is not a valid representation of this class Int32. - """ - if cls._check_native_dtype(dtype): - return cls(endianness=get_endianness_from_numpy_dtype(dtype)) - raise DataTypeValidationError( - f"Invalid data type: {dtype}. Expected an instance of {cls.dtype_cls}" - ) - - def to_native_dtype(self: Self) -> np.dtypes.Int32DType: - """ - Convert the Int32 instance to an np.dtype('int32') instance. - - Returns - ------- - np.dtypes.Int32DType - The np.dtype('int32') instance. - """ - byte_order = endianness_to_numpy_str(self.endianness) - # numpy 2.x stub: newbyteorder widens to base dtype, runtime preserves the concrete subclass - return self.dtype_cls().newbyteorder(byte_order) # type: ignore[return-value] - - @classmethod - def _from_json_v2(cls, data: DTypeJSON) -> Self: - """ - Create an Int32 from Zarr V2-flavored JSON. - - Parameters - ---------- - data : DTypeJSON - The JSON data. - - Returns - ------- - Self - An instance of this data type. - - Raises - ------ - DataTypeValidationError - If the input JSON is not a valid representation of this class Int32. - """ - if cls._check_json_v2(data): - # Going via NumPy ensures that we get the endianness correct without - # annoying string parsing. - name = data["name"] - return cls.from_native_dtype(np.dtype(name)) - msg = f"Invalid JSON representation of {cls.__name__}. Got {data!r}, expected one of the strings {cls._zarr_v2_names!r}." - raise DataTypeValidationError(msg) - - @classmethod - def _from_json_v3(cls, data: DTypeJSON) -> Self: - """ - Create an Int32 from Zarr V3-flavored JSON. - - Parameters - ---------- - data : DTypeJSON - The JSON data. - - Returns - ------- - Self - An instance of this data type. - - Raises - ------ - DataTypeValidationError - If the input JSON is not a valid representation of this class Int32. - """ - if cls._check_json_v3(data): - return cls() - msg = f"Invalid JSON representation of {cls.__name__}. Got {data!r}, expected the string {cls._zarr_v3_name!r}" - raise DataTypeValidationError(msg) - - @overload - def to_json(self, zarr_format: Literal[2]) -> DTypeConfig_V2[Literal[">i4", " Literal["int32"]: ... - - def to_json( - self, zarr_format: ZarrFormat - ) -> DTypeConfig_V2[Literal[">i4", "i4", " int: - """ - The size of a single scalar in bytes. - - Returns - ------- - int - The size of a single scalar in bytes. - """ - return 4 - - -@dataclass(frozen=True, kw_only=True) -class UInt32(BaseInt[np.dtypes.UInt32DType, np.uint32], HasEndianness): - """ - A Zarr data type for arrays containing 32-bit unsigned integers. - - Wraps the [`np.dtypes.UInt32DType`][numpy.dtypes.UInt32DType] data type. Scalars for this data type are instances of - [`np.uint32`][numpy.uint32]. - - Attributes - ---------- - dtype_cls : np.dtypes.UInt32DType - The class of the underlying NumPy dtype. - - References - ---------- - This class implements the 32-bit unsigned integer data type defined in Zarr V2 and V3. - - See the [Zarr V2](https://github.com/zarr-developers/zarr-specs/blob/main/docs/v2/v2.0.rst#data-type-encoding) and [Zarr V3](https://github.com/zarr-developers/zarr-specs/blob/main/docs/v3/data-types/index.rst) specification documents for details. - """ - - dtype_cls = np.dtypes.UInt32DType - _zarr_v3_name: ClassVar[Literal["uint32"]] = "uint32" - _zarr_v2_names: ClassVar[tuple[Literal[">u4"], Literal["u4", " TypeGuard[np.dtypes.UInt32DType]: - """ - A type guard that checks if the input is assignable to the type of ``cls.dtype_class`` - - This method is overridden for this particular data type because of a Windows-specific issue - where ``np.array([1], dtype=np.uint32) & 1`` creates an instance of ``np.dtypes.UIntDType``, - rather than an instance of ``np.dtypes.UInt32DType``, even though both represent 32-bit - unsigned integers. (In contrast to ``np.dtype('i')``, ``np.dtype('u')`` raises an error.) - - Parameters - ---------- - dtype : TDType - The dtype to check. - - Returns - ------- - Bool - True if the dtype matches, False otherwise. - """ - return super()._check_native_dtype(dtype) or dtype == np.dtypes.UInt32DType() - - @classmethod - def from_native_dtype(cls, dtype: TBaseDType) -> Self: - """ - Create a UInt32 from an np.dtype('uint32') instance. - - Parameters - ---------- - dtype : TBaseDType - The NumPy data type. - - Returns - ------- - Self - An instance of this data type. - - Raises - ------ - DataTypeValidationError - If the input data type is not a valid representation of this class 32-bit unsigned - integer. - """ - if cls._check_native_dtype(dtype): - return cls(endianness=get_endianness_from_numpy_dtype(dtype)) - raise DataTypeValidationError( - f"Invalid data type: {dtype}. Expected an instance of {cls.dtype_cls}" - ) - - def to_native_dtype(self) -> np.dtypes.UInt32DType: - """ - Create a NumPy unsigned 32-bit integer dtype instance from this UInt32 ZDType. - - Returns - ------- - np.dtypes.UInt32DType - The NumPy unsigned 32-bit integer dtype. - """ - byte_order = endianness_to_numpy_str(self.endianness) - # numpy 2.x stub: newbyteorder widens to base dtype, runtime preserves the concrete subclass - return self.dtype_cls().newbyteorder(byte_order) # type: ignore[return-value] - - @classmethod - def _from_json_v2(cls, data: DTypeJSON) -> Self: - """ - Create an instance of this data type from Zarr V2-flavored JSON. - - Parameters - ---------- - data : DTypeJSON - The JSON data. - - Returns - ------- - Self - An instance of this data type. - - Raises - ------ - DataTypeValidationError - If the input JSON is not a valid representation of this class 32-bit unsigned - integer. - """ - if cls._check_json_v2(data): - # Going via NumPy ensures that we get the endianness correct without - # annoying string parsing. - name = data["name"] - return cls.from_native_dtype(np.dtype(name)) - msg = f"Invalid JSON representation of {cls.__name__}. Got {data!r}, expected one of the strings {cls._zarr_v2_names}." - raise DataTypeValidationError(msg) - - @classmethod - def _from_json_v3(cls, data: DTypeJSON) -> Self: - """ - Create an instance of this data type from Zarr V3-flavored JSON. - - Parameters - ---------- - data : DTypeJSON - The JSON data. - - Returns - ------- - Self - An instance of this data type. - - Raises - ------ - DataTypeValidationError - If the input JSON is not a valid representation of this class 32-bit unsigned - integer. - """ - if cls._check_json_v3(data): - return cls() - msg = f"Invalid JSON representation of {cls.__name__}. Got {data!r}, expected the string {cls._zarr_v3_name!r}" - raise DataTypeValidationError(msg) - - @overload - def to_json(self, zarr_format: Literal[2]) -> DTypeConfig_V2[Literal[">u4", " Literal["uint32"]: ... - def to_json( - self, zarr_format: ZarrFormat - ) -> DTypeConfig_V2[Literal[">u4", "u4", " int: - """ - The size of a single scalar in bytes. - - Returns - ------- - int - The size of a single scalar in bytes. - """ - return 4 - - -@dataclass(frozen=True, kw_only=True) -class Int64(BaseInt[np.dtypes.Int64DType, np.int64], HasEndianness): - """ - A Zarr data type for arrays containing 64-bit signed integers. - - Wraps the [`np.dtypes.Int64DType`][numpy.dtypes.Int64DType] data type. Scalars for this data type are instances of - [`np.int64`][numpy.int64]. - - Attributes - ---------- - dtype_cls : np.dtypes.Int64DType - The class of the underlying NumPy dtype. - - References - ---------- - This class implements the 64-bit signed integer data type defined in Zarr V2 and V3. - - See the [Zarr V2](https://github.com/zarr-developers/zarr-specs/blob/main/docs/v2/v2.0.rst#data-type-encoding) and [Zarr V3](https://github.com/zarr-developers/zarr-specs/blob/main/docs/v3/data-types/index.rst) specification documents for details. - """ - - dtype_cls = np.dtypes.Int64DType - _zarr_v3_name: ClassVar[Literal["int64"]] = "int64" - _zarr_v2_names: ClassVar[tuple[Literal[">i8"], Literal["i8", " Self: - """ - Create an Int64 from an np.dtype('int64') instance. - - Parameters - ---------- - dtype : TBaseDType - The NumPy data type. - - Returns - ------- - Self - An instance of this data type. - - Raises - ------ - DataTypeValidationError - If the input data type is not a valid representation of this class 64-bit signed - integer. - """ - if cls._check_native_dtype(dtype): - return cls(endianness=get_endianness_from_numpy_dtype(dtype)) - raise DataTypeValidationError( - f"Invalid data type: {dtype}. Expected an instance of {cls.dtype_cls}" - ) - - def to_native_dtype(self) -> np.dtypes.Int64DType: - """ - Create a NumPy signed 64-bit integer dtype instance from this Int64 ZDType. - - Returns - ------- - np.dtypes.Int64DType - The NumPy signed 64-bit integer dtype. - """ - byte_order = endianness_to_numpy_str(self.endianness) - # numpy 2.x stub: newbyteorder widens to base dtype, runtime preserves the concrete subclass - return self.dtype_cls().newbyteorder(byte_order) # type: ignore[return-value] - - @classmethod - def _from_json_v2(cls, data: DTypeJSON) -> Self: - """ - Create an instance of this data type from Zarr V2-flavored JSON. - - Parameters - ---------- - data : DTypeJSON - The JSON data. - - Returns - ------- - Self - An instance of this data type. - - Raises - ------ - DataTypeValidationError - If the input JSON is not a valid representation of this class 64-bit signed - integer. - """ - if cls._check_json_v2(data): - # Going via NumPy ensures that we get the endianness correct without - # annoying string parsing. - name = data["name"] - return cls.from_native_dtype(np.dtype(name)) - msg = f"Invalid JSON representation of {cls.__name__}. Got {data!r}, expected one of the strings {cls._zarr_v2_names}." - raise DataTypeValidationError(msg) - - @classmethod - def _from_json_v3(cls, data: DTypeJSON) -> Self: - """ - Create an instance of this data type from Zarr V3-flavored JSON. - - Parameters - ---------- - data : DTypeJSON - The JSON data. - - Returns - ------- - Self - An instance of this data type. - - Raises - ------ - DataTypeValidationError - If the input JSON is not a valid representation of this class 64-bit signed - integer. - """ - if cls._check_json_v3(data): - return cls() - msg = f"Invalid JSON representation of {cls.__name__}. Got {data!r}, expected the string {cls._zarr_v3_name!r}" - raise DataTypeValidationError(msg) - - @overload - def to_json(self, zarr_format: Literal[2]) -> DTypeConfig_V2[Literal[">i8", " Literal["int64"]: ... - def to_json( - self, zarr_format: ZarrFormat - ) -> DTypeConfig_V2[Literal[">i8", "i8", " int: - """ - The size of a single scalar in bytes. - - Returns - ------- - int - The size of a single scalar in bytes. - """ - return 8 - - -@dataclass(frozen=True, kw_only=True) -class UInt64(BaseInt[np.dtypes.UInt64DType, np.uint64], HasEndianness): - """ - A Zarr data type for arrays containing 64-bit unsigned integers. - - Wraps the [`np.dtypes.UInt64DType`][numpy.dtypes.UInt64DType] data type. Scalars for this data type - are instances of [`np.uint64`][numpy.uint64]. - - Attributes - ---------- - dtype_cls: np.dtypes.UInt64DType - The class of the underlying NumPy dtype. - - References - ---------- - This class implements the unsigned 64-bit integer data type defined in Zarr V2 and V3. + This class implements the unsigned 64-bit integer data type defined in Zarr V2 and V3. See the [Zarr V2](https://github.com/zarr-developers/zarr-specs/blob/main/docs/v2/v2.0.rst#data-type-encoding) and [Zarr V3](https://github.com/zarr-developers/zarr-specs/blob/main/docs/v3/data-types/index.rst) specification documents for details. """ @@ -1413,129 +711,7 @@ class UInt64(BaseInt[np.dtypes.UInt64DType, np.uint64], HasEndianness): dtype_cls = np.dtypes.UInt64DType _zarr_v3_name: ClassVar[Literal["uint64"]] = "uint64" _zarr_v2_names: ClassVar[tuple[Literal[">u8"], Literal["u8", " np.dtypes.UInt64DType: - """ - Convert the data type to a native NumPy dtype. - - Returns - ------- - np.dtypes.UInt64DType - The native NumPy dtype.eeeeeeeeeeeeeeeee - """ - byte_order = endianness_to_numpy_str(self.endianness) - # numpy 2.x stub: newbyteorder widens to base dtype, runtime preserves the concrete subclass - return self.dtype_cls().newbyteorder(byte_order) # type: ignore[return-value] - - @classmethod - def _from_json_v2(cls, data: DTypeJSON) -> Self: - """ - Create an instance of this data type from Zarr V2-flavored JSON. - - Parameters - ---------- - data : DTypeJSON - The JSON data. - - Returns - ------- - Self - An instance of this data type. - - Raises - ------ - DataTypeValidationError - If the input JSON is not a valid representation of this class unsigned 64-bit - integer. - """ - if cls._check_json_v2(data): - # Going via NumPy ensures that we get the endianness correct without - # annoying string parsing. - name = data["name"] - return cls.from_native_dtype(np.dtype(name)) - msg = f"Invalid JSON representation of {cls.__name__}. Got {data!r}, expected one of the strings {cls._zarr_v2_names}." - raise DataTypeValidationError(msg) - - @classmethod - def _from_json_v3(cls, data: DTypeJSON) -> Self: - """ - Create an instance of this data type from Zarr V3-flavored JSON. - - Parameters - ---------- - data : DTypeJSON - The JSON data. - - Returns - ------- - Self - An instance of this data type. - - Raises - ------ - DataTypeValidationError - If the input JSON is not a valid representation of this class unsigned 64-bit - integer. - """ - if cls._check_json_v3(data): - return cls() - msg = f"Invalid JSON representation of {cls.__name__}. Got {data!r}, expected the string {cls._zarr_v3_name!r}" - raise DataTypeValidationError(msg) - - @overload - def to_json(self, zarr_format: Literal[2]) -> DTypeConfig_V2[Literal[">u8", " Literal["uint64"]: ... - - def to_json( - self, zarr_format: ZarrFormat - ) -> DTypeConfig_V2[Literal[">u8", "u8", " Self: - """ - Create an instance of this data type from a native NumPy dtype. - - Parameters - ---------- - dtype : TBaseDType - The native NumPy dtype. - - Returns - ------- - Self - An instance of this data type. - - Raises - ------ - DataTypeValidationError - If the input dtype is not a valid representation of this class unsigned 64-bit - integer. - """ - if cls._check_native_dtype(dtype): - return cls(endianness=get_endianness_from_numpy_dtype(dtype)) - raise DataTypeValidationError( - f"Invalid data type: {dtype}. Expected an instance of {cls.dtype_cls}" - ) + _has_endianness: ClassVar[bool] = True @property def item_size(self) -> int: diff --git a/src/zarr/core/dtype/wrapper.py b/src/zarr/core/dtype/wrapper.py index 42d5d88473..9503b79733 100644 --- a/src/zarr/core/dtype/wrapper.py +++ b/src/zarr/core/dtype/wrapper.py @@ -275,18 +275,3 @@ def to_json_scalar(self, data: object, *, zarr_format: ZarrFormat) -> JSON: The JSON-serialized scalar. """ raise NotImplementedError # pragma: no cover - - -def scalar_failed_type_check_msg( - cls_instance: ZDType[TBaseDType, TBaseScalar], bad_scalar: object -) -> str: - """ - Generate an error message reporting that a particular value failed a type check when attempting - to cast that value to a scalar. - """ - return ( - f"The value {bad_scalar!r} failed a type check. " - f"It cannot be safely cast to a scalar compatible with {cls_instance}. " - f"Consult the documentation for {cls_instance} to determine the possible values that can " - "be cast to scalars of the wrapped data type." - ) diff --git a/src/zarr/core/group.py b/src/zarr/core/group.py index 52eaa3e144..d4fc919a48 100644 --- a/src/zarr/core/group.py +++ b/src/zarr/core/group.py @@ -2400,105 +2400,7 @@ def create( config: ArrayConfigLike | None = None, write_data: bool = True, ) -> AnyArray: - """Create an array within this group. - - This method lightly wraps [`zarr.core.array.create_array`][]. - - Parameters - ---------- - name : str - The name of the array relative to the group. If ``path`` is ``None``, the array will be located - at the root of the store. - shape : ShapeLike, optional - Shape of the array. Must be ``None`` if ``data`` is provided. - dtype : npt.DTypeLike | None - Data type of the array. Must be ``None`` if ``data`` is provided. - data : Array-like data to use for initializing the array. If this parameter is provided, the - ``shape`` and ``dtype`` parameters must be ``None``. - chunks : tuple[int, ...], optional - Chunk shape of the array. - If not specified, default are guessed based on the shape and dtype. - shards : tuple[int, ...], optional - Shard shape of the array. The default value of ``None`` results in no sharding at all. - filters : Iterable[Codec] | Literal["auto"], optional - Iterable of filters to apply to each chunk of the array, in order, before serializing that - chunk to bytes. - - For Zarr format 3, a "filter" is a codec that takes an array and returns an array, - and these values must be instances of [`zarr.abc.codec.ArrayArrayCodec`][], or a - dict representations of [`zarr.abc.codec.ArrayArrayCodec`][]. - - For Zarr format 2, a "filter" can be any numcodecs codec; you should ensure that the - order of your filters is consistent with the behavior of each filter. - - The default value of ``"auto"`` instructs Zarr to use a default based on the data - type of the array and the Zarr format specified. For all data types in Zarr V3, and most - data types in Zarr V2, the default filters are empty. The only cases where default filters - are not empty is when the Zarr format is 2, and the data type is a variable-length data type like - [`zarr.dtype.VariableLengthUTF8`][] or [`zarr.dtype.VariableLengthUTF8`][]. In these cases, - the default filters contains a single element which is a codec specific to that particular data type. - - To create an array with no filters, provide an empty iterable or the value ``None``. - compressors : Iterable[Codec], optional - List of compressors to apply to the array. Compressors are applied in order, and after any - filters are applied (if any are specified) and the data is serialized into bytes. - - For Zarr format 3, a "compressor" is a codec that takes a bytestream, and - returns another bytestream. Multiple compressors may be provided for Zarr format 3. - If no ``compressors`` are provided, a default set of compressors will be used. - These defaults can be changed by modifying the value of ``array.v3_default_compressors`` - in [`zarr.config`][]. - Use ``None`` to omit default compressors. - - For Zarr format 2, a "compressor" can be any numcodecs codec. Only a single compressor may - be provided for Zarr format 2. - If no ``compressor`` is provided, a default compressor will be used. - in [`zarr.config`][]. - Use ``None`` to omit the default compressor. - compressor : Codec, optional - Deprecated in favor of ``compressors``. - serializer : dict[str, JSON] | ArrayBytesCodec, optional - Array-to-bytes codec to use for encoding the array data. - Zarr format 3 only. Zarr format 2 arrays use implicit array-to-bytes conversion. - If no ``serializer`` is provided, a default serializer will be used. - These defaults can be changed by modifying the value of ``array.v3_default_serializer`` - in [`zarr.config`][]. - fill_value : Any, optional - Fill value for the array. - order : {"C", "F"}, optional - The memory order of the array (default is "C"). - For Zarr format 2, this parameter sets the memory order of the array. - For Zarr format 3, this parameter is deprecated, because memory order - is a runtime parameter for Zarr format 3 arrays. The recommended way to specify the memory - order for Zarr format 3 arrays is via the ``config`` parameter, e.g. ``{'config': 'C'}``. - If no ``order`` is provided, a default order will be used. - This default can be changed by modifying the value of ``array.order`` in [`zarr.config`][]. - attributes : dict, optional - Attributes for the array. - chunk_key_encoding : ChunkKeyEncoding, optional - A specification of how the chunk keys are represented in storage. - For Zarr format 3, the default is ``{"name": "default", "separator": "/"}}``. - For Zarr format 2, the default is ``{"name": "v2", "separator": "."}}``. - dimension_names : Iterable[str], optional - The names of the dimensions (default is None). - Zarr format 3 only. Zarr format 2 arrays should not use this parameter. - storage_options : dict, optional - If using an fsspec URL to create the store, these will be passed to the backend implementation. - Ignored otherwise. - overwrite : bool, default False - Whether to overwrite an array with the same name in the store, if one exists. - config : ArrayConfig or ArrayConfigLike, optional - Runtime configuration for the array. - write_data : bool - If a pre-existing array-like object was provided to this function via the ``data`` parameter - then ``write_data`` determines whether the values in that array-like object should be - written to the Zarr array created by this function. If ``write_data`` is ``False``, then the - array will be left empty. - - Returns - ------- - AsyncArray - """ + """Alias for [`zarr.Group.create_array`][].""" return self.create_array( name, shape=shape, @@ -2643,9 +2545,6 @@ def create_array( ------- AsyncArray """ - compressors = _parse_deprecated_compressor( - compressor, compressors, zarr_format=self.metadata.zarr_format - ) return Array( self._sync( self._async_group.create_array( @@ -2659,6 +2558,7 @@ def create_array( attributes=attributes, chunk_key_encoding=chunk_key_encoding, compressors=compressors, + compressor=compressor, serializer=serializer, dimension_names=dimension_names, order=order, @@ -3399,34 +3299,22 @@ async def _read_metadata_v2(store: Store, path: str) -> ArrayV2Metadata | GroupM return _build_metadata_v2(zmeta, zattrs) -async def _read_group_metadata_v2(store: Store, path: str) -> GroupMetadata: - """ - Read group metadata or error - """ - meta = await _read_metadata_v2(store=store, path=path) - if not isinstance(meta, GroupMetadata): - raise FileNotFoundError(f"Group metadata was not found in {store} at {path}") - return meta - - -async def _read_group_metadata_v3(store: Store, path: str) -> GroupMetadata: +async def _read_group_metadata( + store: Store, path: str, *, zarr_format: ZarrFormat +) -> GroupMetadata: """ Read group metadata or error """ - meta = await _read_metadata_v3(store=store, path=path) + meta: ArrayV2Metadata | ArrayV3Metadata | GroupMetadata + if zarr_format == 2: + meta = await _read_metadata_v2(store=store, path=path) + else: + meta = await _read_metadata_v3(store=store, path=path) if not isinstance(meta, GroupMetadata): raise FileNotFoundError(f"Group metadata was not found in {store} at {path}") return meta -async def _read_group_metadata( - store: Store, path: str, *, zarr_format: ZarrFormat -) -> GroupMetadata: - if zarr_format == 2: - return await _read_group_metadata_v2(store=store, path=path) - return await _read_group_metadata_v3(store=store, path=path) - - def _build_metadata_v3(zarr_json: dict[str, JSON]) -> ArrayV3Metadata | GroupMetadata: """ Convert a dict representation of Zarr V3 metadata into the corresponding metadata class. @@ -3486,44 +3374,6 @@ def _build_node( raise ValueError(f"Unexpected metadata type: {type(metadata)}") # pragma: no cover -async def _get_node_v2(store: Store, path: str) -> AsyncArrayV2 | AsyncGroup: - """ - Read a Zarr v2 AsyncArray or AsyncGroup from a path in a Store. - - Parameters - ---------- - store : Store - The store-like object to read from. - path : str - The path to the node to read. - - Returns - ------- - AsyncArray | AsyncGroup - """ - metadata = await _read_metadata_v2(store=store, path=path) - return _build_node(store=store, path=path, metadata=metadata) - - -async def _get_node_v3(store: Store, path: str) -> AsyncArrayV3 | AsyncGroup: - """ - Read a Zarr v3 AsyncArray or AsyncGroup from a path in a Store. - - Parameters - ---------- - store : Store - The store-like object to read from. - path : str - The path to the node to read. - - Returns - ------- - AsyncArray | AsyncGroup - """ - metadata = await _read_metadata_v3(store=store, path=path) - return _build_node(store=store, path=path, metadata=metadata) - - async def get_node(store: Store, path: str, zarr_format: ZarrFormat) -> AnyAsyncArray | AsyncGroup: """ Get an AsyncArray or AsyncGroup from a path in a Store. @@ -3542,13 +3392,15 @@ async def get_node(store: Store, path: str, zarr_format: ZarrFormat) -> AnyAsync AsyncArray | AsyncGroup """ + metadata: ArrayV2Metadata | ArrayV3Metadata | GroupMetadata match zarr_format: case 2: - return await _get_node_v2(store=store, path=path) + metadata = await _read_metadata_v2(store=store, path=path) case 3: - return await _get_node_v3(store=store, path=path) + metadata = await _read_metadata_v3(store=store, path=path) case _: # pragma: no cover raise ValueError(f"Unexpected zarr format: {zarr_format}") # pragma: no cover + return _build_node(store=store, path=path, metadata=metadata) async def _set_return_key( diff --git a/src/zarr/core/indexing.py b/src/zarr/core/indexing.py index cb81164209..ae83719ba9 100644 --- a/src/zarr/core/indexing.py +++ b/src/zarr/core/indexing.py @@ -13,6 +13,7 @@ Any, Literal, NamedTuple, + NoReturn, Protocol, TypeGuard, cast, @@ -544,6 +545,32 @@ class ChunkProjection(NamedTuple): is_complete_chunk: bool +def _iter_chunk_projections( + dim_indexers: Sequence[IntDimIndexer | SliceDimIndexer], +) -> Iterator[ChunkProjection]: + """Yield a ChunkProjection for each chunk touched by the product of ``dim_indexers``. + + Shared by BasicIndexer and BlockIndexer, whose iteration is identical. The four + per-chunk fields (chunk coords, chunk selection, out selection, completeness) are + built in a single pass over each chunk's dim projections rather than four. + """ + for dim_projections in itertools.product(*dim_indexers): + chunk_coords: list[int] = [] + chunk_selection: list[Selector] = [] + out_selection: list[Selector] = [] + is_complete_chunk = True + for p in dim_projections: + chunk_coords.append(p.dim_chunk_ix) + chunk_selection.append(p.dim_chunk_sel) + if p.dim_out_sel is not None: + out_selection.append(p.dim_out_sel) + if not p.is_complete_chunk: + is_complete_chunk = False + yield ChunkProjection( + tuple(chunk_coords), tuple(chunk_selection), tuple(out_selection), is_complete_chunk + ) + + def is_slice(s: Any) -> TypeGuard[slice]: return isinstance(s, slice) @@ -609,14 +636,7 @@ def __init__( object.__setattr__(self, "drop_axes", ()) def __iter__(self) -> Iterator[ChunkProjection]: - for dim_projections in itertools.product(*self.dim_indexers): - chunk_coords = tuple(p.dim_chunk_ix for p in dim_projections) - chunk_selection = tuple(p.dim_chunk_sel for p in dim_projections) - out_selection = tuple( - p.dim_out_sel for p in dim_projections if p.dim_out_sel is not None - ) - is_complete_chunk = all(p.is_complete_chunk for p in dim_projections) - yield ChunkProjection(chunk_coords, chunk_selection, out_selection, is_complete_chunk) + yield from _iter_chunk_projections(self.dim_indexers) @dataclass(frozen=True) @@ -1117,14 +1137,7 @@ def __init__( object.__setattr__(self, "drop_axes", ()) def __iter__(self) -> Iterator[ChunkProjection]: - for dim_projections in itertools.product(*self.dim_indexers): - chunk_coords = tuple(p.dim_chunk_ix for p in dim_projections) - chunk_selection = tuple(p.dim_chunk_sel for p in dim_projections) - out_selection = tuple( - p.dim_out_sel for p in dim_projections if p.dim_out_sel is not None - ) - is_complete_chunk = all(p.is_complete_chunk for p in dim_projections) - yield ChunkProjection(chunk_coords, chunk_selection, out_selection, is_complete_chunk) + yield from _iter_chunk_projections(self.dim_indexers) @dataclass(frozen=True) @@ -1165,6 +1178,16 @@ def is_mask_selection(selection: Selection, shape: tuple[int, ...]) -> TypeGuard ) +def _raise_vindex_invalid_selection(selection: object) -> NoReturn: + """Raise the standard error for a selection that is neither coordinate nor mask.""" + msg = ( + "unsupported selection type for vectorized indexing; only " + "coordinate selection (tuple of integer arrays) and mask selection " + f"(single Boolean array) are supported; got {selection!r}" + ) + raise VindexInvalidSelectionError(msg) + + @dataclass(frozen=True) class CoordinateIndexer(Indexer): sel_shape: tuple[int, ...] @@ -1339,12 +1362,7 @@ def __getitem__( elif is_mask_selection(new_selection, self.array.shape): return self.array.get_mask_selection(new_selection, fields=fields) else: - msg = ( - "unsupported selection type for vectorized indexing; only " - "coordinate selection (tuple of integer arrays) and mask selection " - f"(single Boolean array) are supported; got {new_selection!r}" - ) - raise VindexInvalidSelectionError(msg) + _raise_vindex_invalid_selection(new_selection) def __setitem__( self, selection: CoordinateSelection | MaskSelection, value: npt.ArrayLike @@ -1357,12 +1375,7 @@ def __setitem__( elif is_mask_selection(new_selection, self.array.shape): self.array.set_mask_selection(new_selection, value, fields=fields) else: - msg = ( - "unsupported selection type for vectorized indexing; only " - "coordinate selection (tuple of integer arrays) and mask selection " - f"(single Boolean array) are supported; got {new_selection!r}" - ) - raise VindexInvalidSelectionError(msg) + _raise_vindex_invalid_selection(new_selection) @dataclass(frozen=True) @@ -1388,12 +1401,7 @@ async def getitem( elif is_mask_selection(new_selection, self.array.shape): return await self.array.get_mask_selection(new_selection, fields=fields) else: - msg = ( - "unsupported selection type for vectorized indexing; only " - "coordinate selection (tuple of integer arrays) and mask selection " - f"(single Boolean array) are supported; got {new_selection!r}" - ) - raise VindexInvalidSelectionError(msg) + _raise_vindex_invalid_selection(new_selection) def check_fields(fields: Fields | None, dtype: np.dtype[Any]) -> np.dtype[Any]: @@ -1600,12 +1608,7 @@ def get_indexer( elif is_mask_selection(new_selection, shape): return MaskIndexer(cast("MaskSelection", selection), shape, chunk_grid) else: - msg = ( - "unsupported selection type for vectorized indexing; only " - "coordinate selection (tuple of integer arrays) and mask selection " - f"(single Boolean array) are supported; got {new_selection!r}" - ) - raise VindexInvalidSelectionError(msg) + _raise_vindex_invalid_selection(new_selection) elif is_pure_orthogonal_indexing(pure_selection, len(shape)): return OrthogonalIndexer(cast("OrthogonalSelection", selection), shape, chunk_grid) else: diff --git a/src/zarr/core/sync.py b/src/zarr/core/sync.py index 260d4ad841..9d275d69ca 100644 --- a/src/zarr/core/sync.py +++ b/src/zarr/core/sync.py @@ -13,7 +13,7 @@ from zarr.core.config import config if TYPE_CHECKING: - from collections.abc import AsyncIterator, Awaitable, Callable, Coroutine + from collections.abc import AsyncIterator, Coroutine from typing import Any logger = logging.getLogger(__name__) @@ -210,17 +210,3 @@ async def iter_to_list() -> list[T]: return [item async for item in async_iterator] return self._sync(iter_to_list()) - - -async def _with_semaphore[T]( - func: Callable[[], Awaitable[T]], semaphore: asyncio.Semaphore | None = None -) -> T: - """ - Await the result of invoking the no-argument-callable ``func`` within the context manager - provided by a Semaphore, if one is provided. Otherwise, just await the result of invoking - ``func``. - """ - if semaphore is None: - return await func() - async with semaphore: - return await func() diff --git a/src/zarr/registry.py b/src/zarr/registry.py index 48f60fabd7..49da4993f8 100644 --- a/src/zarr/registry.py +++ b/src/zarr/registry.py @@ -194,26 +194,43 @@ def _resolve_codec(data: dict[str, JSON]) -> Codec: return get_codec_class(data["name"]).from_dict(data) # type: ignore[arg-type] -def _parse_bytes_bytes_codec(data: dict[str, JSON] | Codec) -> BytesBytesCodec: - """ - Normalize the input to a ``BytesBytesCodec`` instance. - If the input is already a ``BytesBytesCodec``, it is returned as is. If the input is a dict, it - is converted to a ``BytesBytesCodec`` instance via the ``_resolve_codec`` function. +def _parse_codec[T: Codec](data: dict[str, JSON] | Codec, cls: type[T], article: str) -> T: """ - from zarr.abc.codec import BytesBytesCodec + Normalize the input to an instance of ``cls``. + If the input is already an instance of ``cls``, it is returned as is. If the input is a dict, it + is converted to a codec instance via the ``_resolve_codec`` function and then type-checked. + + ``article`` is the indefinite article ("a" or "an") that precedes the codec class name in the + error messages, so that the messages read naturally for each codec type. + """ + name = cls.__name__ if isinstance(data, dict): result = _resolve_codec(data) - if not isinstance(result, BytesBytesCodec): - msg = f"Expected a dict representation of a BytesBytesCodec; got a dict representation of a {type(result)} instead." + if not isinstance(result, cls): + msg = ( + f"Expected a dict representation of {article} {name}; got a dict representation " + f"of a {type(result)} instead." + ) raise TypeError(msg) else: - if not isinstance(data, BytesBytesCodec): - raise TypeError(f"Expected a BytesBytesCodec. Got {type(data)} instead.") + if not isinstance(data, cls): + raise TypeError(f"Expected {article} {name}. Got {type(data)} instead.") result = data return result +def _parse_bytes_bytes_codec(data: dict[str, JSON] | Codec) -> BytesBytesCodec: + """ + Normalize the input to a ``BytesBytesCodec`` instance. + If the input is already a ``BytesBytesCodec``, it is returned as is. If the input is a dict, it + is converted to a ``BytesBytesCodec`` instance via the ``_resolve_codec`` function. + """ + from zarr.abc.codec import BytesBytesCodec + + return _parse_codec(data, BytesBytesCodec, "a") # type: ignore[type-abstract] + + def _parse_array_bytes_codec(data: dict[str, JSON] | Codec) -> ArrayBytesCodec: """ Normalize the input to a ``ArrayBytesCodec`` instance. @@ -222,16 +239,7 @@ def _parse_array_bytes_codec(data: dict[str, JSON] | Codec) -> ArrayBytesCodec: """ from zarr.abc.codec import ArrayBytesCodec - if isinstance(data, dict): - result = _resolve_codec(data) - if not isinstance(result, ArrayBytesCodec): - msg = f"Expected a dict representation of an ArrayBytesCodec; got a dict representation of a {type(result)} instead." - raise TypeError(msg) - else: - if not isinstance(data, ArrayBytesCodec): - raise TypeError(f"Expected an ArrayBytesCodec. Got {type(data)} instead.") - result = data - return result + return _parse_codec(data, ArrayBytesCodec, "an") # type: ignore[type-abstract] def _parse_array_array_codec(data: dict[str, JSON] | Codec) -> ArrayArrayCodec: @@ -242,16 +250,7 @@ def _parse_array_array_codec(data: dict[str, JSON] | Codec) -> ArrayArrayCodec: """ from zarr.abc.codec import ArrayArrayCodec - if isinstance(data, dict): - result = _resolve_codec(data) - if not isinstance(result, ArrayArrayCodec): - msg = f"Expected a dict representation of an ArrayArrayCodec; got a dict representation of a {type(result)} instead." - raise TypeError(msg) - else: - if not isinstance(data, ArrayArrayCodec): - raise TypeError(f"Expected an ArrayArrayCodec. Got {type(data)} instead.") - result = data - return result + return _parse_codec(data, ArrayArrayCodec, "an") # type: ignore[type-abstract] def get_pipeline_class(reload_config: bool = False) -> type[CodecPipeline]: diff --git a/src/zarr/storage/_fsspec.py b/src/zarr/storage/_fsspec.py index 29201a6fee..a863de4b4e 100644 --- a/src/zarr/storage/_fsspec.py +++ b/src/zarr/storage/_fsspec.py @@ -15,7 +15,6 @@ Store, SuffixByteRequest, ) -from zarr.core.buffer import Buffer from zarr.errors import ZarrUserWarning from zarr.storage._utils import _dereference_path @@ -28,7 +27,7 @@ from fsspec.asyn import AsyncFileSystem from fsspec.mapping import FSMap - from zarr.core.buffer import BufferPrototype + from zarr.core.buffer import Buffer, BufferPrototype ALLOWED_EXCEPTIONS: tuple[type[Exception], ...] = ( @@ -378,10 +377,7 @@ async def set( if not self._is_open: await self._open() self._check_writable() - if not isinstance(value, Buffer): - raise TypeError( - f"FsspecStore.set(): `value` must be a Buffer instance. Got an instance of {type(value)} instead." - ) + self._check_value(value) path = _dereference_path(self.path, key) # write data if byte_range: diff --git a/src/zarr/storage/_local.py b/src/zarr/storage/_local.py index 3d9882d3db..664cc8bd49 100644 --- a/src/zarr/storage/_local.py +++ b/src/zarr/storage/_local.py @@ -17,14 +17,13 @@ Store, SuffixByteRequest, ) -from zarr.core.buffer import Buffer from zarr.core.buffer.core import default_buffer_prototype from zarr.core.common import AccessModeLiteral, concurrent_map if TYPE_CHECKING: from collections.abc import AsyncIterator, Iterable, Iterator - from zarr.core.buffer import BufferPrototype + from zarr.core.buffer import Buffer, BufferPrototype def _get(path: Path, prototype: BufferPrototype, byte_range: ByteRequest | None) -> Buffer: @@ -220,11 +219,7 @@ def set_sync(self, key: str, value: Buffer) -> None: self._ensure_open_sync() self._check_writable() assert isinstance(key, str) - if not isinstance(value, Buffer): - raise TypeError( - f"LocalStore.set(): `value` must be a Buffer instance. " - f"Got an instance of {type(value)} instead." - ) + self._check_value(value) path = self.root / key _put(path, value) @@ -285,10 +280,7 @@ async def _set(self, key: str, value: Buffer, exclusive: bool = False) -> None: await self._open() self._check_writable() assert isinstance(key, str) - if not isinstance(value, Buffer): - raise TypeError( - f"LocalStore.set(): `value` must be a Buffer instance. Got an instance of {type(value)} instead." - ) + self._check_value(value) path = self.root / key await asyncio.to_thread(_put, path, value, exclusive=exclusive) diff --git a/src/zarr/storage/_memory.py b/src/zarr/storage/_memory.py index e867706155..d8111c4c7e 100644 --- a/src/zarr/storage/_memory.py +++ b/src/zarr/storage/_memory.py @@ -113,10 +113,7 @@ def set_sync(self, key: str, value: Buffer) -> None: if not self._is_open: self._is_open = True assert isinstance(key, str) - if not isinstance(value, Buffer): - raise TypeError( - f"MemoryStore.set(): `value` must be a Buffer instance. Got an instance of {type(value)} instead." - ) + self._check_value(value) self._store_dict[key] = value def delete_sync(self, key: str) -> None: @@ -169,10 +166,7 @@ async def set(self, key: str, value: Buffer, byte_range: tuple[int, int] | None self._check_writable() await self._ensure_open() assert isinstance(key, str) - if not isinstance(value, Buffer): - raise TypeError( - f"MemoryStore.set(): `value` must be a Buffer instance. Got an instance of {type(value)} instead." - ) + self._check_value(value) if byte_range is not None: buf = self._store_dict[key] buf[byte_range[0] : byte_range[1]] = value @@ -288,10 +282,7 @@ async def set(self, key: str, value: Buffer, byte_range: tuple[int, int] | None # docstring inherited self._check_writable() assert isinstance(key, str) - if not isinstance(value, Buffer): - raise TypeError( - f"GpuMemoryStore.set(): `value` must be a Buffer instance. Got an instance of {type(value)} instead." - ) + self._check_value(value) # Convert to gpu.Buffer gpu_value = value if isinstance(value, gpu.Buffer) else gpu.Buffer.from_buffer(value) await super().set(key, gpu_value, byte_range=byte_range) diff --git a/src/zarr/storage/_zip.py b/src/zarr/storage/_zip.py index 897797e999..d4ba627ff2 100644 --- a/src/zarr/storage/_zip.py +++ b/src/zarr/storage/_zip.py @@ -15,11 +15,12 @@ Store, SuffixByteRequest, ) -from zarr.core.buffer import Buffer, BufferPrototype if TYPE_CHECKING: from collections.abc import AsyncIterator, Iterable + from zarr.core.buffer import Buffer, BufferPrototype + ZipStoreAccessModeLiteral = Literal["r", "w", "a"] @@ -213,10 +214,7 @@ async def set(self, key: str, value: Buffer) -> None: if not self._is_open: self._sync_open() assert isinstance(key, str) - if not isinstance(value, Buffer): - raise TypeError( - f"ZipStore.set(): `value` must be a Buffer instance. Got an instance of {type(value)} instead." - ) + self._check_value(value) with self._lock: self._set(key, value)