From 176b041d6104fe43d22cbfbc387212f309352dae Mon Sep 17 00:00:00 2001 From: Duo-ee <113885053+Duo-ee@users.noreply.github.com> Date: Thu, 22 Oct 2026 11:16:43 +0800 Subject: [PATCH 01/12] fix: reject reserved Zarr v3 node names Validate node names before writing Zarr v3 metadata. --- src/zarr/core/common.py | 918 ++--- src/zarr/core/group.py | 7896 ++++++++++++++++++++------------------- 2 files changed, 4416 insertions(+), 4398 deletions(-) diff --git a/src/zarr/core/common.py b/src/zarr/core/common.py index 40046124ec..0f514b4349 100644 --- a/src/zarr/core/common.py +++ b/src/zarr/core/common.py @@ -1,452 +1,466 @@ -from __future__ import annotations - -import asyncio -import math -import warnings -from collections.abc import Iterable, Mapping, Sequence -from enum import Enum -from typing import ( - TYPE_CHECKING, - Any, - Final, - Literal, - NotRequired, - TypedDict, - cast, - overload, -) - -import numpy as np -from typing_extensions import ReadOnly - -from zarr.core.config import config as zarr_config -from zarr.core.json_parse import convert, parse_field -from zarr.errors import ZarrRuntimeWarning - -if TYPE_CHECKING: - from collections.abc import Awaitable, Callable, Iterator - - from zarr.core.metadata.v3 import ChunkGridMetadata - - -ZARR_JSON = "zarr.json" -ZARRAY_JSON = ".zarray" -ZGROUP_JSON = ".zgroup" -ZATTRS_JSON = ".zattrs" -ZMETADATA_V2_JSON = ".zmetadata" - -BytesLike = bytes | bytearray | memoryview -ShapeLike = Iterable[int | np.integer[Any]] | int | np.integer[Any] -# Per-dimension chunk specs may mix a bare int (uniform chunk size, the -# rectilinear spec's step-size shorthand) with explicit edge-length sequences. -# A stored chunk grid (`ChunkGridMetadata`) is also accepted and is used -# verbatim, under the tolerant stored-metadata validation rules (e.g. trailing -# edges beyond the array extent, as left behind by a shrinking resize). -type ChunksLike = ShapeLike | Iterable[int | Iterable[int]] | ChunkGridMetadata -# For backwards compatibility -ChunkCoords = tuple[int, ...] -ZarrFormat = Literal[2, 3] -NodeType = Literal["array", "group"] -JSON = str | int | float | bool | Mapping[str, "JSON"] | Sequence["JSON"] | None -MemoryOrder = Literal["C", "F"] -AccessModeLiteral = Literal["r", "r+", "a", "w", "w-"] -ANY_ACCESS_MODE: Final = "r", "r+", "a", "w", "w-" -DimensionNamesLike = Iterable[str | None] | None -DimensionNames = DimensionNamesLike # for backwards compatibility - - -class NamedConfig[TName: str, TConfig: Mapping[str, object]](TypedDict): - """ - A typed dictionary representing an object with a name and configuration, where the configuration - is an optional mapping of string keys to values, e.g. another typed dictionary or a JSON object. - - This class is generic with two type parameters: the type of the name (``TName``) and the type of - the configuration (``TConfig``). - """ - - name: ReadOnly[TName] - """The name of the object.""" - - configuration: NotRequired[ReadOnly[TConfig]] - """The configuration of the object. Not required.""" - - -class NamedRequiredConfig[TName: str, TConfig: Mapping[str, object]](TypedDict): - """ - A typed dictionary representing an object with a name and configuration, where the configuration - is a mapping of string keys to values, e.g. another typed dictionary or a JSON object. - - This class is generic with two type parameters: the type of the name (``TName``) and the type of - the configuration (``TConfig``). - """ - - name: ReadOnly[TName] - """The name of the object.""" - - configuration: ReadOnly[TConfig] - """The configuration of the object.""" - - -def product(tup: tuple[int, ...]) -> int: - return math.prod(tup) - - -def ceildiv(a: float, b: float) -> int: - """Ceiling of ``a / b`` using floating-point division; zero when ``a`` is zero.""" - if a == 0: - return 0 - return math.ceil(a / b) - - -def ceildiv_int(a: int, b: int) -> int: - """Ceiling of integer division using exact Python integer arithmetic.""" - return -(-int(a) // int(b)) - - -def concurrent_iter[T: tuple[Any, ...], V]( - items: Iterable[T], - func: Callable[..., Awaitable[V]], - limit: int | None = None, -) -> list[asyncio.Task[V]]: - """Launch `func(*item)` for each item concurrently, returning the tasks. - - When `limit` is set, no more than `limit` calls are in flight at once. - Tasks are returned in input order; callers that want completion order - should wrap the result in `asyncio.as_completed`. - - Every task is scheduled (via `ensure_future`) before this function - returns, not on first iteration of the result. That matters for callers - that await the returned tasks one at a time — without eager scheduling, - each coroutine would only start when individually awaited, serializing - the work and defeating the semaphore. It also makes the return type - honest (real `Task`s support `.cancel()`, `.done()`, callbacks) rather - than bare coroutines. - - See https://docs.python.org/3/library/asyncio-task.html#coroutines: - "Note that simply calling a coroutine will not schedule it to be executed:" - """ - if limit is None: - return [asyncio.ensure_future(func(*item)) for item in items] - - sem = asyncio.Semaphore(limit) - - async def run(item: T) -> V: - async with sem: - return await func(*item) - - return [asyncio.ensure_future(run(item)) for item in items] - - -async def concurrent_map[T: tuple[Any, ...], V]( - items: Iterable[T], - func: Callable[..., Awaitable[V]], - limit: int | None = None, -) -> list[V]: - return await asyncio.gather(*concurrent_iter(items, func, limit)) - - -async def concurrent_foreach[T: tuple[Any, ...]]( - items: Iterable[T], - func: Callable[..., Awaitable[object]], - limit: int, -) -> None: - """Run `func(*item)` for every item with at most `limit` calls in flight. - - Unlike `concurrent_map`, `items` is consumed lazily: `limit` workers share - one iterator and each takes the next item as soon as it finishes the - previous one, so auxiliary memory is bounded by `limit`, not by the number - of items. Results are discarded. - - Once a call fails, no further items are taken and the calls still in - flight are cancelled. The failure is re-raised as itself, so callers catch - the store's own exception type. If other in-flight calls failed in the - same event-loop iteration, the exception raised is the one from the - lowest-numbered worker and the `BaseExceptionGroup` holding every failure - is attached as its `__cause__`, so nothing is lost from tracebacks and - logs. Cancellation of the caller propagates as `CancelledError`. - """ - if limit < 1: - raise ValueError(f"limit must be at least 1, got {limit}") - shared = iter(items) - failed = False - - async def worker() -> None: - nonlocal failed - # `next()` on the shared iterator is synchronous, so workers never - # re-enter it: a worker only yields to the loop while awaiting `func`. - while not failed: - try: - item = next(shared) - except StopIteration: - return - try: - await func(*item) - except BaseException: - failed = True - raise - - try: - async with asyncio.TaskGroup() as tg: - for _ in range(limit): - tg.create_task(worker()) - except BaseExceptionGroup as eg: - if len(eg.exceptions) == 1: - raise eg.exceptions[0] from None - raise eg.exceptions[0] from eg - - -def enum_names[E: Enum](enum: type[E]) -> Iterator[str]: - for item in enum: - yield item.name - - -def parse_enum[E: Enum](data: object, cls: type[E]) -> E: - if isinstance(data, cls): - return data - if not isinstance(data, str): - raise TypeError(f"Expected str, got {type(data)}") - if data in enum_names(cls): - return cls(data) - raise ValueError(f"Value must be one of {list(enum_names(cls))!r}. Got {data} instead.") - - -def parse_name(data: JSON, expected: str | None = None) -> str: - try: - data = cast("str", convert(data, str)) - except (ValueError, TypeError) as exc: - raise TypeError(f"Expected a string, got an instance of {type(data)}.") from exc - if expected is None or data == expected: - return data - raise ValueError(f"Expected '{expected}'. Got {data} instead.") - - -def parse_configuration(data: JSON) -> JSON: - if not isinstance(data, dict): - raise TypeError(f"Expected dict, got {type(data)}") - return data - - -@overload -def parse_named_configuration( - data: JSON | NamedConfig[str, Any], expected_name: str | None = None -) -> tuple[str, dict[str, JSON]]: ... - - -@overload -def parse_named_configuration( - data: JSON | NamedConfig[str, Any], - expected_name: str | None = None, - *, - require_configuration: bool = True, -) -> tuple[str, dict[str, JSON] | None]: ... - - -def parse_named_configuration( - data: JSON | NamedConfig[str, Any], - expected_name: str | None = None, - *, - require_configuration: bool = True, -) -> tuple[str, JSON | None]: - if not isinstance(data, dict): - raise TypeError(f"Expected dict, got {type(data)}") - if "name" not in data: - raise ValueError(f"Named configuration does not have a 'name' key. Got {data}.") - name_parsed = parse_name(data["name"], expected_name) - if "configuration" in data: - configuration_parsed = parse_configuration(data["configuration"]) - elif require_configuration: - raise ValueError(f"Named configuration does not have a 'configuration' key. Got {data}.") - else: - configuration_parsed = None - return name_parsed, configuration_parsed - - -def parse_shapelike(data: ShapeLike) -> tuple[int, ...]: - """ - Parse a shape-like input into an explicit shape. - """ - if isinstance(data, int | np.integer): - if data < 0: - raise ValueError(f"Expected a non-negative integer. Got {data} instead") - return (int(data),) - try: - data_tuple = tuple(data) - except TypeError as e: - msg = f"Expected an integer or an iterable of integers. Got {data} instead." - raise TypeError(msg) from e - - if not all(isinstance(v, int | np.integer) for v in data_tuple): - msg = f"Expected an iterable of integers. Got {data} instead." - raise TypeError(msg) - if not all(v > -1 for v in data_tuple): - msg = f"Expected all values to be non-negative. Got {data} instead." - raise ValueError(msg) - - # cast NumPy scalars to plain python ints - return tuple(int(x) for x in data_tuple) - - -def parse_fill_value(data: Any) -> Any: - # todo: real validation - return data - - -def parse_order(data: Any) -> Literal["C", "F"]: - return cast("Literal['C', 'F']", parse_field(data, Literal["C", "F"], "order")) - - -def parse_bool(data: Any) -> bool: - return cast("bool", convert(data, bool)) - - -def parse_int(data: Any) -> int: - if isinstance(data, int) and not isinstance(data, bool): - return data - raise ValueError(f"Expected int, got {data} instead.") - - -def _warn_write_empty_chunks_kwarg() -> None: - # TODO: link to docs page on array configuration in this message - msg = ( - "The `write_empty_chunks` keyword argument is deprecated and will be removed in future versions. " - "To control whether empty chunks are written to storage, either use the `config` keyword " - "argument, as in `config={'write_empty_chunks': True}`," - "or change the global 'array.write_empty_chunks' configuration variable." - ) - warnings.warn(msg, ZarrRuntimeWarning, stacklevel=2) - - -def _warn_order_kwarg() -> None: - # TODO: link to docs page on array configuration in this message - msg = ( - "The `order` keyword argument has no effect for Zarr format 3 arrays. " - "To control the memory layout of the array, either use the `config` keyword " - "argument, as in `config={'order': 'C'}`," - "or change the global 'array.order' configuration variable." - ) - warnings.warn(msg, ZarrRuntimeWarning, stacklevel=2) - - -def _default_zarr_format() -> ZarrFormat: - """Return the default zarr_format.""" - return cast("ZarrFormat", int(zarr_config.get("default_zarr_format", 3))) - - -def _subject(name: str, axis: int | None) -> str: - """`name` as the subject of an error message, prefixed by the dimension `axis`.""" - return name[0].upper() + name[1:] if axis is None else f"Dimension {axis}: {name}" - - -def _parse_positive_int(value: object, name: str, axis: int | None) -> int: - """`value` as an `int` of at least 1. A `bool` is read as the `int` it equals; any - other type, a NumPy integer or a float (even an integral one: stored documents with - integral floats are read by `zarr.core.metadata.repair`), is rejected.""" - subject = _subject(name, axis) - if not isinstance(value, int): - raise TypeError(f"{subject} must be an int, got {value!r}") - if value < 1: - raise ValueError(f"{subject} must be >= 1, got {value!r}") - return int(value) - - -def parse_chunk_edge(size: object, axis: int | None = None) -> int: - """Check that `size` is a chunk edge length: an `int` of at least 1 (a `bool` is - read as the `int` it equals). - - This is the one rule for chunk edge lengths in metadata: bare chunk sizes, explicit - edges and run-length encoded sizes. `axis`, when given, is named in the error. - """ - return _parse_positive_int(size, "chunk edge length", axis) - - -def parse_chunk_shape(data: object) -> tuple[int, ...]: - """Check a regular chunk shape: an iterable, other than a string or a mapping, of one - chunk edge length per axis (see `parse_chunk_edge`).""" - match data: - case str() | Mapping(): - pass - case Iterable(): - return tuple(parse_chunk_edge(size, axis) for axis, size in enumerate(data)) - raise TypeError(f"A chunk shape must be an iterable of chunk edge lengths, got {data!r}") - - -def expand_rle(data: Sequence[object], axis: int | None = None) -> list[int]: - """Expand a mixed array of bare integers and RLE pairs, the edges of dimension - `axis` (named in errors, when given). - - Per the rectilinear chunk grid spec, each element can be: - - a bare integer (an explicit edge length) - - a two-element array ``[value, count]`` (run-length encoded) - """ - result: list[int] = [] - for item in data: - if isinstance(item, list): - if len(item) != 2: - subject = _subject("RLE entries", axis) - raise ValueError(f"{subject} must be an integer or [size, count], got {item}") - size, count = item - repeat = _parse_positive_int(count, "RLE repeat count", axis) - result.extend([parse_chunk_edge(size, axis)] * repeat) - else: - result.append(parse_chunk_edge(item, axis)) - return result - - -def compress_rle(sizes: Sequence[int]) -> list[int | list[int]]: - """Compress chunk sizes to mixed RLE format per the rectilinear spec. - - Runs of length > 1 are emitted as ``[value, count]`` pairs; runs of - length 1 are emitted as bare integers:: - - [10, 10, 10, 5] -> [[10, 3], 5] - """ - if not sizes: - return [] - result: list[int | list[int]] = [] - current = sizes[0] - count = 1 - for s in sizes[1:]: - if s == current: - count += 1 - else: - result.append([current, count] if count > 1 else current) - current = s - count = 1 - result.append([current, count] if count > 1 else current) - return result - - -def validate_rectilinear_kind(kind: str | None) -> None: - """Validate the ``kind`` field of a rectilinear chunk grid configuration. - - The rectilinear spec requires ``kind: "inline"``. - """ - if kind is None: - raise ValueError( - "Rectilinear chunk grid configuration requires a 'kind' field. " - "Only 'inline' is currently supported." - ) - if kind != "inline": - raise ValueError( - f"Unsupported rectilinear chunk grid kind: {kind!r}. " - "Only 'inline' is currently supported." - ) - - -def validate_rectilinear_edges( - chunk_shapes: Sequence[int | Sequence[int]], array_shape: Sequence[int] -) -> None: - """Validate that rectilinear chunk edges cover the array extent per dimension. - - Bare-int dimensions (regular step) always cover any extent, so they are - skipped. Explicit edge lists must sum to at least the array extent. - """ - for i, (dim_spec, extent) in enumerate(zip(chunk_shapes, array_shape, strict=True)): - if isinstance(dim_spec, int): - continue - edge_sum = sum(dim_spec) - if edge_sum < extent: - raise ValueError( - f"Rectilinear chunk edges for dimension {i} sum to {edge_sum} " - f"but array shape extent is {extent} (edge sum must be >= extent)" - ) +from __future__ import annotations + +import asyncio +import math +import warnings +from collections.abc import Iterable, Mapping, Sequence +from enum import Enum +from typing import ( + TYPE_CHECKING, + Any, + Final, + Literal, + NotRequired, + TypedDict, + cast, + overload, +) + +import numpy as np +from typing_extensions import ReadOnly + +from zarr.core.config import config as zarr_config +from zarr.core.json_parse import convert, parse_field +from zarr.errors import ZarrRuntimeWarning + +if TYPE_CHECKING: + from collections.abc import Awaitable, Callable, Iterator + + from zarr.core.metadata.v3 import ChunkGridMetadata + + +ZARR_JSON = "zarr.json" +ZARRAY_JSON = ".zarray" +ZGROUP_JSON = ".zgroup" +ZATTRS_JSON = ".zattrs" +ZMETADATA_V2_JSON = ".zmetadata" + +BytesLike = bytes | bytearray | memoryview +ShapeLike = Iterable[int | np.integer[Any]] | int | np.integer[Any] +# Per-dimension chunk specs may mix a bare int (uniform chunk size, the +# rectilinear spec's step-size shorthand) with explicit edge-length sequences. +# A stored chunk grid (`ChunkGridMetadata`) is also accepted and is used +# verbatim, under the tolerant stored-metadata validation rules (e.g. trailing +# edges beyond the array extent, as left behind by a shrinking resize). +type ChunksLike = ShapeLike | Iterable[int | Iterable[int]] | ChunkGridMetadata +# For backwards compatibility +ChunkCoords = tuple[int, ...] +ZarrFormat = Literal[2, 3] +NodeType = Literal["array", "group"] +JSON = str | int | float | bool | Mapping[str, "JSON"] | Sequence["JSON"] | None +MemoryOrder = Literal["C", "F"] +AccessModeLiteral = Literal["r", "r+", "a", "w", "w-"] +ANY_ACCESS_MODE: Final = "r", "r+", "a", "w", "w-" +DimensionNamesLike = Iterable[str | None] | None +DimensionNames = DimensionNamesLike # for backwards compatibility + + +def validate_node_path(path: str, zarr_format: ZarrFormat) -> None: + """Validate node names in a normalized hierarchy path before creating them.""" + if zarr_format != 3 or not path: + return + + for name in path.split("/"): + if all(char == "." for char in name) or name.startswith("__") or name == ZARR_JSON: + raise ValueError( + f"Invalid Zarr format 3 node name {name!r} in path {path!r}. " + "Node names must not consist only of periods, start with '__', " + "or equal 'zarr.json'." + ) + + +class NamedConfig[TName: str, TConfig: Mapping[str, object]](TypedDict): + """ + A typed dictionary representing an object with a name and configuration, where the configuration + is an optional mapping of string keys to values, e.g. another typed dictionary or a JSON object. + + This class is generic with two type parameters: the type of the name (``TName``) and the type of + the configuration (``TConfig``). + """ + + name: ReadOnly[TName] + """The name of the object.""" + + configuration: NotRequired[ReadOnly[TConfig]] + """The configuration of the object. Not required.""" + + +class NamedRequiredConfig[TName: str, TConfig: Mapping[str, object]](TypedDict): + """ + A typed dictionary representing an object with a name and configuration, where the configuration + is a mapping of string keys to values, e.g. another typed dictionary or a JSON object. + + This class is generic with two type parameters: the type of the name (``TName``) and the type of + the configuration (``TConfig``). + """ + + name: ReadOnly[TName] + """The name of the object.""" + + configuration: ReadOnly[TConfig] + """The configuration of the object.""" + + +def product(tup: tuple[int, ...]) -> int: + return math.prod(tup) + + +def ceildiv(a: float, b: float) -> int: + """Ceiling of ``a / b`` using floating-point division; zero when ``a`` is zero.""" + if a == 0: + return 0 + return math.ceil(a / b) + + +def ceildiv_int(a: int, b: int) -> int: + """Ceiling of integer division using exact Python integer arithmetic.""" + return -(-int(a) // int(b)) + + +def concurrent_iter[T: tuple[Any, ...], V]( + items: Iterable[T], + func: Callable[..., Awaitable[V]], + limit: int | None = None, +) -> list[asyncio.Task[V]]: + """Launch `func(*item)` for each item concurrently, returning the tasks. + + When `limit` is set, no more than `limit` calls are in flight at once. + Tasks are returned in input order; callers that want completion order + should wrap the result in `asyncio.as_completed`. + + Every task is scheduled (via `ensure_future`) before this function + returns, not on first iteration of the result. That matters for callers + that await the returned tasks one at a time — without eager scheduling, + each coroutine would only start when individually awaited, serializing + the work and defeating the semaphore. It also makes the return type + honest (real `Task`s support `.cancel()`, `.done()`, callbacks) rather + than bare coroutines. + + See https://docs.python.org/3/library/asyncio-task.html#coroutines: + "Note that simply calling a coroutine will not schedule it to be executed:" + """ + if limit is None: + return [asyncio.ensure_future(func(*item)) for item in items] + + sem = asyncio.Semaphore(limit) + + async def run(item: T) -> V: + async with sem: + return await func(*item) + + return [asyncio.ensure_future(run(item)) for item in items] + + +async def concurrent_map[T: tuple[Any, ...], V]( + items: Iterable[T], + func: Callable[..., Awaitable[V]], + limit: int | None = None, +) -> list[V]: + return await asyncio.gather(*concurrent_iter(items, func, limit)) + + +async def concurrent_foreach[T: tuple[Any, ...]]( + items: Iterable[T], + func: Callable[..., Awaitable[object]], + limit: int, +) -> None: + """Run `func(*item)` for every item with at most `limit` calls in flight. + + Unlike `concurrent_map`, `items` is consumed lazily: `limit` workers share + one iterator and each takes the next item as soon as it finishes the + previous one, so auxiliary memory is bounded by `limit`, not by the number + of items. Results are discarded. + + Once a call fails, no further items are taken and the calls still in + flight are cancelled. The failure is re-raised as itself, so callers catch + the store's own exception type. If other in-flight calls failed in the + same event-loop iteration, the exception raised is the one from the + lowest-numbered worker and the `BaseExceptionGroup` holding every failure + is attached as its `__cause__`, so nothing is lost from tracebacks and + logs. Cancellation of the caller propagates as `CancelledError`. + """ + if limit < 1: + raise ValueError(f"limit must be at least 1, got {limit}") + shared = iter(items) + failed = False + + async def worker() -> None: + nonlocal failed + # `next()` on the shared iterator is synchronous, so workers never + # re-enter it: a worker only yields to the loop while awaiting `func`. + while not failed: + try: + item = next(shared) + except StopIteration: + return + try: + await func(*item) + except BaseException: + failed = True + raise + + try: + async with asyncio.TaskGroup() as tg: + for _ in range(limit): + tg.create_task(worker()) + except BaseExceptionGroup as eg: + if len(eg.exceptions) == 1: + raise eg.exceptions[0] from None + raise eg.exceptions[0] from eg + + +def enum_names[E: Enum](enum: type[E]) -> Iterator[str]: + for item in enum: + yield item.name + + +def parse_enum[E: Enum](data: object, cls: type[E]) -> E: + if isinstance(data, cls): + return data + if not isinstance(data, str): + raise TypeError(f"Expected str, got {type(data)}") + if data in enum_names(cls): + return cls(data) + raise ValueError(f"Value must be one of {list(enum_names(cls))!r}. Got {data} instead.") + + +def parse_name(data: JSON, expected: str | None = None) -> str: + try: + data = cast("str", convert(data, str)) + except (ValueError, TypeError) as exc: + raise TypeError(f"Expected a string, got an instance of {type(data)}.") from exc + if expected is None or data == expected: + return data + raise ValueError(f"Expected '{expected}'. Got {data} instead.") + + +def parse_configuration(data: JSON) -> JSON: + if not isinstance(data, dict): + raise TypeError(f"Expected dict, got {type(data)}") + return data + + +@overload +def parse_named_configuration( + data: JSON | NamedConfig[str, Any], expected_name: str | None = None +) -> tuple[str, dict[str, JSON]]: ... + + +@overload +def parse_named_configuration( + data: JSON | NamedConfig[str, Any], + expected_name: str | None = None, + *, + require_configuration: bool = True, +) -> tuple[str, dict[str, JSON] | None]: ... + + +def parse_named_configuration( + data: JSON | NamedConfig[str, Any], + expected_name: str | None = None, + *, + require_configuration: bool = True, +) -> tuple[str, JSON | None]: + if not isinstance(data, dict): + raise TypeError(f"Expected dict, got {type(data)}") + if "name" not in data: + raise ValueError(f"Named configuration does not have a 'name' key. Got {data}.") + name_parsed = parse_name(data["name"], expected_name) + if "configuration" in data: + configuration_parsed = parse_configuration(data["configuration"]) + elif require_configuration: + raise ValueError(f"Named configuration does not have a 'configuration' key. Got {data}.") + else: + configuration_parsed = None + return name_parsed, configuration_parsed + + +def parse_shapelike(data: ShapeLike) -> tuple[int, ...]: + """ + Parse a shape-like input into an explicit shape. + """ + if isinstance(data, int | np.integer): + if data < 0: + raise ValueError(f"Expected a non-negative integer. Got {data} instead") + return (int(data),) + try: + data_tuple = tuple(data) + except TypeError as e: + msg = f"Expected an integer or an iterable of integers. Got {data} instead." + raise TypeError(msg) from e + + if not all(isinstance(v, int | np.integer) for v in data_tuple): + msg = f"Expected an iterable of integers. Got {data} instead." + raise TypeError(msg) + if not all(v > -1 for v in data_tuple): + msg = f"Expected all values to be non-negative. Got {data} instead." + raise ValueError(msg) + + # cast NumPy scalars to plain python ints + return tuple(int(x) for x in data_tuple) + + +def parse_fill_value(data: Any) -> Any: + # todo: real validation + return data + + +def parse_order(data: Any) -> Literal["C", "F"]: + return cast("Literal['C', 'F']", parse_field(data, Literal["C", "F"], "order")) + + +def parse_bool(data: Any) -> bool: + return cast("bool", convert(data, bool)) + + +def parse_int(data: Any) -> int: + if isinstance(data, int) and not isinstance(data, bool): + return data + raise ValueError(f"Expected int, got {data} instead.") + + +def _warn_write_empty_chunks_kwarg() -> None: + # TODO: link to docs page on array configuration in this message + msg = ( + "The `write_empty_chunks` keyword argument is deprecated and will be removed in future versions. " + "To control whether empty chunks are written to storage, either use the `config` keyword " + "argument, as in `config={'write_empty_chunks': True}`," + "or change the global 'array.write_empty_chunks' configuration variable." + ) + warnings.warn(msg, ZarrRuntimeWarning, stacklevel=2) + + +def _warn_order_kwarg() -> None: + # TODO: link to docs page on array configuration in this message + msg = ( + "The `order` keyword argument has no effect for Zarr format 3 arrays. " + "To control the memory layout of the array, either use the `config` keyword " + "argument, as in `config={'order': 'C'}`," + "or change the global 'array.order' configuration variable." + ) + warnings.warn(msg, ZarrRuntimeWarning, stacklevel=2) + + +def _default_zarr_format() -> ZarrFormat: + """Return the default zarr_format.""" + return cast("ZarrFormat", int(zarr_config.get("default_zarr_format", 3))) + + +def _subject(name: str, axis: int | None) -> str: + """`name` as the subject of an error message, prefixed by the dimension `axis`.""" + return name[0].upper() + name[1:] if axis is None else f"Dimension {axis}: {name}" + + +def _parse_positive_int(value: object, name: str, axis: int | None) -> int: + """`value` as an `int` of at least 1. A `bool` is read as the `int` it equals; any + other type, a NumPy integer or a float (even an integral one: stored documents with + integral floats are read by `zarr.core.metadata.repair`), is rejected.""" + subject = _subject(name, axis) + if not isinstance(value, int): + raise TypeError(f"{subject} must be an int, got {value!r}") + if value < 1: + raise ValueError(f"{subject} must be >= 1, got {value!r}") + return int(value) + + +def parse_chunk_edge(size: object, axis: int | None = None) -> int: + """Check that `size` is a chunk edge length: an `int` of at least 1 (a `bool` is + read as the `int` it equals). + + This is the one rule for chunk edge lengths in metadata: bare chunk sizes, explicit + edges and run-length encoded sizes. `axis`, when given, is named in the error. + """ + return _parse_positive_int(size, "chunk edge length", axis) + + +def parse_chunk_shape(data: object) -> tuple[int, ...]: + """Check a regular chunk shape: an iterable, other than a string or a mapping, of one + chunk edge length per axis (see `parse_chunk_edge`).""" + match data: + case str() | Mapping(): + pass + case Iterable(): + return tuple(parse_chunk_edge(size, axis) for axis, size in enumerate(data)) + raise TypeError(f"A chunk shape must be an iterable of chunk edge lengths, got {data!r}") + + +def expand_rle(data: Sequence[object], axis: int | None = None) -> list[int]: + """Expand a mixed array of bare integers and RLE pairs, the edges of dimension + `axis` (named in errors, when given). + + Per the rectilinear chunk grid spec, each element can be: + - a bare integer (an explicit edge length) + - a two-element array ``[value, count]`` (run-length encoded) + """ + result: list[int] = [] + for item in data: + if isinstance(item, list): + if len(item) != 2: + subject = _subject("RLE entries", axis) + raise ValueError(f"{subject} must be an integer or [size, count], got {item}") + size, count = item + repeat = _parse_positive_int(count, "RLE repeat count", axis) + result.extend([parse_chunk_edge(size, axis)] * repeat) + else: + result.append(parse_chunk_edge(item, axis)) + return result + + +def compress_rle(sizes: Sequence[int]) -> list[int | list[int]]: + """Compress chunk sizes to mixed RLE format per the rectilinear spec. + + Runs of length > 1 are emitted as ``[value, count]`` pairs; runs of + length 1 are emitted as bare integers:: + + [10, 10, 10, 5] -> [[10, 3], 5] + """ + if not sizes: + return [] + result: list[int | list[int]] = [] + current = sizes[0] + count = 1 + for s in sizes[1:]: + if s == current: + count += 1 + else: + result.append([current, count] if count > 1 else current) + current = s + count = 1 + result.append([current, count] if count > 1 else current) + return result + + +def validate_rectilinear_kind(kind: str | None) -> None: + """Validate the ``kind`` field of a rectilinear chunk grid configuration. + + The rectilinear spec requires ``kind: "inline"``. + """ + if kind is None: + raise ValueError( + "Rectilinear chunk grid configuration requires a 'kind' field. " + "Only 'inline' is currently supported." + ) + if kind != "inline": + raise ValueError( + f"Unsupported rectilinear chunk grid kind: {kind!r}. " + "Only 'inline' is currently supported." + ) + + +def validate_rectilinear_edges( + chunk_shapes: Sequence[int | Sequence[int]], array_shape: Sequence[int] +) -> None: + """Validate that rectilinear chunk edges cover the array extent per dimension. + + Bare-int dimensions (regular step) always cover any extent, so they are + skipped. Explicit edge lists must sum to at least the array extent. + """ + for i, (dim_spec, extent) in enumerate(zip(chunk_shapes, array_shape, strict=True)): + if isinstance(dim_spec, int): + continue + edge_sum = sum(dim_spec) + if edge_sum < extent: + raise ValueError( + f"Rectilinear chunk edges for dimension {i} sum to {edge_sum} " + f"but array shape extent is {extent} (edge sum must be >= extent)" + ) diff --git a/src/zarr/core/group.py b/src/zarr/core/group.py index 7792fa58de..2932aeb6a7 100644 --- a/src/zarr/core/group.py +++ b/src/zarr/core/group.py @@ -1,3946 +1,3950 @@ -from __future__ import annotations - -import asyncio -import logging -import unicodedata -import warnings -from collections import defaultdict -from dataclasses import asdict, dataclass, field, replace -from itertools import accumulate -from typing import TYPE_CHECKING, Final, Literal, assert_never, cast, overload - -import numpy as np - -import zarr.api.asynchronous as async_api -from zarr.abc.metadata import Metadata -from zarr.abc.store import Store, set_or_delete -from zarr.core._info import GroupInfo -from zarr.core._json import buffer_to_json_object, json_to_buffer -from zarr.core.array import ( - DEFAULT_FILL_VALUE, - Array, - AsyncArray, - CompressorLike, - CompressorsLike, - FiltersLike, - SerializerLike, - ShardsLike, - _parse_deprecated_compressor, - create_array, -) -from zarr.core.array_spec import parse_array_config -from zarr.core.attributes import Attributes -from zarr.core.buffer import default_buffer_prototype -from zarr.core.common import ( - JSON, - ZARR_JSON, - ZARRAY_JSON, - ZATTRS_JSON, - ZGROUP_JSON, - ZMETADATA_V2_JSON, - ChunksLike, - DimensionNamesLike, - NodeType, - ShapeLike, - ZarrFormat, - parse_shapelike, -) -from zarr.core.config import config -from zarr.core.dtype import parse_data_type -from zarr.core.json_parse import parse_field -from zarr.core.metadata import ArrayV2Metadata, ArrayV3Metadata -from zarr.core.metadata.io import ( - encode_documents, - save_metadata, - save_new_metadata, - store_documents, -) -from zarr.core.metadata.v3 import AllowedExtraField, check_storable, parse_extra_fields -from zarr.core.sync import SyncMixin, sync -from zarr.errors import ( - ArrayNotFoundError, - ContainsArrayError, - ContainsGroupError, - GroupNotFoundError, - MetadataValidationError, - NodeTypeValidationError, - ZarrUserWarning, -) -from zarr.storage import StoreLike, StorePath -from zarr.storage._common import make_store_path -from zarr.storage._utils import _join_paths, _normalize_path_keys, normalize_path - -if TYPE_CHECKING: - from collections.abc import ( - AsyncGenerator, - AsyncIterator, - Coroutine, - Generator, - Iterable, - Iterator, - Mapping, - ) - from typing import Any - - from zarr.core.array_spec import ArrayConfigLike - from zarr.core.buffer import Buffer, BufferPrototype - from zarr.core.chunk_key_encodings import ChunkKeyEncodingLike - from zarr.core.common import MemoryOrder - from zarr.core.dtype import ZDTypeLike - from zarr.types import AnyArray, AnyAsyncArray, ArrayV2, ArrayV3, AsyncArrayV2, AsyncArrayV3 - -logger = logging.getLogger("zarr.group") - - -def parse_zarr_format(data: Any) -> ZarrFormat: - """Parse the zarr_format field from metadata.""" - return cast("ZarrFormat", parse_field(data, Literal[2, 3], "zarr_format")) - - -def parse_node_type(data: Any) -> NodeType: - """Parse the node_type field from metadata.""" - return cast( - "Literal['array', 'group']", - parse_field(data, Literal["array", "group"], "node_type", error=MetadataValidationError), - ) - - -# todo: convert None to empty dict -def parse_attributes(data: Any) -> dict[str, Any]: - """Parse the attributes field from metadata.""" - if data is None: - return {} - elif isinstance(data, dict) and all(isinstance(k, str) for k in data): - return dict(data) - msg = f"Expected dict with string keys. Got {type(data)} instead." - raise TypeError(msg) - - -@overload -def _parse_async_node(node: AsyncArrayV3) -> ArrayV3: ... - - -@overload -def _parse_async_node(node: AsyncArrayV2) -> ArrayV2: ... - - -@overload -def _parse_async_node(node: AsyncGroup) -> Group: ... - - -def _parse_async_node( - node: AnyAsyncArray | AsyncGroup, -) -> AnyArray | Group: - """Wrap an AsyncArray in an Array, or an AsyncGroup in a Group.""" - if isinstance(node, AsyncArray): - return Array(node) - elif isinstance(node, AsyncGroup): - return Group(node) - else: - raise TypeError(f"Unknown node type, got {type(node)}") - - -@dataclass(frozen=True) -class ConsolidatedMetadata: - """ - Consolidated Metadata for this Group. - - This stores the metadata of child nodes below this group. Any child groups - will have their consolidated metadata set appropriately. - """ - - metadata: dict[str, ArrayV2Metadata | ArrayV3Metadata | GroupMetadata] - kind: Literal["inline"] = "inline" - must_understand: Literal[False] = False - - def to_dict(self) -> dict[str, JSON]: - """The consolidated metadata document. An array read from a stored document that - had to be repaired is written as that document was stored, so every reader of - the consolidated metadata reads it as repaired again (see `mark_repaired`).""" - return { - "kind": self.kind, - "must_understand": self.must_understand, - "metadata": { - k: v.to_dict() - if isinstance(v, GroupMetadata) or v._stored_document is None - else dict(v._stored_document) - for k, v in sorted( - self.flattened_metadata.items(), - key=lambda item: ( - item[0].count("/"), - unicodedata.normalize("NFKC", item[0]).casefold(), - ), - ) - }, - } - - @classmethod - def from_dict(cls, data: dict[str, JSON], *, path: str | None = None) -> ConsolidatedMetadata: - """Read consolidated metadata, naming each member by its path under the group at - `path` (or relative to that group, when `path` is not given) in warnings.""" - data = dict(data) - - kind = data.get("kind") - if kind != "inline": - raise ValueError(f"Consolidated metadata kind='{kind}' is not supported.") - - raw_metadata = data.get("metadata") - if not isinstance(raw_metadata, dict): - raise TypeError(f"Unexpected type for 'metadata': {type(raw_metadata)}") - - metadata: dict[str, ArrayV2Metadata | ArrayV3Metadata | GroupMetadata] = {} - if raw_metadata: - for k, v in raw_metadata.items(): - member = k if path is None else _join_paths([path, k]) - if not isinstance(v, dict): - raise TypeError( - f"Invalid value for metadata items. key='{k}', type='{type(v).__name__}'" - ) - - # zarr_format is present in v2 and v3. - zarr_format = parse_zarr_format(v["zarr_format"]) - - if zarr_format == 3: - node_type = parse_node_type(v.get("node_type", None)) - if node_type == "group": - metadata[k] = GroupMetadata.from_dict(v, path=member) - elif node_type == "array": - metadata[k] = ArrayV3Metadata.from_dict(v, path=member) - else: - assert_never(node_type) - elif zarr_format == 2: - if "shape" in v: - metadata[k] = ArrayV2Metadata.from_dict(v, path=member) - else: - metadata[k] = GroupMetadata.from_dict(v, path=member) - else: - assert_never(zarr_format) - - cls._flat_to_nested(metadata) - - return cls(metadata=metadata) - - @staticmethod - def _flat_to_nested( - metadata: dict[str, ArrayV2Metadata | ArrayV3Metadata | GroupMetadata], - ) -> None: - """ - Convert a flat metadata representation to a nested one. - - Notes - ----- - Flat metadata is used when persisting the consolidated metadata. The keys - include the full path, not just the node name. The key prefixes can be - used to determine which nodes are children of which other nodes. - - Nested metadata is used in-memory. The outermost level will only have the - *immediate* children of the Group. All nested child groups will be stored - under the consolidated metadata of their immediate parent. - """ - # We have a flat mapping from {k: v} where the keys include the *full* - # path segment: - # { - # "/a/b": { group_metadata }, - # "/a/b/array-0": { array_metadata }, - # "/a/b/array-1": { array_metadata }, - # } - # - # We want to reorganize the metadata such that each Group contains the - # array metadata of its immediate children. - # In the example, the group at `/a/b` will have consolidated metadata - # for its children `array-0` and `array-1`. - - # Group keys by their parent path. This must not rely on same-parent keys - # being adjacent: the persisted key order is arbitrary, so accumulate - # instead of using itertools.groupby, which only groups consecutive runs. - grouped: dict[str, list[str]] = defaultdict(list) - for k in sorted(metadata, key=lambda k: k.count("/")): - grouped[k.rsplit("/", 1)[0]].append(k) - - # we go top down and directly manipulate metadata. - for key, children_keys in grouped.items(): - # key is a key like "a", "a/b", "a/b/c" - # The basic idea is to find the immediate parent (so "", "a", or "a/b") - # and update that node's consolidated metadata to include the metadata - # in children_keys - *prefixes, name = key.split("/") - parent = metadata - - while prefixes: - # e.g. a/b/c has a parent "a/b". Walk through to get - # metadata["a"]["b"] - part = prefixes.pop(0) - # we can assume that parent[part] here is a group - # otherwise we wouldn't have a node with this `part` prefix. - # We can also assume that the parent node will have consolidated metadata, - # because we're walking top to bottom. - parent = parent[part].consolidated_metadata.metadata # type: ignore[union-attr] - - node = parent[name] - children_keys = list(children_keys) - - if isinstance(node, ArrayV2Metadata | ArrayV3Metadata): - # These are already present, either thanks to being an array in the - # root, or by being collected as a child in the else clause - continue - children: dict[str, ArrayV2Metadata | ArrayV3Metadata | GroupMetadata] = {} - # We pop from metadata, since we're *moving* this under group. - # While doing this, normalize leaf groups to carry empty consolidated metadata. - for child_key in children_keys: - if child_key == key: - continue - child = metadata.pop(child_key) - if isinstance(child, GroupMetadata) and child.consolidated_metadata is None: - child = replace(child, consolidated_metadata=ConsolidatedMetadata(metadata={})) - children[child_key.split("/")[-1]] = child - - parent[name] = replace( - node, consolidated_metadata=ConsolidatedMetadata(metadata=children) - ) - - @property - def flattened_metadata(self) -> dict[str, ArrayV2Metadata | ArrayV3Metadata | GroupMetadata]: - """ - Return the flattened representation of Consolidated Metadata. - - The returned dictionary will have a key for each child node in the hierarchy - under this group. Under the default (nested) representation available through - ``self.metadata``, the dictionary only contains keys for immediate children. - - The keys of the dictionary will include the full path to a child node from - the current group, where segments are joined by ``/``. - - Examples - -------- - ```python - from zarr.core.group import ConsolidatedMetadata, GroupMetadata - cm = ConsolidatedMetadata( - metadata={ - "group-0": GroupMetadata( - consolidated_metadata=ConsolidatedMetadata( - { - "group-0-0": GroupMetadata(), - } - ) - ), - "group-1": GroupMetadata(), - } - ) - # {'group-0': GroupMetadata(attributes={}, zarr_format=3, consolidated_metadata=None, node_type='group', extra_fields={}), - # 'group-0/group-0-0': GroupMetadata(attributes={}, zarr_format=3, consolidated_metadata=None, node_type='group', extra_fields={}), - # 'group-1': GroupMetadata(attributes={}, zarr_format=3, consolidated_metadata=None, node_type='group', extra_fields={})} - ``` - """ - metadata = {} - - def flatten( - key: str, group: GroupMetadata | ArrayV2Metadata | ArrayV3Metadata - ) -> dict[str, ArrayV2Metadata | ArrayV3Metadata | GroupMetadata]: - children: dict[str, ArrayV2Metadata | ArrayV3Metadata | GroupMetadata] = {} - if isinstance(group, ArrayV2Metadata | ArrayV3Metadata): - children[key] = group - else: - if group.consolidated_metadata and group.consolidated_metadata.metadata is not None: - children[key] = replace( - group, consolidated_metadata=ConsolidatedMetadata(metadata={}) - ) - for name, val in group.consolidated_metadata.metadata.items(): - full_key = f"{key}/{name}" - if isinstance(val, GroupMetadata): - children.update(flatten(full_key, val)) - else: - children[full_key] = val - else: - children[key] = replace(group, consolidated_metadata=None) - return children - - for k, v in self.metadata.items(): - metadata.update(flatten(k, v)) - - return metadata - - -GROUP_METADATA_KEYS: Final[set[str]] = { - "zarr_format", - "node_type", - "attributes", - "consolidated_metadata", -} -""" -The names of the fields of the group metadata document. `consolidated_metadata` is not in the -Zarr V3 spec; Zarr-Python writes it. -""" - - -@dataclass(frozen=True) -class GroupMetadata(Metadata): - """ - Metadata for a Group. - """ - - attributes: dict[str, Any] = field(default_factory=dict) - zarr_format: ZarrFormat = 3 - consolidated_metadata: ConsolidatedMetadata | None = None - node_type: Literal["group"] = field(default="group", init=False) - extra_fields: dict[str, AllowedExtraField] = field(default_factory=dict) - - def to_buffer_dict(self, prototype: BufferPrototype) -> dict[str, Buffer]: - if self.consolidated_metadata is not None: - for path, member in self.consolidated_metadata.flattened_metadata.items(): - # A member read from a document that had to be repaired is stored as it - # was stored (see `ConsolidatedMetadata.to_dict`). - if isinstance(member, ArrayV3Metadata) and member._stored_document is None: - try: - check_storable(member) - except ValueError as e: - e.add_note(f"Array {path!r} in the consolidated metadata.") - raise - indent = config.get("json_indent") - if self.zarr_format == 3: - return {ZARR_JSON: json_to_buffer(self.to_dict(), prototype=prototype, indent=indent)} - else: - items = { - ZGROUP_JSON: json_to_buffer( - {"zarr_format": self.zarr_format}, prototype=prototype, indent=indent - ), - ZATTRS_JSON: json_to_buffer(self.attributes, prototype=prototype, indent=indent), - } - if self.consolidated_metadata: - d = { - ZGROUP_JSON: {"zarr_format": self.zarr_format}, - ZATTRS_JSON: self.attributes, - } - consolidated_metadata = self.consolidated_metadata.to_dict()["metadata"] - if not isinstance(consolidated_metadata, dict): - raise TypeError( - "Expected consolidated metadata to serialize to a dict, " - f"got {type(consolidated_metadata).__name__}." - ) - for k, v in consolidated_metadata.items(): - attrs = v.pop("attributes", {}) - d[f"{k}/{ZATTRS_JSON}"] = attrs - if "shape" in v: - # it's an array - d[f"{k}/{ZARRAY_JSON}"] = v - else: - d[f"{k}/{ZGROUP_JSON}"] = { - "zarr_format": self.zarr_format, - "consolidated_metadata": { - "metadata": {}, - "must_understand": False, - "kind": "inline", - }, - } - - # The consolidated metadata blob is written compactly (no indent). - items[ZMETADATA_V2_JSON] = json_to_buffer( - {"metadata": d, "zarr_consolidated_format": 1}, prototype=prototype - ) - - return items - - def __init__( - self, - attributes: dict[str, Any] | None = None, - zarr_format: ZarrFormat = 3, - consolidated_metadata: ConsolidatedMetadata | None = None, - extra_fields: Mapping[str, AllowedExtraField] | None = None, - ) -> None: - attributes_parsed = parse_attributes(attributes) - zarr_format_parsed = parse_zarr_format(zarr_format) - if zarr_format_parsed == 2 and extra_fields: - raise ValueError( - "Invalid extra fields. Zarr format 2 group metadata does not support extra fields." - ) - extra_fields_parsed = parse_extra_fields( - extra_fields, reserved_keys=GROUP_METADATA_KEYS, node_type="group" - ) - - object.__setattr__(self, "attributes", attributes_parsed) - object.__setattr__(self, "zarr_format", zarr_format_parsed) - object.__setattr__(self, "consolidated_metadata", consolidated_metadata) - object.__setattr__(self, "extra_fields", extra_fields_parsed) - - @classmethod - def from_dict(cls, data: dict[str, Any], *, path: str | None = None) -> GroupMetadata: - """Read a stored group document; `path` names the group in warnings about the - consolidated metadata it holds.""" - data = dict(data) - node_type = data.pop("node_type", None) - if node_type not in ("group", None): - raise NodeTypeValidationError( - f"Invalid value for 'node_type'. Expected 'group' or None. Got {node_type!r}." - ) - consolidated_metadata = data.pop("consolidated_metadata", None) - if consolidated_metadata: - data["consolidated_metadata"] = ConsolidatedMetadata.from_dict( - consolidated_metadata, path=path - ) - - zarr_format = data.get("zarr_format") - if zarr_format == 3: - # Zarr v3 allows an extra key only if it is an object with "must_understand": false. - extra_fields = parse_extra_fields( - {k: v for k, v in data.items() if k not in GROUP_METADATA_KEYS}, - reserved_keys=GROUP_METADATA_KEYS, - node_type="group", - ) - else: - # Zarr v2 allowed arbitrary keys; they are dropped rather than kept. - extra_fields = {} - data = {k: v for k, v in data.items() if k in GROUP_METADATA_KEYS} - return cls(**data, extra_fields=extra_fields) - - def to_dict(self) -> dict[str, Any]: - result = asdict(replace(self, consolidated_metadata=None)) - result.update(result.pop("extra_fields")) - if self.consolidated_metadata is not None: - result["consolidated_metadata"] = self.consolidated_metadata.to_dict() - else: - # Leave consolidated metadata unset if it's None - result.pop("consolidated_metadata") - return result - - -@dataclass(frozen=True) -class ImplicitGroupMarker(GroupMetadata): - """ - Marker for an implicit group. Instances of this class are only used in the context of group - creation as a placeholder to represent groups that should only be created if they do not - already exist in storage - """ - - -@dataclass(frozen=True) -class AsyncGroup: - """ - Asynchronous Group object. - """ - - metadata: GroupMetadata - store_path: StorePath - - # TODO: make this correct and work - # TODO: ensure that this can be bound properly to subclass of AsyncGroup - - @classmethod - async def from_store( - cls, - store: StoreLike, - *, - attributes: dict[str, Any] | None = None, - overwrite: bool = False, - zarr_format: ZarrFormat = 3, - ) -> AsyncGroup: - store_path = await make_store_path(store) - attributes = attributes or {} - group = cls( - metadata=GroupMetadata(attributes=attributes, zarr_format=zarr_format), - store_path=store_path, - ) - await save_new_metadata( - store_path, group.metadata, overwrite=overwrite, ensure_parents=True - ) - return group - - @classmethod - async def open( - cls, - store: StoreLike, - zarr_format: ZarrFormat | None = 3, - use_consolidated: bool | str | None = None, - ) -> AsyncGroup: - """Open a new AsyncGroup - - Parameters - ---------- - store : StoreLike - zarr_format : {2, 3}, optional - use_consolidated : bool or str, default None - Whether to use consolidated metadata. - - By default, consolidated metadata is used if it's present in the - store (in the ``zarr.json`` for Zarr format 3 and in the ``.zmetadata`` file - for Zarr format 2) and the Store supports it. - - To explicitly require consolidated metadata, set ``use_consolidated=True``. - In this case, if the Store doesn't support consolidation or consolidated metadata is - not found, a ``ValueError`` exception is raised. - - To explicitly *not* use consolidated metadata, set ``use_consolidated=False``, - which will fall back to using the regular, non consolidated metadata. - - Zarr format 2 allowed configuring the key storing the consolidated metadata - (``.zmetadata`` by default). Specify the custom key as ``use_consolidated`` - to load consolidated metadata from a non-default key. - """ - store_path = await make_store_path(store) - if not store_path.store.supports_consolidated_metadata: - # Fail if consolidated metadata was requested but the Store doesn't support it - if use_consolidated: - store_name = type(store_path.store).__name__ - raise ValueError( - f"The Zarr store in use ({store_name}) doesn't support consolidated metadata." - ) - - # if use_consolidated was None (optional), the Store dictates it doesn't want consolidation - use_consolidated = False - - consolidated_key = ZMETADATA_V2_JSON - - if (zarr_format == 2 or zarr_format is None) and isinstance(use_consolidated, str): - consolidated_key = use_consolidated - - if zarr_format == 2: - paths = [store_path / ZGROUP_JSON, store_path / ZATTRS_JSON] - if use_consolidated or use_consolidated is None: - paths.append(store_path / consolidated_key) - - zgroup_bytes, zattrs_bytes, *rest = await asyncio.gather( - *[path.get() for path in paths] - ) - if zgroup_bytes is None: - raise FileNotFoundError(store_path) - - if use_consolidated or use_consolidated is None: - maybe_consolidated_metadata_bytes = rest[0] - - else: - maybe_consolidated_metadata_bytes = None - - elif zarr_format == 3: - zarr_json_bytes = await (store_path / ZARR_JSON).get() - if zarr_json_bytes is None: - raise FileNotFoundError(store_path) - elif zarr_format is None: - ( - zarr_json_bytes, - zgroup_bytes, - zattrs_bytes, - maybe_consolidated_metadata_bytes, - ) = await asyncio.gather( - (store_path / ZARR_JSON).get(), - (store_path / ZGROUP_JSON).get(), - (store_path / ZATTRS_JSON).get(), - (store_path / str(consolidated_key)).get(), - ) - if zarr_json_bytes is not None and zgroup_bytes is not None: - # warn and favor v3 - msg = f"Both zarr.json (Zarr format 3) and .zgroup (Zarr format 2) metadata objects exist at {store_path}. Zarr format 3 will be used." - warnings.warn(msg, category=ZarrUserWarning, stacklevel=1) - if zarr_json_bytes is None and zgroup_bytes is None: - raise FileNotFoundError( - f"could not find zarr.json or .zgroup objects in {store_path}" - ) - # set zarr_format based on which keys were found - if zarr_json_bytes is not None: - zarr_format = 3 - else: - zarr_format = 2 - else: - msg = f"Invalid value for 'zarr_format'. Expected 2, 3, or None. Got '{zarr_format}'." # type: ignore[unreachable] - raise MetadataValidationError(msg) - - if zarr_format == 2: - if zgroup_bytes is None: - raise FileNotFoundError(store_path) - - if use_consolidated and maybe_consolidated_metadata_bytes is None: - # the user requested consolidated metadata, but it was missing - raise ValueError(consolidated_key) - - elif use_consolidated is False: - # the user explicitly opted out of consolidated_metadata. - # Discard anything we might have read. - maybe_consolidated_metadata_bytes = None - - return cls._from_bytes_v2( - store_path, zgroup_bytes, zattrs_bytes, maybe_consolidated_metadata_bytes - ) - else: - # V3 groups are comprised of a zarr.json object - if zarr_json_bytes is None: - raise FileNotFoundError(store_path) - if not isinstance(use_consolidated, bool | None): - raise TypeError("use_consolidated must be a bool or None for Zarr format 3.") - - return cls._from_bytes_v3( - store_path, - zarr_json_bytes, - use_consolidated=use_consolidated, - ) - - @classmethod - def _from_bytes_v2( - cls, - store_path: StorePath, - zgroup_bytes: Buffer, - zattrs_bytes: Buffer | None, - consolidated_metadata_bytes: Buffer | None, - ) -> AsyncGroup: - # V2 groups are comprised of a .zgroup and .zattrs objects - zgroup = buffer_to_json_object(zgroup_bytes) - zattrs = buffer_to_json_object(zattrs_bytes) if zattrs_bytes is not None else {} - group_metadata: dict[str, Any] = {**zgroup, "attributes": zattrs} - - if consolidated_metadata_bytes is not None: - v2_consolidated_doc = buffer_to_json_object(consolidated_metadata_bytes) - v2_consolidated_metadata = cast("dict[str, Any]", v2_consolidated_doc["metadata"]) - # We already read zattrs and zgroup. Should we ignore these? - v2_consolidated_metadata.pop(".zattrs", None) - v2_consolidated_metadata.pop(".zgroup", None) - - consolidated_metadata: defaultdict[str, dict[str, Any]] = defaultdict(dict) - - # keys like air/.zarray, air/.zattrs - for k, v in v2_consolidated_metadata.items(): - path, kind = k.rsplit("/.", 1) - - if kind == "zarray": - consolidated_metadata[path].update(v) - elif kind == "zattrs": - consolidated_metadata[path]["attributes"] = v - elif kind == "zgroup": - consolidated_metadata[path].update(v) - else: - raise ValueError(f"Invalid file type '{kind}' at path '{path}") - - group_metadata["consolidated_metadata"] = { - "metadata": dict(consolidated_metadata), - "kind": "inline", - "must_understand": False, - } - - return cls.from_dict(store_path, group_metadata) - - @classmethod - def _from_bytes_v3( - cls, - store_path: StorePath, - zarr_json_bytes: Buffer, - use_consolidated: bool | None, - ) -> AsyncGroup: - group_metadata = buffer_to_json_object(zarr_json_bytes) - if use_consolidated and group_metadata.get("consolidated_metadata") is None: - msg = f"Consolidated metadata requested with 'use_consolidated=True' but not found in '{store_path.path}'." - raise ValueError(msg) - - elif use_consolidated is False: - # Drop consolidated metadata if it's there. - group_metadata.pop("consolidated_metadata", None) - - return cls.from_dict(store_path, group_metadata) - - @classmethod - def from_dict( - cls, - store_path: StorePath, - data: dict[str, Any], - ) -> AsyncGroup: - node_type = data.pop("node_type", None) - if node_type == "array": - msg = f"An array already exists in store {store_path.store} at path {store_path.path}." - raise ContainsArrayError(msg) - elif node_type not in ("group", None): - msg = f"Node type in metadata ({node_type}) is not 'group'" - raise GroupNotFoundError(msg) - return cls( - metadata=GroupMetadata.from_dict(data, path=str(store_path)), - store_path=store_path, - ) - - async def setitem(self, key: str, value: Any) -> None: - """ - Fastpath for creating a new array - New arrays will be created with default array settings for the array type. - - Parameters - ---------- - key : str - Array name - value : array-like - Array data - """ - path = self.store_path / key - await async_api.save_array( - store=path, arr=value, zarr_format=self.metadata.zarr_format, overwrite=True - ) - - async def getitem( - self, - key: str, - ) -> AnyAsyncArray | AsyncGroup: - """ - Get a subarray or subgroup from the group. - - Parameters - ---------- - key : str - Array or group name - - Returns - ------- - AsyncArray or AsyncGroup - """ - store_path = self.store_path / key - logger.debug("key=%s, store_path=%s", key, store_path) - - # Consolidated metadata lets us avoid some I/O operations so try that first. - if self.metadata.consolidated_metadata is not None: - return self._getitem_consolidated(store_path, key, prefix=self.name) - try: - return await get_node( - store=store_path.store, path=store_path.path, zarr_format=self.metadata.zarr_format - ) - except FileNotFoundError as e: - raise KeyError(key) from e - - def _getitem_consolidated( - self, store_path: StorePath, key: str, prefix: str - ) -> AnyAsyncArray | AsyncGroup: - # getitem, in the special case where we have consolidated metadata. - # Note that this is a regular def (non async) function. - # This shouldn't do any additional I/O. - - # we support nested getitems like group/subgroup/array - indexers = normalize_path(key).split("/") - indexers.reverse() - metadata: ArrayV2Metadata | ArrayV3Metadata | GroupMetadata = self.metadata - - while indexers: - indexer = indexers.pop() - if isinstance(metadata, ArrayV2Metadata | ArrayV3Metadata): - # we've indexed into an array with group["array/subarray"]. Invalid. - raise KeyError(key) - if metadata.consolidated_metadata is None: - # we've indexed into a group without consolidated metadata. - # This isn't normal; typically, consolidated metadata - # will include explicit markers for when there are no child - # nodes as metadata={}. - # We have some freedom in exactly how we interpret this case. - # For now, we treat None as the same as {}, i.e. we don't - # have any children. - raise KeyError(key) - try: - metadata = metadata.consolidated_metadata.metadata[indexer] - except KeyError as e: - # The Group Metadata has consolidated metadata, but the key - # isn't present. We trust this to mean that the key isn't in - # the hierarchy, and *don't* fall back to checking the store. - msg = f"'{key}' not found in consolidated metadata." - raise KeyError(msg) from e - - # update store_path to ensure that AsyncArray/Group.name is correct - if prefix != "/": - key = "/".join([prefix.lstrip("/"), key]) - store_path = StorePath(store=store_path.store, path=key) - - if isinstance(metadata, GroupMetadata): - return AsyncGroup(metadata=metadata, store_path=store_path) - else: - return AsyncArray(metadata=metadata, store_path=store_path) - - async def delitem(self, key: str) -> None: - """Delete a group member. - - Parameters - ---------- - key : str - Array or group name - """ - store_path = self.store_path / key - consolidated = self.metadata.consolidated_metadata - if consolidated is None: - await store_path.delete_dir() - return - # Encode the group metadata without the member before deleting it: metadata that - # cannot be stored then fails with the store and this group untouched. What is - # stored is encoded after the deletion, from the metadata as it then is, so - # concurrent deletions each store the deletions made before them. - members = {name: node for name, node in consolidated.metadata.items() if name != key} - encode_documents( - self.store_path, - replace(self.metadata, consolidated_metadata=replace(consolidated, metadata=members)), - ) - await store_path.delete_dir() - # In place, so every handle sharing this consolidated metadata (a parent's or a - # subgroup's) sees the deletion. - consolidated.metadata.pop(key, None) - await store_documents(self.store_path, encode_documents(self.store_path, self.metadata)) - - async def get[DefaultT]( - self, key: str, default: DefaultT | None = None - ) -> AnyAsyncArray | AsyncGroup | DefaultT | None: - """Obtain a group member, returning default if not found. - - Parameters - ---------- - key : str - Group member name. - default : object - Default value to return if key is not found (default: None). - - Returns - ------- - object - Group member (AsyncArray or AsyncGroup) or default if not found. - """ - try: - return await self.getitem(key) - except KeyError: - return default - - async def get_array(self, path: str) -> AnyAsyncArray: - """Obtain an array member of this group, raising if it is absent or not an array. - - Parameters - ---------- - path : str - Path of the array relative to this group. May contain `/` to reference - a member of a subgroup, e.g. `subgroup/subarray`. - - Returns - ------- - AsyncArray - The array at the given path. - - Raises - ------ - ArrayNotFoundError - If no node exists at the given path. - ContainsGroupError - If the node at the given path is a group rather than an array. - """ - store_path = self.store_path / path - try: - node = await self.getitem(path) - except KeyError as e: - msg = f"No array found in store {store_path.store!r} at path {store_path.path!r}" - raise ArrayNotFoundError(msg) from e - if isinstance(node, AsyncGroup): - msg = f"A group exists in store {store_path.store!r} at path {store_path.path!r}." - raise ContainsGroupError(msg) - return node - - async def get_group(self, path: str) -> AsyncGroup: - """Obtain a group member of this group, raising if it is absent or not a group. - - Parameters - ---------- - path : str - Path of the group relative to this group. May contain `/` to reference - a member of a subgroup, e.g. `subgroup/subsubgroup`. - - Returns - ------- - AsyncGroup - The group at the given path. - - Raises - ------ - GroupNotFoundError - If no node exists at the given path. - ContainsArrayError - If the node at the given path is an array rather than a group. - """ - store_path = self.store_path / path - try: - node = await self.getitem(path) - except KeyError as e: - msg = f"No group found in store {store_path.store!r} at path {store_path.path!r}" - raise GroupNotFoundError(msg) from e - if isinstance(node, AsyncArray): - msg = f"An array exists in store {store_path.store!r} at path {store_path.path!r}." - raise ContainsArrayError(msg) - return node - - async def _save_metadata(self, ensure_parents: bool = False) -> None: - await save_metadata(self.store_path, self.metadata, ensure_parents=ensure_parents) - - @property - def path(self) -> str: - """Storage path.""" - return self.store_path.path - - @property - def name(self) -> str: - """Group name following h5py convention.""" - if self.path: - # follow h5py convention: add leading slash - name = self.path - if name[0] != "/": - name = f"/{name}" - return name - return "/" - - @property - def basename(self) -> str: - """Final component of name.""" - return self.name.split("/")[-1] - - @property - def attrs(self) -> dict[str, Any]: - return self.metadata.attributes - - @property - def info(self) -> Any: - """ - Return a visual representation of the statically known information about a group. - - Note that this doesn't include dynamic information, like the number of child - Groups or Arrays. - - Returns - ------- - GroupInfo - - Related - ------- - [zarr.AsyncGroup.info_complete][] - All information about a group, including dynamic information - """ - - if self.metadata.consolidated_metadata: - members = list(self.metadata.consolidated_metadata.flattened_metadata.values()) - else: - members = None - return self._info(members=members) - - async def info_complete(self) -> Any: - """ - Return all the information for a group. - - This includes dynamic information like the number - of child Groups or Arrays. If this group doesn't contain consolidated - metadata then this will need to read from the backing Store. - - Returns - ------- - GroupInfo - - Related - ------- - [zarr.AsyncGroup.info][] - """ - members = [x[1].metadata async for x in self.members(max_depth=None)] - return self._info(members=members) - - def _info( - self, members: list[ArrayV2Metadata | ArrayV3Metadata | GroupMetadata] | None = None - ) -> Any: - kwargs = {} - if members is not None: - kwargs["_count_members"] = len(members) - count_arrays = 0 - count_groups = 0 - for member in members: - if isinstance(member, GroupMetadata): - count_groups += 1 - else: - count_arrays += 1 - kwargs["_count_arrays"] = count_arrays - kwargs["_count_groups"] = count_groups - - return GroupInfo( - _name=self.store_path.path, - _read_only=self.read_only, - _store_type=type(self.store_path.store).__name__, - _zarr_format=self.metadata.zarr_format, - # maybe do a typeddict - **kwargs, # type: ignore[arg-type] - ) - - @property - def store(self) -> Store: - return self.store_path.store - - @property - def read_only(self) -> bool: - # Backwards compatibility for 2.x - return self.store_path.read_only - - @property - def synchronizer(self) -> None: - # Backwards compatibility for 2.x - # Not implemented in 3.x yet. - return None - - async def create_group( - self, - name: str, - *, - overwrite: bool = False, - attributes: dict[str, Any] | None = None, - ) -> AsyncGroup: - """Create a sub-group. - - Parameters - ---------- - name : str - Group name. - overwrite : bool, optional - If True, do not raise an error if the group already exists. - attributes : dict, optional - Group attributes. - - Returns - ------- - g : AsyncGroup - """ - attributes = attributes or {} - return await type(self).from_store( - self.store_path / name, - attributes=attributes, - overwrite=overwrite, - zarr_format=self.metadata.zarr_format, - ) - - async def require_group(self, name: str, overwrite: bool = False) -> AsyncGroup: - """Obtain a sub-group, creating one if it doesn't exist. - - Parameters - ---------- - name : str - Group name. - overwrite : bool, optional - Overwrite any existing group with given `name` if present. - - Returns - ------- - g : AsyncGroup - """ - if overwrite: - # TODO: check that overwrite=True errors if an array exists where the group is being created - grp = await self.create_group(name, overwrite=True) - else: - try: - item: AsyncGroup | AnyAsyncArray = await self.getitem(name) - if not isinstance(item, AsyncGroup): - raise TypeError( - f"Incompatible object ({item.__class__.__name__}) already exists" - ) - grp = item - except KeyError: - grp = await self.create_group(name) - return grp - - async def require_groups(self, *names: str) -> tuple[AsyncGroup, ...]: - """Convenience method to require multiple groups in a single call. - - Parameters - ---------- - *names : str - Group names. - - Returns - ------- - Tuple[AsyncGroup, ...] - """ - if not names: - return () - return tuple(await asyncio.gather(*(self.require_group(name) for name in names))) - - async def create_array( - self, - name: str, - *, - shape: ShapeLike | None = None, - dtype: ZDTypeLike | None = None, - data: np.ndarray[Any, np.dtype[Any]] | None = None, - chunks: ChunksLike | Literal["auto"] = "auto", - shards: ShardsLike | None = None, - filters: FiltersLike = "auto", - compressors: CompressorsLike = "auto", - compressor: CompressorLike = "auto", - serializer: SerializerLike = "auto", - fill_value: Any | None = DEFAULT_FILL_VALUE, - order: MemoryOrder | None = None, - attributes: dict[str, JSON] | None = None, - chunk_key_encoding: ChunkKeyEncodingLike | None = None, - dimension_names: DimensionNamesLike = None, - storage_options: dict[str, Any] | None = None, - overwrite: bool = False, - config: ArrayConfigLike | None = None, - write_data: bool = True, - ) -> AnyAsyncArray: - """Create an array within this group. - - This method lightly wraps [zarr.core.array.create_array][]. - - Parameters - ---------- - name : str - The name of the array relative to the group. If `path` is `None`, the array will be located - at the root of the store. - shape : ShapeLike, optional - Shape of the array. Must be `None` if `data` is provided. - dtype : ZDTypeLike | None - Data type of the array. Must be `None` if `data` is provided. - data : np.ndarray, optional - Array-like data to use for initializing the array. If this parameter is provided, the - `shape` and `dtype` parameters must be `None`. - chunks : tuple[int, ...] | Sequence[Sequence[int]] | Literal["auto"], default="auto" - Chunk shape of the array. - If chunks is "auto", a chunk shape is guessed based on the shape of the array and the dtype. - A nested list of per-dimension edge sizes creates a rectilinear grid. - Rectilinear chunk grids are experimental and must be explicitly enabled - with `zarr.config.set({'array.rectilinear_chunks': True})` while the - feature is stabilizing. - shards : tuple[int, ...], optional - Shard shape of the array. The default value of `None` results in no sharding at all. - filters : Iterable[Codec] | Literal["auto"], optional - Iterable of filters to apply to each chunk of the array, in order, before serializing that - chunk to bytes. - - For Zarr format 3, a "filter" is a codec that takes an array and returns an array, - and these values must be instances of [`zarr.abc.codec.ArrayArrayCodec`][], or a - dict representations of [`zarr.abc.codec.ArrayArrayCodec`][]. - - For Zarr format 2, a "filter" can be any numcodecs codec; you should ensure that the - order of your filters is consistent with the behavior of each filter. - - The default value of `"auto"` instructs Zarr to use a default based on the data - type of the array and the Zarr format specified. For all data types in Zarr V3, and most - data types in Zarr V2, the default filters are empty. The only cases where default filters - are not empty is when the Zarr format is 2, and the data type is a variable-length data type like - [`zarr.dtype.VariableLengthUTF8`][] or [`zarr.dtype.VariableLengthBytes`][]. In these cases, - the default filters contains a single element which is a codec specific to that particular data type. - - To create an array with no filters, provide an empty iterable or the value `None`. - compressors : Iterable[Codec], optional - List of compressors to apply to the array. Compressors are applied in order, and after any - filters are applied (if any are specified) and the data is serialized into bytes. - - For Zarr format 3, a "compressor" is a codec that takes a bytestream, and - returns another bytestream. Multiple compressors may be provided for Zarr format 3. - If no `compressors` are provided, a default set of compressors will be used. - These defaults can be changed by modifying the value of `array.v3_default_compressors` - in [`zarr.config`][zarr.config]. - Use `None` to omit default compressors. - - For Zarr format 2, a "compressor" can be any numcodecs codec. Only a single compressor may - be provided for Zarr format 2. - If no `compressor` is provided, a default compressor will be used. - This default can be changed by modifying the value of `array.v2_default_compressor` - in [`zarr.config`][zarr.config]. - Use `None` to omit the default compressor. - compressor : Codec, optional - Deprecated in favor of `compressors`. - serializer : dict[str, JSON] | ArrayBytesCodec, optional - Array-to-bytes codec to use for encoding the array data. - Zarr format 3 only. Zarr format 2 arrays use implicit array-to-bytes conversion. - If no `serializer` is provided, a default serializer will be used. - These defaults can be changed by modifying the value of `array.v3_default_serializer` - in [`zarr.config`][zarr.config]. - fill_value : Any, optional - Fill value for the array. - order : {"C", "F"}, optional - The memory order of the array (default is "C"). - For Zarr format 2, this parameter sets the memory order of the array. - For Zarr format 3, this parameter is deprecated, because memory order - is a runtime parameter for Zarr format 3 arrays. The recommended way to specify the memory - order for Zarr format 3 arrays is via the `config` parameter, e.g. `{'config': 'C'}`. - If no `order` is provided, a default order will be used. - This default can be changed by modifying the value of `array.order` in [`zarr.config`][zarr.config]. - attributes : dict, optional - Attributes for the array. - chunk_key_encoding : ChunkKeyEncodingLike, optional - A specification of how the chunk keys are represented in storage. - For Zarr format 3, the default is `{"name": "default", "separator": "/"}`. - For Zarr format 2, the default is `{"name": "v2", "separator": "."}`. - dimension_names : Iterable[str], optional - The names of the dimensions (default is None). - Zarr format 3 only. Zarr format 2 arrays should not use this parameter. - storage_options : dict, optional - If using an fsspec URL to create the store, these will be passed to the backend implementation. - Ignored otherwise. - overwrite : bool, default False - Whether to overwrite an array with the same name in the store, if one exists. - If `True`, any existing keys under that path are deleted first. - config : ArrayConfigLike, optional - Runtime configuration for the array. - write_data : bool - If a pre-existing array-like object was provided to this function via the `data` parameter - then `write_data` determines whether the values in that array-like object should be - written to the Zarr array created by this function. If `write_data` is `False`, then the - array will be left empty. - - Returns - ------- - AsyncArray - - """ - compressors = _parse_deprecated_compressor( - compressor, compressors, zarr_format=self.metadata.zarr_format - ) - return await create_array( - store=self.store_path, - name=name, - shape=shape, - dtype=dtype, - data=data, - chunks=chunks, - shards=shards, - filters=filters, - compressors=compressors, - serializer=serializer, - fill_value=fill_value, - order=order, - zarr_format=self.metadata.zarr_format, - attributes=attributes, - chunk_key_encoding=chunk_key_encoding, - dimension_names=dimension_names, - storage_options=storage_options, - overwrite=overwrite, - config=config, - write_data=write_data, - ) - - async def require_array( - self, - name: str, - *, - shape: ShapeLike, - dtype: ZDTypeLike | None = None, - exact: bool = False, - config: ArrayConfigLike | None = None, - **kwargs: Any, - ) -> AnyAsyncArray: - """Obtain an array, creating if it doesn't exist. - - Other `kwargs` are as per [zarr.AsyncGroup.create_array][]. - - Parameters - ---------- - name : str - Array name. - shape : int or tuple of ints - Array shape. - dtype : ZDTypeLike, optional - The data type of the array, given as a string, a NumPy dtype, or a - Zarr data type. - exact : bool, optional - If True, require `dtype` to match exactly. If false, require - `dtype` can be cast from array dtype. - config : ArrayConfigLike or None, default=None - Runtime configuration for the array, whether it is created or already exists. - Keys not specified are taken from the global configuration. - - Returns - ------- - a : AsyncArray - """ - try: - ds = await self.getitem(name) - if not isinstance(ds, AsyncArray): - raise TypeError(f"Incompatible object ({ds.__class__.__name__}) already exists") - - shape = parse_shapelike(shape) - if shape != ds.shape: - raise TypeError(f"Incompatible shape ({ds.shape} vs {shape})") - - # `np.dtype(None)` used to resolve to float64 here; keep that default. - dtype = parse_data_type( - "float64" if dtype is None else dtype, - zarr_format=self.metadata.zarr_format, - ).to_native_dtype() - if exact: - if ds.dtype != dtype: - raise TypeError(f"Incompatible dtype ({ds.dtype} vs {dtype})") - else: - if not np.can_cast(ds.dtype, dtype): - raise TypeError(f"Incompatible dtype ({ds.dtype} vs {dtype})") - except KeyError: - return await self.create_array(name, shape=shape, dtype=dtype, config=config, **kwargs) - - # `config` is the runtime configuration of the returned array, not stored metadata, - # so it applies to an existing array too. It is parsed the same way as on the create - # branch: missing keys come from the global configuration, unknown keys raise. - if config is not None: - return ds.with_config(parse_array_config(config)) - return ds - - async def update_attributes(self, new_attributes: dict[str, Any]) -> AsyncGroup: - """Update group attributes. - - Parameters - ---------- - new_attributes : dict - New attributes to set on the group. - - Returns - ------- - self : AsyncGroup - """ - self.metadata.attributes.update(new_attributes) - - # Write new metadata - await self._save_metadata() - - return self - - def __repr__(self) -> str: - return f"" - - async def nmembers( - self, - max_depth: int | None = 0, - ) -> int: - """Count the number of members in this group. - - Parameters - ---------- - max_depth : int, default 0 - The maximum number of levels of the hierarchy to include. By - default, (``max_depth=0``) only immediate children are included. Set - ``max_depth=None`` to include all nodes, and some positive integer - to consider children within that many levels of the root Group. - - Returns - ------- - count : int - """ - # check if we can use consolidated metadata, which requires that we have non-None - # consolidated metadata at all points in the hierarchy. - if self.metadata.consolidated_metadata is not None: - if max_depth is not None and max_depth < 0: - raise ValueError(f"max_depth must be None or >= 0. Got '{max_depth}' instead") - if max_depth is None: - return len(self.metadata.consolidated_metadata.flattened_metadata) - else: - return len( - [ - x - for x in self.metadata.consolidated_metadata.flattened_metadata - if x.count("/") <= max_depth - ] - ) - # TODO: consider using aioitertools.builtins.sum for this - # return await aioitertools.builtins.sum((1 async for _ in self.members()), start=0) - n = 0 - async for _ in self.members(max_depth=max_depth): - n += 1 - return n - - async def members( - self, - max_depth: int | None = 0, - *, - use_consolidated_for_children: bool = True, - ) -> AsyncGenerator[ - tuple[str, AnyAsyncArray | AsyncGroup], - None, - ]: - """ - Returns an AsyncGenerator over the arrays and groups contained in this group. - This method requires that `store_path.store` supports directory listing. - - The results are not guaranteed to be ordered. - - Parameters - ---------- - max_depth : int, default 0 - The maximum number of levels of the hierarchy to include. By - default, (``max_depth=0``) only immediate children are included. Set - ``max_depth=None`` to include all nodes, and some positive integer - to consider children within that many levels of the root Group. - use_consolidated_for_children : bool, default True - Whether to use the consolidated metadata of child groups loaded - from the store. Note that this only affects groups loaded from the - store. If the current Group already has consolidated metadata, it - will always be used. - - Returns - ------- - path: - A string giving the path to the target, relative to the Group ``self``. - value: AsyncArray or AsyncGroup - The AsyncArray or AsyncGroup that is a child of ``self``. - """ - if max_depth is not None and max_depth < 0: - raise ValueError(f"max_depth must be None or >= 0. Got '{max_depth}' instead") - async for item in self._members( - max_depth=max_depth, use_consolidated_for_children=use_consolidated_for_children - ): - yield item - - def _members_consolidated( - self, max_depth: int | None, prefix: str = "" - ) -> Generator[ - tuple[str, AnyAsyncArray | AsyncGroup], - None, - ]: - consolidated_metadata = self.metadata.consolidated_metadata - - do_recursion = max_depth is None or max_depth > 0 - - # we kind of just want the top-level keys. - if consolidated_metadata is not None: - for key in consolidated_metadata.metadata: - obj = self._getitem_consolidated( - self.store_path, key, prefix=self.name - ) # Metadata -> Group/Array - key = f"{prefix}/{key}".lstrip("/") - yield key, obj - - if do_recursion and isinstance(obj, AsyncGroup): - if max_depth is None: - new_depth = None - else: - new_depth = max_depth - 1 - yield from obj._members_consolidated(new_depth, prefix=key) - - async def _members( - self, max_depth: int | None, *, use_consolidated_for_children: bool = True - ) -> AsyncGenerator[tuple[str, AnyAsyncArray | AsyncGroup], None]: - skip_keys: tuple[str, ...] - if self.metadata.zarr_format == 2: - skip_keys = (".zattrs", ".zgroup", ".zarray", ".zmetadata") - elif self.metadata.zarr_format == 3: - skip_keys = ("zarr.json",) - else: - raise ValueError(f"Unknown Zarr format: {self.metadata.zarr_format}") - - if self.metadata.consolidated_metadata is not None: - members = self._members_consolidated(max_depth=max_depth) - for member in members: - yield member - return - - if not self.store_path.store.supports_listing: - msg = ( - f"The store associated with this group ({type(self.store_path.store)}) " - "does not support listing, " - "specifically via the `list_dir` method. " - "This function requires a store that supports listing." - ) - - raise ValueError(msg) - # enforce a concurrency limit by passing a semaphore to all the recursive functions - semaphore = asyncio.Semaphore(config.get("async.concurrency")) - async for member in _iter_members_deep( - self, - max_depth=max_depth, - skip_keys=skip_keys, - semaphore=semaphore, - use_consolidated_for_children=use_consolidated_for_children, - ): - yield member - - async def create_hierarchy( - self, - nodes: dict[str, ArrayV2Metadata | ArrayV3Metadata | GroupMetadata], - *, - overwrite: bool = False, - ) -> AsyncIterator[tuple[str, AsyncGroup | AnyAsyncArray]]: - """ - Create a hierarchy of arrays or groups rooted at this group. - - This function will parse its input to ensure that the hierarchy is complete. Any implicit groups - will be inserted as needed. For example, an input like - ```{'a/b': GroupMetadata}``` will be parsed to - ```{'': GroupMetadata, 'a': GroupMetadata, 'b': Groupmetadata}```. - - Explicitly specifying a root group, e.g. with ``nodes = {'': GroupMetadata()}`` is an error - because this group instance is the root group. - - After input parsing, this function then creates all the nodes in the hierarchy concurrently. - - Arrays and Groups are yielded in the order they are created. This order is not stable and - should not be relied on. - - Parameters - ---------- - nodes : dict[str, GroupMetadata | ArrayV3Metadata | ArrayV2Metadata] - A dictionary defining the hierarchy. The keys are the paths of the nodes in the hierarchy, - relative to the path of the group. The values are instances of ``GroupMetadata`` or ``ArrayMetadata``. Note that - all values must have the same ``zarr_format`` as the parent group -- it is an error to mix zarr versions in the - same hierarchy. - - Leading "/" characters from keys will be removed. - overwrite : bool - Whether to overwrite existing nodes. Defaults to ``False``, in which case an error is - raised instead of overwriting an existing array or group. - - This function will not erase an existing group unless that group is explicitly named in - ``nodes``. If ``nodes`` defines implicit groups, e.g. ``{`'a/b/c': GroupMetadata}``, and a - group already exists at path ``a``, then this function will leave the group at ``a`` as-is. - - Yields - ------ - tuple[str, AsyncArray | AsyncGroup]. - """ - # check that all the nodes have the same zarr_format as Self - prefix = self.path - nodes_parsed = {} - for key, value in nodes.items(): - if value.zarr_format != self.metadata.zarr_format: - msg = ( - "The zarr_format of the nodes must be the same as the parent group. " - f"The node at {key} has zarr_format {value.zarr_format}, but the parent group" - f" has zarr_format {self.metadata.zarr_format}." - ) - raise ValueError(msg) - if normalize_path(key) == "": - msg = ( - "The input defines a root node, but a root node already exists, namely this Group instance." - "It is an error to use this method to create a root node. " - "Remove the root node from the input dict, or use a function like " - "create_rooted_hierarchy to create a rooted hierarchy." - ) - raise ValueError(msg) - else: - nodes_parsed[_join_paths([prefix, key])] = value - - async for key, node in create_hierarchy( - store=self.store, - nodes=nodes_parsed, - overwrite=overwrite, - ): - if prefix == "": - out_key = key - else: - out_key = key.removeprefix(prefix + "/") - yield out_key, node - - async def keys(self) -> AsyncGenerator[str, None]: - """Iterate over member names.""" - async for key, _ in self.members(): - yield key - - async def contains(self, member: str) -> bool: - """Check if a member exists in the group. - - Parameters - ---------- - member : str - Member name. - - Returns - ------- - bool - """ - # TODO: this can be made more efficient. - try: - await self.getitem(member) - except KeyError: - return False - else: - return True - - async def groups(self) -> AsyncGenerator[tuple[str, AsyncGroup], None]: - """Iterate over subgroups.""" - async for name, value in self.members(): - if isinstance(value, AsyncGroup): - yield name, value - - async def group_keys(self) -> AsyncGenerator[str, None]: - """Iterate over group names.""" - async for key, _ in self.groups(): - yield key - - async def group_values(self) -> AsyncGenerator[AsyncGroup, None]: - """Iterate over group values.""" - async for _, group in self.groups(): - yield group - - async def arrays( - self, - ) -> AsyncGenerator[tuple[str, AnyAsyncArray], None]: - """Iterate over arrays.""" - async for key, value in self.members(): - if isinstance(value, AsyncArray): - yield key, value - - async def array_keys(self) -> AsyncGenerator[str, None]: - """Iterate over array names.""" - async for key, _ in self.arrays(): - yield key - - async def array_values( - self, - ) -> AsyncGenerator[AnyAsyncArray, None]: - """Iterate over array values.""" - async for _, array in self.arrays(): - yield array - - async def tree( - self, - expand: bool | None = None, - level: int | None = None, - *, - max_nodes: int = 500, - plain: bool = False, - ) -> Any: - """ - Return a tree-like representation of a hierarchy. - - Parameters - ---------- - expand : bool, optional - This keyword is not yet supported. A NotImplementedError is raised if - it's used. - level : int, optional - The maximum depth below this Group to display in the tree. - max_nodes : int - Maximum number of nodes to display before truncating. Default is 500. - plain : bool, optional - If True, return a plain-text tree without ANSI styling. This is - useful when the output will be consumed by an LLM or written to a - file. Default is False. - - Returns - ------- - TreeRepr - A pretty-printable object displaying the hierarchy. - """ - from zarr.core._tree import group_tree_async - - if expand is not None: - raise NotImplementedError("'expand' is not yet implemented.") - return await group_tree_async(self, max_depth=level, max_nodes=max_nodes, plain=plain) - - def _member_zarr_format(self, zarr_format: ZarrFormat | None) -> ZarrFormat: - """ - The zarr format of a new array in this group, which is the format of the group. - - A group only lists members of its own format, so an array of another format - would be written into the group without becoming a member of it. - """ - if zarr_format is not None and zarr_format != self.metadata.zarr_format: - raise ValueError( - f"Cannot create a zarr_format={zarr_format} array in a " - f"zarr_format={self.metadata.zarr_format} group." - ) - return self.metadata.zarr_format - - async def empty(self, *, name: str, shape: tuple[int, ...], **kwargs: Any) -> AnyAsyncArray: - """Create an empty array with the specified shape in this Group. The contents will - be filled with the array's fill value or zeros if no fill value is provided. - - Parameters - ---------- - name : str - Name of the array. - shape : int or tuple of int - Shape of the empty array. - **kwargs - Keyword arguments passed to [zarr.api.asynchronous.create][]. - - Notes - ----- - The contents of an empty Zarr array are not defined. On attempting to - retrieve data from an empty Zarr array, any values may be returned, - and these are not guaranteed to be stable from one access to the next. - """ - return await async_api.empty( - shape=shape, - store=self.store_path, - path=name, - zarr_format=self._member_zarr_format(kwargs.pop("zarr_format", None)), - **kwargs, - ) - - async def zeros(self, *, name: str, shape: tuple[int, ...], **kwargs: Any) -> AnyAsyncArray: - """Create an array, with zero being used as the default value for uninitialized portions of the array. - - Parameters - ---------- - name : str - Name of the array. - shape : int or tuple of int - Shape of the empty array. - **kwargs - Keyword arguments passed to [zarr.api.asynchronous.create][]. - - Returns - ------- - AsyncArray - The new array. - """ - return await async_api.zeros( - shape=shape, - store=self.store_path, - path=name, - zarr_format=self._member_zarr_format(kwargs.pop("zarr_format", None)), - **kwargs, - ) - - async def ones(self, *, name: str, shape: tuple[int, ...], **kwargs: Any) -> AnyAsyncArray: - """Create an array, with one being used as the default value for uninitialized portions of the array. - - Parameters - ---------- - name : str - Name of the array. - shape : int or tuple of int - Shape of the empty array. - **kwargs - Keyword arguments passed to [zarr.api.asynchronous.create][]. - - Returns - ------- - AsyncArray - The new array. - """ - return await async_api.ones( - shape=shape, - store=self.store_path, - path=name, - zarr_format=self._member_zarr_format(kwargs.pop("zarr_format", None)), - **kwargs, - ) - - async def full( - self, *, name: str, shape: tuple[int, ...], fill_value: Any | None, **kwargs: Any - ) -> AnyAsyncArray: - """Create an array, with "fill_value" being used as the default value for uninitialized portions of the array. - - Parameters - ---------- - name : str - Name of the array. - shape : int or tuple of int - Shape of the empty array. - fill_value : scalar - Value to fill the array with. - **kwargs - Keyword arguments passed to [zarr.api.asynchronous.create][]. - - Returns - ------- - AsyncArray - The new array. - """ - return await async_api.full( - shape=shape, - fill_value=fill_value, - store=self.store_path, - path=name, - zarr_format=self._member_zarr_format(kwargs.pop("zarr_format", None)), - **kwargs, - ) - - async def empty_like( - self, *, name: str, data: async_api.ArrayLike, **kwargs: Any - ) -> AnyAsyncArray: - """Create an empty sub-array like `data`. The contents will be filled with - the array's fill value or zeros if no fill value is provided. - - Parameters - ---------- - name : str - Name of the array. - data : array-like - The array to create an empty array like. - **kwargs - Keyword arguments passed to [zarr.api.asynchronous.create][]. - - Returns - ------- - AsyncArray - The new array. - """ - return await async_api.empty_like( - a=data, - store=self.store_path, - path=name, - zarr_format=self._member_zarr_format(kwargs.pop("zarr_format", None)), - **kwargs, - ) - - async def zeros_like( - self, *, name: str, data: async_api.ArrayLike, **kwargs: Any - ) -> AnyAsyncArray: - """Create a sub-array of zeros like `data`. - - Parameters - ---------- - name : str - Name of the array. - data : array-like - The array to create the new array like. - **kwargs - Keyword arguments passed to [zarr.api.asynchronous.create][]. - - Returns - ------- - AsyncArray - The new array. - """ - return await async_api.zeros_like( - a=data, - store=self.store_path, - path=name, - zarr_format=self._member_zarr_format(kwargs.pop("zarr_format", None)), - **kwargs, - ) - - async def ones_like( - self, *, name: str, data: async_api.ArrayLike, **kwargs: Any - ) -> AnyAsyncArray: - """Create a sub-array of ones like `data`. - - Parameters - ---------- - name : str - Name of the array. - data : array-like - The array to create the new array like. - **kwargs - Keyword arguments passed to [zarr.api.asynchronous.create][]. - - Returns - ------- - AsyncArray - The new array. - """ - return await async_api.ones_like( - a=data, - store=self.store_path, - path=name, - zarr_format=self._member_zarr_format(kwargs.pop("zarr_format", None)), - **kwargs, - ) - - async def full_like( - self, *, name: str, data: async_api.ArrayLike, **kwargs: Any - ) -> AnyAsyncArray: - """Create a sub-array like `data` filled with the `fill_value` of `data` . - - Parameters - ---------- - name : str - Name of the array. - data : array-like - The array to create the new array like. - **kwargs - Keyword arguments passed to [zarr.api.asynchronous.create][]. - - Returns - ------- - AsyncArray - The new array. - """ - return await async_api.full_like( - a=data, - store=self.store_path, - path=name, - zarr_format=self._member_zarr_format(kwargs.pop("zarr_format", None)), - **kwargs, - ) - - async def move(self, source: str, dest: str) -> None: - """Move a sub-group or sub-array from one path to another. - - Notes - ----- - Not implemented - """ - raise NotImplementedError - - -@dataclass(frozen=True) -class Group(SyncMixin): - """ - A Zarr group. - """ - - _async_group: AsyncGroup - - @classmethod - def from_store( - cls, - store: StoreLike, - *, - attributes: dict[str, Any] | None = None, - zarr_format: ZarrFormat = 3, - overwrite: bool = False, - ) -> Group: - """Instantiate a group from an initialized store. - - Parameters - ---------- - store : StoreLike - StoreLike containing the Group. See the - [storage documentation in the user guide][user-guide-store-like] - for a description of all valid StoreLike values. - attributes : dict, optional - A dictionary of JSON-serializable values with user-defined attributes. - zarr_format : {2, 3}, optional - Zarr storage format version. - overwrite : bool, optional - If True, do not raise an error if the group already exists. - - Returns - ------- - Group - Group instantiated from the store. - - Raises - ------ - ContainsArrayError, ContainsGroupError, ContainsArrayAndGroupError - """ - attributes = attributes or {} - obj = sync( - AsyncGroup.from_store( - store, - attributes=attributes, - overwrite=overwrite, - zarr_format=zarr_format, - ), - ) - - return cls(obj) - - @classmethod - def open( - cls, - store: StoreLike, - zarr_format: ZarrFormat | None = 3, - ) -> Group: - """Open a group from an initialized store. - - Parameters - ---------- - store : StoreLike - Store containing the Group. See the - [storage documentation in the user guide][user-guide-store-like] - for a description of all valid StoreLike values. - zarr_format : {2, 3, None}, optional - Zarr storage format version. - - Returns - ------- - Group - Group instantiated from the store. - """ - obj = sync(AsyncGroup.open(store, zarr_format=zarr_format)) - return cls(obj) - - def __getitem__(self, path: str) -> AnyArray | Group: - """Obtain a group member. - - Parameters - ---------- - path : str - Group member name. - - Returns - ------- - Array | Group - Group member (Array or Group) at the specified key - - Examples - -------- - ```python - import zarr - from zarr.core.group import Group - group = Group.from_store(zarr.storage.MemoryStore()) - group.create_array(name="subarray", shape=(10,), chunks=(10,), dtype="float64") - group.create_group(name="subgroup").create_array(name="subarray", shape=(10,), chunks=(10,), dtype="float64") - group["subarray"] - # - group["subgroup"] - # - group["subgroup"]["subarray"] - # - ``` - - """ - obj = self._sync(self._async_group.getitem(path)) - if isinstance(obj, AsyncArray): - return Array(obj) - else: - return Group(obj) - - def get[DefaultT]( - self, path: str, default: DefaultT | None = None - ) -> AnyArray | Group | DefaultT | None: - """Obtain a group member, returning default if not found. - - Parameters - ---------- - path : str - Group member name. - default : object - Default value to return if key is not found (default: None). - - Returns - ------- - object - Group member (Array or Group) or default if not found. - - Examples - -------- - ```python - import zarr - from zarr.core.group import Group - group = Group.from_store(zarr.storage.MemoryStore()) - group.create_array(name="subarray", shape=(10,), chunks=(10,), dtype="float64") - group.create_group(name="subgroup") - group.get("subarray") - # - group.get("subgroup") - # - group.get("nonexistent", None) - # None - ``` - - """ - try: - return self[path] - except KeyError: - return default - - def get_array(self, path: str) -> AnyArray: - """Obtain an array member of this group, raising if it is absent or not an array. - - Parameters - ---------- - path : str - Path of the array relative to this group. May contain `/` to reference - a member of a subgroup, e.g. `subgroup/subarray`. - - Returns - ------- - Array - The array at the given path. - - Raises - ------ - ArrayNotFoundError - If no node exists at the given path. - ContainsGroupError - If the node at the given path is a group rather than an array. - - Examples - -------- - ```python - import zarr - from zarr.core.group import Group - group = Group.from_store(zarr.storage.MemoryStore()) - group.create_array(name="subarray", shape=(10,), chunks=(10,), dtype="float64") - group.get_array("subarray") - # - ``` - """ - return Array(self._sync(self._async_group.get_array(path))) - - def get_group(self, path: str) -> Group: - """Obtain a group member of this group, raising if it is absent or not a group. - - Parameters - ---------- - path : str - Path of the group relative to this group. May contain `/` to reference - a member of a subgroup, e.g. `subgroup/subsubgroup`. - - Returns - ------- - Group - The group at the given path. - - Raises - ------ - GroupNotFoundError - If no node exists at the given path. - ContainsArrayError - If the node at the given path is an array rather than a group. - - Examples - -------- - ```python - import zarr - from zarr.core.group import Group - group = Group.from_store(zarr.storage.MemoryStore()) - group.create_group(name="subgroup") - group.get_group("subgroup") - # - ``` - """ - return Group(self._sync(self._async_group.get_group(path))) - - def __delitem__(self, key: str) -> None: - """Delete a group member. - - Parameters - ---------- - key : str - Group member name. - - Examples - -------- - >>> import zarr - >>> group = Group.from_store(zarr.storage.MemoryStore()) - >>> a = group.create_array(name="subarray", dtype="i1", shape=(10,), chunks=(10,)) - >>> del group["subarray"] - >>> "subarray" in group - False - """ - - self._sync(self._async_group.delitem(key)) - - def __iter__(self) -> Iterator[str]: - """Return an iterator over group member names. - Examples - -------- - >>> import zarr - >>> g1 = zarr.group() - >>> g2 = g1.create_group('foo') - >>> g3 = g1.create_group('bar') - >>> d1 = g1.create_array('baz', dtype="i1", shape=(10,), chunks=(10,)) - >>> d2 = g1.create_array('quux', dtype="i1", shape=(10,), chunks=(10,)) - >>> sorted(g1) - ['bar', 'baz', 'foo', 'quux'] - """ - yield from self.keys() - - def __len__(self) -> int: - """Number of members.""" - return self.nmembers() - - def __setitem__(self, key: str, value: Any) -> None: - """Fastpath for creating a new array. - - New arrays will be created using default settings for the array type. - If you need to create an array with custom settings, use the `create_array` method. - - Parameters - ---------- - key : str - Array name. - value : Any - Array data. - - Examples - -------- - >>> import numpy as np - >>> import zarr - >>> group = zarr.group() - >>> group["foo"] = np.array(zarr.zeros((10,))) - >>> group["foo"] - - """ - self._sync(self._async_group.setitem(key, value)) - - def __repr__(self) -> str: - return f"" - - async def update_attributes_async(self, new_attributes: dict[str, Any]) -> Group: - """Update the attributes of this group. - - Examples - -------- - >>> async def example(): - ... import zarr - ... - ... group = zarr.group() - ... new_group = await group.update_attributes_async({"foo": "bar"}) - ... return new_group.attrs.asdict() - - >>> import asyncio - >>> asyncio.run(example()) - {'foo': 'bar'} - """ - new_metadata = replace(self.metadata, attributes=new_attributes) - - # Write new metadata - to_save = new_metadata.to_buffer_dict(default_buffer_prototype()) - awaitables = [set_or_delete(self.store_path / key, value) for key, value in to_save.items()] - await asyncio.gather(*awaitables) - - async_group = replace(self._async_group, metadata=new_metadata) - return replace(self, _async_group=async_group) - - @property - def store_path(self) -> StorePath: - """Path-like interface for the Store.""" - return self._async_group.store_path - - @property - def metadata(self) -> GroupMetadata: - """Group metadata.""" - return self._async_group.metadata - - @property - def path(self) -> str: - """Storage path.""" - return self._async_group.path - - @property - def name(self) -> str: - """Group name following h5py convention.""" - return self._async_group.name - - @property - def basename(self) -> str: - """Final component of name.""" - return self._async_group.basename - - @property - def attrs(self) -> Attributes: - """Attributes of this Group""" - return Attributes(self) - - @property - def info(self) -> Any: - """ - Return the statically known information for a group. - - Returns - ------- - GroupInfo - - Related - ------- - [zarr.Group.info_complete][] - All information about a group, including dynamic information - like the children members. - """ - return self._async_group.info - - def info_complete(self) -> Any: - """ - Return information for a group. - - If this group doesn't contain consolidated metadata then - this will need to read from the backing Store. - - Returns - ------- - GroupInfo - - Related - ------- - [zarr.Group.info][] - """ - return self._sync(self._async_group.info_complete()) - - @property - def store(self) -> Store: - # Backwards compatibility for 2.x - return self._async_group.store - - @property - def read_only(self) -> bool: - # Backwards compatibility for 2.x - return self._async_group.read_only - - @property - def synchronizer(self) -> None: - # Backwards compatibility for 2.x - # Not implemented in 3.x yet. - return self._async_group.synchronizer - - def update_attributes(self, new_attributes: dict[str, Any]) -> Group: - """Update the attributes of this group. - - Examples - -------- - >>> import zarr - >>> group = zarr.group().update_attributes({"foo": "bar"}) - >>> group.attrs.asdict() - {'foo': 'bar'} - """ - self._sync(self._async_group.update_attributes(new_attributes)) - return self - - def nmembers(self, max_depth: int | None = 0) -> int: - """Count the number of members in this group. - - Parameters - ---------- - max_depth : int, default 0 - The maximum number of levels of the hierarchy to include. By - default, (``max_depth=0``) only immediate children are included. Set - ``max_depth=None`` to include all nodes, and some positive integer - to consider children within that many levels of the root Group. - - Returns - ------- - count : int - """ - - return self._sync(self._async_group.nmembers(max_depth=max_depth)) - - def members( - self, max_depth: int | None = 0, *, use_consolidated_for_children: bool = True - ) -> tuple[tuple[str, AnyArray | Group], ...]: - """ - Returns an AsyncGenerator over the arrays and groups contained in this group. - This method requires that `store_path.store` supports directory listing. - - The results are not guaranteed to be ordered. - - Parameters - ---------- - max_depth : int, default 0 - The maximum number of levels of the hierarchy to include. By - default, (``max_depth=0``) only immediate children are included. Set - ``max_depth=None`` to include all nodes, and some positive integer - to consider children within that many levels of the root Group. - use_consolidated_for_children : bool, default True - Whether to use the consolidated metadata of child groups loaded - from the store. Note that this only affects groups loaded from the - store. If the current Group already has consolidated metadata, it - will always be used. - - Returns - ------- - path: - A string giving the path to the target, relative to the Group ``self``. - value: AsyncArray or AsyncGroup - The AsyncArray or AsyncGroup that is a child of ``self``. - """ - _members = self._sync_iter(self._async_group.members(max_depth=max_depth)) - - return tuple((kv[0], _parse_async_node(kv[1])) for kv in _members) - - def create_hierarchy( - self, - nodes: dict[str, ArrayV2Metadata | ArrayV3Metadata | GroupMetadata], - *, - overwrite: bool = False, - ) -> Iterator[tuple[str, Group | AnyArray]]: - """ - Create a hierarchy of arrays or groups rooted at this group. - - This function will parse its input to ensure that the hierarchy is complete. Any implicit groups - will be inserted as needed. For example, an input like - ```{'a/b': GroupMetadata}``` will be parsed to - ```{'': GroupMetadata, 'a': GroupMetadata, 'b': Groupmetadata}```. - - Explicitly specifying a root group, e.g. with ``nodes = {'': GroupMetadata()}`` is an error - because this group instance is the root group. - - After input parsing, this function then creates all the nodes in the hierarchy concurrently. - - Arrays and Groups are yielded in the order they are created. This order is not stable and - should not be relied on. - - Parameters - ---------- - nodes : dict[str, GroupMetadata | ArrayV3Metadata | ArrayV2Metadata] - A dictionary defining the hierarchy. The keys are the paths of the nodes in the hierarchy, - relative to the path of the group. The values are instances of ``GroupMetadata`` or ``ArrayMetadata``. Note that - all values must have the same ``zarr_format`` as the parent group -- it is an error to mix zarr versions in the - same hierarchy. - - Leading "/" characters from keys will be removed. - overwrite : bool - Whether to overwrite existing nodes. Defaults to ``False``, in which case an error is - raised instead of overwriting an existing array or group. - - This function will not erase an existing group unless that group is explicitly named in - ``nodes``. If ``nodes`` defines implicit groups, e.g. ``{`'a/b/c': GroupMetadata}``, and a - group already exists at path ``a``, then this function will leave the group at ``a`` as-is. - - Yields - ------ - tuple[str, Array | Group]. - - Examples - -------- - >>> import zarr - >>> from zarr.core.group import GroupMetadata - >>> root = zarr.create_group(store={}) - >>> sorted(root.create_hierarchy({'a/b/c': GroupMetadata()})) - [('a', ), - ('a/b', ), - ('a/b/c', )] - """ - for key, node in self._sync_iter( - self._async_group.create_hierarchy(nodes, overwrite=overwrite) - ): - yield (key, _parse_async_node(node)) - - def keys(self) -> Generator[str]: - """Return an iterator over group member names. - - Examples - -------- - >>> import zarr - >>> g1 = zarr.group() - >>> g2 = g1.create_group('foo') - >>> g3 = g1.create_group('bar') - >>> d1 = g1.create_array('baz', dtype="i1", shape=(10,), chunks=(10,)) - >>> d2 = g1.create_array('quux', dtype="i1", shape=(10,), chunks=(10,)) - >>> sorted(g1.keys()) - ['bar', 'baz', 'foo', 'quux'] - """ - yield from self._sync_iter(self._async_group.keys()) - - def __contains__(self, member: str) -> bool: - """Test for group membership. - - Examples - -------- - >>> import zarr - >>> g1 = zarr.group() - >>> g2 = g1.create_group('foo') - >>> d1 = g1.create_array('bar', dtype="i1", shape=(10,), chunks=(10,)) - >>> 'foo' in g1 - True - >>> 'bar' in g1 - True - >>> 'baz' in g1 - False - """ - return self._sync(self._async_group.contains(member)) - - def groups(self) -> Generator[tuple[str, Group], None]: - """Return the sub-groups of this group as a generator of (name, group) pairs. - - Examples - -------- - >>> import zarr - >>> group = zarr.group() - >>> subgroup = group.create_group("subgroup") - >>> list(group.groups()) - [('subgroup', )] - """ - for name, async_group in self._sync_iter(self._async_group.groups()): - yield name, Group(async_group) - - def group_keys(self) -> Generator[str, None]: - """Return an iterator over group member names. - - Examples - -------- - >>> import zarr - >>> group = zarr.group() - >>> subgroup = group.create_group("subgroup") - >>> list(group.group_keys()) - ['subgroup'] - """ - for name, _ in self.groups(): - yield name - - def group_values(self) -> Generator[Group, None]: - """Return an iterator over group members. - - Examples - -------- - >>> import zarr - >>> group = zarr.group() - >>> subgroup = group.create_group("subgroup") - >>> list(group.group_values()) - [] - """ - for _, group in self.groups(): - yield group - - def arrays(self) -> Generator[tuple[str, AnyArray], None]: - """Return the sub-arrays of this group as a generator of (name, array) pairs - - Examples - -------- - >>> import zarr - >>> group = zarr.group() - >>> subarray = group.create_array("subarray", dtype="i1", shape=(10,), chunks=(10,)) - >>> list(group.arrays()) - [('subarray', )] - """ - for name, async_array in self._sync_iter(self._async_group.arrays()): - yield name, Array(async_array) - - def array_keys(self) -> Generator[str, None]: - """Return an iterator over group member names. - - Examples - -------- - >>> import zarr - >>> group = zarr.group() - >>> subarray = group.create_array("subarray", dtype="i1", shape=(10,), chunks=(10,)) - >>> list(group.array_keys()) - ['subarray'] - """ - - for name, _ in self.arrays(): - yield name - - def array_values(self) -> Generator[AnyArray, None]: - """Return an iterator over group members. - - Examples - -------- - >>> import zarr - >>> group = zarr.group() - >>> subarray = group.create_array("subarray", dtype="i1", shape=(10,), chunks=(10,)) - >>> list(group.array_values()) - [] - """ - for _, array in self.arrays(): - yield array - - def tree( - self, - expand: bool | None = None, - level: int | None = None, - *, - max_nodes: int = 500, - plain: bool = False, - ) -> Any: - """ - Return a tree-like representation of a hierarchy. - - Parameters - ---------- - expand : bool, optional - This keyword is not yet supported. A NotImplementedError is raised if - it's used. - level : int, optional - The maximum depth below this Group to display in the tree. - max_nodes : int - Maximum number of nodes to display before truncating. Default is 500. - plain : bool, optional - If True, return a plain-text tree without ANSI styling. This is - useful when the output will be consumed by an LLM or written to a - file. Default is False. - - Returns - ------- - TreeRepr - A pretty-printable object displaying the hierarchy. - """ - return self._sync( - self._async_group.tree(expand=expand, level=level, max_nodes=max_nodes, plain=plain) - ) - - def create_group(self, name: str, **kwargs: Any) -> Group: - """Create a sub-group. - - Parameters - ---------- - name : str - Name of the new subgroup. - - Returns - ------- - Group - - Examples - -------- - >>> import zarr - >>> group = zarr.group() - >>> subgroup = group.create_group("subgroup") - >>> subgroup - - """ - return Group(self._sync(self._async_group.create_group(name, **kwargs))) - - def require_group(self, name: str, **kwargs: Any) -> Group: - """Obtain a sub-group, creating one if it doesn't exist. - - Parameters - ---------- - name : str - Group name. - - Returns - ------- - g : Group - """ - return Group(self._sync(self._async_group.require_group(name, **kwargs))) - - def require_groups(self, *names: str) -> tuple[Group, ...]: - """Convenience method to require multiple groups in a single call. - - Parameters - ---------- - *names : str - Group names. - - Returns - ------- - groups : tuple of Groups - """ - return tuple(map(Group, self._sync(self._async_group.require_groups(*names)))) - - def create( - self, - name: str, - *, - shape: ShapeLike | None = None, - dtype: ZDTypeLike | None = None, - data: np.ndarray[Any, np.dtype[Any]] | None = None, - chunks: ChunksLike | Literal["auto"] = "auto", - shards: ShardsLike | None = None, - filters: FiltersLike = "auto", - compressors: CompressorsLike = "auto", - compressor: CompressorLike = "auto", - serializer: SerializerLike = "auto", - fill_value: Any | None = DEFAULT_FILL_VALUE, - order: MemoryOrder | None = None, - attributes: dict[str, JSON] | None = None, - chunk_key_encoding: ChunkKeyEncodingLike | None = None, - dimension_names: DimensionNamesLike = None, - storage_options: dict[str, Any] | None = None, - overwrite: bool = False, - config: ArrayConfigLike | None = None, - write_data: bool = True, - ) -> AnyArray: - """Create an array within this group. - - This method lightly wraps [`zarr.core.array.create_array`][]. - - Parameters - ---------- - name : str - The name of the array relative to the group. If `path` is `None`, the array will be located - at the root of the store. - shape : ShapeLike, optional - Shape of the array. Must be `None` if `data` is provided. - dtype : ZDTypeLike | None - Data type of the array. Must be `None` if `data` is provided. - data : np.ndarray, optional - Array-like data to use for initializing the array. If this parameter is provided, the - `shape` and `dtype` parameters must be `None`. - chunks : tuple[int, ...] | Sequence[Sequence[int]] | Literal["auto"], default="auto" - Chunk shape of the array. - If chunks is "auto", a chunk shape is guessed based on the shape of the array and the dtype. - A nested list of per-dimension edge sizes creates a rectilinear grid. - Rectilinear chunk grids are experimental and must be explicitly enabled - with `zarr.config.set({'array.rectilinear_chunks': True})` while the - feature is stabilizing. - shards : tuple[int, ...], optional - Shard shape of the array. The default value of `None` results in no sharding at all. - filters : Iterable[Codec] | Literal["auto"], optional - Iterable of filters to apply to each chunk of the array, in order, before serializing that - chunk to bytes. - - For Zarr format 3, a "filter" is a codec that takes an array and returns an array, - and these values must be instances of [`zarr.abc.codec.ArrayArrayCodec`][], or a - dict representations of [`zarr.abc.codec.ArrayArrayCodec`][]. - - For Zarr format 2, a "filter" can be any numcodecs codec; you should ensure that the - order of your filters is consistent with the behavior of each filter. - - The default value of `"auto"` instructs Zarr to use a default based on the data - type of the array and the Zarr format specified. For all data types in Zarr V3, and most - data types in Zarr V2, the default filters are empty. The only cases where default filters - are not empty is when the Zarr format is 2, and the data type is a variable-length data type like - [`zarr.dtype.VariableLengthUTF8`][] or [`zarr.dtype.VariableLengthBytes`][]. In these cases, - the default filters contains a single element which is a codec specific to that particular data type. - - To create an array with no filters, provide an empty iterable or the value `None`. - compressors : Iterable[Codec], optional - List of compressors to apply to the array. Compressors are applied in order, and after any - filters are applied (if any are specified) and the data is serialized into bytes. - - For Zarr format 3, a "compressor" is a codec that takes a bytestream, and - returns another bytestream. Multiple compressors may be provided for Zarr format 3. - If no `compressors` are provided, a default set of compressors will be used. - These defaults can be changed by modifying the value of `array.v3_default_compressors` - in [`zarr.config`][zarr.config]. - Use `None` to omit default compressors. - - For Zarr format 2, a "compressor" can be any numcodecs codec. Only a single compressor may - be provided for Zarr format 2. - If no `compressor` is provided, a default compressor will be used. - This default can be changed by modifying the value of `array.v2_default_compressor` - in [`zarr.config`][zarr.config]. - Use `None` to omit the default compressor. - compressor : Codec, optional - Deprecated in favor of `compressors`. - serializer : dict[str, JSON] | ArrayBytesCodec, optional - Array-to-bytes codec to use for encoding the array data. - Zarr format 3 only. Zarr format 2 arrays use implicit array-to-bytes conversion. - If no `serializer` is provided, a default serializer will be used. - These defaults can be changed by modifying the value of `array.v3_default_serializer` - in [`zarr.config`][zarr.config]. - fill_value : Any, optional - Fill value for the array. - order : {"C", "F"}, optional - The memory order of the array (default is "C"). - For Zarr format 2, this parameter sets the memory order of the array. - For Zarr format 3, this parameter is deprecated, because memory order - is a runtime parameter for Zarr format 3 arrays. The recommended way to specify the memory - order for Zarr format 3 arrays is via the `config` parameter, e.g. `{'config': 'C'}`. - If no `order` is provided, a default order will be used. - This default can be changed by modifying the value of `array.order` in [`zarr.config`][zarr.config]. - attributes : dict, optional - Attributes for the array. - chunk_key_encoding : ChunkKeyEncodingLike, optional - A specification of how the chunk keys are represented in storage. - For Zarr format 3, the default is `{"name": "default", "separator": "/"}`. - For Zarr format 2, the default is `{"name": "v2", "separator": "."}`. - dimension_names : Iterable[str], optional - The names of the dimensions (default is None). - Zarr format 3 only. Zarr format 2 arrays should not use this parameter. - storage_options : dict, optional - If using an fsspec URL to create the store, these will be passed to the backend implementation. - Ignored otherwise. - overwrite : bool, default False - Whether to overwrite an array with the same name in the store, if one exists. - If `True`, any existing keys under that path are deleted first. - config : ArrayConfigLike, optional - Runtime configuration for the array. - write_data : bool - If a pre-existing array-like object was provided to this function via the `data` parameter - then `write_data` determines whether the values in that array-like object should be - written to the Zarr array created by this function. If `write_data` is `False`, then the - array will be left empty. - - Returns - ------- - AsyncArray - """ - return self.create_array( - name, - shape=shape, - dtype=dtype, - data=data, - chunks=chunks, - shards=shards, - filters=filters, - compressors=compressors, - compressor=compressor, - serializer=serializer, - fill_value=fill_value, - order=order, - attributes=attributes, - chunk_key_encoding=chunk_key_encoding, - dimension_names=dimension_names, - storage_options=storage_options, - overwrite=overwrite, - config=config, - write_data=write_data, - ) - - def create_array( - self, - name: str, - *, - shape: ShapeLike | None = None, - dtype: ZDTypeLike | None = None, - data: np.ndarray[Any, np.dtype[Any]] | None = None, - chunks: ChunksLike | Literal["auto"] = "auto", - shards: ShardsLike | None = None, - filters: FiltersLike = "auto", - compressors: CompressorsLike = "auto", - compressor: CompressorLike = "auto", - serializer: SerializerLike = "auto", - fill_value: Any | None = DEFAULT_FILL_VALUE, - order: MemoryOrder | None = None, - attributes: dict[str, JSON] | None = None, - chunk_key_encoding: ChunkKeyEncodingLike | None = None, - dimension_names: DimensionNamesLike = None, - storage_options: dict[str, Any] | None = None, - overwrite: bool = False, - config: ArrayConfigLike | None = None, - write_data: bool = True, - ) -> AnyArray: - """Create an array within this group. - - This method lightly wraps [zarr.core.array.create_array][]. - - Parameters - ---------- - name : str - The name of the array relative to the group. If `path` is `None`, the array will be located - at the root of the store. - shape : ShapeLike, optional - Shape of the array. Must be `None` if `data` is provided. - dtype : ZDTypeLike | None - Data type of the array. Must be `None` if `data` is provided. - data : np.ndarray, optional - Array-like data to use for initializing the array. If this parameter is provided, the - `shape` and `dtype` parameters must be `None`. - chunks : tuple[int, ...] | Sequence[Sequence[int]] | Literal["auto"], default="auto" - Chunk shape of the array. - If chunks is "auto", a chunk shape is guessed based on the shape of the array and the dtype. - A nested list of per-dimension edge sizes creates a rectilinear grid. - Rectilinear chunk grids are experimental and must be explicitly enabled - with `zarr.config.set({'array.rectilinear_chunks': True})` while the - feature is stabilizing. - shards : tuple[int, ...], optional - Shard shape of the array. The default value of `None` results in no sharding at all. - filters : Iterable[Codec] | Literal["auto"], optional - Iterable of filters to apply to each chunk of the array, in order, before serializing that - chunk to bytes. - - For Zarr format 3, a "filter" is a codec that takes an array and returns an array, - and these values must be instances of [`zarr.abc.codec.ArrayArrayCodec`][], or a - dict representations of [`zarr.abc.codec.ArrayArrayCodec`][]. - - For Zarr format 2, a "filter" can be any numcodecs codec; you should ensure that the - order of your filters is consistent with the behavior of each filter. - - The default value of `"auto"` instructs Zarr to use a default based on the data - type of the array and the Zarr format specified. For all data types in Zarr V3, and most - data types in Zarr V2, the default filters are empty. The only cases where default filters - are not empty is when the Zarr format is 2, and the data type is a variable-length data type like - [`zarr.dtype.VariableLengthUTF8`][] or [`zarr.dtype.VariableLengthBytes`][]. In these cases, - the default filters contains a single element which is a codec specific to that particular data type. - - To create an array with no filters, provide an empty iterable or the value `None`. - compressors : Iterable[Codec], optional - List of compressors to apply to the array. Compressors are applied in order, and after any - filters are applied (if any are specified) and the data is serialized into bytes. - - For Zarr format 3, a "compressor" is a codec that takes a bytestream, and - returns another bytestream. Multiple compressors may be provided for Zarr format 3. - If no `compressors` are provided, a default set of compressors will be used. - These defaults can be changed by modifying the value of `array.v3_default_compressors` - in [`zarr.config`][zarr.config]. - Use `None` to omit default compressors. - - For Zarr format 2, a "compressor" can be any numcodecs codec. Only a single compressor may - be provided for Zarr format 2. - If no `compressor` is provided, a default compressor will be used. - This default can be changed by modifying the value of `array.v2_default_compressor` - in [`zarr.config`][zarr.config]. - Use `None` to omit the default compressor. - compressor : Codec, optional - Deprecated in favor of `compressors`. - serializer : dict[str, JSON] | ArrayBytesCodec, optional - Array-to-bytes codec to use for encoding the array data. - Zarr format 3 only. Zarr format 2 arrays use implicit array-to-bytes conversion. - If no `serializer` is provided, a default serializer will be used. - These defaults can be changed by modifying the value of `array.v3_default_serializer` - in [`zarr.config`][zarr.config]. - fill_value : Any, optional - Fill value for the array. - order : {"C", "F"}, optional - The memory order of the array (default is "C"). - For Zarr format 2, this parameter sets the memory order of the array. - For Zarr format 3, this parameter is deprecated, because memory order - is a runtime parameter for Zarr format 3 arrays. The recommended way to specify the memory - order for Zarr format 3 arrays is via the `config` parameter, e.g. `{'config': 'C'}`. - If no `order` is provided, a default order will be used. - This default can be changed by modifying the value of `array.order` in [`zarr.config`][zarr.config]. - attributes : dict, optional - Attributes for the array. - chunk_key_encoding : ChunkKeyEncodingLike, optional - A specification of how the chunk keys are represented in storage. - For Zarr format 3, the default is `{"name": "default", "separator": "/"}`. - For Zarr format 2, the default is `{"name": "v2", "separator": "."}`. - dimension_names : Iterable[str], optional - The names of the dimensions (default is None). - Zarr format 3 only. Zarr format 2 arrays should not use this parameter. - storage_options : dict, optional - If using an fsspec URL to create the store, these will be passed to the backend implementation. - Ignored otherwise. - overwrite : bool, default False - Whether to overwrite an array with the same name in the store, if one exists. - If `True`, any existing keys under that path are deleted first. - config : ArrayConfigLike, optional - Runtime configuration for the array. - write_data : bool - If a pre-existing array-like object was provided to this function via the `data` parameter - then `write_data` determines whether the values in that array-like object should be - written to the Zarr array created by this function. If `write_data` is `False`, then the - array will be left empty. - - Returns - ------- - AsyncArray - """ - compressors = _parse_deprecated_compressor( - compressor, compressors, zarr_format=self.metadata.zarr_format - ) - return Array( - self._sync( - self._async_group.create_array( - name=name, - shape=shape, - dtype=dtype, - data=data, - chunks=chunks, - shards=shards, - fill_value=fill_value, - attributes=attributes, - chunk_key_encoding=chunk_key_encoding, - compressors=compressors, - serializer=serializer, - dimension_names=dimension_names, - order=order, - filters=filters, - overwrite=overwrite, - storage_options=storage_options, - config=config, - write_data=write_data, - ) - ) - ) - - def require_array( - self, - name: str, - *, - shape: ShapeLike, - config: ArrayConfigLike | None = None, - **kwargs: Any, - ) -> AnyArray: - """Obtain an array, creating if it doesn't exist. - - Other `kwargs` are as per [zarr.Group.create_array][]. - - Parameters - ---------- - name : str - Array name. - config : ArrayConfigLike or None, default=None - Runtime configuration for the array, whether it is created or already exists. - Keys not specified are taken from the global configuration. - **kwargs : - See [zarr.Group.create_array][]. - - Returns - ------- - a : Array - """ - return Array( - self._sync(self._async_group.require_array(name, shape=shape, config=config, **kwargs)) - ) - - def empty(self, *, name: str, shape: tuple[int, ...], **kwargs: Any) -> AnyArray: - """Create an empty array with the specified shape in this Group. The contents will be filled with - the array's fill value or zeros if no fill value is provided. - - Parameters - ---------- - name : str - Name of the array. - shape : int or tuple of int - Shape of the empty array. - **kwargs - Keyword arguments passed to [zarr.api.asynchronous.create][]. - - Notes - ----- - The contents of an empty Zarr array are not defined. On attempting to - retrieve data from an empty Zarr array, any values may be returned, - and these are not guaranteed to be stable from one access to the next. - """ - return Array(self._sync(self._async_group.empty(name=name, shape=shape, **kwargs))) - - def zeros(self, *, name: str, shape: tuple[int, ...], **kwargs: Any) -> AnyArray: - """Create an array, with zero being used as the default value for uninitialized portions of the array. - - Parameters - ---------- - name : str - Name of the array. - shape : int or tuple of int - Shape of the empty array. - **kwargs - Keyword arguments passed to [zarr.api.asynchronous.create][]. - - Returns - ------- - Array - The new array. - """ - return Array(self._sync(self._async_group.zeros(name=name, shape=shape, **kwargs))) - - def ones(self, *, name: str, shape: tuple[int, ...], **kwargs: Any) -> AnyArray: - """Create an array, with one being used as the default value for uninitialized portions of the array. - - Parameters - ---------- - name : str - Name of the array. - shape : int or tuple of int - Shape of the empty array. - **kwargs - Keyword arguments passed to [zarr.api.asynchronous.create][]. - - Returns - ------- - Array - The new array. - """ - return Array(self._sync(self._async_group.ones(name=name, shape=shape, **kwargs))) - - def full( - self, *, name: str, shape: tuple[int, ...], fill_value: Any | None, **kwargs: Any - ) -> AnyArray: - """Create an array, with "fill_value" being used as the default value for uninitialized portions of the array. - - Parameters - ---------- - name : str - Name of the array. - shape : int or tuple of int - Shape of the empty array. - fill_value : scalar - Value to fill the array with. - **kwargs - Keyword arguments passed to [zarr.api.asynchronous.create][]. - - Returns - ------- - Array - The new array. - """ - return Array( - self._sync( - self._async_group.full(name=name, shape=shape, fill_value=fill_value, **kwargs) - ) - ) - - def empty_like(self, *, name: str, data: async_api.ArrayLike, **kwargs: Any) -> AnyArray: - """Create an empty sub-array like `data`. The contents will be filled - with the array's fill value or zeros if no fill value is provided. - - Parameters - ---------- - name : str - Name of the array. - data : array-like - The array to create an empty array like. - **kwargs - Keyword arguments passed to [zarr.api.asynchronous.create][]. - - Returns - ------- - Array - The new array. - - Notes - ----- - The contents of an empty Zarr array are not defined. On attempting to - retrieve data from an empty Zarr array, any values may be returned, - and these are not guaranteed to be stable from one access to the next. - """ - return Array(self._sync(self._async_group.empty_like(name=name, data=data, **kwargs))) - - def zeros_like(self, *, name: str, data: async_api.ArrayLike, **kwargs: Any) -> AnyArray: - """Create a sub-array of zeros like `data`. - - Parameters - ---------- - name : str - Name of the array. - data : array-like - The array to create the new array like. - **kwargs - Keyword arguments passed to [zarr.api.asynchronous.create][]. - - Returns - ------- - Array - The new array. - """ - - return Array(self._sync(self._async_group.zeros_like(name=name, data=data, **kwargs))) - - def ones_like(self, *, name: str, data: async_api.ArrayLike, **kwargs: Any) -> AnyArray: - """Create a sub-array of ones like `data`. - - Parameters - ---------- - name : str - Name of the array. - data : array-like - The array to create the new array like. - **kwargs - Keyword arguments passed to [zarr.api.asynchronous.create][]. - - Returns - ------- - Array - The new array. - """ - return Array(self._sync(self._async_group.ones_like(name=name, data=data, **kwargs))) - - def full_like(self, *, name: str, data: async_api.ArrayLike, **kwargs: Any) -> AnyArray: - """Create a sub-array like `data` filled with the `fill_value` of `data` . - - Parameters - ---------- - name : str - Name of the array. - data : array-like - The array to create the new array like. - **kwargs - Keyword arguments passed to [zarr.api.asynchronous.create][]. - - Returns - ------- - Array - The new array. - """ - return Array(self._sync(self._async_group.full_like(name=name, data=data, **kwargs))) - - def move(self, source: str, dest: str) -> None: - """Move a sub-group or sub-array from one path to another. - - Notes - ----- - Not implemented - """ - return self._sync(self._async_group.move(source, dest)) - - -async def create_hierarchy( - *, - store: Store, - nodes: dict[str, GroupMetadata | ArrayV2Metadata | ArrayV3Metadata], - overwrite: bool = False, -) -> AsyncIterator[tuple[str, AsyncGroup | AnyAsyncArray]]: - """ - Create a complete zarr hierarchy from a collection of metadata objects. - - This function will parse its input to ensure that the hierarchy is complete. Any implicit groups - will be inserted as needed. For example, an input like - ```{'a/b': GroupMetadata}``` will be parsed to - ```{'': GroupMetadata, 'a': GroupMetadata, 'b': Groupmetadata}``` - - After input parsing, this function then creates all the nodes in the hierarchy concurrently. - - Arrays and Groups are yielded in the order they are created. This order is not stable and - should not be relied on. - - Parameters - ---------- - store : Store - The storage backend to use. - nodes : dict[str, GroupMetadata | ArrayV3Metadata | ArrayV2Metadata] - A dictionary defining the hierarchy. The keys are the paths of the nodes in the hierarchy, - relative to the root of the ``Store``. The root of the store can be specified with the empty - string ``''``. The values are instances of ``GroupMetadata`` or ``ArrayMetadata``. Note that - all values must have the same ``zarr_format`` -- it is an error to mix zarr versions in the - same hierarchy. - - Leading "/" characters from keys will be removed. - overwrite : bool - Whether to overwrite existing nodes. Defaults to ``False``, in which case an error is - raised instead of overwriting an existing array or group. - - This function will not erase an existing group unless that group is explicitly named in - ``nodes``. If ``nodes`` defines implicit groups, e.g. ``{`'a/b/c': GroupMetadata}``, and a - group already exists at path ``a``, then this function will leave the group at ``a`` as-is. - - Yields - ------ - tuple[str, AsyncGroup | AsyncArray] - Yields (path, node) pairs, in the order the nodes were created. - - Examples - -------- - >>> async def example(): - ... from zarr.api.asynchronous import create_hierarchy - ... from zarr.core.group import GroupMetadata - ... from zarr.storage import MemoryStore - ... - ... store = MemoryStore() - ... nodes = {'a': GroupMetadata(attributes={'name': 'leaf'})} - ... return sorted([x async for x in create_hierarchy(store=store, nodes=nodes)]) - - >>> import asyncio - >>> asyncio.run(example()) - [('', ), ('a', )] - """ - - # normalize the keys to be valid paths - nodes_normed_keys = _normalize_path_keys(nodes) - - # ensure that all nodes have the same zarr_format, and add implicit groups as needed - nodes_parsed = _parse_hierarchy_dict(data=nodes_normed_keys) - redundant_implicit_groups = [] - to_delete_keys: list[str] = [] - - # empty hierarchies should be a no-op - if len(nodes_parsed) > 0: - # figure out which zarr format we are using - zarr_format = next(iter(nodes_parsed.values())).zarr_format - - # check which implicit groups will require materialization - implicit_group_keys = tuple( - filter(lambda k: isinstance(nodes_parsed[k], ImplicitGroupMarker), nodes_parsed) - ) - # read potential group metadata for each implicit group - maybe_extant_group_coros = ( - _read_group_metadata(store, k, zarr_format=zarr_format) for k in implicit_group_keys - ) - maybe_extant_groups = await asyncio.gather( - *maybe_extant_group_coros, return_exceptions=True - ) - - for key, value in zip(implicit_group_keys, maybe_extant_groups, strict=True): - if isinstance(value, BaseException): - if isinstance(value, FileNotFoundError): - # this is fine -- there was no group there, so we will create one - pass - else: - raise value - else: - # a loop exists already at ``key``, so we can avoid creating anything there - redundant_implicit_groups.append(key) - - if overwrite: - # we will remove any nodes that collide with arrays and non-implicit groups defined in - # nodes - to_delete_keys = [k for k in nodes_parsed if k not in implicit_group_keys] - else: - # This type is long. - coros: ( - Generator[Coroutine[Any, Any, ArrayV2Metadata | GroupMetadata], None, None] - | Generator[Coroutine[Any, Any, ArrayV3Metadata | GroupMetadata], None, None] - ) - if zarr_format == 2: - coros = (_read_metadata_v2(store=store, path=key) for key in nodes_parsed) - elif zarr_format == 3: - coros = (_read_metadata_v3(store=store, path=key) for key in nodes_parsed) - else: # pragma: no cover - raise ValueError(f"Invalid zarr_format: {zarr_format}") # pragma: no cover - - extant_node_query = dict( - zip( - nodes_parsed.keys(), - await asyncio.gather(*coros, return_exceptions=True), - strict=False, - ) - ) - # iterate over the existing arrays / groups and figure out which of them conflict - # with the arrays / groups we want to create - for key, extant_node in extant_node_query.items(): - proposed_node = nodes_parsed[key] - if isinstance(extant_node, BaseException): - if isinstance(extant_node, FileNotFoundError): - # ignore FileNotFoundError, because they represent nodes we can safely create - pass - else: - # Any other exception is a real error - raise extant_node - else: - # this is a node that already exists, but a node with the same key was specified - # in nodes_parsed. - if isinstance(extant_node, GroupMetadata): - # a group already exists where we want to create a group - if isinstance(proposed_node, ImplicitGroupMarker): - # we have proposed an implicit group, which is OK -- we will just skip - # creating this particular metadata document - redundant_implicit_groups.append(key) - else: - # we have proposed an explicit group, which is an error, given that a - # group already exists. - msg = f"A group exists in store {store!r} at path {key!r}." - raise ContainsGroupError(msg) - elif isinstance(extant_node, ArrayV2Metadata | ArrayV3Metadata): - # we are trying to overwrite an existing array. this is an error. - msg = f"An array exists in store {store!r} at path {key!r}." - raise ContainsArrayError(msg) - - nodes_explicit: dict[str, GroupMetadata | ArrayV2Metadata | ArrayV3Metadata] = {} - - for k, v in nodes_parsed.items(): - if k not in redundant_implicit_groups: - if isinstance(v, ImplicitGroupMarker): - nodes_explicit[k] = GroupMetadata(zarr_format=v.zarr_format) - else: - nodes_explicit[k] = v - - # Build and encode every node before deleting anything: a node that cannot be built - # or whose metadata cannot be stored then fails with the store untouched. - built, documents = _prepare_nodes(store, nodes_explicit) - await asyncio.gather(*(store.delete_dir(key) for key in to_delete_keys)) - async for key, node in _store_nodes(store, nodes_explicit, built, documents): - yield key, node - - -async def create_nodes( - *, - store: Store, - nodes: dict[str, GroupMetadata | ArrayV2Metadata | ArrayV3Metadata], -) -> AsyncIterator[tuple[str, AsyncGroup | AnyAsyncArray]]: - """Create a collection of arrays and / or groups concurrently. - - Note: no attempt is made to validate that these arrays and / or groups collectively form a - valid Zarr hierarchy. It is the responsibility of the caller of this function to ensure that - the ``nodes`` parameter satisfies any correctness constraints. - - Parameters - ---------- - store : Store - The storage backend to use. - nodes : dict[str, GroupMetadata | ArrayV3Metadata | ArrayV2Metadata] - A dictionary defining the hierarchy. The keys are the paths of the nodes - in the hierarchy, and the values are the metadata of the nodes. The - metadata must be either an instance of GroupMetadata, ArrayV3Metadata - or ArrayV2Metadata. - - Yields - ------ - AsyncGroup | AsyncArray - The created nodes in the order they are created. - """ - async for key, node in _store_nodes(store, nodes, *_prepare_nodes(store, nodes)): - yield key, node - - -def _prepare_nodes( - store: Store, nodes: Mapping[str, GroupMetadata | ArrayV2Metadata | ArrayV3Metadata] -) -> tuple[dict[str, AsyncGroup | AnyAsyncArray], dict[str, Buffer]]: - """The array or group each of `nodes` describes, at its path in `store`, and the - metadata documents of `nodes`, by their keys in the store: every node is built and - encoded before anything is stored.""" - built = { - path: _build_node(store=store, path=path, metadata=meta) for path, meta in nodes.items() - } - documents = { - _join_paths([path, key]): value - for path, metadata in nodes.items() - for key, value in encode_documents(StorePath(store, path), metadata).items() - } - return built, documents - - -async def _store_nodes( - store: Store, - nodes: Mapping[str, GroupMetadata | ArrayV2Metadata | ArrayV3Metadata], - built: Mapping[str, AsyncGroup | AnyAsyncArray], - documents: Mapping[str, Buffer], -) -> AsyncIterator[tuple[str, AsyncGroup | AnyAsyncArray]]: - """Store the `documents` encoded from `nodes` and yield the nodes built from them, as - `_prepare_nodes` returns them (see `create_nodes`).""" - # Note: the only way to alter this value is via the config. If that's undesirable for some reason, - # then we should consider adding a keyword argument to this function - semaphore = asyncio.Semaphore(config.get("async.concurrency")) - create_tasks = [ - _set_return_key(store=store, key=key, value=value, semaphore=semaphore) - for key, value in documents.items() - ] - - created_object_keys = [] - - for coro in asyncio.as_completed(create_tasks): - created_key = await coro - # we need this to track which metadata documents were written so that we can yield a - # complete v2 Array / Group class after both .zattrs and the metadata JSON was created. - created_object_keys.append(created_key) - - # get the node name from the object key - if len(created_key.split("/")) == 1: - # this is the root node - meta_out = nodes[""] - node_name = "" - else: - # turn "foo/" into "foo" - node_name = created_key[: created_key.rfind("/")] - meta_out = nodes[node_name] - if meta_out.zarr_format == 3: - yield node_name, built[node_name] - else: - # For zarr v2 - # we only want to yield when both the metadata and attributes are created - # so we track which keys have been created, and wait for both the meta key and - # the attrs key to be created before yielding back the AsyncArray / AsyncGroup - - attrs_done = _join_paths([node_name, ZATTRS_JSON]) in created_object_keys - - if isinstance(meta_out, GroupMetadata): - meta_done = _join_paths([node_name, ZGROUP_JSON]) in created_object_keys - else: - meta_done = _join_paths([node_name, ZARRAY_JSON]) in created_object_keys - - if meta_done and attrs_done: - yield node_name, built[node_name] - - continue - - -def _get_roots( - data: Iterable[str], -) -> tuple[str, ...]: - """ - Return the keys of the root(s) of the hierarchy. A root is a key with the fewest number of - path segments. - """ - if "" in data: - return ("",) - keys_split = sorted((key.split("/") for key in data), key=len) - groups: defaultdict[int, list[str]] = defaultdict(list) - for key_split in keys_split: - groups[len(key_split)].append("/".join(key_split)) - return tuple(groups[min(groups.keys())]) - - -def _parse_hierarchy_dict( - *, - data: Mapping[str, ImplicitGroupMarker | GroupMetadata | ArrayV2Metadata | ArrayV3Metadata], -) -> dict[str, ImplicitGroupMarker | GroupMetadata | ArrayV2Metadata | ArrayV3Metadata]: - """ - Take an input with type Mapping[str, ArrayMetadata | GroupMetadata] and parse it into - a dict of str: node pairs that models a valid, complete Zarr hierarchy. - - If the input represents a complete Zarr hierarchy, i.e. one with no implicit groups, - then return a dict with the exact same data as the input. - - Otherwise, return a dict derived from the input with GroupMetadata inserted as needed to make - the hierarchy complete. - - For example, an input of {'a/b': ArrayMetadata} is incomplete, because it references two - groups (the root group '' and a group at 'a') that are not specified in the input. Applying this function - to that input will result in a return value of - {'': GroupMetadata, 'a': GroupMetadata, 'a/b': ArrayMetadata}, i.e. the implied groups - were added. - - The input is also checked for the following conditions; an error is raised if any are violated: - - - No arrays can contain group or arrays (i.e., all arrays must be leaf nodes). - - All arrays and groups must have the same ``zarr_format`` value. - - This function ensures that the input is transformed into a specification of a complete and valid - Zarr hierarchy. - """ - - # ensure that all nodes have the same zarr format - data_purified = _ensure_consistent_zarr_format(data) - - # ensure that keys are normalized to zarr paths - data_normed_keys = _normalize_path_keys(data_purified) - - # insert an implicit root group if a root was not specified - # but not if an empty dict was provided, because any empty hierarchy has no nodes - if len(data_normed_keys) > 0 and "" not in data_normed_keys: - z_format = next(iter(data_normed_keys.values())).zarr_format - data_normed_keys = data_normed_keys | {"": ImplicitGroupMarker(zarr_format=z_format)} - - out: dict[str, GroupMetadata | ArrayV2Metadata | ArrayV3Metadata] = {**data_normed_keys} - - for k, v in data_normed_keys.items(): - key_split = k.split("/") - - # get every parent path - *subpaths, _ = accumulate(key_split, lambda a, b: _join_paths([a, b])) - - for subpath in subpaths: - # If a component is not already in the output dict, add ImplicitGroupMetadata - if subpath not in out: - out[subpath] = ImplicitGroupMarker(zarr_format=v.zarr_format) - else: - if not isinstance(out[subpath], GroupMetadata | ImplicitGroupMarker): - msg = ( - f"The node at {subpath} contains other nodes, but it is not a Zarr group. " - "This is invalid. Only Zarr groups can contain other nodes." - ) - raise ValueError(msg) - return out - - -def _ensure_consistent_zarr_format( - data: Mapping[str, GroupMetadata | ArrayV2Metadata | ArrayV3Metadata], -) -> Mapping[str, GroupMetadata | ArrayV2Metadata] | Mapping[str, GroupMetadata | ArrayV3Metadata]: - """ - Ensure that all values of the input dict have the same zarr format. If any do not, - then a value error is raised. - """ - observed_zarr_formats: dict[ZarrFormat, list[str]] = {2: [], 3: []} - - for k, v in data.items(): - observed_zarr_formats[v.zarr_format].append(k) - - if len(observed_zarr_formats[2]) > 0 and len(observed_zarr_formats[3]) > 0: - msg = ( - "Got data with both Zarr v2 and Zarr v3 nodes, which is invalid. " - f"The following keys map to Zarr v2 nodes: {observed_zarr_formats.get(2)}. " - f"The following keys map to Zarr v3 nodes: {observed_zarr_formats.get(3)}." - "Ensure that all nodes have the same Zarr format." - ) - raise ValueError(msg) - - return cast( - "Mapping[str, GroupMetadata | ArrayV2Metadata] | Mapping[str, GroupMetadata | ArrayV3Metadata]", - data, - ) - - -async def _getitem_semaphore( - node: AsyncGroup, key: str, semaphore: asyncio.Semaphore | None -) -> AnyAsyncArray | AsyncGroup: - """ - Wrap Group.getitem with an optional semaphore. - - If the semaphore parameter is an - asyncio.Semaphore instance, then the getitem operation is performed inside an async context - manager provided by that semaphore. If the semaphore parameter is None, then getitem is invoked - without a context manager. - """ - if semaphore is not None: - async with semaphore: - return await node.getitem(key) - else: - return await node.getitem(key) - - -async def _iter_members( - node: AsyncGroup, - skip_keys: tuple[str, ...], - semaphore: asyncio.Semaphore | None, -) -> AsyncGenerator[tuple[str, AnyAsyncArray | AsyncGroup], None]: - """ - Iterate over the arrays and groups contained in a group. - - Parameters - ---------- - node : AsyncGroup - The group to traverse. - skip_keys : tuple[str, ...] - A tuple of keys to skip when iterating over the possible members of the group. - semaphore : asyncio.Semaphore | None - An optional semaphore to use for concurrency control. - - Yields - ------ - tuple[str, AnyAsyncArray | AsyncGroup] - """ - - # retrieve keys from storage - keys = [key async for key in node.store.list_dir(node.path)] - keys_filtered = tuple(filter(lambda v: v not in skip_keys, keys)) - - node_tasks = tuple( - asyncio.create_task(_getitem_semaphore(node, key, semaphore), name=key) - for key in keys_filtered - ) - - for fetched_node_coro in asyncio.as_completed(node_tasks): - try: - fetched_node = await fetched_node_coro - except KeyError as e: - # keyerror is raised when `key` names an object (in the object storage sense), - # as opposed to a prefix, in the store under the prefix associated with this group - # in which case `key` cannot be the name of a sub-array or sub-group. - warnings.warn( - f"Object at {e.args[0]} is not recognized as a component of a Zarr hierarchy.", - ZarrUserWarning, - stacklevel=1, - ) - continue - match fetched_node: - case AsyncArray() | AsyncGroup(): - yield fetched_node.basename, fetched_node - case _: - raise ValueError(f"Unexpected type: {type(fetched_node)}") - - -async def _iter_members_deep( - group: AsyncGroup, - *, - max_depth: int | None, - skip_keys: tuple[str, ...], - semaphore: asyncio.Semaphore | None = None, - use_consolidated_for_children: bool = True, -) -> AsyncGenerator[tuple[str, AnyAsyncArray | AsyncGroup], None]: - """ - Iterate over the arrays and groups contained in a group, and optionally the - arrays and groups contained in those groups. - - Parameters - ---------- - group : AsyncGroup - The group to traverse. - max_depth : int | None - The maximum depth of recursion. - skip_keys : tuple[str, ...] - A tuple of keys to skip when iterating over the possible members of the group. - semaphore : asyncio.Semaphore | None - An optional semaphore to use for concurrency control. - use_consolidated_for_children : bool, default True - Whether to use the consolidated metadata of child groups loaded - from the store. Note that this only affects groups loaded from the - store. If the current Group already has consolidated metadata, it - will always be used. - - Yields - ------ - tuple[str, AnyAsyncArray | AsyncGroup] - """ - - to_recurse = {} - do_recursion = max_depth is None or max_depth > 0 - - if max_depth is None: - new_depth = None - else: - new_depth = max_depth - 1 - async for name, node in _iter_members(group, skip_keys=skip_keys, semaphore=semaphore): - is_group = isinstance(node, AsyncGroup) - if ( - is_group - and not use_consolidated_for_children - and node.metadata.consolidated_metadata is not None - ): - node = cast("AsyncGroup", node) - # We've decided not to trust consolidated metadata at this point, because we're - # reconsolidating the metadata, for example. - node = replace(node, metadata=replace(node.metadata, consolidated_metadata=None)) - yield name, node - if is_group and do_recursion: - node = cast("AsyncGroup", node) - to_recurse[name] = _iter_members_deep( - node, max_depth=new_depth, skip_keys=skip_keys, semaphore=semaphore - ) - - for prefix, subgroup_iter in to_recurse.items(): - async for name, node in subgroup_iter: - key = f"{prefix}/{name}".lstrip("/") - yield key, node - - -async def _read_metadata_v3(store: Store, path: str) -> ArrayV3Metadata | GroupMetadata: - """ - Given a store_path, return ArrayV3Metadata or GroupMetadata defined by the metadata - document stored at store_path.path / zarr.json. If no such document is found, raise a - FileNotFoundError. - """ - zarr_json_bytes = await store.get( - _join_paths([path, ZARR_JSON]), prototype=default_buffer_prototype() - ) - if zarr_json_bytes is None: - raise FileNotFoundError(path) - return _build_metadata_v3( - buffer_to_json_object(zarr_json_bytes), path=str(StorePath(store, path)) - ) - - -async def _read_metadata_v2(store: Store, path: str) -> ArrayV2Metadata | GroupMetadata: - """ - Given a store_path, return ArrayV2Metadata or GroupMetadata defined by the metadata - document stored at store_path.path / (.zgroup | .zarray). If no such document is found, - raise a FileNotFoundError. - """ - # TODO: consider first fetching array metadata, and only fetching group metadata when we don't - # find an array - zarray_bytes, zgroup_bytes, zattrs_bytes = await asyncio.gather( - store.get(_join_paths([path, ZARRAY_JSON]), prototype=default_buffer_prototype()), - store.get(_join_paths([path, ZGROUP_JSON]), prototype=default_buffer_prototype()), - store.get(_join_paths([path, ZATTRS_JSON]), prototype=default_buffer_prototype()), - ) - - zattrs: dict[str, JSON] - if zattrs_bytes is None: - zattrs = {} - else: - zattrs = buffer_to_json_object(zattrs_bytes) - - # TODO: decide how to handle finding both array and group metadata. The spec does not seem to - # consider this situation. A practical approach would be to ignore that combination, and only - # return the array metadata. - if zarray_bytes is not None: - zmeta = buffer_to_json_object(zarray_bytes) - else: - if zgroup_bytes is None: - # neither .zarray or .zgroup were found results in KeyError - raise FileNotFoundError(path) - else: - zmeta = buffer_to_json_object(zgroup_bytes) - - return _build_metadata_v2(zmeta, zattrs, path=str(StorePath(store, path))) - - -async def _read_group_metadata_v2(store: Store, path: str) -> GroupMetadata: - """ - Read group metadata or error - """ - meta = await _read_metadata_v2(store=store, path=path) - if not isinstance(meta, GroupMetadata): - raise FileNotFoundError(f"Group metadata was not found in {store} at {path}") - return meta - - -async def _read_group_metadata_v3(store: Store, path: str) -> GroupMetadata: - """ - Read group metadata or error - """ - meta = await _read_metadata_v3(store=store, path=path) - if not isinstance(meta, GroupMetadata): - raise FileNotFoundError(f"Group metadata was not found in {store} at {path}") - return meta - - -async def _read_group_metadata( - store: Store, path: str, *, zarr_format: ZarrFormat -) -> GroupMetadata: - if zarr_format == 2: - return await _read_group_metadata_v2(store=store, path=path) - return await _read_group_metadata_v3(store=store, path=path) - - -def _build_metadata_v3( - zarr_json: dict[str, JSON], *, path: str | None = None -) -> ArrayV3Metadata | GroupMetadata: - """ - Convert a dict representation of Zarr V3 metadata into the corresponding metadata class. - """ - if "node_type" not in zarr_json: - msg = "Required key 'node_type' is missing from the provided metadata document." - raise MetadataValidationError(msg) - match zarr_json: - case {"node_type": "array"}: - return ArrayV3Metadata.from_dict(zarr_json, path=path) - case {"node_type": "group"}: - return GroupMetadata.from_dict(zarr_json, path=path) - case _: # pragma: no cover - raise ValueError( - "invalid value for `node_type` key in metadata document" - ) # pragma: no cover - - -def _build_metadata_v2( - zarr_json: dict[str, JSON], attrs_json: dict[str, JSON], *, path: str | None = None -) -> ArrayV2Metadata | GroupMetadata: - """ - Convert a dict representation of Zarr V2 metadata into the corresponding metadata class. - """ - match zarr_json: - case {"shape": _}: - return ArrayV2Metadata.from_dict(zarr_json | {"attributes": attrs_json}, path=path) - case _: # pragma: no cover - return GroupMetadata.from_dict(zarr_json | {"attributes": attrs_json}, path=path) - - -@overload -def _build_node(*, store: Store, path: str, metadata: ArrayV2Metadata) -> AsyncArrayV2: ... - - -@overload -def _build_node(*, store: Store, path: str, metadata: ArrayV3Metadata) -> AsyncArrayV3: ... - - -@overload -def _build_node(*, store: Store, path: str, metadata: GroupMetadata) -> AsyncGroup: ... - - -def _build_node( - *, store: Store, path: str, metadata: ArrayV3Metadata | ArrayV2Metadata | GroupMetadata -) -> AnyAsyncArray | AsyncGroup: - """ - Take a metadata object and return a node (AsyncArray or AsyncGroup). - """ - store_path = StorePath(store=store, path=path) - match metadata: - case ArrayV2Metadata() | ArrayV3Metadata(): - return AsyncArray(metadata, store_path=store_path) - case GroupMetadata(): - return AsyncGroup(metadata, store_path=store_path) - case _: # pragma: no cover - raise ValueError(f"Unexpected metadata type: {type(metadata)}") # pragma: no cover - - -async def _get_node_v2(store: Store, path: str) -> AsyncArrayV2 | AsyncGroup: - """ - Read a Zarr v2 AsyncArray or AsyncGroup from a path in a Store. - - Parameters - ---------- - store : Store - The store-like object to read from. - path : str - The path to the node to read. - - Returns - ------- - AsyncArray | AsyncGroup - """ - metadata = await _read_metadata_v2(store=store, path=path) - return _build_node(store=store, path=path, metadata=metadata) - - -async def _get_node_v3(store: Store, path: str) -> AsyncArrayV3 | AsyncGroup: - """ - Read a Zarr v3 AsyncArray or AsyncGroup from a path in a Store. - - Parameters - ---------- - store : Store - The store-like object to read from. - path : str - The path to the node to read. - - Returns - ------- - AsyncArray | AsyncGroup - """ - metadata = await _read_metadata_v3(store=store, path=path) - return _build_node(store=store, path=path, metadata=metadata) - - -async def get_node(store: Store, path: str, zarr_format: ZarrFormat) -> AnyAsyncArray | AsyncGroup: - """ - Get an AsyncArray or AsyncGroup from a path in a Store. - - Parameters - ---------- - store : Store - The store-like object to read from. - path : str - The path to the node to read. - zarr_format : {2, 3} - The zarr format of the node to read. - - Returns - ------- - AsyncArray | AsyncGroup - """ - - match zarr_format: - case 2: - return await _get_node_v2(store=store, path=path) - case 3: - return await _get_node_v3(store=store, path=path) - case _: # pragma: no cover - raise ValueError(f"Unexpected zarr format: {zarr_format}") # pragma: no cover - - -async def _set_return_key( - *, store: Store, key: str, value: Buffer, semaphore: asyncio.Semaphore | None = None -) -> str: - """ - Write a value to storage at the given key. The key is returned. - Useful when saving values via routines that return results in execution order, - like asyncio.as_completed, because in this case we need to know which key was saved in order - to yield the right object to the caller. - - Parameters - ---------- - store : Store - The store to save the value to. - key : str - The key to save the value to. - value : Buffer - The value to save. - semaphore : asyncio.Semaphore | None - An optional semaphore to use to limit the number of concurrent writes. - """ - - if semaphore is not None: - async with semaphore: - await store.set(key, value) - else: - await store.set(key, value) - return key - - -async def create_rooted_hierarchy( - *, - store: Store, - nodes: dict[str, GroupMetadata | ArrayV2Metadata | ArrayV3Metadata], - overwrite: bool = False, -) -> AsyncGroup | AnyAsyncArray: - """ - Create an ``AsyncGroup`` or ``AsyncArray`` from a store and a dict of metadata documents. - This function ensures that its input contains a specification of a root node, - calls ``create_hierarchy`` to create nodes, and returns the root node of the hierarchy. - """ - roots = _get_roots(nodes.keys()) - if len(roots) != 1: - msg = ( - "The input does not specify a root node. " - "This function can only create hierarchies that contain a root node, which is " - "defined as a group that is ancestral to all the other arrays and " - "groups in the hierarchy, or a single array." - ) - raise ValueError(msg) - else: - root_key = roots[0] - - nodes_created = [ - x async for x in create_hierarchy(store=store, nodes=nodes, overwrite=overwrite) - ] - return dict(nodes_created)[root_key] +from __future__ import annotations + +import asyncio +import logging +import unicodedata +import warnings +from collections import defaultdict +from dataclasses import asdict, dataclass, field, replace +from itertools import accumulate +from typing import TYPE_CHECKING, Final, Literal, assert_never, cast, overload + +import numpy as np + +import zarr.api.asynchronous as async_api +from zarr.abc.metadata import Metadata +from zarr.abc.store import Store, set_or_delete +from zarr.core._info import GroupInfo +from zarr.core._json import buffer_to_json_object, json_to_buffer +from zarr.core.array import ( + DEFAULT_FILL_VALUE, + Array, + AsyncArray, + CompressorLike, + CompressorsLike, + FiltersLike, + SerializerLike, + ShardsLike, + _parse_deprecated_compressor, + create_array, +) +from zarr.core.array_spec import parse_array_config +from zarr.core.attributes import Attributes +from zarr.core.buffer import default_buffer_prototype +from zarr.core.common import ( + JSON, + ZARR_JSON, + ZARRAY_JSON, + ZATTRS_JSON, + ZGROUP_JSON, + ZMETADATA_V2_JSON, + ChunksLike, + DimensionNamesLike, + NodeType, + ShapeLike, + ZarrFormat, + parse_shapelike, + validate_node_path, +) +from zarr.core.config import config +from zarr.core.dtype import parse_data_type +from zarr.core.json_parse import parse_field +from zarr.core.metadata import ArrayV2Metadata, ArrayV3Metadata +from zarr.core.metadata.io import ( + encode_documents, + save_metadata, + save_new_metadata, + store_documents, +) +from zarr.core.metadata.v3 import AllowedExtraField, check_storable, parse_extra_fields +from zarr.core.sync import SyncMixin, sync +from zarr.errors import ( + ArrayNotFoundError, + ContainsArrayError, + ContainsGroupError, + GroupNotFoundError, + MetadataValidationError, + NodeTypeValidationError, + ZarrUserWarning, +) +from zarr.storage import StoreLike, StorePath +from zarr.storage._common import make_store_path +from zarr.storage._utils import _join_paths, _normalize_path_keys, normalize_path + +if TYPE_CHECKING: + from collections.abc import ( + AsyncGenerator, + AsyncIterator, + Coroutine, + Generator, + Iterable, + Iterator, + Mapping, + ) + from typing import Any + + from zarr.core.array_spec import ArrayConfigLike + from zarr.core.buffer import Buffer, BufferPrototype + from zarr.core.chunk_key_encodings import ChunkKeyEncodingLike + from zarr.core.common import MemoryOrder + from zarr.core.dtype import ZDTypeLike + from zarr.types import AnyArray, AnyAsyncArray, ArrayV2, ArrayV3, AsyncArrayV2, AsyncArrayV3 + +logger = logging.getLogger("zarr.group") + + +def parse_zarr_format(data: Any) -> ZarrFormat: + """Parse the zarr_format field from metadata.""" + return cast("ZarrFormat", parse_field(data, Literal[2, 3], "zarr_format")) + + +def parse_node_type(data: Any) -> NodeType: + """Parse the node_type field from metadata.""" + return cast( + "Literal['array', 'group']", + parse_field(data, Literal["array", "group"], "node_type", error=MetadataValidationError), + ) + + +# todo: convert None to empty dict +def parse_attributes(data: Any) -> dict[str, Any]: + """Parse the attributes field from metadata.""" + if data is None: + return {} + elif isinstance(data, dict) and all(isinstance(k, str) for k in data): + return dict(data) + msg = f"Expected dict with string keys. Got {type(data)} instead." + raise TypeError(msg) + + +@overload +def _parse_async_node(node: AsyncArrayV3) -> ArrayV3: ... + + +@overload +def _parse_async_node(node: AsyncArrayV2) -> ArrayV2: ... + + +@overload +def _parse_async_node(node: AsyncGroup) -> Group: ... + + +def _parse_async_node( + node: AnyAsyncArray | AsyncGroup, +) -> AnyArray | Group: + """Wrap an AsyncArray in an Array, or an AsyncGroup in a Group.""" + if isinstance(node, AsyncArray): + return Array(node) + elif isinstance(node, AsyncGroup): + return Group(node) + else: + raise TypeError(f"Unknown node type, got {type(node)}") + + +@dataclass(frozen=True) +class ConsolidatedMetadata: + """ + Consolidated Metadata for this Group. + + This stores the metadata of child nodes below this group. Any child groups + will have their consolidated metadata set appropriately. + """ + + metadata: dict[str, ArrayV2Metadata | ArrayV3Metadata | GroupMetadata] + kind: Literal["inline"] = "inline" + must_understand: Literal[False] = False + + def to_dict(self) -> dict[str, JSON]: + """The consolidated metadata document. An array read from a stored document that + had to be repaired is written as that document was stored, so every reader of + the consolidated metadata reads it as repaired again (see `mark_repaired`).""" + return { + "kind": self.kind, + "must_understand": self.must_understand, + "metadata": { + k: v.to_dict() + if isinstance(v, GroupMetadata) or v._stored_document is None + else dict(v._stored_document) + for k, v in sorted( + self.flattened_metadata.items(), + key=lambda item: ( + item[0].count("/"), + unicodedata.normalize("NFKC", item[0]).casefold(), + ), + ) + }, + } + + @classmethod + def from_dict(cls, data: dict[str, JSON], *, path: str | None = None) -> ConsolidatedMetadata: + """Read consolidated metadata, naming each member by its path under the group at + `path` (or relative to that group, when `path` is not given) in warnings.""" + data = dict(data) + + kind = data.get("kind") + if kind != "inline": + raise ValueError(f"Consolidated metadata kind='{kind}' is not supported.") + + raw_metadata = data.get("metadata") + if not isinstance(raw_metadata, dict): + raise TypeError(f"Unexpected type for 'metadata': {type(raw_metadata)}") + + metadata: dict[str, ArrayV2Metadata | ArrayV3Metadata | GroupMetadata] = {} + if raw_metadata: + for k, v in raw_metadata.items(): + member = k if path is None else _join_paths([path, k]) + if not isinstance(v, dict): + raise TypeError( + f"Invalid value for metadata items. key='{k}', type='{type(v).__name__}'" + ) + + # zarr_format is present in v2 and v3. + zarr_format = parse_zarr_format(v["zarr_format"]) + + if zarr_format == 3: + node_type = parse_node_type(v.get("node_type", None)) + if node_type == "group": + metadata[k] = GroupMetadata.from_dict(v, path=member) + elif node_type == "array": + metadata[k] = ArrayV3Metadata.from_dict(v, path=member) + else: + assert_never(node_type) + elif zarr_format == 2: + if "shape" in v: + metadata[k] = ArrayV2Metadata.from_dict(v, path=member) + else: + metadata[k] = GroupMetadata.from_dict(v, path=member) + else: + assert_never(zarr_format) + + cls._flat_to_nested(metadata) + + return cls(metadata=metadata) + + @staticmethod + def _flat_to_nested( + metadata: dict[str, ArrayV2Metadata | ArrayV3Metadata | GroupMetadata], + ) -> None: + """ + Convert a flat metadata representation to a nested one. + + Notes + ----- + Flat metadata is used when persisting the consolidated metadata. The keys + include the full path, not just the node name. The key prefixes can be + used to determine which nodes are children of which other nodes. + + Nested metadata is used in-memory. The outermost level will only have the + *immediate* children of the Group. All nested child groups will be stored + under the consolidated metadata of their immediate parent. + """ + # We have a flat mapping from {k: v} where the keys include the *full* + # path segment: + # { + # "/a/b": { group_metadata }, + # "/a/b/array-0": { array_metadata }, + # "/a/b/array-1": { array_metadata }, + # } + # + # We want to reorganize the metadata such that each Group contains the + # array metadata of its immediate children. + # In the example, the group at `/a/b` will have consolidated metadata + # for its children `array-0` and `array-1`. + + # Group keys by their parent path. This must not rely on same-parent keys + # being adjacent: the persisted key order is arbitrary, so accumulate + # instead of using itertools.groupby, which only groups consecutive runs. + grouped: dict[str, list[str]] = defaultdict(list) + for k in sorted(metadata, key=lambda k: k.count("/")): + grouped[k.rsplit("/", 1)[0]].append(k) + + # we go top down and directly manipulate metadata. + for key, children_keys in grouped.items(): + # key is a key like "a", "a/b", "a/b/c" + # The basic idea is to find the immediate parent (so "", "a", or "a/b") + # and update that node's consolidated metadata to include the metadata + # in children_keys + *prefixes, name = key.split("/") + parent = metadata + + while prefixes: + # e.g. a/b/c has a parent "a/b". Walk through to get + # metadata["a"]["b"] + part = prefixes.pop(0) + # we can assume that parent[part] here is a group + # otherwise we wouldn't have a node with this `part` prefix. + # We can also assume that the parent node will have consolidated metadata, + # because we're walking top to bottom. + parent = parent[part].consolidated_metadata.metadata # type: ignore[union-attr] + + node = parent[name] + children_keys = list(children_keys) + + if isinstance(node, ArrayV2Metadata | ArrayV3Metadata): + # These are already present, either thanks to being an array in the + # root, or by being collected as a child in the else clause + continue + children: dict[str, ArrayV2Metadata | ArrayV3Metadata | GroupMetadata] = {} + # We pop from metadata, since we're *moving* this under group. + # While doing this, normalize leaf groups to carry empty consolidated metadata. + for child_key in children_keys: + if child_key == key: + continue + child = metadata.pop(child_key) + if isinstance(child, GroupMetadata) and child.consolidated_metadata is None: + child = replace(child, consolidated_metadata=ConsolidatedMetadata(metadata={})) + children[child_key.split("/")[-1]] = child + + parent[name] = replace( + node, consolidated_metadata=ConsolidatedMetadata(metadata=children) + ) + + @property + def flattened_metadata(self) -> dict[str, ArrayV2Metadata | ArrayV3Metadata | GroupMetadata]: + """ + Return the flattened representation of Consolidated Metadata. + + The returned dictionary will have a key for each child node in the hierarchy + under this group. Under the default (nested) representation available through + ``self.metadata``, the dictionary only contains keys for immediate children. + + The keys of the dictionary will include the full path to a child node from + the current group, where segments are joined by ``/``. + + Examples + -------- + ```python + from zarr.core.group import ConsolidatedMetadata, GroupMetadata + cm = ConsolidatedMetadata( + metadata={ + "group-0": GroupMetadata( + consolidated_metadata=ConsolidatedMetadata( + { + "group-0-0": GroupMetadata(), + } + ) + ), + "group-1": GroupMetadata(), + } + ) + # {'group-0': GroupMetadata(attributes={}, zarr_format=3, consolidated_metadata=None, node_type='group', extra_fields={}), + # 'group-0/group-0-0': GroupMetadata(attributes={}, zarr_format=3, consolidated_metadata=None, node_type='group', extra_fields={}), + # 'group-1': GroupMetadata(attributes={}, zarr_format=3, consolidated_metadata=None, node_type='group', extra_fields={})} + ``` + """ + metadata = {} + + def flatten( + key: str, group: GroupMetadata | ArrayV2Metadata | ArrayV3Metadata + ) -> dict[str, ArrayV2Metadata | ArrayV3Metadata | GroupMetadata]: + children: dict[str, ArrayV2Metadata | ArrayV3Metadata | GroupMetadata] = {} + if isinstance(group, ArrayV2Metadata | ArrayV3Metadata): + children[key] = group + else: + if group.consolidated_metadata and group.consolidated_metadata.metadata is not None: + children[key] = replace( + group, consolidated_metadata=ConsolidatedMetadata(metadata={}) + ) + for name, val in group.consolidated_metadata.metadata.items(): + full_key = f"{key}/{name}" + if isinstance(val, GroupMetadata): + children.update(flatten(full_key, val)) + else: + children[full_key] = val + else: + children[key] = replace(group, consolidated_metadata=None) + return children + + for k, v in self.metadata.items(): + metadata.update(flatten(k, v)) + + return metadata + + +GROUP_METADATA_KEYS: Final[set[str]] = { + "zarr_format", + "node_type", + "attributes", + "consolidated_metadata", +} +""" +The names of the fields of the group metadata document. `consolidated_metadata` is not in the +Zarr V3 spec; Zarr-Python writes it. +""" + + +@dataclass(frozen=True) +class GroupMetadata(Metadata): + """ + Metadata for a Group. + """ + + attributes: dict[str, Any] = field(default_factory=dict) + zarr_format: ZarrFormat = 3 + consolidated_metadata: ConsolidatedMetadata | None = None + node_type: Literal["group"] = field(default="group", init=False) + extra_fields: dict[str, AllowedExtraField] = field(default_factory=dict) + + def to_buffer_dict(self, prototype: BufferPrototype) -> dict[str, Buffer]: + if self.consolidated_metadata is not None: + for path, member in self.consolidated_metadata.flattened_metadata.items(): + # A member read from a document that had to be repaired is stored as it + # was stored (see `ConsolidatedMetadata.to_dict`). + if isinstance(member, ArrayV3Metadata) and member._stored_document is None: + try: + check_storable(member) + except ValueError as e: + e.add_note(f"Array {path!r} in the consolidated metadata.") + raise + indent = config.get("json_indent") + if self.zarr_format == 3: + return {ZARR_JSON: json_to_buffer(self.to_dict(), prototype=prototype, indent=indent)} + else: + items = { + ZGROUP_JSON: json_to_buffer( + {"zarr_format": self.zarr_format}, prototype=prototype, indent=indent + ), + ZATTRS_JSON: json_to_buffer(self.attributes, prototype=prototype, indent=indent), + } + if self.consolidated_metadata: + d = { + ZGROUP_JSON: {"zarr_format": self.zarr_format}, + ZATTRS_JSON: self.attributes, + } + consolidated_metadata = self.consolidated_metadata.to_dict()["metadata"] + if not isinstance(consolidated_metadata, dict): + raise TypeError( + "Expected consolidated metadata to serialize to a dict, " + f"got {type(consolidated_metadata).__name__}." + ) + for k, v in consolidated_metadata.items(): + attrs = v.pop("attributes", {}) + d[f"{k}/{ZATTRS_JSON}"] = attrs + if "shape" in v: + # it's an array + d[f"{k}/{ZARRAY_JSON}"] = v + else: + d[f"{k}/{ZGROUP_JSON}"] = { + "zarr_format": self.zarr_format, + "consolidated_metadata": { + "metadata": {}, + "must_understand": False, + "kind": "inline", + }, + } + + # The consolidated metadata blob is written compactly (no indent). + items[ZMETADATA_V2_JSON] = json_to_buffer( + {"metadata": d, "zarr_consolidated_format": 1}, prototype=prototype + ) + + return items + + def __init__( + self, + attributes: dict[str, Any] | None = None, + zarr_format: ZarrFormat = 3, + consolidated_metadata: ConsolidatedMetadata | None = None, + extra_fields: Mapping[str, AllowedExtraField] | None = None, + ) -> None: + attributes_parsed = parse_attributes(attributes) + zarr_format_parsed = parse_zarr_format(zarr_format) + if zarr_format_parsed == 2 and extra_fields: + raise ValueError( + "Invalid extra fields. Zarr format 2 group metadata does not support extra fields." + ) + extra_fields_parsed = parse_extra_fields( + extra_fields, reserved_keys=GROUP_METADATA_KEYS, node_type="group" + ) + + object.__setattr__(self, "attributes", attributes_parsed) + object.__setattr__(self, "zarr_format", zarr_format_parsed) + object.__setattr__(self, "consolidated_metadata", consolidated_metadata) + object.__setattr__(self, "extra_fields", extra_fields_parsed) + + @classmethod + def from_dict(cls, data: dict[str, Any], *, path: str | None = None) -> GroupMetadata: + """Read a stored group document; `path` names the group in warnings about the + consolidated metadata it holds.""" + data = dict(data) + node_type = data.pop("node_type", None) + if node_type not in ("group", None): + raise NodeTypeValidationError( + f"Invalid value for 'node_type'. Expected 'group' or None. Got {node_type!r}." + ) + consolidated_metadata = data.pop("consolidated_metadata", None) + if consolidated_metadata: + data["consolidated_metadata"] = ConsolidatedMetadata.from_dict( + consolidated_metadata, path=path + ) + + zarr_format = data.get("zarr_format") + if zarr_format == 3: + # Zarr v3 allows an extra key only if it is an object with "must_understand": false. + extra_fields = parse_extra_fields( + {k: v for k, v in data.items() if k not in GROUP_METADATA_KEYS}, + reserved_keys=GROUP_METADATA_KEYS, + node_type="group", + ) + else: + # Zarr v2 allowed arbitrary keys; they are dropped rather than kept. + extra_fields = {} + data = {k: v for k, v in data.items() if k in GROUP_METADATA_KEYS} + return cls(**data, extra_fields=extra_fields) + + def to_dict(self) -> dict[str, Any]: + result = asdict(replace(self, consolidated_metadata=None)) + result.update(result.pop("extra_fields")) + if self.consolidated_metadata is not None: + result["consolidated_metadata"] = self.consolidated_metadata.to_dict() + else: + # Leave consolidated metadata unset if it's None + result.pop("consolidated_metadata") + return result + + +@dataclass(frozen=True) +class ImplicitGroupMarker(GroupMetadata): + """ + Marker for an implicit group. Instances of this class are only used in the context of group + creation as a placeholder to represent groups that should only be created if they do not + already exist in storage + """ + + +@dataclass(frozen=True) +class AsyncGroup: + """ + Asynchronous Group object. + """ + + metadata: GroupMetadata + store_path: StorePath + + # TODO: make this correct and work + # TODO: ensure that this can be bound properly to subclass of AsyncGroup + + @classmethod + async def from_store( + cls, + store: StoreLike, + *, + attributes: dict[str, Any] | None = None, + overwrite: bool = False, + zarr_format: ZarrFormat = 3, + ) -> AsyncGroup: + store_path = await make_store_path(store) + attributes = attributes or {} + group = cls( + metadata=GroupMetadata(attributes=attributes, zarr_format=zarr_format), + store_path=store_path, + ) + await save_new_metadata( + store_path, group.metadata, overwrite=overwrite, ensure_parents=True + ) + return group + + @classmethod + async def open( + cls, + store: StoreLike, + zarr_format: ZarrFormat | None = 3, + use_consolidated: bool | str | None = None, + ) -> AsyncGroup: + """Open a new AsyncGroup + + Parameters + ---------- + store : StoreLike + zarr_format : {2, 3}, optional + use_consolidated : bool or str, default None + Whether to use consolidated metadata. + + By default, consolidated metadata is used if it's present in the + store (in the ``zarr.json`` for Zarr format 3 and in the ``.zmetadata`` file + for Zarr format 2) and the Store supports it. + + To explicitly require consolidated metadata, set ``use_consolidated=True``. + In this case, if the Store doesn't support consolidation or consolidated metadata is + not found, a ``ValueError`` exception is raised. + + To explicitly *not* use consolidated metadata, set ``use_consolidated=False``, + which will fall back to using the regular, non consolidated metadata. + + Zarr format 2 allowed configuring the key storing the consolidated metadata + (``.zmetadata`` by default). Specify the custom key as ``use_consolidated`` + to load consolidated metadata from a non-default key. + """ + store_path = await make_store_path(store) + if not store_path.store.supports_consolidated_metadata: + # Fail if consolidated metadata was requested but the Store doesn't support it + if use_consolidated: + store_name = type(store_path.store).__name__ + raise ValueError( + f"The Zarr store in use ({store_name}) doesn't support consolidated metadata." + ) + + # if use_consolidated was None (optional), the Store dictates it doesn't want consolidation + use_consolidated = False + + consolidated_key = ZMETADATA_V2_JSON + + if (zarr_format == 2 or zarr_format is None) and isinstance(use_consolidated, str): + consolidated_key = use_consolidated + + if zarr_format == 2: + paths = [store_path / ZGROUP_JSON, store_path / ZATTRS_JSON] + if use_consolidated or use_consolidated is None: + paths.append(store_path / consolidated_key) + + zgroup_bytes, zattrs_bytes, *rest = await asyncio.gather( + *[path.get() for path in paths] + ) + if zgroup_bytes is None: + raise FileNotFoundError(store_path) + + if use_consolidated or use_consolidated is None: + maybe_consolidated_metadata_bytes = rest[0] + + else: + maybe_consolidated_metadata_bytes = None + + elif zarr_format == 3: + zarr_json_bytes = await (store_path / ZARR_JSON).get() + if zarr_json_bytes is None: + raise FileNotFoundError(store_path) + elif zarr_format is None: + ( + zarr_json_bytes, + zgroup_bytes, + zattrs_bytes, + maybe_consolidated_metadata_bytes, + ) = await asyncio.gather( + (store_path / ZARR_JSON).get(), + (store_path / ZGROUP_JSON).get(), + (store_path / ZATTRS_JSON).get(), + (store_path / str(consolidated_key)).get(), + ) + if zarr_json_bytes is not None and zgroup_bytes is not None: + # warn and favor v3 + msg = f"Both zarr.json (Zarr format 3) and .zgroup (Zarr format 2) metadata objects exist at {store_path}. Zarr format 3 will be used." + warnings.warn(msg, category=ZarrUserWarning, stacklevel=1) + if zarr_json_bytes is None and zgroup_bytes is None: + raise FileNotFoundError( + f"could not find zarr.json or .zgroup objects in {store_path}" + ) + # set zarr_format based on which keys were found + if zarr_json_bytes is not None: + zarr_format = 3 + else: + zarr_format = 2 + else: + msg = f"Invalid value for 'zarr_format'. Expected 2, 3, or None. Got '{zarr_format}'." # type: ignore[unreachable] + raise MetadataValidationError(msg) + + if zarr_format == 2: + if zgroup_bytes is None: + raise FileNotFoundError(store_path) + + if use_consolidated and maybe_consolidated_metadata_bytes is None: + # the user requested consolidated metadata, but it was missing + raise ValueError(consolidated_key) + + elif use_consolidated is False: + # the user explicitly opted out of consolidated_metadata. + # Discard anything we might have read. + maybe_consolidated_metadata_bytes = None + + return cls._from_bytes_v2( + store_path, zgroup_bytes, zattrs_bytes, maybe_consolidated_metadata_bytes + ) + else: + # V3 groups are comprised of a zarr.json object + if zarr_json_bytes is None: + raise FileNotFoundError(store_path) + if not isinstance(use_consolidated, bool | None): + raise TypeError("use_consolidated must be a bool or None for Zarr format 3.") + + return cls._from_bytes_v3( + store_path, + zarr_json_bytes, + use_consolidated=use_consolidated, + ) + + @classmethod + def _from_bytes_v2( + cls, + store_path: StorePath, + zgroup_bytes: Buffer, + zattrs_bytes: Buffer | None, + consolidated_metadata_bytes: Buffer | None, + ) -> AsyncGroup: + # V2 groups are comprised of a .zgroup and .zattrs objects + zgroup = buffer_to_json_object(zgroup_bytes) + zattrs = buffer_to_json_object(zattrs_bytes) if zattrs_bytes is not None else {} + group_metadata: dict[str, Any] = {**zgroup, "attributes": zattrs} + + if consolidated_metadata_bytes is not None: + v2_consolidated_doc = buffer_to_json_object(consolidated_metadata_bytes) + v2_consolidated_metadata = cast("dict[str, Any]", v2_consolidated_doc["metadata"]) + # We already read zattrs and zgroup. Should we ignore these? + v2_consolidated_metadata.pop(".zattrs", None) + v2_consolidated_metadata.pop(".zgroup", None) + + consolidated_metadata: defaultdict[str, dict[str, Any]] = defaultdict(dict) + + # keys like air/.zarray, air/.zattrs + for k, v in v2_consolidated_metadata.items(): + path, kind = k.rsplit("/.", 1) + + if kind == "zarray": + consolidated_metadata[path].update(v) + elif kind == "zattrs": + consolidated_metadata[path]["attributes"] = v + elif kind == "zgroup": + consolidated_metadata[path].update(v) + else: + raise ValueError(f"Invalid file type '{kind}' at path '{path}") + + group_metadata["consolidated_metadata"] = { + "metadata": dict(consolidated_metadata), + "kind": "inline", + "must_understand": False, + } + + return cls.from_dict(store_path, group_metadata) + + @classmethod + def _from_bytes_v3( + cls, + store_path: StorePath, + zarr_json_bytes: Buffer, + use_consolidated: bool | None, + ) -> AsyncGroup: + group_metadata = buffer_to_json_object(zarr_json_bytes) + if use_consolidated and group_metadata.get("consolidated_metadata") is None: + msg = f"Consolidated metadata requested with 'use_consolidated=True' but not found in '{store_path.path}'." + raise ValueError(msg) + + elif use_consolidated is False: + # Drop consolidated metadata if it's there. + group_metadata.pop("consolidated_metadata", None) + + return cls.from_dict(store_path, group_metadata) + + @classmethod + def from_dict( + cls, + store_path: StorePath, + data: dict[str, Any], + ) -> AsyncGroup: + node_type = data.pop("node_type", None) + if node_type == "array": + msg = f"An array already exists in store {store_path.store} at path {store_path.path}." + raise ContainsArrayError(msg) + elif node_type not in ("group", None): + msg = f"Node type in metadata ({node_type}) is not 'group'" + raise GroupNotFoundError(msg) + return cls( + metadata=GroupMetadata.from_dict(data, path=str(store_path)), + store_path=store_path, + ) + + async def setitem(self, key: str, value: Any) -> None: + """ + Fastpath for creating a new array + New arrays will be created with default array settings for the array type. + + Parameters + ---------- + key : str + Array name + value : array-like + Array data + """ + path = self.store_path / key + await async_api.save_array( + store=path, arr=value, zarr_format=self.metadata.zarr_format, overwrite=True + ) + + async def getitem( + self, + key: str, + ) -> AnyAsyncArray | AsyncGroup: + """ + Get a subarray or subgroup from the group. + + Parameters + ---------- + key : str + Array or group name + + Returns + ------- + AsyncArray or AsyncGroup + """ + store_path = self.store_path / key + logger.debug("key=%s, store_path=%s", key, store_path) + + # Consolidated metadata lets us avoid some I/O operations so try that first. + if self.metadata.consolidated_metadata is not None: + return self._getitem_consolidated(store_path, key, prefix=self.name) + try: + return await get_node( + store=store_path.store, path=store_path.path, zarr_format=self.metadata.zarr_format + ) + except FileNotFoundError as e: + raise KeyError(key) from e + + def _getitem_consolidated( + self, store_path: StorePath, key: str, prefix: str + ) -> AnyAsyncArray | AsyncGroup: + # getitem, in the special case where we have consolidated metadata. + # Note that this is a regular def (non async) function. + # This shouldn't do any additional I/O. + + # we support nested getitems like group/subgroup/array + indexers = normalize_path(key).split("/") + indexers.reverse() + metadata: ArrayV2Metadata | ArrayV3Metadata | GroupMetadata = self.metadata + + while indexers: + indexer = indexers.pop() + if isinstance(metadata, ArrayV2Metadata | ArrayV3Metadata): + # we've indexed into an array with group["array/subarray"]. Invalid. + raise KeyError(key) + if metadata.consolidated_metadata is None: + # we've indexed into a group without consolidated metadata. + # This isn't normal; typically, consolidated metadata + # will include explicit markers for when there are no child + # nodes as metadata={}. + # We have some freedom in exactly how we interpret this case. + # For now, we treat None as the same as {}, i.e. we don't + # have any children. + raise KeyError(key) + try: + metadata = metadata.consolidated_metadata.metadata[indexer] + except KeyError as e: + # The Group Metadata has consolidated metadata, but the key + # isn't present. We trust this to mean that the key isn't in + # the hierarchy, and *don't* fall back to checking the store. + msg = f"'{key}' not found in consolidated metadata." + raise KeyError(msg) from e + + # update store_path to ensure that AsyncArray/Group.name is correct + if prefix != "/": + key = "/".join([prefix.lstrip("/"), key]) + store_path = StorePath(store=store_path.store, path=key) + + if isinstance(metadata, GroupMetadata): + return AsyncGroup(metadata=metadata, store_path=store_path) + else: + return AsyncArray(metadata=metadata, store_path=store_path) + + async def delitem(self, key: str) -> None: + """Delete a group member. + + Parameters + ---------- + key : str + Array or group name + """ + store_path = self.store_path / key + consolidated = self.metadata.consolidated_metadata + if consolidated is None: + await store_path.delete_dir() + return + # Encode the group metadata without the member before deleting it: metadata that + # cannot be stored then fails with the store and this group untouched. What is + # stored is encoded after the deletion, from the metadata as it then is, so + # concurrent deletions each store the deletions made before them. + members = {name: node for name, node in consolidated.metadata.items() if name != key} + encode_documents( + self.store_path, + replace(self.metadata, consolidated_metadata=replace(consolidated, metadata=members)), + ) + await store_path.delete_dir() + # In place, so every handle sharing this consolidated metadata (a parent's or a + # subgroup's) sees the deletion. + consolidated.metadata.pop(key, None) + await store_documents(self.store_path, encode_documents(self.store_path, self.metadata)) + + async def get[DefaultT]( + self, key: str, default: DefaultT | None = None + ) -> AnyAsyncArray | AsyncGroup | DefaultT | None: + """Obtain a group member, returning default if not found. + + Parameters + ---------- + key : str + Group member name. + default : object + Default value to return if key is not found (default: None). + + Returns + ------- + object + Group member (AsyncArray or AsyncGroup) or default if not found. + """ + try: + return await self.getitem(key) + except KeyError: + return default + + async def get_array(self, path: str) -> AnyAsyncArray: + """Obtain an array member of this group, raising if it is absent or not an array. + + Parameters + ---------- + path : str + Path of the array relative to this group. May contain `/` to reference + a member of a subgroup, e.g. `subgroup/subarray`. + + Returns + ------- + AsyncArray + The array at the given path. + + Raises + ------ + ArrayNotFoundError + If no node exists at the given path. + ContainsGroupError + If the node at the given path is a group rather than an array. + """ + store_path = self.store_path / path + try: + node = await self.getitem(path) + except KeyError as e: + msg = f"No array found in store {store_path.store!r} at path {store_path.path!r}" + raise ArrayNotFoundError(msg) from e + if isinstance(node, AsyncGroup): + msg = f"A group exists in store {store_path.store!r} at path {store_path.path!r}." + raise ContainsGroupError(msg) + return node + + async def get_group(self, path: str) -> AsyncGroup: + """Obtain a group member of this group, raising if it is absent or not a group. + + Parameters + ---------- + path : str + Path of the group relative to this group. May contain `/` to reference + a member of a subgroup, e.g. `subgroup/subsubgroup`. + + Returns + ------- + AsyncGroup + The group at the given path. + + Raises + ------ + GroupNotFoundError + If no node exists at the given path. + ContainsArrayError + If the node at the given path is an array rather than a group. + """ + store_path = self.store_path / path + try: + node = await self.getitem(path) + except KeyError as e: + msg = f"No group found in store {store_path.store!r} at path {store_path.path!r}" + raise GroupNotFoundError(msg) from e + if isinstance(node, AsyncArray): + msg = f"An array exists in store {store_path.store!r} at path {store_path.path!r}." + raise ContainsArrayError(msg) + return node + + async def _save_metadata(self, ensure_parents: bool = False) -> None: + await save_metadata(self.store_path, self.metadata, ensure_parents=ensure_parents) + + @property + def path(self) -> str: + """Storage path.""" + return self.store_path.path + + @property + def name(self) -> str: + """Group name following h5py convention.""" + if self.path: + # follow h5py convention: add leading slash + name = self.path + if name[0] != "/": + name = f"/{name}" + return name + return "/" + + @property + def basename(self) -> str: + """Final component of name.""" + return self.name.split("/")[-1] + + @property + def attrs(self) -> dict[str, Any]: + return self.metadata.attributes + + @property + def info(self) -> Any: + """ + Return a visual representation of the statically known information about a group. + + Note that this doesn't include dynamic information, like the number of child + Groups or Arrays. + + Returns + ------- + GroupInfo + + Related + ------- + [zarr.AsyncGroup.info_complete][] + All information about a group, including dynamic information + """ + + if self.metadata.consolidated_metadata: + members = list(self.metadata.consolidated_metadata.flattened_metadata.values()) + else: + members = None + return self._info(members=members) + + async def info_complete(self) -> Any: + """ + Return all the information for a group. + + This includes dynamic information like the number + of child Groups or Arrays. If this group doesn't contain consolidated + metadata then this will need to read from the backing Store. + + Returns + ------- + GroupInfo + + Related + ------- + [zarr.AsyncGroup.info][] + """ + members = [x[1].metadata async for x in self.members(max_depth=None)] + return self._info(members=members) + + def _info( + self, members: list[ArrayV2Metadata | ArrayV3Metadata | GroupMetadata] | None = None + ) -> Any: + kwargs = {} + if members is not None: + kwargs["_count_members"] = len(members) + count_arrays = 0 + count_groups = 0 + for member in members: + if isinstance(member, GroupMetadata): + count_groups += 1 + else: + count_arrays += 1 + kwargs["_count_arrays"] = count_arrays + kwargs["_count_groups"] = count_groups + + return GroupInfo( + _name=self.store_path.path, + _read_only=self.read_only, + _store_type=type(self.store_path.store).__name__, + _zarr_format=self.metadata.zarr_format, + # maybe do a typeddict + **kwargs, # type: ignore[arg-type] + ) + + @property + def store(self) -> Store: + return self.store_path.store + + @property + def read_only(self) -> bool: + # Backwards compatibility for 2.x + return self.store_path.read_only + + @property + def synchronizer(self) -> None: + # Backwards compatibility for 2.x + # Not implemented in 3.x yet. + return None + + async def create_group( + self, + name: str, + *, + overwrite: bool = False, + attributes: dict[str, Any] | None = None, + ) -> AsyncGroup: + """Create a sub-group. + + Parameters + ---------- + name : str + Group name. + overwrite : bool, optional + If True, do not raise an error if the group already exists. + attributes : dict, optional + Group attributes. + + Returns + ------- + g : AsyncGroup + """ + attributes = attributes or {} + return await type(self).from_store( + self.store_path / name, + attributes=attributes, + overwrite=overwrite, + zarr_format=self.metadata.zarr_format, + ) + + async def require_group(self, name: str, overwrite: bool = False) -> AsyncGroup: + """Obtain a sub-group, creating one if it doesn't exist. + + Parameters + ---------- + name : str + Group name. + overwrite : bool, optional + Overwrite any existing group with given `name` if present. + + Returns + ------- + g : AsyncGroup + """ + if overwrite: + # TODO: check that overwrite=True errors if an array exists where the group is being created + grp = await self.create_group(name, overwrite=True) + else: + try: + item: AsyncGroup | AnyAsyncArray = await self.getitem(name) + if not isinstance(item, AsyncGroup): + raise TypeError( + f"Incompatible object ({item.__class__.__name__}) already exists" + ) + grp = item + except KeyError: + grp = await self.create_group(name) + return grp + + async def require_groups(self, *names: str) -> tuple[AsyncGroup, ...]: + """Convenience method to require multiple groups in a single call. + + Parameters + ---------- + *names : str + Group names. + + Returns + ------- + Tuple[AsyncGroup, ...] + """ + if not names: + return () + return tuple(await asyncio.gather(*(self.require_group(name) for name in names))) + + async def create_array( + self, + name: str, + *, + shape: ShapeLike | None = None, + dtype: ZDTypeLike | None = None, + data: np.ndarray[Any, np.dtype[Any]] | None = None, + chunks: ChunksLike | Literal["auto"] = "auto", + shards: ShardsLike | None = None, + filters: FiltersLike = "auto", + compressors: CompressorsLike = "auto", + compressor: CompressorLike = "auto", + serializer: SerializerLike = "auto", + fill_value: Any | None = DEFAULT_FILL_VALUE, + order: MemoryOrder | None = None, + attributes: dict[str, JSON] | None = None, + chunk_key_encoding: ChunkKeyEncodingLike | None = None, + dimension_names: DimensionNamesLike = None, + storage_options: dict[str, Any] | None = None, + overwrite: bool = False, + config: ArrayConfigLike | None = None, + write_data: bool = True, + ) -> AnyAsyncArray: + """Create an array within this group. + + This method lightly wraps [zarr.core.array.create_array][]. + + Parameters + ---------- + name : str + The name of the array relative to the group. If `path` is `None`, the array will be located + at the root of the store. + shape : ShapeLike, optional + Shape of the array. Must be `None` if `data` is provided. + dtype : ZDTypeLike | None + Data type of the array. Must be `None` if `data` is provided. + data : np.ndarray, optional + Array-like data to use for initializing the array. If this parameter is provided, the + `shape` and `dtype` parameters must be `None`. + chunks : tuple[int, ...] | Sequence[Sequence[int]] | Literal["auto"], default="auto" + Chunk shape of the array. + If chunks is "auto", a chunk shape is guessed based on the shape of the array and the dtype. + A nested list of per-dimension edge sizes creates a rectilinear grid. + Rectilinear chunk grids are experimental and must be explicitly enabled + with `zarr.config.set({'array.rectilinear_chunks': True})` while the + feature is stabilizing. + shards : tuple[int, ...], optional + Shard shape of the array. The default value of `None` results in no sharding at all. + filters : Iterable[Codec] | Literal["auto"], optional + Iterable of filters to apply to each chunk of the array, in order, before serializing that + chunk to bytes. + + For Zarr format 3, a "filter" is a codec that takes an array and returns an array, + and these values must be instances of [`zarr.abc.codec.ArrayArrayCodec`][], or a + dict representations of [`zarr.abc.codec.ArrayArrayCodec`][]. + + For Zarr format 2, a "filter" can be any numcodecs codec; you should ensure that the + order of your filters is consistent with the behavior of each filter. + + The default value of `"auto"` instructs Zarr to use a default based on the data + type of the array and the Zarr format specified. For all data types in Zarr V3, and most + data types in Zarr V2, the default filters are empty. The only cases where default filters + are not empty is when the Zarr format is 2, and the data type is a variable-length data type like + [`zarr.dtype.VariableLengthUTF8`][] or [`zarr.dtype.VariableLengthBytes`][]. In these cases, + the default filters contains a single element which is a codec specific to that particular data type. + + To create an array with no filters, provide an empty iterable or the value `None`. + compressors : Iterable[Codec], optional + List of compressors to apply to the array. Compressors are applied in order, and after any + filters are applied (if any are specified) and the data is serialized into bytes. + + For Zarr format 3, a "compressor" is a codec that takes a bytestream, and + returns another bytestream. Multiple compressors may be provided for Zarr format 3. + If no `compressors` are provided, a default set of compressors will be used. + These defaults can be changed by modifying the value of `array.v3_default_compressors` + in [`zarr.config`][zarr.config]. + Use `None` to omit default compressors. + + For Zarr format 2, a "compressor" can be any numcodecs codec. Only a single compressor may + be provided for Zarr format 2. + If no `compressor` is provided, a default compressor will be used. + This default can be changed by modifying the value of `array.v2_default_compressor` + in [`zarr.config`][zarr.config]. + Use `None` to omit the default compressor. + compressor : Codec, optional + Deprecated in favor of `compressors`. + serializer : dict[str, JSON] | ArrayBytesCodec, optional + Array-to-bytes codec to use for encoding the array data. + Zarr format 3 only. Zarr format 2 arrays use implicit array-to-bytes conversion. + If no `serializer` is provided, a default serializer will be used. + These defaults can be changed by modifying the value of `array.v3_default_serializer` + in [`zarr.config`][zarr.config]. + fill_value : Any, optional + Fill value for the array. + order : {"C", "F"}, optional + The memory order of the array (default is "C"). + For Zarr format 2, this parameter sets the memory order of the array. + For Zarr format 3, this parameter is deprecated, because memory order + is a runtime parameter for Zarr format 3 arrays. The recommended way to specify the memory + order for Zarr format 3 arrays is via the `config` parameter, e.g. `{'config': 'C'}`. + If no `order` is provided, a default order will be used. + This default can be changed by modifying the value of `array.order` in [`zarr.config`][zarr.config]. + attributes : dict, optional + Attributes for the array. + chunk_key_encoding : ChunkKeyEncodingLike, optional + A specification of how the chunk keys are represented in storage. + For Zarr format 3, the default is `{"name": "default", "separator": "/"}`. + For Zarr format 2, the default is `{"name": "v2", "separator": "."}`. + dimension_names : Iterable[str], optional + The names of the dimensions (default is None). + Zarr format 3 only. Zarr format 2 arrays should not use this parameter. + storage_options : dict, optional + If using an fsspec URL to create the store, these will be passed to the backend implementation. + Ignored otherwise. + overwrite : bool, default False + Whether to overwrite an array with the same name in the store, if one exists. + If `True`, any existing keys under that path are deleted first. + config : ArrayConfigLike, optional + Runtime configuration for the array. + write_data : bool + If a pre-existing array-like object was provided to this function via the `data` parameter + then `write_data` determines whether the values in that array-like object should be + written to the Zarr array created by this function. If `write_data` is `False`, then the + array will be left empty. + + Returns + ------- + AsyncArray + + """ + compressors = _parse_deprecated_compressor( + compressor, compressors, zarr_format=self.metadata.zarr_format + ) + return await create_array( + store=self.store_path, + name=name, + shape=shape, + dtype=dtype, + data=data, + chunks=chunks, + shards=shards, + filters=filters, + compressors=compressors, + serializer=serializer, + fill_value=fill_value, + order=order, + zarr_format=self.metadata.zarr_format, + attributes=attributes, + chunk_key_encoding=chunk_key_encoding, + dimension_names=dimension_names, + storage_options=storage_options, + overwrite=overwrite, + config=config, + write_data=write_data, + ) + + async def require_array( + self, + name: str, + *, + shape: ShapeLike, + dtype: ZDTypeLike | None = None, + exact: bool = False, + config: ArrayConfigLike | None = None, + **kwargs: Any, + ) -> AnyAsyncArray: + """Obtain an array, creating if it doesn't exist. + + Other `kwargs` are as per [zarr.AsyncGroup.create_array][]. + + Parameters + ---------- + name : str + Array name. + shape : int or tuple of ints + Array shape. + dtype : ZDTypeLike, optional + The data type of the array, given as a string, a NumPy dtype, or a + Zarr data type. + exact : bool, optional + If True, require `dtype` to match exactly. If false, require + `dtype` can be cast from array dtype. + config : ArrayConfigLike or None, default=None + Runtime configuration for the array, whether it is created or already exists. + Keys not specified are taken from the global configuration. + + Returns + ------- + a : AsyncArray + """ + try: + ds = await self.getitem(name) + if not isinstance(ds, AsyncArray): + raise TypeError(f"Incompatible object ({ds.__class__.__name__}) already exists") + + shape = parse_shapelike(shape) + if shape != ds.shape: + raise TypeError(f"Incompatible shape ({ds.shape} vs {shape})") + + # `np.dtype(None)` used to resolve to float64 here; keep that default. + dtype = parse_data_type( + "float64" if dtype is None else dtype, + zarr_format=self.metadata.zarr_format, + ).to_native_dtype() + if exact: + if ds.dtype != dtype: + raise TypeError(f"Incompatible dtype ({ds.dtype} vs {dtype})") + else: + if not np.can_cast(ds.dtype, dtype): + raise TypeError(f"Incompatible dtype ({ds.dtype} vs {dtype})") + except KeyError: + return await self.create_array(name, shape=shape, dtype=dtype, config=config, **kwargs) + + # `config` is the runtime configuration of the returned array, not stored metadata, + # so it applies to an existing array too. It is parsed the same way as on the create + # branch: missing keys come from the global configuration, unknown keys raise. + if config is not None: + return ds.with_config(parse_array_config(config)) + return ds + + async def update_attributes(self, new_attributes: dict[str, Any]) -> AsyncGroup: + """Update group attributes. + + Parameters + ---------- + new_attributes : dict + New attributes to set on the group. + + Returns + ------- + self : AsyncGroup + """ + self.metadata.attributes.update(new_attributes) + + # Write new metadata + await self._save_metadata() + + return self + + def __repr__(self) -> str: + return f"" + + async def nmembers( + self, + max_depth: int | None = 0, + ) -> int: + """Count the number of members in this group. + + Parameters + ---------- + max_depth : int, default 0 + The maximum number of levels of the hierarchy to include. By + default, (``max_depth=0``) only immediate children are included. Set + ``max_depth=None`` to include all nodes, and some positive integer + to consider children within that many levels of the root Group. + + Returns + ------- + count : int + """ + # check if we can use consolidated metadata, which requires that we have non-None + # consolidated metadata at all points in the hierarchy. + if self.metadata.consolidated_metadata is not None: + if max_depth is not None and max_depth < 0: + raise ValueError(f"max_depth must be None or >= 0. Got '{max_depth}' instead") + if max_depth is None: + return len(self.metadata.consolidated_metadata.flattened_metadata) + else: + return len( + [ + x + for x in self.metadata.consolidated_metadata.flattened_metadata + if x.count("/") <= max_depth + ] + ) + # TODO: consider using aioitertools.builtins.sum for this + # return await aioitertools.builtins.sum((1 async for _ in self.members()), start=0) + n = 0 + async for _ in self.members(max_depth=max_depth): + n += 1 + return n + + async def members( + self, + max_depth: int | None = 0, + *, + use_consolidated_for_children: bool = True, + ) -> AsyncGenerator[ + tuple[str, AnyAsyncArray | AsyncGroup], + None, + ]: + """ + Returns an AsyncGenerator over the arrays and groups contained in this group. + This method requires that `store_path.store` supports directory listing. + + The results are not guaranteed to be ordered. + + Parameters + ---------- + max_depth : int, default 0 + The maximum number of levels of the hierarchy to include. By + default, (``max_depth=0``) only immediate children are included. Set + ``max_depth=None`` to include all nodes, and some positive integer + to consider children within that many levels of the root Group. + use_consolidated_for_children : bool, default True + Whether to use the consolidated metadata of child groups loaded + from the store. Note that this only affects groups loaded from the + store. If the current Group already has consolidated metadata, it + will always be used. + + Returns + ------- + path: + A string giving the path to the target, relative to the Group ``self``. + value: AsyncArray or AsyncGroup + The AsyncArray or AsyncGroup that is a child of ``self``. + """ + if max_depth is not None and max_depth < 0: + raise ValueError(f"max_depth must be None or >= 0. Got '{max_depth}' instead") + async for item in self._members( + max_depth=max_depth, use_consolidated_for_children=use_consolidated_for_children + ): + yield item + + def _members_consolidated( + self, max_depth: int | None, prefix: str = "" + ) -> Generator[ + tuple[str, AnyAsyncArray | AsyncGroup], + None, + ]: + consolidated_metadata = self.metadata.consolidated_metadata + + do_recursion = max_depth is None or max_depth > 0 + + # we kind of just want the top-level keys. + if consolidated_metadata is not None: + for key in consolidated_metadata.metadata: + obj = self._getitem_consolidated( + self.store_path, key, prefix=self.name + ) # Metadata -> Group/Array + key = f"{prefix}/{key}".lstrip("/") + yield key, obj + + if do_recursion and isinstance(obj, AsyncGroup): + if max_depth is None: + new_depth = None + else: + new_depth = max_depth - 1 + yield from obj._members_consolidated(new_depth, prefix=key) + + async def _members( + self, max_depth: int | None, *, use_consolidated_for_children: bool = True + ) -> AsyncGenerator[tuple[str, AnyAsyncArray | AsyncGroup], None]: + skip_keys: tuple[str, ...] + if self.metadata.zarr_format == 2: + skip_keys = (".zattrs", ".zgroup", ".zarray", ".zmetadata") + elif self.metadata.zarr_format == 3: + skip_keys = ("zarr.json",) + else: + raise ValueError(f"Unknown Zarr format: {self.metadata.zarr_format}") + + if self.metadata.consolidated_metadata is not None: + members = self._members_consolidated(max_depth=max_depth) + for member in members: + yield member + return + + if not self.store_path.store.supports_listing: + msg = ( + f"The store associated with this group ({type(self.store_path.store)}) " + "does not support listing, " + "specifically via the `list_dir` method. " + "This function requires a store that supports listing." + ) + + raise ValueError(msg) + # enforce a concurrency limit by passing a semaphore to all the recursive functions + semaphore = asyncio.Semaphore(config.get("async.concurrency")) + async for member in _iter_members_deep( + self, + max_depth=max_depth, + skip_keys=skip_keys, + semaphore=semaphore, + use_consolidated_for_children=use_consolidated_for_children, + ): + yield member + + async def create_hierarchy( + self, + nodes: dict[str, ArrayV2Metadata | ArrayV3Metadata | GroupMetadata], + *, + overwrite: bool = False, + ) -> AsyncIterator[tuple[str, AsyncGroup | AnyAsyncArray]]: + """ + Create a hierarchy of arrays or groups rooted at this group. + + This function will parse its input to ensure that the hierarchy is complete. Any implicit groups + will be inserted as needed. For example, an input like + ```{'a/b': GroupMetadata}``` will be parsed to + ```{'': GroupMetadata, 'a': GroupMetadata, 'b': Groupmetadata}```. + + Explicitly specifying a root group, e.g. with ``nodes = {'': GroupMetadata()}`` is an error + because this group instance is the root group. + + After input parsing, this function then creates all the nodes in the hierarchy concurrently. + + Arrays and Groups are yielded in the order they are created. This order is not stable and + should not be relied on. + + Parameters + ---------- + nodes : dict[str, GroupMetadata | ArrayV3Metadata | ArrayV2Metadata] + A dictionary defining the hierarchy. The keys are the paths of the nodes in the hierarchy, + relative to the path of the group. The values are instances of ``GroupMetadata`` or ``ArrayMetadata``. Note that + all values must have the same ``zarr_format`` as the parent group -- it is an error to mix zarr versions in the + same hierarchy. + + Leading "/" characters from keys will be removed. + overwrite : bool + Whether to overwrite existing nodes. Defaults to ``False``, in which case an error is + raised instead of overwriting an existing array or group. + + This function will not erase an existing group unless that group is explicitly named in + ``nodes``. If ``nodes`` defines implicit groups, e.g. ``{`'a/b/c': GroupMetadata}``, and a + group already exists at path ``a``, then this function will leave the group at ``a`` as-is. + + Yields + ------ + tuple[str, AsyncArray | AsyncGroup]. + """ + # check that all the nodes have the same zarr_format as Self + prefix = self.path + nodes_parsed = {} + for key, value in nodes.items(): + if value.zarr_format != self.metadata.zarr_format: + msg = ( + "The zarr_format of the nodes must be the same as the parent group. " + f"The node at {key} has zarr_format {value.zarr_format}, but the parent group" + f" has zarr_format {self.metadata.zarr_format}." + ) + raise ValueError(msg) + if normalize_path(key) == "": + msg = ( + "The input defines a root node, but a root node already exists, namely this Group instance." + "It is an error to use this method to create a root node. " + "Remove the root node from the input dict, or use a function like " + "create_rooted_hierarchy to create a rooted hierarchy." + ) + raise ValueError(msg) + else: + nodes_parsed[_join_paths([prefix, key])] = value + + async for key, node in create_hierarchy( + store=self.store, + nodes=nodes_parsed, + overwrite=overwrite, + ): + if prefix == "": + out_key = key + else: + out_key = key.removeprefix(prefix + "/") + yield out_key, node + + async def keys(self) -> AsyncGenerator[str, None]: + """Iterate over member names.""" + async for key, _ in self.members(): + yield key + + async def contains(self, member: str) -> bool: + """Check if a member exists in the group. + + Parameters + ---------- + member : str + Member name. + + Returns + ------- + bool + """ + # TODO: this can be made more efficient. + try: + await self.getitem(member) + except KeyError: + return False + else: + return True + + async def groups(self) -> AsyncGenerator[tuple[str, AsyncGroup], None]: + """Iterate over subgroups.""" + async for name, value in self.members(): + if isinstance(value, AsyncGroup): + yield name, value + + async def group_keys(self) -> AsyncGenerator[str, None]: + """Iterate over group names.""" + async for key, _ in self.groups(): + yield key + + async def group_values(self) -> AsyncGenerator[AsyncGroup, None]: + """Iterate over group values.""" + async for _, group in self.groups(): + yield group + + async def arrays( + self, + ) -> AsyncGenerator[tuple[str, AnyAsyncArray], None]: + """Iterate over arrays.""" + async for key, value in self.members(): + if isinstance(value, AsyncArray): + yield key, value + + async def array_keys(self) -> AsyncGenerator[str, None]: + """Iterate over array names.""" + async for key, _ in self.arrays(): + yield key + + async def array_values( + self, + ) -> AsyncGenerator[AnyAsyncArray, None]: + """Iterate over array values.""" + async for _, array in self.arrays(): + yield array + + async def tree( + self, + expand: bool | None = None, + level: int | None = None, + *, + max_nodes: int = 500, + plain: bool = False, + ) -> Any: + """ + Return a tree-like representation of a hierarchy. + + Parameters + ---------- + expand : bool, optional + This keyword is not yet supported. A NotImplementedError is raised if + it's used. + level : int, optional + The maximum depth below this Group to display in the tree. + max_nodes : int + Maximum number of nodes to display before truncating. Default is 500. + plain : bool, optional + If True, return a plain-text tree without ANSI styling. This is + useful when the output will be consumed by an LLM or written to a + file. Default is False. + + Returns + ------- + TreeRepr + A pretty-printable object displaying the hierarchy. + """ + from zarr.core._tree import group_tree_async + + if expand is not None: + raise NotImplementedError("'expand' is not yet implemented.") + return await group_tree_async(self, max_depth=level, max_nodes=max_nodes, plain=plain) + + def _member_zarr_format(self, zarr_format: ZarrFormat | None) -> ZarrFormat: + """ + The zarr format of a new array in this group, which is the format of the group. + + A group only lists members of its own format, so an array of another format + would be written into the group without becoming a member of it. + """ + if zarr_format is not None and zarr_format != self.metadata.zarr_format: + raise ValueError( + f"Cannot create a zarr_format={zarr_format} array in a " + f"zarr_format={self.metadata.zarr_format} group." + ) + return self.metadata.zarr_format + + async def empty(self, *, name: str, shape: tuple[int, ...], **kwargs: Any) -> AnyAsyncArray: + """Create an empty array with the specified shape in this Group. The contents will + be filled with the array's fill value or zeros if no fill value is provided. + + Parameters + ---------- + name : str + Name of the array. + shape : int or tuple of int + Shape of the empty array. + **kwargs + Keyword arguments passed to [zarr.api.asynchronous.create][]. + + Notes + ----- + The contents of an empty Zarr array are not defined. On attempting to + retrieve data from an empty Zarr array, any values may be returned, + and these are not guaranteed to be stable from one access to the next. + """ + return await async_api.empty( + shape=shape, + store=self.store_path, + path=name, + zarr_format=self._member_zarr_format(kwargs.pop("zarr_format", None)), + **kwargs, + ) + + async def zeros(self, *, name: str, shape: tuple[int, ...], **kwargs: Any) -> AnyAsyncArray: + """Create an array, with zero being used as the default value for uninitialized portions of the array. + + Parameters + ---------- + name : str + Name of the array. + shape : int or tuple of int + Shape of the empty array. + **kwargs + Keyword arguments passed to [zarr.api.asynchronous.create][]. + + Returns + ------- + AsyncArray + The new array. + """ + return await async_api.zeros( + shape=shape, + store=self.store_path, + path=name, + zarr_format=self._member_zarr_format(kwargs.pop("zarr_format", None)), + **kwargs, + ) + + async def ones(self, *, name: str, shape: tuple[int, ...], **kwargs: Any) -> AnyAsyncArray: + """Create an array, with one being used as the default value for uninitialized portions of the array. + + Parameters + ---------- + name : str + Name of the array. + shape : int or tuple of int + Shape of the empty array. + **kwargs + Keyword arguments passed to [zarr.api.asynchronous.create][]. + + Returns + ------- + AsyncArray + The new array. + """ + return await async_api.ones( + shape=shape, + store=self.store_path, + path=name, + zarr_format=self._member_zarr_format(kwargs.pop("zarr_format", None)), + **kwargs, + ) + + async def full( + self, *, name: str, shape: tuple[int, ...], fill_value: Any | None, **kwargs: Any + ) -> AnyAsyncArray: + """Create an array, with "fill_value" being used as the default value for uninitialized portions of the array. + + Parameters + ---------- + name : str + Name of the array. + shape : int or tuple of int + Shape of the empty array. + fill_value : scalar + Value to fill the array with. + **kwargs + Keyword arguments passed to [zarr.api.asynchronous.create][]. + + Returns + ------- + AsyncArray + The new array. + """ + return await async_api.full( + shape=shape, + fill_value=fill_value, + store=self.store_path, + path=name, + zarr_format=self._member_zarr_format(kwargs.pop("zarr_format", None)), + **kwargs, + ) + + async def empty_like( + self, *, name: str, data: async_api.ArrayLike, **kwargs: Any + ) -> AnyAsyncArray: + """Create an empty sub-array like `data`. The contents will be filled with + the array's fill value or zeros if no fill value is provided. + + Parameters + ---------- + name : str + Name of the array. + data : array-like + The array to create an empty array like. + **kwargs + Keyword arguments passed to [zarr.api.asynchronous.create][]. + + Returns + ------- + AsyncArray + The new array. + """ + return await async_api.empty_like( + a=data, + store=self.store_path, + path=name, + zarr_format=self._member_zarr_format(kwargs.pop("zarr_format", None)), + **kwargs, + ) + + async def zeros_like( + self, *, name: str, data: async_api.ArrayLike, **kwargs: Any + ) -> AnyAsyncArray: + """Create a sub-array of zeros like `data`. + + Parameters + ---------- + name : str + Name of the array. + data : array-like + The array to create the new array like. + **kwargs + Keyword arguments passed to [zarr.api.asynchronous.create][]. + + Returns + ------- + AsyncArray + The new array. + """ + return await async_api.zeros_like( + a=data, + store=self.store_path, + path=name, + zarr_format=self._member_zarr_format(kwargs.pop("zarr_format", None)), + **kwargs, + ) + + async def ones_like( + self, *, name: str, data: async_api.ArrayLike, **kwargs: Any + ) -> AnyAsyncArray: + """Create a sub-array of ones like `data`. + + Parameters + ---------- + name : str + Name of the array. + data : array-like + The array to create the new array like. + **kwargs + Keyword arguments passed to [zarr.api.asynchronous.create][]. + + Returns + ------- + AsyncArray + The new array. + """ + return await async_api.ones_like( + a=data, + store=self.store_path, + path=name, + zarr_format=self._member_zarr_format(kwargs.pop("zarr_format", None)), + **kwargs, + ) + + async def full_like( + self, *, name: str, data: async_api.ArrayLike, **kwargs: Any + ) -> AnyAsyncArray: + """Create a sub-array like `data` filled with the `fill_value` of `data` . + + Parameters + ---------- + name : str + Name of the array. + data : array-like + The array to create the new array like. + **kwargs + Keyword arguments passed to [zarr.api.asynchronous.create][]. + + Returns + ------- + AsyncArray + The new array. + """ + return await async_api.full_like( + a=data, + store=self.store_path, + path=name, + zarr_format=self._member_zarr_format(kwargs.pop("zarr_format", None)), + **kwargs, + ) + + async def move(self, source: str, dest: str) -> None: + """Move a sub-group or sub-array from one path to another. + + Notes + ----- + Not implemented + """ + raise NotImplementedError + + +@dataclass(frozen=True) +class Group(SyncMixin): + """ + A Zarr group. + """ + + _async_group: AsyncGroup + + @classmethod + def from_store( + cls, + store: StoreLike, + *, + attributes: dict[str, Any] | None = None, + zarr_format: ZarrFormat = 3, + overwrite: bool = False, + ) -> Group: + """Instantiate a group from an initialized store. + + Parameters + ---------- + store : StoreLike + StoreLike containing the Group. See the + [storage documentation in the user guide][user-guide-store-like] + for a description of all valid StoreLike values. + attributes : dict, optional + A dictionary of JSON-serializable values with user-defined attributes. + zarr_format : {2, 3}, optional + Zarr storage format version. + overwrite : bool, optional + If True, do not raise an error if the group already exists. + + Returns + ------- + Group + Group instantiated from the store. + + Raises + ------ + ContainsArrayError, ContainsGroupError, ContainsArrayAndGroupError + """ + attributes = attributes or {} + obj = sync( + AsyncGroup.from_store( + store, + attributes=attributes, + overwrite=overwrite, + zarr_format=zarr_format, + ), + ) + + return cls(obj) + + @classmethod + def open( + cls, + store: StoreLike, + zarr_format: ZarrFormat | None = 3, + ) -> Group: + """Open a group from an initialized store. + + Parameters + ---------- + store : StoreLike + Store containing the Group. See the + [storage documentation in the user guide][user-guide-store-like] + for a description of all valid StoreLike values. + zarr_format : {2, 3, None}, optional + Zarr storage format version. + + Returns + ------- + Group + Group instantiated from the store. + """ + obj = sync(AsyncGroup.open(store, zarr_format=zarr_format)) + return cls(obj) + + def __getitem__(self, path: str) -> AnyArray | Group: + """Obtain a group member. + + Parameters + ---------- + path : str + Group member name. + + Returns + ------- + Array | Group + Group member (Array or Group) at the specified key + + Examples + -------- + ```python + import zarr + from zarr.core.group import Group + group = Group.from_store(zarr.storage.MemoryStore()) + group.create_array(name="subarray", shape=(10,), chunks=(10,), dtype="float64") + group.create_group(name="subgroup").create_array(name="subarray", shape=(10,), chunks=(10,), dtype="float64") + group["subarray"] + # + group["subgroup"] + # + group["subgroup"]["subarray"] + # + ``` + + """ + obj = self._sync(self._async_group.getitem(path)) + if isinstance(obj, AsyncArray): + return Array(obj) + else: + return Group(obj) + + def get[DefaultT]( + self, path: str, default: DefaultT | None = None + ) -> AnyArray | Group | DefaultT | None: + """Obtain a group member, returning default if not found. + + Parameters + ---------- + path : str + Group member name. + default : object + Default value to return if key is not found (default: None). + + Returns + ------- + object + Group member (Array or Group) or default if not found. + + Examples + -------- + ```python + import zarr + from zarr.core.group import Group + group = Group.from_store(zarr.storage.MemoryStore()) + group.create_array(name="subarray", shape=(10,), chunks=(10,), dtype="float64") + group.create_group(name="subgroup") + group.get("subarray") + # + group.get("subgroup") + # + group.get("nonexistent", None) + # None + ``` + + """ + try: + return self[path] + except KeyError: + return default + + def get_array(self, path: str) -> AnyArray: + """Obtain an array member of this group, raising if it is absent or not an array. + + Parameters + ---------- + path : str + Path of the array relative to this group. May contain `/` to reference + a member of a subgroup, e.g. `subgroup/subarray`. + + Returns + ------- + Array + The array at the given path. + + Raises + ------ + ArrayNotFoundError + If no node exists at the given path. + ContainsGroupError + If the node at the given path is a group rather than an array. + + Examples + -------- + ```python + import zarr + from zarr.core.group import Group + group = Group.from_store(zarr.storage.MemoryStore()) + group.create_array(name="subarray", shape=(10,), chunks=(10,), dtype="float64") + group.get_array("subarray") + # + ``` + """ + return Array(self._sync(self._async_group.get_array(path))) + + def get_group(self, path: str) -> Group: + """Obtain a group member of this group, raising if it is absent or not a group. + + Parameters + ---------- + path : str + Path of the group relative to this group. May contain `/` to reference + a member of a subgroup, e.g. `subgroup/subsubgroup`. + + Returns + ------- + Group + The group at the given path. + + Raises + ------ + GroupNotFoundError + If no node exists at the given path. + ContainsArrayError + If the node at the given path is an array rather than a group. + + Examples + -------- + ```python + import zarr + from zarr.core.group import Group + group = Group.from_store(zarr.storage.MemoryStore()) + group.create_group(name="subgroup") + group.get_group("subgroup") + # + ``` + """ + return Group(self._sync(self._async_group.get_group(path))) + + def __delitem__(self, key: str) -> None: + """Delete a group member. + + Parameters + ---------- + key : str + Group member name. + + Examples + -------- + >>> import zarr + >>> group = Group.from_store(zarr.storage.MemoryStore()) + >>> a = group.create_array(name="subarray", dtype="i1", shape=(10,), chunks=(10,)) + >>> del group["subarray"] + >>> "subarray" in group + False + """ + + self._sync(self._async_group.delitem(key)) + + def __iter__(self) -> Iterator[str]: + """Return an iterator over group member names. + Examples + -------- + >>> import zarr + >>> g1 = zarr.group() + >>> g2 = g1.create_group('foo') + >>> g3 = g1.create_group('bar') + >>> d1 = g1.create_array('baz', dtype="i1", shape=(10,), chunks=(10,)) + >>> d2 = g1.create_array('quux', dtype="i1", shape=(10,), chunks=(10,)) + >>> sorted(g1) + ['bar', 'baz', 'foo', 'quux'] + """ + yield from self.keys() + + def __len__(self) -> int: + """Number of members.""" + return self.nmembers() + + def __setitem__(self, key: str, value: Any) -> None: + """Fastpath for creating a new array. + + New arrays will be created using default settings for the array type. + If you need to create an array with custom settings, use the `create_array` method. + + Parameters + ---------- + key : str + Array name. + value : Any + Array data. + + Examples + -------- + >>> import numpy as np + >>> import zarr + >>> group = zarr.group() + >>> group["foo"] = np.array(zarr.zeros((10,))) + >>> group["foo"] + + """ + self._sync(self._async_group.setitem(key, value)) + + def __repr__(self) -> str: + return f"" + + async def update_attributes_async(self, new_attributes: dict[str, Any]) -> Group: + """Update the attributes of this group. + + Examples + -------- + >>> async def example(): + ... import zarr + ... + ... group = zarr.group() + ... new_group = await group.update_attributes_async({"foo": "bar"}) + ... return new_group.attrs.asdict() + + >>> import asyncio + >>> asyncio.run(example()) + {'foo': 'bar'} + """ + new_metadata = replace(self.metadata, attributes=new_attributes) + + # Write new metadata + to_save = new_metadata.to_buffer_dict(default_buffer_prototype()) + awaitables = [set_or_delete(self.store_path / key, value) for key, value in to_save.items()] + await asyncio.gather(*awaitables) + + async_group = replace(self._async_group, metadata=new_metadata) + return replace(self, _async_group=async_group) + + @property + def store_path(self) -> StorePath: + """Path-like interface for the Store.""" + return self._async_group.store_path + + @property + def metadata(self) -> GroupMetadata: + """Group metadata.""" + return self._async_group.metadata + + @property + def path(self) -> str: + """Storage path.""" + return self._async_group.path + + @property + def name(self) -> str: + """Group name following h5py convention.""" + return self._async_group.name + + @property + def basename(self) -> str: + """Final component of name.""" + return self._async_group.basename + + @property + def attrs(self) -> Attributes: + """Attributes of this Group""" + return Attributes(self) + + @property + def info(self) -> Any: + """ + Return the statically known information for a group. + + Returns + ------- + GroupInfo + + Related + ------- + [zarr.Group.info_complete][] + All information about a group, including dynamic information + like the children members. + """ + return self._async_group.info + + def info_complete(self) -> Any: + """ + Return information for a group. + + If this group doesn't contain consolidated metadata then + this will need to read from the backing Store. + + Returns + ------- + GroupInfo + + Related + ------- + [zarr.Group.info][] + """ + return self._sync(self._async_group.info_complete()) + + @property + def store(self) -> Store: + # Backwards compatibility for 2.x + return self._async_group.store + + @property + def read_only(self) -> bool: + # Backwards compatibility for 2.x + return self._async_group.read_only + + @property + def synchronizer(self) -> None: + # Backwards compatibility for 2.x + # Not implemented in 3.x yet. + return self._async_group.synchronizer + + def update_attributes(self, new_attributes: dict[str, Any]) -> Group: + """Update the attributes of this group. + + Examples + -------- + >>> import zarr + >>> group = zarr.group().update_attributes({"foo": "bar"}) + >>> group.attrs.asdict() + {'foo': 'bar'} + """ + self._sync(self._async_group.update_attributes(new_attributes)) + return self + + def nmembers(self, max_depth: int | None = 0) -> int: + """Count the number of members in this group. + + Parameters + ---------- + max_depth : int, default 0 + The maximum number of levels of the hierarchy to include. By + default, (``max_depth=0``) only immediate children are included. Set + ``max_depth=None`` to include all nodes, and some positive integer + to consider children within that many levels of the root Group. + + Returns + ------- + count : int + """ + + return self._sync(self._async_group.nmembers(max_depth=max_depth)) + + def members( + self, max_depth: int | None = 0, *, use_consolidated_for_children: bool = True + ) -> tuple[tuple[str, AnyArray | Group], ...]: + """ + Returns an AsyncGenerator over the arrays and groups contained in this group. + This method requires that `store_path.store` supports directory listing. + + The results are not guaranteed to be ordered. + + Parameters + ---------- + max_depth : int, default 0 + The maximum number of levels of the hierarchy to include. By + default, (``max_depth=0``) only immediate children are included. Set + ``max_depth=None`` to include all nodes, and some positive integer + to consider children within that many levels of the root Group. + use_consolidated_for_children : bool, default True + Whether to use the consolidated metadata of child groups loaded + from the store. Note that this only affects groups loaded from the + store. If the current Group already has consolidated metadata, it + will always be used. + + Returns + ------- + path: + A string giving the path to the target, relative to the Group ``self``. + value: AsyncArray or AsyncGroup + The AsyncArray or AsyncGroup that is a child of ``self``. + """ + _members = self._sync_iter(self._async_group.members(max_depth=max_depth)) + + return tuple((kv[0], _parse_async_node(kv[1])) for kv in _members) + + def create_hierarchy( + self, + nodes: dict[str, ArrayV2Metadata | ArrayV3Metadata | GroupMetadata], + *, + overwrite: bool = False, + ) -> Iterator[tuple[str, Group | AnyArray]]: + """ + Create a hierarchy of arrays or groups rooted at this group. + + This function will parse its input to ensure that the hierarchy is complete. Any implicit groups + will be inserted as needed. For example, an input like + ```{'a/b': GroupMetadata}``` will be parsed to + ```{'': GroupMetadata, 'a': GroupMetadata, 'b': Groupmetadata}```. + + Explicitly specifying a root group, e.g. with ``nodes = {'': GroupMetadata()}`` is an error + because this group instance is the root group. + + After input parsing, this function then creates all the nodes in the hierarchy concurrently. + + Arrays and Groups are yielded in the order they are created. This order is not stable and + should not be relied on. + + Parameters + ---------- + nodes : dict[str, GroupMetadata | ArrayV3Metadata | ArrayV2Metadata] + A dictionary defining the hierarchy. The keys are the paths of the nodes in the hierarchy, + relative to the path of the group. The values are instances of ``GroupMetadata`` or ``ArrayMetadata``. Note that + all values must have the same ``zarr_format`` as the parent group -- it is an error to mix zarr versions in the + same hierarchy. + + Leading "/" characters from keys will be removed. + overwrite : bool + Whether to overwrite existing nodes. Defaults to ``False``, in which case an error is + raised instead of overwriting an existing array or group. + + This function will not erase an existing group unless that group is explicitly named in + ``nodes``. If ``nodes`` defines implicit groups, e.g. ``{`'a/b/c': GroupMetadata}``, and a + group already exists at path ``a``, then this function will leave the group at ``a`` as-is. + + Yields + ------ + tuple[str, Array | Group]. + + Examples + -------- + >>> import zarr + >>> from zarr.core.group import GroupMetadata + >>> root = zarr.create_group(store={}) + >>> sorted(root.create_hierarchy({'a/b/c': GroupMetadata()})) + [('a', ), + ('a/b', ), + ('a/b/c', )] + """ + for key, node in self._sync_iter( + self._async_group.create_hierarchy(nodes, overwrite=overwrite) + ): + yield (key, _parse_async_node(node)) + + def keys(self) -> Generator[str]: + """Return an iterator over group member names. + + Examples + -------- + >>> import zarr + >>> g1 = zarr.group() + >>> g2 = g1.create_group('foo') + >>> g3 = g1.create_group('bar') + >>> d1 = g1.create_array('baz', dtype="i1", shape=(10,), chunks=(10,)) + >>> d2 = g1.create_array('quux', dtype="i1", shape=(10,), chunks=(10,)) + >>> sorted(g1.keys()) + ['bar', 'baz', 'foo', 'quux'] + """ + yield from self._sync_iter(self._async_group.keys()) + + def __contains__(self, member: str) -> bool: + """Test for group membership. + + Examples + -------- + >>> import zarr + >>> g1 = zarr.group() + >>> g2 = g1.create_group('foo') + >>> d1 = g1.create_array('bar', dtype="i1", shape=(10,), chunks=(10,)) + >>> 'foo' in g1 + True + >>> 'bar' in g1 + True + >>> 'baz' in g1 + False + """ + return self._sync(self._async_group.contains(member)) + + def groups(self) -> Generator[tuple[str, Group], None]: + """Return the sub-groups of this group as a generator of (name, group) pairs. + + Examples + -------- + >>> import zarr + >>> group = zarr.group() + >>> subgroup = group.create_group("subgroup") + >>> list(group.groups()) + [('subgroup', )] + """ + for name, async_group in self._sync_iter(self._async_group.groups()): + yield name, Group(async_group) + + def group_keys(self) -> Generator[str, None]: + """Return an iterator over group member names. + + Examples + -------- + >>> import zarr + >>> group = zarr.group() + >>> subgroup = group.create_group("subgroup") + >>> list(group.group_keys()) + ['subgroup'] + """ + for name, _ in self.groups(): + yield name + + def group_values(self) -> Generator[Group, None]: + """Return an iterator over group members. + + Examples + -------- + >>> import zarr + >>> group = zarr.group() + >>> subgroup = group.create_group("subgroup") + >>> list(group.group_values()) + [] + """ + for _, group in self.groups(): + yield group + + def arrays(self) -> Generator[tuple[str, AnyArray], None]: + """Return the sub-arrays of this group as a generator of (name, array) pairs + + Examples + -------- + >>> import zarr + >>> group = zarr.group() + >>> subarray = group.create_array("subarray", dtype="i1", shape=(10,), chunks=(10,)) + >>> list(group.arrays()) + [('subarray', )] + """ + for name, async_array in self._sync_iter(self._async_group.arrays()): + yield name, Array(async_array) + + def array_keys(self) -> Generator[str, None]: + """Return an iterator over group member names. + + Examples + -------- + >>> import zarr + >>> group = zarr.group() + >>> subarray = group.create_array("subarray", dtype="i1", shape=(10,), chunks=(10,)) + >>> list(group.array_keys()) + ['subarray'] + """ + + for name, _ in self.arrays(): + yield name + + def array_values(self) -> Generator[AnyArray, None]: + """Return an iterator over group members. + + Examples + -------- + >>> import zarr + >>> group = zarr.group() + >>> subarray = group.create_array("subarray", dtype="i1", shape=(10,), chunks=(10,)) + >>> list(group.array_values()) + [] + """ + for _, array in self.arrays(): + yield array + + def tree( + self, + expand: bool | None = None, + level: int | None = None, + *, + max_nodes: int = 500, + plain: bool = False, + ) -> Any: + """ + Return a tree-like representation of a hierarchy. + + Parameters + ---------- + expand : bool, optional + This keyword is not yet supported. A NotImplementedError is raised if + it's used. + level : int, optional + The maximum depth below this Group to display in the tree. + max_nodes : int + Maximum number of nodes to display before truncating. Default is 500. + plain : bool, optional + If True, return a plain-text tree without ANSI styling. This is + useful when the output will be consumed by an LLM or written to a + file. Default is False. + + Returns + ------- + TreeRepr + A pretty-printable object displaying the hierarchy. + """ + return self._sync( + self._async_group.tree(expand=expand, level=level, max_nodes=max_nodes, plain=plain) + ) + + def create_group(self, name: str, **kwargs: Any) -> Group: + """Create a sub-group. + + Parameters + ---------- + name : str + Name of the new subgroup. + + Returns + ------- + Group + + Examples + -------- + >>> import zarr + >>> group = zarr.group() + >>> subgroup = group.create_group("subgroup") + >>> subgroup + + """ + return Group(self._sync(self._async_group.create_group(name, **kwargs))) + + def require_group(self, name: str, **kwargs: Any) -> Group: + """Obtain a sub-group, creating one if it doesn't exist. + + Parameters + ---------- + name : str + Group name. + + Returns + ------- + g : Group + """ + return Group(self._sync(self._async_group.require_group(name, **kwargs))) + + def require_groups(self, *names: str) -> tuple[Group, ...]: + """Convenience method to require multiple groups in a single call. + + Parameters + ---------- + *names : str + Group names. + + Returns + ------- + groups : tuple of Groups + """ + return tuple(map(Group, self._sync(self._async_group.require_groups(*names)))) + + def create( + self, + name: str, + *, + shape: ShapeLike | None = None, + dtype: ZDTypeLike | None = None, + data: np.ndarray[Any, np.dtype[Any]] | None = None, + chunks: ChunksLike | Literal["auto"] = "auto", + shards: ShardsLike | None = None, + filters: FiltersLike = "auto", + compressors: CompressorsLike = "auto", + compressor: CompressorLike = "auto", + serializer: SerializerLike = "auto", + fill_value: Any | None = DEFAULT_FILL_VALUE, + order: MemoryOrder | None = None, + attributes: dict[str, JSON] | None = None, + chunk_key_encoding: ChunkKeyEncodingLike | None = None, + dimension_names: DimensionNamesLike = None, + storage_options: dict[str, Any] | None = None, + overwrite: bool = False, + config: ArrayConfigLike | None = None, + write_data: bool = True, + ) -> AnyArray: + """Create an array within this group. + + This method lightly wraps [`zarr.core.array.create_array`][]. + + Parameters + ---------- + name : str + The name of the array relative to the group. If `path` is `None`, the array will be located + at the root of the store. + shape : ShapeLike, optional + Shape of the array. Must be `None` if `data` is provided. + dtype : ZDTypeLike | None + Data type of the array. Must be `None` if `data` is provided. + data : np.ndarray, optional + Array-like data to use for initializing the array. If this parameter is provided, the + `shape` and `dtype` parameters must be `None`. + chunks : tuple[int, ...] | Sequence[Sequence[int]] | Literal["auto"], default="auto" + Chunk shape of the array. + If chunks is "auto", a chunk shape is guessed based on the shape of the array and the dtype. + A nested list of per-dimension edge sizes creates a rectilinear grid. + Rectilinear chunk grids are experimental and must be explicitly enabled + with `zarr.config.set({'array.rectilinear_chunks': True})` while the + feature is stabilizing. + shards : tuple[int, ...], optional + Shard shape of the array. The default value of `None` results in no sharding at all. + filters : Iterable[Codec] | Literal["auto"], optional + Iterable of filters to apply to each chunk of the array, in order, before serializing that + chunk to bytes. + + For Zarr format 3, a "filter" is a codec that takes an array and returns an array, + and these values must be instances of [`zarr.abc.codec.ArrayArrayCodec`][], or a + dict representations of [`zarr.abc.codec.ArrayArrayCodec`][]. + + For Zarr format 2, a "filter" can be any numcodecs codec; you should ensure that the + order of your filters is consistent with the behavior of each filter. + + The default value of `"auto"` instructs Zarr to use a default based on the data + type of the array and the Zarr format specified. For all data types in Zarr V3, and most + data types in Zarr V2, the default filters are empty. The only cases where default filters + are not empty is when the Zarr format is 2, and the data type is a variable-length data type like + [`zarr.dtype.VariableLengthUTF8`][] or [`zarr.dtype.VariableLengthBytes`][]. In these cases, + the default filters contains a single element which is a codec specific to that particular data type. + + To create an array with no filters, provide an empty iterable or the value `None`. + compressors : Iterable[Codec], optional + List of compressors to apply to the array. Compressors are applied in order, and after any + filters are applied (if any are specified) and the data is serialized into bytes. + + For Zarr format 3, a "compressor" is a codec that takes a bytestream, and + returns another bytestream. Multiple compressors may be provided for Zarr format 3. + If no `compressors` are provided, a default set of compressors will be used. + These defaults can be changed by modifying the value of `array.v3_default_compressors` + in [`zarr.config`][zarr.config]. + Use `None` to omit default compressors. + + For Zarr format 2, a "compressor" can be any numcodecs codec. Only a single compressor may + be provided for Zarr format 2. + If no `compressor` is provided, a default compressor will be used. + This default can be changed by modifying the value of `array.v2_default_compressor` + in [`zarr.config`][zarr.config]. + Use `None` to omit the default compressor. + compressor : Codec, optional + Deprecated in favor of `compressors`. + serializer : dict[str, JSON] | ArrayBytesCodec, optional + Array-to-bytes codec to use for encoding the array data. + Zarr format 3 only. Zarr format 2 arrays use implicit array-to-bytes conversion. + If no `serializer` is provided, a default serializer will be used. + These defaults can be changed by modifying the value of `array.v3_default_serializer` + in [`zarr.config`][zarr.config]. + fill_value : Any, optional + Fill value for the array. + order : {"C", "F"}, optional + The memory order of the array (default is "C"). + For Zarr format 2, this parameter sets the memory order of the array. + For Zarr format 3, this parameter is deprecated, because memory order + is a runtime parameter for Zarr format 3 arrays. The recommended way to specify the memory + order for Zarr format 3 arrays is via the `config` parameter, e.g. `{'config': 'C'}`. + If no `order` is provided, a default order will be used. + This default can be changed by modifying the value of `array.order` in [`zarr.config`][zarr.config]. + attributes : dict, optional + Attributes for the array. + chunk_key_encoding : ChunkKeyEncodingLike, optional + A specification of how the chunk keys are represented in storage. + For Zarr format 3, the default is `{"name": "default", "separator": "/"}`. + For Zarr format 2, the default is `{"name": "v2", "separator": "."}`. + dimension_names : Iterable[str], optional + The names of the dimensions (default is None). + Zarr format 3 only. Zarr format 2 arrays should not use this parameter. + storage_options : dict, optional + If using an fsspec URL to create the store, these will be passed to the backend implementation. + Ignored otherwise. + overwrite : bool, default False + Whether to overwrite an array with the same name in the store, if one exists. + If `True`, any existing keys under that path are deleted first. + config : ArrayConfigLike, optional + Runtime configuration for the array. + write_data : bool + If a pre-existing array-like object was provided to this function via the `data` parameter + then `write_data` determines whether the values in that array-like object should be + written to the Zarr array created by this function. If `write_data` is `False`, then the + array will be left empty. + + Returns + ------- + AsyncArray + """ + return self.create_array( + name, + shape=shape, + dtype=dtype, + data=data, + chunks=chunks, + shards=shards, + filters=filters, + compressors=compressors, + compressor=compressor, + serializer=serializer, + fill_value=fill_value, + order=order, + attributes=attributes, + chunk_key_encoding=chunk_key_encoding, + dimension_names=dimension_names, + storage_options=storage_options, + overwrite=overwrite, + config=config, + write_data=write_data, + ) + + def create_array( + self, + name: str, + *, + shape: ShapeLike | None = None, + dtype: ZDTypeLike | None = None, + data: np.ndarray[Any, np.dtype[Any]] | None = None, + chunks: ChunksLike | Literal["auto"] = "auto", + shards: ShardsLike | None = None, + filters: FiltersLike = "auto", + compressors: CompressorsLike = "auto", + compressor: CompressorLike = "auto", + serializer: SerializerLike = "auto", + fill_value: Any | None = DEFAULT_FILL_VALUE, + order: MemoryOrder | None = None, + attributes: dict[str, JSON] | None = None, + chunk_key_encoding: ChunkKeyEncodingLike | None = None, + dimension_names: DimensionNamesLike = None, + storage_options: dict[str, Any] | None = None, + overwrite: bool = False, + config: ArrayConfigLike | None = None, + write_data: bool = True, + ) -> AnyArray: + """Create an array within this group. + + This method lightly wraps [zarr.core.array.create_array][]. + + Parameters + ---------- + name : str + The name of the array relative to the group. If `path` is `None`, the array will be located + at the root of the store. + shape : ShapeLike, optional + Shape of the array. Must be `None` if `data` is provided. + dtype : ZDTypeLike | None + Data type of the array. Must be `None` if `data` is provided. + data : np.ndarray, optional + Array-like data to use for initializing the array. If this parameter is provided, the + `shape` and `dtype` parameters must be `None`. + chunks : tuple[int, ...] | Sequence[Sequence[int]] | Literal["auto"], default="auto" + Chunk shape of the array. + If chunks is "auto", a chunk shape is guessed based on the shape of the array and the dtype. + A nested list of per-dimension edge sizes creates a rectilinear grid. + Rectilinear chunk grids are experimental and must be explicitly enabled + with `zarr.config.set({'array.rectilinear_chunks': True})` while the + feature is stabilizing. + shards : tuple[int, ...], optional + Shard shape of the array. The default value of `None` results in no sharding at all. + filters : Iterable[Codec] | Literal["auto"], optional + Iterable of filters to apply to each chunk of the array, in order, before serializing that + chunk to bytes. + + For Zarr format 3, a "filter" is a codec that takes an array and returns an array, + and these values must be instances of [`zarr.abc.codec.ArrayArrayCodec`][], or a + dict representations of [`zarr.abc.codec.ArrayArrayCodec`][]. + + For Zarr format 2, a "filter" can be any numcodecs codec; you should ensure that the + order of your filters is consistent with the behavior of each filter. + + The default value of `"auto"` instructs Zarr to use a default based on the data + type of the array and the Zarr format specified. For all data types in Zarr V3, and most + data types in Zarr V2, the default filters are empty. The only cases where default filters + are not empty is when the Zarr format is 2, and the data type is a variable-length data type like + [`zarr.dtype.VariableLengthUTF8`][] or [`zarr.dtype.VariableLengthBytes`][]. In these cases, + the default filters contains a single element which is a codec specific to that particular data type. + + To create an array with no filters, provide an empty iterable or the value `None`. + compressors : Iterable[Codec], optional + List of compressors to apply to the array. Compressors are applied in order, and after any + filters are applied (if any are specified) and the data is serialized into bytes. + + For Zarr format 3, a "compressor" is a codec that takes a bytestream, and + returns another bytestream. Multiple compressors may be provided for Zarr format 3. + If no `compressors` are provided, a default set of compressors will be used. + These defaults can be changed by modifying the value of `array.v3_default_compressors` + in [`zarr.config`][zarr.config]. + Use `None` to omit default compressors. + + For Zarr format 2, a "compressor" can be any numcodecs codec. Only a single compressor may + be provided for Zarr format 2. + If no `compressor` is provided, a default compressor will be used. + This default can be changed by modifying the value of `array.v2_default_compressor` + in [`zarr.config`][zarr.config]. + Use `None` to omit the default compressor. + compressor : Codec, optional + Deprecated in favor of `compressors`. + serializer : dict[str, JSON] | ArrayBytesCodec, optional + Array-to-bytes codec to use for encoding the array data. + Zarr format 3 only. Zarr format 2 arrays use implicit array-to-bytes conversion. + If no `serializer` is provided, a default serializer will be used. + These defaults can be changed by modifying the value of `array.v3_default_serializer` + in [`zarr.config`][zarr.config]. + fill_value : Any, optional + Fill value for the array. + order : {"C", "F"}, optional + The memory order of the array (default is "C"). + For Zarr format 2, this parameter sets the memory order of the array. + For Zarr format 3, this parameter is deprecated, because memory order + is a runtime parameter for Zarr format 3 arrays. The recommended way to specify the memory + order for Zarr format 3 arrays is via the `config` parameter, e.g. `{'config': 'C'}`. + If no `order` is provided, a default order will be used. + This default can be changed by modifying the value of `array.order` in [`zarr.config`][zarr.config]. + attributes : dict, optional + Attributes for the array. + chunk_key_encoding : ChunkKeyEncodingLike, optional + A specification of how the chunk keys are represented in storage. + For Zarr format 3, the default is `{"name": "default", "separator": "/"}`. + For Zarr format 2, the default is `{"name": "v2", "separator": "."}`. + dimension_names : Iterable[str], optional + The names of the dimensions (default is None). + Zarr format 3 only. Zarr format 2 arrays should not use this parameter. + storage_options : dict, optional + If using an fsspec URL to create the store, these will be passed to the backend implementation. + Ignored otherwise. + overwrite : bool, default False + Whether to overwrite an array with the same name in the store, if one exists. + If `True`, any existing keys under that path are deleted first. + config : ArrayConfigLike, optional + Runtime configuration for the array. + write_data : bool + If a pre-existing array-like object was provided to this function via the `data` parameter + then `write_data` determines whether the values in that array-like object should be + written to the Zarr array created by this function. If `write_data` is `False`, then the + array will be left empty. + + Returns + ------- + AsyncArray + """ + compressors = _parse_deprecated_compressor( + compressor, compressors, zarr_format=self.metadata.zarr_format + ) + return Array( + self._sync( + self._async_group.create_array( + name=name, + shape=shape, + dtype=dtype, + data=data, + chunks=chunks, + shards=shards, + fill_value=fill_value, + attributes=attributes, + chunk_key_encoding=chunk_key_encoding, + compressors=compressors, + serializer=serializer, + dimension_names=dimension_names, + order=order, + filters=filters, + overwrite=overwrite, + storage_options=storage_options, + config=config, + write_data=write_data, + ) + ) + ) + + def require_array( + self, + name: str, + *, + shape: ShapeLike, + config: ArrayConfigLike | None = None, + **kwargs: Any, + ) -> AnyArray: + """Obtain an array, creating if it doesn't exist. + + Other `kwargs` are as per [zarr.Group.create_array][]. + + Parameters + ---------- + name : str + Array name. + config : ArrayConfigLike or None, default=None + Runtime configuration for the array, whether it is created or already exists. + Keys not specified are taken from the global configuration. + **kwargs : + See [zarr.Group.create_array][]. + + Returns + ------- + a : Array + """ + return Array( + self._sync(self._async_group.require_array(name, shape=shape, config=config, **kwargs)) + ) + + def empty(self, *, name: str, shape: tuple[int, ...], **kwargs: Any) -> AnyArray: + """Create an empty array with the specified shape in this Group. The contents will be filled with + the array's fill value or zeros if no fill value is provided. + + Parameters + ---------- + name : str + Name of the array. + shape : int or tuple of int + Shape of the empty array. + **kwargs + Keyword arguments passed to [zarr.api.asynchronous.create][]. + + Notes + ----- + The contents of an empty Zarr array are not defined. On attempting to + retrieve data from an empty Zarr array, any values may be returned, + and these are not guaranteed to be stable from one access to the next. + """ + return Array(self._sync(self._async_group.empty(name=name, shape=shape, **kwargs))) + + def zeros(self, *, name: str, shape: tuple[int, ...], **kwargs: Any) -> AnyArray: + """Create an array, with zero being used as the default value for uninitialized portions of the array. + + Parameters + ---------- + name : str + Name of the array. + shape : int or tuple of int + Shape of the empty array. + **kwargs + Keyword arguments passed to [zarr.api.asynchronous.create][]. + + Returns + ------- + Array + The new array. + """ + return Array(self._sync(self._async_group.zeros(name=name, shape=shape, **kwargs))) + + def ones(self, *, name: str, shape: tuple[int, ...], **kwargs: Any) -> AnyArray: + """Create an array, with one being used as the default value for uninitialized portions of the array. + + Parameters + ---------- + name : str + Name of the array. + shape : int or tuple of int + Shape of the empty array. + **kwargs + Keyword arguments passed to [zarr.api.asynchronous.create][]. + + Returns + ------- + Array + The new array. + """ + return Array(self._sync(self._async_group.ones(name=name, shape=shape, **kwargs))) + + def full( + self, *, name: str, shape: tuple[int, ...], fill_value: Any | None, **kwargs: Any + ) -> AnyArray: + """Create an array, with "fill_value" being used as the default value for uninitialized portions of the array. + + Parameters + ---------- + name : str + Name of the array. + shape : int or tuple of int + Shape of the empty array. + fill_value : scalar + Value to fill the array with. + **kwargs + Keyword arguments passed to [zarr.api.asynchronous.create][]. + + Returns + ------- + Array + The new array. + """ + return Array( + self._sync( + self._async_group.full(name=name, shape=shape, fill_value=fill_value, **kwargs) + ) + ) + + def empty_like(self, *, name: str, data: async_api.ArrayLike, **kwargs: Any) -> AnyArray: + """Create an empty sub-array like `data`. The contents will be filled + with the array's fill value or zeros if no fill value is provided. + + Parameters + ---------- + name : str + Name of the array. + data : array-like + The array to create an empty array like. + **kwargs + Keyword arguments passed to [zarr.api.asynchronous.create][]. + + Returns + ------- + Array + The new array. + + Notes + ----- + The contents of an empty Zarr array are not defined. On attempting to + retrieve data from an empty Zarr array, any values may be returned, + and these are not guaranteed to be stable from one access to the next. + """ + return Array(self._sync(self._async_group.empty_like(name=name, data=data, **kwargs))) + + def zeros_like(self, *, name: str, data: async_api.ArrayLike, **kwargs: Any) -> AnyArray: + """Create a sub-array of zeros like `data`. + + Parameters + ---------- + name : str + Name of the array. + data : array-like + The array to create the new array like. + **kwargs + Keyword arguments passed to [zarr.api.asynchronous.create][]. + + Returns + ------- + Array + The new array. + """ + + return Array(self._sync(self._async_group.zeros_like(name=name, data=data, **kwargs))) + + def ones_like(self, *, name: str, data: async_api.ArrayLike, **kwargs: Any) -> AnyArray: + """Create a sub-array of ones like `data`. + + Parameters + ---------- + name : str + Name of the array. + data : array-like + The array to create the new array like. + **kwargs + Keyword arguments passed to [zarr.api.asynchronous.create][]. + + Returns + ------- + Array + The new array. + """ + return Array(self._sync(self._async_group.ones_like(name=name, data=data, **kwargs))) + + def full_like(self, *, name: str, data: async_api.ArrayLike, **kwargs: Any) -> AnyArray: + """Create a sub-array like `data` filled with the `fill_value` of `data` . + + Parameters + ---------- + name : str + Name of the array. + data : array-like + The array to create the new array like. + **kwargs + Keyword arguments passed to [zarr.api.asynchronous.create][]. + + Returns + ------- + Array + The new array. + """ + return Array(self._sync(self._async_group.full_like(name=name, data=data, **kwargs))) + + def move(self, source: str, dest: str) -> None: + """Move a sub-group or sub-array from one path to another. + + Notes + ----- + Not implemented + """ + return self._sync(self._async_group.move(source, dest)) + + +async def create_hierarchy( + *, + store: Store, + nodes: dict[str, GroupMetadata | ArrayV2Metadata | ArrayV3Metadata], + overwrite: bool = False, +) -> AsyncIterator[tuple[str, AsyncGroup | AnyAsyncArray]]: + """ + Create a complete zarr hierarchy from a collection of metadata objects. + + This function will parse its input to ensure that the hierarchy is complete. Any implicit groups + will be inserted as needed. For example, an input like + ```{'a/b': GroupMetadata}``` will be parsed to + ```{'': GroupMetadata, 'a': GroupMetadata, 'b': Groupmetadata}``` + + After input parsing, this function then creates all the nodes in the hierarchy concurrently. + + Arrays and Groups are yielded in the order they are created. This order is not stable and + should not be relied on. + + Parameters + ---------- + store : Store + The storage backend to use. + nodes : dict[str, GroupMetadata | ArrayV3Metadata | ArrayV2Metadata] + A dictionary defining the hierarchy. The keys are the paths of the nodes in the hierarchy, + relative to the root of the ``Store``. The root of the store can be specified with the empty + string ``''``. The values are instances of ``GroupMetadata`` or ``ArrayMetadata``. Note that + all values must have the same ``zarr_format`` -- it is an error to mix zarr versions in the + same hierarchy. + + Leading "/" characters from keys will be removed. + overwrite : bool + Whether to overwrite existing nodes. Defaults to ``False``, in which case an error is + raised instead of overwriting an existing array or group. + + This function will not erase an existing group unless that group is explicitly named in + ``nodes``. If ``nodes`` defines implicit groups, e.g. ``{`'a/b/c': GroupMetadata}``, and a + group already exists at path ``a``, then this function will leave the group at ``a`` as-is. + + Yields + ------ + tuple[str, AsyncGroup | AsyncArray] + Yields (path, node) pairs, in the order the nodes were created. + + Examples + -------- + >>> async def example(): + ... from zarr.api.asynchronous import create_hierarchy + ... from zarr.core.group import GroupMetadata + ... from zarr.storage import MemoryStore + ... + ... store = MemoryStore() + ... nodes = {'a': GroupMetadata(attributes={'name': 'leaf'})} + ... return sorted([x async for x in create_hierarchy(store=store, nodes=nodes)]) + + >>> import asyncio + >>> asyncio.run(example()) + [('', ), ('a', )] + """ + + # normalize the keys to be valid paths + nodes_normed_keys = _normalize_path_keys(nodes) + + # ensure that all nodes have the same zarr_format, and add implicit groups as needed + nodes_parsed = _parse_hierarchy_dict(data=nodes_normed_keys) + redundant_implicit_groups = [] + to_delete_keys: list[str] = [] + + # empty hierarchies should be a no-op + if len(nodes_parsed) > 0: + # figure out which zarr format we are using + zarr_format = next(iter(nodes_parsed.values())).zarr_format + + # check which implicit groups will require materialization + implicit_group_keys = tuple( + filter(lambda k: isinstance(nodes_parsed[k], ImplicitGroupMarker), nodes_parsed) + ) + # read potential group metadata for each implicit group + maybe_extant_group_coros = ( + _read_group_metadata(store, k, zarr_format=zarr_format) for k in implicit_group_keys + ) + maybe_extant_groups = await asyncio.gather( + *maybe_extant_group_coros, return_exceptions=True + ) + + for key, value in zip(implicit_group_keys, maybe_extant_groups, strict=True): + if isinstance(value, BaseException): + if isinstance(value, FileNotFoundError): + # this is fine -- there was no group there, so we will create one + pass + else: + raise value + else: + # a loop exists already at ``key``, so we can avoid creating anything there + redundant_implicit_groups.append(key) + + if overwrite: + # we will remove any nodes that collide with arrays and non-implicit groups defined in + # nodes + to_delete_keys = [k for k in nodes_parsed if k not in implicit_group_keys] + else: + # This type is long. + coros: ( + Generator[Coroutine[Any, Any, ArrayV2Metadata | GroupMetadata], None, None] + | Generator[Coroutine[Any, Any, ArrayV3Metadata | GroupMetadata], None, None] + ) + if zarr_format == 2: + coros = (_read_metadata_v2(store=store, path=key) for key in nodes_parsed) + elif zarr_format == 3: + coros = (_read_metadata_v3(store=store, path=key) for key in nodes_parsed) + else: # pragma: no cover + raise ValueError(f"Invalid zarr_format: {zarr_format}") # pragma: no cover + + extant_node_query = dict( + zip( + nodes_parsed.keys(), + await asyncio.gather(*coros, return_exceptions=True), + strict=False, + ) + ) + # iterate over the existing arrays / groups and figure out which of them conflict + # with the arrays / groups we want to create + for key, extant_node in extant_node_query.items(): + proposed_node = nodes_parsed[key] + if isinstance(extant_node, BaseException): + if isinstance(extant_node, FileNotFoundError): + # ignore FileNotFoundError, because they represent nodes we can safely create + pass + else: + # Any other exception is a real error + raise extant_node + else: + # this is a node that already exists, but a node with the same key was specified + # in nodes_parsed. + if isinstance(extant_node, GroupMetadata): + # a group already exists where we want to create a group + if isinstance(proposed_node, ImplicitGroupMarker): + # we have proposed an implicit group, which is OK -- we will just skip + # creating this particular metadata document + redundant_implicit_groups.append(key) + else: + # we have proposed an explicit group, which is an error, given that a + # group already exists. + msg = f"A group exists in store {store!r} at path {key!r}." + raise ContainsGroupError(msg) + elif isinstance(extant_node, ArrayV2Metadata | ArrayV3Metadata): + # we are trying to overwrite an existing array. this is an error. + msg = f"An array exists in store {store!r} at path {key!r}." + raise ContainsArrayError(msg) + + nodes_explicit: dict[str, GroupMetadata | ArrayV2Metadata | ArrayV3Metadata] = {} + + for k, v in nodes_parsed.items(): + if k not in redundant_implicit_groups: + if isinstance(v, ImplicitGroupMarker): + nodes_explicit[k] = GroupMetadata(zarr_format=v.zarr_format) + else: + nodes_explicit[k] = v + + # Build and encode every node before deleting anything: a node that cannot be built + # or whose metadata cannot be stored then fails with the store untouched. + built, documents = _prepare_nodes(store, nodes_explicit) + await asyncio.gather(*(store.delete_dir(key) for key in to_delete_keys)) + async for key, node in _store_nodes(store, nodes_explicit, built, documents): + yield key, node + + +async def create_nodes( + *, + store: Store, + nodes: dict[str, GroupMetadata | ArrayV2Metadata | ArrayV3Metadata], +) -> AsyncIterator[tuple[str, AsyncGroup | AnyAsyncArray]]: + """Create a collection of arrays and / or groups concurrently. + + Note: no attempt is made to validate that these arrays and / or groups collectively form a + valid Zarr hierarchy. It is the responsibility of the caller of this function to ensure that + the ``nodes`` parameter satisfies any correctness constraints. + + Parameters + ---------- + store : Store + The storage backend to use. + nodes : dict[str, GroupMetadata | ArrayV3Metadata | ArrayV2Metadata] + A dictionary defining the hierarchy. The keys are the paths of the nodes + in the hierarchy, and the values are the metadata of the nodes. The + metadata must be either an instance of GroupMetadata, ArrayV3Metadata + or ArrayV2Metadata. + + Yields + ------ + AsyncGroup | AsyncArray + The created nodes in the order they are created. + """ + async for key, node in _store_nodes(store, nodes, *_prepare_nodes(store, nodes)): + yield key, node + + +def _prepare_nodes( + store: Store, nodes: Mapping[str, GroupMetadata | ArrayV2Metadata | ArrayV3Metadata] +) -> tuple[dict[str, AsyncGroup | AnyAsyncArray], dict[str, Buffer]]: + """The array or group each of `nodes` describes, at its path in `store`, and the + metadata documents of `nodes`, by their keys in the store: every node is built and + encoded before anything is stored.""" + for path, metadata in nodes.items(): + validate_node_path(path, metadata.zarr_format) + + built = { + path: _build_node(store=store, path=path, metadata=meta) for path, meta in nodes.items() + } + documents = { + _join_paths([path, key]): value + for path, metadata in nodes.items() + for key, value in encode_documents(StorePath(store, path), metadata).items() + } + return built, documents + + +async def _store_nodes( + store: Store, + nodes: Mapping[str, GroupMetadata | ArrayV2Metadata | ArrayV3Metadata], + built: Mapping[str, AsyncGroup | AnyAsyncArray], + documents: Mapping[str, Buffer], +) -> AsyncIterator[tuple[str, AsyncGroup | AnyAsyncArray]]: + """Store the `documents` encoded from `nodes` and yield the nodes built from them, as + `_prepare_nodes` returns them (see `create_nodes`).""" + # Note: the only way to alter this value is via the config. If that's undesirable for some reason, + # then we should consider adding a keyword argument to this function + semaphore = asyncio.Semaphore(config.get("async.concurrency")) + create_tasks = [ + _set_return_key(store=store, key=key, value=value, semaphore=semaphore) + for key, value in documents.items() + ] + + created_object_keys = [] + + for coro in asyncio.as_completed(create_tasks): + created_key = await coro + # we need this to track which metadata documents were written so that we can yield a + # complete v2 Array / Group class after both .zattrs and the metadata JSON was created. + created_object_keys.append(created_key) + + # get the node name from the object key + if len(created_key.split("/")) == 1: + # this is the root node + meta_out = nodes[""] + node_name = "" + else: + # turn "foo/" into "foo" + node_name = created_key[: created_key.rfind("/")] + meta_out = nodes[node_name] + if meta_out.zarr_format == 3: + yield node_name, built[node_name] + else: + # For zarr v2 + # we only want to yield when both the metadata and attributes are created + # so we track which keys have been created, and wait for both the meta key and + # the attrs key to be created before yielding back the AsyncArray / AsyncGroup + + attrs_done = _join_paths([node_name, ZATTRS_JSON]) in created_object_keys + + if isinstance(meta_out, GroupMetadata): + meta_done = _join_paths([node_name, ZGROUP_JSON]) in created_object_keys + else: + meta_done = _join_paths([node_name, ZARRAY_JSON]) in created_object_keys + + if meta_done and attrs_done: + yield node_name, built[node_name] + + continue + + +def _get_roots( + data: Iterable[str], +) -> tuple[str, ...]: + """ + Return the keys of the root(s) of the hierarchy. A root is a key with the fewest number of + path segments. + """ + if "" in data: + return ("",) + keys_split = sorted((key.split("/") for key in data), key=len) + groups: defaultdict[int, list[str]] = defaultdict(list) + for key_split in keys_split: + groups[len(key_split)].append("/".join(key_split)) + return tuple(groups[min(groups.keys())]) + + +def _parse_hierarchy_dict( + *, + data: Mapping[str, ImplicitGroupMarker | GroupMetadata | ArrayV2Metadata | ArrayV3Metadata], +) -> dict[str, ImplicitGroupMarker | GroupMetadata | ArrayV2Metadata | ArrayV3Metadata]: + """ + Take an input with type Mapping[str, ArrayMetadata | GroupMetadata] and parse it into + a dict of str: node pairs that models a valid, complete Zarr hierarchy. + + If the input represents a complete Zarr hierarchy, i.e. one with no implicit groups, + then return a dict with the exact same data as the input. + + Otherwise, return a dict derived from the input with GroupMetadata inserted as needed to make + the hierarchy complete. + + For example, an input of {'a/b': ArrayMetadata} is incomplete, because it references two + groups (the root group '' and a group at 'a') that are not specified in the input. Applying this function + to that input will result in a return value of + {'': GroupMetadata, 'a': GroupMetadata, 'a/b': ArrayMetadata}, i.e. the implied groups + were added. + + The input is also checked for the following conditions; an error is raised if any are violated: + + - No arrays can contain group or arrays (i.e., all arrays must be leaf nodes). + - All arrays and groups must have the same ``zarr_format`` value. + + This function ensures that the input is transformed into a specification of a complete and valid + Zarr hierarchy. + """ + + # ensure that all nodes have the same zarr format + data_purified = _ensure_consistent_zarr_format(data) + + # ensure that keys are normalized to zarr paths + data_normed_keys = _normalize_path_keys(data_purified) + + # insert an implicit root group if a root was not specified + # but not if an empty dict was provided, because any empty hierarchy has no nodes + if len(data_normed_keys) > 0 and "" not in data_normed_keys: + z_format = next(iter(data_normed_keys.values())).zarr_format + data_normed_keys = data_normed_keys | {"": ImplicitGroupMarker(zarr_format=z_format)} + + out: dict[str, GroupMetadata | ArrayV2Metadata | ArrayV3Metadata] = {**data_normed_keys} + + for k, v in data_normed_keys.items(): + key_split = k.split("/") + + # get every parent path + *subpaths, _ = accumulate(key_split, lambda a, b: _join_paths([a, b])) + + for subpath in subpaths: + # If a component is not already in the output dict, add ImplicitGroupMetadata + if subpath not in out: + out[subpath] = ImplicitGroupMarker(zarr_format=v.zarr_format) + else: + if not isinstance(out[subpath], GroupMetadata | ImplicitGroupMarker): + msg = ( + f"The node at {subpath} contains other nodes, but it is not a Zarr group. " + "This is invalid. Only Zarr groups can contain other nodes." + ) + raise ValueError(msg) + return out + + +def _ensure_consistent_zarr_format( + data: Mapping[str, GroupMetadata | ArrayV2Metadata | ArrayV3Metadata], +) -> Mapping[str, GroupMetadata | ArrayV2Metadata] | Mapping[str, GroupMetadata | ArrayV3Metadata]: + """ + Ensure that all values of the input dict have the same zarr format. If any do not, + then a value error is raised. + """ + observed_zarr_formats: dict[ZarrFormat, list[str]] = {2: [], 3: []} + + for k, v in data.items(): + observed_zarr_formats[v.zarr_format].append(k) + + if len(observed_zarr_formats[2]) > 0 and len(observed_zarr_formats[3]) > 0: + msg = ( + "Got data with both Zarr v2 and Zarr v3 nodes, which is invalid. " + f"The following keys map to Zarr v2 nodes: {observed_zarr_formats.get(2)}. " + f"The following keys map to Zarr v3 nodes: {observed_zarr_formats.get(3)}." + "Ensure that all nodes have the same Zarr format." + ) + raise ValueError(msg) + + return cast( + "Mapping[str, GroupMetadata | ArrayV2Metadata] | Mapping[str, GroupMetadata | ArrayV3Metadata]", + data, + ) + + +async def _getitem_semaphore( + node: AsyncGroup, key: str, semaphore: asyncio.Semaphore | None +) -> AnyAsyncArray | AsyncGroup: + """ + Wrap Group.getitem with an optional semaphore. + + If the semaphore parameter is an + asyncio.Semaphore instance, then the getitem operation is performed inside an async context + manager provided by that semaphore. If the semaphore parameter is None, then getitem is invoked + without a context manager. + """ + if semaphore is not None: + async with semaphore: + return await node.getitem(key) + else: + return await node.getitem(key) + + +async def _iter_members( + node: AsyncGroup, + skip_keys: tuple[str, ...], + semaphore: asyncio.Semaphore | None, +) -> AsyncGenerator[tuple[str, AnyAsyncArray | AsyncGroup], None]: + """ + Iterate over the arrays and groups contained in a group. + + Parameters + ---------- + node : AsyncGroup + The group to traverse. + skip_keys : tuple[str, ...] + A tuple of keys to skip when iterating over the possible members of the group. + semaphore : asyncio.Semaphore | None + An optional semaphore to use for concurrency control. + + Yields + ------ + tuple[str, AnyAsyncArray | AsyncGroup] + """ + + # retrieve keys from storage + keys = [key async for key in node.store.list_dir(node.path)] + keys_filtered = tuple(filter(lambda v: v not in skip_keys, keys)) + + node_tasks = tuple( + asyncio.create_task(_getitem_semaphore(node, key, semaphore), name=key) + for key in keys_filtered + ) + + for fetched_node_coro in asyncio.as_completed(node_tasks): + try: + fetched_node = await fetched_node_coro + except KeyError as e: + # keyerror is raised when `key` names an object (in the object storage sense), + # as opposed to a prefix, in the store under the prefix associated with this group + # in which case `key` cannot be the name of a sub-array or sub-group. + warnings.warn( + f"Object at {e.args[0]} is not recognized as a component of a Zarr hierarchy.", + ZarrUserWarning, + stacklevel=1, + ) + continue + match fetched_node: + case AsyncArray() | AsyncGroup(): + yield fetched_node.basename, fetched_node + case _: + raise ValueError(f"Unexpected type: {type(fetched_node)}") + + +async def _iter_members_deep( + group: AsyncGroup, + *, + max_depth: int | None, + skip_keys: tuple[str, ...], + semaphore: asyncio.Semaphore | None = None, + use_consolidated_for_children: bool = True, +) -> AsyncGenerator[tuple[str, AnyAsyncArray | AsyncGroup], None]: + """ + Iterate over the arrays and groups contained in a group, and optionally the + arrays and groups contained in those groups. + + Parameters + ---------- + group : AsyncGroup + The group to traverse. + max_depth : int | None + The maximum depth of recursion. + skip_keys : tuple[str, ...] + A tuple of keys to skip when iterating over the possible members of the group. + semaphore : asyncio.Semaphore | None + An optional semaphore to use for concurrency control. + use_consolidated_for_children : bool, default True + Whether to use the consolidated metadata of child groups loaded + from the store. Note that this only affects groups loaded from the + store. If the current Group already has consolidated metadata, it + will always be used. + + Yields + ------ + tuple[str, AnyAsyncArray | AsyncGroup] + """ + + to_recurse = {} + do_recursion = max_depth is None or max_depth > 0 + + if max_depth is None: + new_depth = None + else: + new_depth = max_depth - 1 + async for name, node in _iter_members(group, skip_keys=skip_keys, semaphore=semaphore): + is_group = isinstance(node, AsyncGroup) + if ( + is_group + and not use_consolidated_for_children + and node.metadata.consolidated_metadata is not None + ): + node = cast("AsyncGroup", node) + # We've decided not to trust consolidated metadata at this point, because we're + # reconsolidating the metadata, for example. + node = replace(node, metadata=replace(node.metadata, consolidated_metadata=None)) + yield name, node + if is_group and do_recursion: + node = cast("AsyncGroup", node) + to_recurse[name] = _iter_members_deep( + node, max_depth=new_depth, skip_keys=skip_keys, semaphore=semaphore + ) + + for prefix, subgroup_iter in to_recurse.items(): + async for name, node in subgroup_iter: + key = f"{prefix}/{name}".lstrip("/") + yield key, node + + +async def _read_metadata_v3(store: Store, path: str) -> ArrayV3Metadata | GroupMetadata: + """ + Given a store_path, return ArrayV3Metadata or GroupMetadata defined by the metadata + document stored at store_path.path / zarr.json. If no such document is found, raise a + FileNotFoundError. + """ + zarr_json_bytes = await store.get( + _join_paths([path, ZARR_JSON]), prototype=default_buffer_prototype() + ) + if zarr_json_bytes is None: + raise FileNotFoundError(path) + return _build_metadata_v3( + buffer_to_json_object(zarr_json_bytes), path=str(StorePath(store, path)) + ) + + +async def _read_metadata_v2(store: Store, path: str) -> ArrayV2Metadata | GroupMetadata: + """ + Given a store_path, return ArrayV2Metadata or GroupMetadata defined by the metadata + document stored at store_path.path / (.zgroup | .zarray). If no such document is found, + raise a FileNotFoundError. + """ + # TODO: consider first fetching array metadata, and only fetching group metadata when we don't + # find an array + zarray_bytes, zgroup_bytes, zattrs_bytes = await asyncio.gather( + store.get(_join_paths([path, ZARRAY_JSON]), prototype=default_buffer_prototype()), + store.get(_join_paths([path, ZGROUP_JSON]), prototype=default_buffer_prototype()), + store.get(_join_paths([path, ZATTRS_JSON]), prototype=default_buffer_prototype()), + ) + + zattrs: dict[str, JSON] + if zattrs_bytes is None: + zattrs = {} + else: + zattrs = buffer_to_json_object(zattrs_bytes) + + # TODO: decide how to handle finding both array and group metadata. The spec does not seem to + # consider this situation. A practical approach would be to ignore that combination, and only + # return the array metadata. + if zarray_bytes is not None: + zmeta = buffer_to_json_object(zarray_bytes) + else: + if zgroup_bytes is None: + # neither .zarray or .zgroup were found results in KeyError + raise FileNotFoundError(path) + else: + zmeta = buffer_to_json_object(zgroup_bytes) + + return _build_metadata_v2(zmeta, zattrs, path=str(StorePath(store, path))) + + +async def _read_group_metadata_v2(store: Store, path: str) -> GroupMetadata: + """ + Read group metadata or error + """ + meta = await _read_metadata_v2(store=store, path=path) + if not isinstance(meta, GroupMetadata): + raise FileNotFoundError(f"Group metadata was not found in {store} at {path}") + return meta + + +async def _read_group_metadata_v3(store: Store, path: str) -> GroupMetadata: + """ + Read group metadata or error + """ + meta = await _read_metadata_v3(store=store, path=path) + if not isinstance(meta, GroupMetadata): + raise FileNotFoundError(f"Group metadata was not found in {store} at {path}") + return meta + + +async def _read_group_metadata( + store: Store, path: str, *, zarr_format: ZarrFormat +) -> GroupMetadata: + if zarr_format == 2: + return await _read_group_metadata_v2(store=store, path=path) + return await _read_group_metadata_v3(store=store, path=path) + + +def _build_metadata_v3( + zarr_json: dict[str, JSON], *, path: str | None = None +) -> ArrayV3Metadata | GroupMetadata: + """ + Convert a dict representation of Zarr V3 metadata into the corresponding metadata class. + """ + if "node_type" not in zarr_json: + msg = "Required key 'node_type' is missing from the provided metadata document." + raise MetadataValidationError(msg) + match zarr_json: + case {"node_type": "array"}: + return ArrayV3Metadata.from_dict(zarr_json, path=path) + case {"node_type": "group"}: + return GroupMetadata.from_dict(zarr_json, path=path) + case _: # pragma: no cover + raise ValueError( + "invalid value for `node_type` key in metadata document" + ) # pragma: no cover + + +def _build_metadata_v2( + zarr_json: dict[str, JSON], attrs_json: dict[str, JSON], *, path: str | None = None +) -> ArrayV2Metadata | GroupMetadata: + """ + Convert a dict representation of Zarr V2 metadata into the corresponding metadata class. + """ + match zarr_json: + case {"shape": _}: + return ArrayV2Metadata.from_dict(zarr_json | {"attributes": attrs_json}, path=path) + case _: # pragma: no cover + return GroupMetadata.from_dict(zarr_json | {"attributes": attrs_json}, path=path) + + +@overload +def _build_node(*, store: Store, path: str, metadata: ArrayV2Metadata) -> AsyncArrayV2: ... + + +@overload +def _build_node(*, store: Store, path: str, metadata: ArrayV3Metadata) -> AsyncArrayV3: ... + + +@overload +def _build_node(*, store: Store, path: str, metadata: GroupMetadata) -> AsyncGroup: ... + + +def _build_node( + *, store: Store, path: str, metadata: ArrayV3Metadata | ArrayV2Metadata | GroupMetadata +) -> AnyAsyncArray | AsyncGroup: + """ + Take a metadata object and return a node (AsyncArray or AsyncGroup). + """ + store_path = StorePath(store=store, path=path) + match metadata: + case ArrayV2Metadata() | ArrayV3Metadata(): + return AsyncArray(metadata, store_path=store_path) + case GroupMetadata(): + return AsyncGroup(metadata, store_path=store_path) + case _: # pragma: no cover + raise ValueError(f"Unexpected metadata type: {type(metadata)}") # pragma: no cover + + +async def _get_node_v2(store: Store, path: str) -> AsyncArrayV2 | AsyncGroup: + """ + Read a Zarr v2 AsyncArray or AsyncGroup from a path in a Store. + + Parameters + ---------- + store : Store + The store-like object to read from. + path : str + The path to the node to read. + + Returns + ------- + AsyncArray | AsyncGroup + """ + metadata = await _read_metadata_v2(store=store, path=path) + return _build_node(store=store, path=path, metadata=metadata) + + +async def _get_node_v3(store: Store, path: str) -> AsyncArrayV3 | AsyncGroup: + """ + Read a Zarr v3 AsyncArray or AsyncGroup from a path in a Store. + + Parameters + ---------- + store : Store + The store-like object to read from. + path : str + The path to the node to read. + + Returns + ------- + AsyncArray | AsyncGroup + """ + metadata = await _read_metadata_v3(store=store, path=path) + return _build_node(store=store, path=path, metadata=metadata) + + +async def get_node(store: Store, path: str, zarr_format: ZarrFormat) -> AnyAsyncArray | AsyncGroup: + """ + Get an AsyncArray or AsyncGroup from a path in a Store. + + Parameters + ---------- + store : Store + The store-like object to read from. + path : str + The path to the node to read. + zarr_format : {2, 3} + The zarr format of the node to read. + + Returns + ------- + AsyncArray | AsyncGroup + """ + + match zarr_format: + case 2: + return await _get_node_v2(store=store, path=path) + case 3: + return await _get_node_v3(store=store, path=path) + case _: # pragma: no cover + raise ValueError(f"Unexpected zarr format: {zarr_format}") # pragma: no cover + + +async def _set_return_key( + *, store: Store, key: str, value: Buffer, semaphore: asyncio.Semaphore | None = None +) -> str: + """ + Write a value to storage at the given key. The key is returned. + Useful when saving values via routines that return results in execution order, + like asyncio.as_completed, because in this case we need to know which key was saved in order + to yield the right object to the caller. + + Parameters + ---------- + store : Store + The store to save the value to. + key : str + The key to save the value to. + value : Buffer + The value to save. + semaphore : asyncio.Semaphore | None + An optional semaphore to use to limit the number of concurrent writes. + """ + + if semaphore is not None: + async with semaphore: + await store.set(key, value) + else: + await store.set(key, value) + return key + + +async def create_rooted_hierarchy( + *, + store: Store, + nodes: dict[str, GroupMetadata | ArrayV2Metadata | ArrayV3Metadata], + overwrite: bool = False, +) -> AsyncGroup | AnyAsyncArray: + """ + Create an ``AsyncGroup`` or ``AsyncArray`` from a store and a dict of metadata documents. + This function ensures that its input contains a specification of a root node, + calls ``create_hierarchy`` to create nodes, and returns the root node of the hierarchy. + """ + roots = _get_roots(nodes.keys()) + if len(roots) != 1: + msg = ( + "The input does not specify a root node. " + "This function can only create hierarchies that contain a root node, which is " + "defined as a group that is ancestral to all the other arrays and " + "groups in the hierarchy, or a single array." + ) + raise ValueError(msg) + else: + root_key = roots[0] + + nodes_created = [ + x async for x in create_hierarchy(store=store, nodes=nodes, overwrite=overwrite) + ] + return dict(nodes_created)[root_key] From 949e5780140676f4d16b2b7b13a23c98ecbcf4a8 Mon Sep 17 00:00:00 2001 From: Duo-ee <113885053+Duo-ee@users.noreply.github.com> Date: Thu, 22 Oct 2026 11:17:43 +0800 Subject: [PATCH 02/12] fix: normalize Zarr v3 node validation files Keep the branch diff focused by preserving repository line endings. --- src/zarr/core/common.py | 932 ++--- src/zarr/core/group.py | 7900 +++++++++++++++++++-------------------- 2 files changed, 4416 insertions(+), 4416 deletions(-) diff --git a/src/zarr/core/common.py b/src/zarr/core/common.py index 0f514b4349..74f862bcc3 100644 --- a/src/zarr/core/common.py +++ b/src/zarr/core/common.py @@ -1,466 +1,466 @@ -from __future__ import annotations - -import asyncio -import math -import warnings -from collections.abc import Iterable, Mapping, Sequence -from enum import Enum -from typing import ( - TYPE_CHECKING, - Any, - Final, - Literal, - NotRequired, - TypedDict, - cast, - overload, -) - -import numpy as np -from typing_extensions import ReadOnly - -from zarr.core.config import config as zarr_config -from zarr.core.json_parse import convert, parse_field -from zarr.errors import ZarrRuntimeWarning - -if TYPE_CHECKING: - from collections.abc import Awaitable, Callable, Iterator - - from zarr.core.metadata.v3 import ChunkGridMetadata - - -ZARR_JSON = "zarr.json" -ZARRAY_JSON = ".zarray" -ZGROUP_JSON = ".zgroup" -ZATTRS_JSON = ".zattrs" -ZMETADATA_V2_JSON = ".zmetadata" - -BytesLike = bytes | bytearray | memoryview -ShapeLike = Iterable[int | np.integer[Any]] | int | np.integer[Any] -# Per-dimension chunk specs may mix a bare int (uniform chunk size, the -# rectilinear spec's step-size shorthand) with explicit edge-length sequences. -# A stored chunk grid (`ChunkGridMetadata`) is also accepted and is used -# verbatim, under the tolerant stored-metadata validation rules (e.g. trailing -# edges beyond the array extent, as left behind by a shrinking resize). -type ChunksLike = ShapeLike | Iterable[int | Iterable[int]] | ChunkGridMetadata -# For backwards compatibility -ChunkCoords = tuple[int, ...] -ZarrFormat = Literal[2, 3] -NodeType = Literal["array", "group"] -JSON = str | int | float | bool | Mapping[str, "JSON"] | Sequence["JSON"] | None -MemoryOrder = Literal["C", "F"] -AccessModeLiteral = Literal["r", "r+", "a", "w", "w-"] -ANY_ACCESS_MODE: Final = "r", "r+", "a", "w", "w-" -DimensionNamesLike = Iterable[str | None] | None -DimensionNames = DimensionNamesLike # for backwards compatibility - - -def validate_node_path(path: str, zarr_format: ZarrFormat) -> None: - """Validate node names in a normalized hierarchy path before creating them.""" - if zarr_format != 3 or not path: - return - - for name in path.split("/"): - if all(char == "." for char in name) or name.startswith("__") or name == ZARR_JSON: - raise ValueError( - f"Invalid Zarr format 3 node name {name!r} in path {path!r}. " - "Node names must not consist only of periods, start with '__', " - "or equal 'zarr.json'." - ) - - -class NamedConfig[TName: str, TConfig: Mapping[str, object]](TypedDict): - """ - A typed dictionary representing an object with a name and configuration, where the configuration - is an optional mapping of string keys to values, e.g. another typed dictionary or a JSON object. - - This class is generic with two type parameters: the type of the name (``TName``) and the type of - the configuration (``TConfig``). - """ - - name: ReadOnly[TName] - """The name of the object.""" - - configuration: NotRequired[ReadOnly[TConfig]] - """The configuration of the object. Not required.""" - - -class NamedRequiredConfig[TName: str, TConfig: Mapping[str, object]](TypedDict): - """ - A typed dictionary representing an object with a name and configuration, where the configuration - is a mapping of string keys to values, e.g. another typed dictionary or a JSON object. - - This class is generic with two type parameters: the type of the name (``TName``) and the type of - the configuration (``TConfig``). - """ - - name: ReadOnly[TName] - """The name of the object.""" - - configuration: ReadOnly[TConfig] - """The configuration of the object.""" - - -def product(tup: tuple[int, ...]) -> int: - return math.prod(tup) - - -def ceildiv(a: float, b: float) -> int: - """Ceiling of ``a / b`` using floating-point division; zero when ``a`` is zero.""" - if a == 0: - return 0 - return math.ceil(a / b) - - -def ceildiv_int(a: int, b: int) -> int: - """Ceiling of integer division using exact Python integer arithmetic.""" - return -(-int(a) // int(b)) - - -def concurrent_iter[T: tuple[Any, ...], V]( - items: Iterable[T], - func: Callable[..., Awaitable[V]], - limit: int | None = None, -) -> list[asyncio.Task[V]]: - """Launch `func(*item)` for each item concurrently, returning the tasks. - - When `limit` is set, no more than `limit` calls are in flight at once. - Tasks are returned in input order; callers that want completion order - should wrap the result in `asyncio.as_completed`. - - Every task is scheduled (via `ensure_future`) before this function - returns, not on first iteration of the result. That matters for callers - that await the returned tasks one at a time — without eager scheduling, - each coroutine would only start when individually awaited, serializing - the work and defeating the semaphore. It also makes the return type - honest (real `Task`s support `.cancel()`, `.done()`, callbacks) rather - than bare coroutines. - - See https://docs.python.org/3/library/asyncio-task.html#coroutines: - "Note that simply calling a coroutine will not schedule it to be executed:" - """ - if limit is None: - return [asyncio.ensure_future(func(*item)) for item in items] - - sem = asyncio.Semaphore(limit) - - async def run(item: T) -> V: - async with sem: - return await func(*item) - - return [asyncio.ensure_future(run(item)) for item in items] - - -async def concurrent_map[T: tuple[Any, ...], V]( - items: Iterable[T], - func: Callable[..., Awaitable[V]], - limit: int | None = None, -) -> list[V]: - return await asyncio.gather(*concurrent_iter(items, func, limit)) - - -async def concurrent_foreach[T: tuple[Any, ...]]( - items: Iterable[T], - func: Callable[..., Awaitable[object]], - limit: int, -) -> None: - """Run `func(*item)` for every item with at most `limit` calls in flight. - - Unlike `concurrent_map`, `items` is consumed lazily: `limit` workers share - one iterator and each takes the next item as soon as it finishes the - previous one, so auxiliary memory is bounded by `limit`, not by the number - of items. Results are discarded. - - Once a call fails, no further items are taken and the calls still in - flight are cancelled. The failure is re-raised as itself, so callers catch - the store's own exception type. If other in-flight calls failed in the - same event-loop iteration, the exception raised is the one from the - lowest-numbered worker and the `BaseExceptionGroup` holding every failure - is attached as its `__cause__`, so nothing is lost from tracebacks and - logs. Cancellation of the caller propagates as `CancelledError`. - """ - if limit < 1: - raise ValueError(f"limit must be at least 1, got {limit}") - shared = iter(items) - failed = False - - async def worker() -> None: - nonlocal failed - # `next()` on the shared iterator is synchronous, so workers never - # re-enter it: a worker only yields to the loop while awaiting `func`. - while not failed: - try: - item = next(shared) - except StopIteration: - return - try: - await func(*item) - except BaseException: - failed = True - raise - - try: - async with asyncio.TaskGroup() as tg: - for _ in range(limit): - tg.create_task(worker()) - except BaseExceptionGroup as eg: - if len(eg.exceptions) == 1: - raise eg.exceptions[0] from None - raise eg.exceptions[0] from eg - - -def enum_names[E: Enum](enum: type[E]) -> Iterator[str]: - for item in enum: - yield item.name - - -def parse_enum[E: Enum](data: object, cls: type[E]) -> E: - if isinstance(data, cls): - return data - if not isinstance(data, str): - raise TypeError(f"Expected str, got {type(data)}") - if data in enum_names(cls): - return cls(data) - raise ValueError(f"Value must be one of {list(enum_names(cls))!r}. Got {data} instead.") - - -def parse_name(data: JSON, expected: str | None = None) -> str: - try: - data = cast("str", convert(data, str)) - except (ValueError, TypeError) as exc: - raise TypeError(f"Expected a string, got an instance of {type(data)}.") from exc - if expected is None or data == expected: - return data - raise ValueError(f"Expected '{expected}'. Got {data} instead.") - - -def parse_configuration(data: JSON) -> JSON: - if not isinstance(data, dict): - raise TypeError(f"Expected dict, got {type(data)}") - return data - - -@overload -def parse_named_configuration( - data: JSON | NamedConfig[str, Any], expected_name: str | None = None -) -> tuple[str, dict[str, JSON]]: ... - - -@overload -def parse_named_configuration( - data: JSON | NamedConfig[str, Any], - expected_name: str | None = None, - *, - require_configuration: bool = True, -) -> tuple[str, dict[str, JSON] | None]: ... - - -def parse_named_configuration( - data: JSON | NamedConfig[str, Any], - expected_name: str | None = None, - *, - require_configuration: bool = True, -) -> tuple[str, JSON | None]: - if not isinstance(data, dict): - raise TypeError(f"Expected dict, got {type(data)}") - if "name" not in data: - raise ValueError(f"Named configuration does not have a 'name' key. Got {data}.") - name_parsed = parse_name(data["name"], expected_name) - if "configuration" in data: - configuration_parsed = parse_configuration(data["configuration"]) - elif require_configuration: - raise ValueError(f"Named configuration does not have a 'configuration' key. Got {data}.") - else: - configuration_parsed = None - return name_parsed, configuration_parsed - - -def parse_shapelike(data: ShapeLike) -> tuple[int, ...]: - """ - Parse a shape-like input into an explicit shape. - """ - if isinstance(data, int | np.integer): - if data < 0: - raise ValueError(f"Expected a non-negative integer. Got {data} instead") - return (int(data),) - try: - data_tuple = tuple(data) - except TypeError as e: - msg = f"Expected an integer or an iterable of integers. Got {data} instead." - raise TypeError(msg) from e - - if not all(isinstance(v, int | np.integer) for v in data_tuple): - msg = f"Expected an iterable of integers. Got {data} instead." - raise TypeError(msg) - if not all(v > -1 for v in data_tuple): - msg = f"Expected all values to be non-negative. Got {data} instead." - raise ValueError(msg) - - # cast NumPy scalars to plain python ints - return tuple(int(x) for x in data_tuple) - - -def parse_fill_value(data: Any) -> Any: - # todo: real validation - return data - - -def parse_order(data: Any) -> Literal["C", "F"]: - return cast("Literal['C', 'F']", parse_field(data, Literal["C", "F"], "order")) - - -def parse_bool(data: Any) -> bool: - return cast("bool", convert(data, bool)) - - -def parse_int(data: Any) -> int: - if isinstance(data, int) and not isinstance(data, bool): - return data - raise ValueError(f"Expected int, got {data} instead.") - - -def _warn_write_empty_chunks_kwarg() -> None: - # TODO: link to docs page on array configuration in this message - msg = ( - "The `write_empty_chunks` keyword argument is deprecated and will be removed in future versions. " - "To control whether empty chunks are written to storage, either use the `config` keyword " - "argument, as in `config={'write_empty_chunks': True}`," - "or change the global 'array.write_empty_chunks' configuration variable." - ) - warnings.warn(msg, ZarrRuntimeWarning, stacklevel=2) - - -def _warn_order_kwarg() -> None: - # TODO: link to docs page on array configuration in this message - msg = ( - "The `order` keyword argument has no effect for Zarr format 3 arrays. " - "To control the memory layout of the array, either use the `config` keyword " - "argument, as in `config={'order': 'C'}`," - "or change the global 'array.order' configuration variable." - ) - warnings.warn(msg, ZarrRuntimeWarning, stacklevel=2) - - -def _default_zarr_format() -> ZarrFormat: - """Return the default zarr_format.""" - return cast("ZarrFormat", int(zarr_config.get("default_zarr_format", 3))) - - -def _subject(name: str, axis: int | None) -> str: - """`name` as the subject of an error message, prefixed by the dimension `axis`.""" - return name[0].upper() + name[1:] if axis is None else f"Dimension {axis}: {name}" - - -def _parse_positive_int(value: object, name: str, axis: int | None) -> int: - """`value` as an `int` of at least 1. A `bool` is read as the `int` it equals; any - other type, a NumPy integer or a float (even an integral one: stored documents with - integral floats are read by `zarr.core.metadata.repair`), is rejected.""" - subject = _subject(name, axis) - if not isinstance(value, int): - raise TypeError(f"{subject} must be an int, got {value!r}") - if value < 1: - raise ValueError(f"{subject} must be >= 1, got {value!r}") - return int(value) - - -def parse_chunk_edge(size: object, axis: int | None = None) -> int: - """Check that `size` is a chunk edge length: an `int` of at least 1 (a `bool` is - read as the `int` it equals). - - This is the one rule for chunk edge lengths in metadata: bare chunk sizes, explicit - edges and run-length encoded sizes. `axis`, when given, is named in the error. - """ - return _parse_positive_int(size, "chunk edge length", axis) - - -def parse_chunk_shape(data: object) -> tuple[int, ...]: - """Check a regular chunk shape: an iterable, other than a string or a mapping, of one - chunk edge length per axis (see `parse_chunk_edge`).""" - match data: - case str() | Mapping(): - pass - case Iterable(): - return tuple(parse_chunk_edge(size, axis) for axis, size in enumerate(data)) - raise TypeError(f"A chunk shape must be an iterable of chunk edge lengths, got {data!r}") - - -def expand_rle(data: Sequence[object], axis: int | None = None) -> list[int]: - """Expand a mixed array of bare integers and RLE pairs, the edges of dimension - `axis` (named in errors, when given). - - Per the rectilinear chunk grid spec, each element can be: - - a bare integer (an explicit edge length) - - a two-element array ``[value, count]`` (run-length encoded) - """ - result: list[int] = [] - for item in data: - if isinstance(item, list): - if len(item) != 2: - subject = _subject("RLE entries", axis) - raise ValueError(f"{subject} must be an integer or [size, count], got {item}") - size, count = item - repeat = _parse_positive_int(count, "RLE repeat count", axis) - result.extend([parse_chunk_edge(size, axis)] * repeat) - else: - result.append(parse_chunk_edge(item, axis)) - return result - - -def compress_rle(sizes: Sequence[int]) -> list[int | list[int]]: - """Compress chunk sizes to mixed RLE format per the rectilinear spec. - - Runs of length > 1 are emitted as ``[value, count]`` pairs; runs of - length 1 are emitted as bare integers:: - - [10, 10, 10, 5] -> [[10, 3], 5] - """ - if not sizes: - return [] - result: list[int | list[int]] = [] - current = sizes[0] - count = 1 - for s in sizes[1:]: - if s == current: - count += 1 - else: - result.append([current, count] if count > 1 else current) - current = s - count = 1 - result.append([current, count] if count > 1 else current) - return result - - -def validate_rectilinear_kind(kind: str | None) -> None: - """Validate the ``kind`` field of a rectilinear chunk grid configuration. - - The rectilinear spec requires ``kind: "inline"``. - """ - if kind is None: - raise ValueError( - "Rectilinear chunk grid configuration requires a 'kind' field. " - "Only 'inline' is currently supported." - ) - if kind != "inline": - raise ValueError( - f"Unsupported rectilinear chunk grid kind: {kind!r}. " - "Only 'inline' is currently supported." - ) - - -def validate_rectilinear_edges( - chunk_shapes: Sequence[int | Sequence[int]], array_shape: Sequence[int] -) -> None: - """Validate that rectilinear chunk edges cover the array extent per dimension. - - Bare-int dimensions (regular step) always cover any extent, so they are - skipped. Explicit edge lists must sum to at least the array extent. - """ - for i, (dim_spec, extent) in enumerate(zip(chunk_shapes, array_shape, strict=True)): - if isinstance(dim_spec, int): - continue - edge_sum = sum(dim_spec) - if edge_sum < extent: - raise ValueError( - f"Rectilinear chunk edges for dimension {i} sum to {edge_sum} " - f"but array shape extent is {extent} (edge sum must be >= extent)" - ) +from __future__ import annotations + +import asyncio +import math +import warnings +from collections.abc import Iterable, Mapping, Sequence +from enum import Enum +from typing import ( + TYPE_CHECKING, + Any, + Final, + Literal, + NotRequired, + TypedDict, + cast, + overload, +) + +import numpy as np +from typing_extensions import ReadOnly + +from zarr.core.config import config as zarr_config +from zarr.core.json_parse import convert, parse_field +from zarr.errors import ZarrRuntimeWarning + +if TYPE_CHECKING: + from collections.abc import Awaitable, Callable, Iterator + + from zarr.core.metadata.v3 import ChunkGridMetadata + + +ZARR_JSON = "zarr.json" +ZARRAY_JSON = ".zarray" +ZGROUP_JSON = ".zgroup" +ZATTRS_JSON = ".zattrs" +ZMETADATA_V2_JSON = ".zmetadata" + +BytesLike = bytes | bytearray | memoryview +ShapeLike = Iterable[int | np.integer[Any]] | int | np.integer[Any] +# Per-dimension chunk specs may mix a bare int (uniform chunk size, the +# rectilinear spec's step-size shorthand) with explicit edge-length sequences. +# A stored chunk grid (`ChunkGridMetadata`) is also accepted and is used +# verbatim, under the tolerant stored-metadata validation rules (e.g. trailing +# edges beyond the array extent, as left behind by a shrinking resize). +type ChunksLike = ShapeLike | Iterable[int | Iterable[int]] | ChunkGridMetadata +# For backwards compatibility +ChunkCoords = tuple[int, ...] +ZarrFormat = Literal[2, 3] +NodeType = Literal["array", "group"] +JSON = str | int | float | bool | Mapping[str, "JSON"] | Sequence["JSON"] | None +MemoryOrder = Literal["C", "F"] +AccessModeLiteral = Literal["r", "r+", "a", "w", "w-"] +ANY_ACCESS_MODE: Final = "r", "r+", "a", "w", "w-" +DimensionNamesLike = Iterable[str | None] | None +DimensionNames = DimensionNamesLike # for backwards compatibility + + +def validate_node_path(path: str, zarr_format: ZarrFormat) -> None: + """Validate node names in a normalized hierarchy path before creating them.""" + if zarr_format != 3 or not path: + return + + for name in path.split("/"): + if all(char == "." for char in name) or name.startswith("__") or name == ZARR_JSON: + raise ValueError( + f"Invalid Zarr format 3 node name {name!r} in path {path!r}. " + "Node names must not consist only of periods, start with '__', " + "or equal 'zarr.json'." + ) + + +class NamedConfig[TName: str, TConfig: Mapping[str, object]](TypedDict): + """ + A typed dictionary representing an object with a name and configuration, where the configuration + is an optional mapping of string keys to values, e.g. another typed dictionary or a JSON object. + + This class is generic with two type parameters: the type of the name (``TName``) and the type of + the configuration (``TConfig``). + """ + + name: ReadOnly[TName] + """The name of the object.""" + + configuration: NotRequired[ReadOnly[TConfig]] + """The configuration of the object. Not required.""" + + +class NamedRequiredConfig[TName: str, TConfig: Mapping[str, object]](TypedDict): + """ + A typed dictionary representing an object with a name and configuration, where the configuration + is a mapping of string keys to values, e.g. another typed dictionary or a JSON object. + + This class is generic with two type parameters: the type of the name (``TName``) and the type of + the configuration (``TConfig``). + """ + + name: ReadOnly[TName] + """The name of the object.""" + + configuration: ReadOnly[TConfig] + """The configuration of the object.""" + + +def product(tup: tuple[int, ...]) -> int: + return math.prod(tup) + + +def ceildiv(a: float, b: float) -> int: + """Ceiling of ``a / b`` using floating-point division; zero when ``a`` is zero.""" + if a == 0: + return 0 + return math.ceil(a / b) + + +def ceildiv_int(a: int, b: int) -> int: + """Ceiling of integer division using exact Python integer arithmetic.""" + return -(-int(a) // int(b)) + + +def concurrent_iter[T: tuple[Any, ...], V]( + items: Iterable[T], + func: Callable[..., Awaitable[V]], + limit: int | None = None, +) -> list[asyncio.Task[V]]: + """Launch `func(*item)` for each item concurrently, returning the tasks. + + When `limit` is set, no more than `limit` calls are in flight at once. + Tasks are returned in input order; callers that want completion order + should wrap the result in `asyncio.as_completed`. + + Every task is scheduled (via `ensure_future`) before this function + returns, not on first iteration of the result. That matters for callers + that await the returned tasks one at a time — without eager scheduling, + each coroutine would only start when individually awaited, serializing + the work and defeating the semaphore. It also makes the return type + honest (real `Task`s support `.cancel()`, `.done()`, callbacks) rather + than bare coroutines. + + See https://docs.python.org/3/library/asyncio-task.html#coroutines: + "Note that simply calling a coroutine will not schedule it to be executed:" + """ + if limit is None: + return [asyncio.ensure_future(func(*item)) for item in items] + + sem = asyncio.Semaphore(limit) + + async def run(item: T) -> V: + async with sem: + return await func(*item) + + return [asyncio.ensure_future(run(item)) for item in items] + + +async def concurrent_map[T: tuple[Any, ...], V]( + items: Iterable[T], + func: Callable[..., Awaitable[V]], + limit: int | None = None, +) -> list[V]: + return await asyncio.gather(*concurrent_iter(items, func, limit)) + + +async def concurrent_foreach[T: tuple[Any, ...]]( + items: Iterable[T], + func: Callable[..., Awaitable[object]], + limit: int, +) -> None: + """Run `func(*item)` for every item with at most `limit` calls in flight. + + Unlike `concurrent_map`, `items` is consumed lazily: `limit` workers share + one iterator and each takes the next item as soon as it finishes the + previous one, so auxiliary memory is bounded by `limit`, not by the number + of items. Results are discarded. + + Once a call fails, no further items are taken and the calls still in + flight are cancelled. The failure is re-raised as itself, so callers catch + the store's own exception type. If other in-flight calls failed in the + same event-loop iteration, the exception raised is the one from the + lowest-numbered worker and the `BaseExceptionGroup` holding every failure + is attached as its `__cause__`, so nothing is lost from tracebacks and + logs. Cancellation of the caller propagates as `CancelledError`. + """ + if limit < 1: + raise ValueError(f"limit must be at least 1, got {limit}") + shared = iter(items) + failed = False + + async def worker() -> None: + nonlocal failed + # `next()` on the shared iterator is synchronous, so workers never + # re-enter it: a worker only yields to the loop while awaiting `func`. + while not failed: + try: + item = next(shared) + except StopIteration: + return + try: + await func(*item) + except BaseException: + failed = True + raise + + try: + async with asyncio.TaskGroup() as tg: + for _ in range(limit): + tg.create_task(worker()) + except BaseExceptionGroup as eg: + if len(eg.exceptions) == 1: + raise eg.exceptions[0] from None + raise eg.exceptions[0] from eg + + +def enum_names[E: Enum](enum: type[E]) -> Iterator[str]: + for item in enum: + yield item.name + + +def parse_enum[E: Enum](data: object, cls: type[E]) -> E: + if isinstance(data, cls): + return data + if not isinstance(data, str): + raise TypeError(f"Expected str, got {type(data)}") + if data in enum_names(cls): + return cls(data) + raise ValueError(f"Value must be one of {list(enum_names(cls))!r}. Got {data} instead.") + + +def parse_name(data: JSON, expected: str | None = None) -> str: + try: + data = cast("str", convert(data, str)) + except (ValueError, TypeError) as exc: + raise TypeError(f"Expected a string, got an instance of {type(data)}.") from exc + if expected is None or data == expected: + return data + raise ValueError(f"Expected '{expected}'. Got {data} instead.") + + +def parse_configuration(data: JSON) -> JSON: + if not isinstance(data, dict): + raise TypeError(f"Expected dict, got {type(data)}") + return data + + +@overload +def parse_named_configuration( + data: JSON | NamedConfig[str, Any], expected_name: str | None = None +) -> tuple[str, dict[str, JSON]]: ... + + +@overload +def parse_named_configuration( + data: JSON | NamedConfig[str, Any], + expected_name: str | None = None, + *, + require_configuration: bool = True, +) -> tuple[str, dict[str, JSON] | None]: ... + + +def parse_named_configuration( + data: JSON | NamedConfig[str, Any], + expected_name: str | None = None, + *, + require_configuration: bool = True, +) -> tuple[str, JSON | None]: + if not isinstance(data, dict): + raise TypeError(f"Expected dict, got {type(data)}") + if "name" not in data: + raise ValueError(f"Named configuration does not have a 'name' key. Got {data}.") + name_parsed = parse_name(data["name"], expected_name) + if "configuration" in data: + configuration_parsed = parse_configuration(data["configuration"]) + elif require_configuration: + raise ValueError(f"Named configuration does not have a 'configuration' key. Got {data}.") + else: + configuration_parsed = None + return name_parsed, configuration_parsed + + +def parse_shapelike(data: ShapeLike) -> tuple[int, ...]: + """ + Parse a shape-like input into an explicit shape. + """ + if isinstance(data, int | np.integer): + if data < 0: + raise ValueError(f"Expected a non-negative integer. Got {data} instead") + return (int(data),) + try: + data_tuple = tuple(data) + except TypeError as e: + msg = f"Expected an integer or an iterable of integers. Got {data} instead." + raise TypeError(msg) from e + + if not all(isinstance(v, int | np.integer) for v in data_tuple): + msg = f"Expected an iterable of integers. Got {data} instead." + raise TypeError(msg) + if not all(v > -1 for v in data_tuple): + msg = f"Expected all values to be non-negative. Got {data} instead." + raise ValueError(msg) + + # cast NumPy scalars to plain python ints + return tuple(int(x) for x in data_tuple) + + +def parse_fill_value(data: Any) -> Any: + # todo: real validation + return data + + +def parse_order(data: Any) -> Literal["C", "F"]: + return cast("Literal['C', 'F']", parse_field(data, Literal["C", "F"], "order")) + + +def parse_bool(data: Any) -> bool: + return cast("bool", convert(data, bool)) + + +def parse_int(data: Any) -> int: + if isinstance(data, int) and not isinstance(data, bool): + return data + raise ValueError(f"Expected int, got {data} instead.") + + +def _warn_write_empty_chunks_kwarg() -> None: + # TODO: link to docs page on array configuration in this message + msg = ( + "The `write_empty_chunks` keyword argument is deprecated and will be removed in future versions. " + "To control whether empty chunks are written to storage, either use the `config` keyword " + "argument, as in `config={'write_empty_chunks': True}`," + "or change the global 'array.write_empty_chunks' configuration variable." + ) + warnings.warn(msg, ZarrRuntimeWarning, stacklevel=2) + + +def _warn_order_kwarg() -> None: + # TODO: link to docs page on array configuration in this message + msg = ( + "The `order` keyword argument has no effect for Zarr format 3 arrays. " + "To control the memory layout of the array, either use the `config` keyword " + "argument, as in `config={'order': 'C'}`," + "or change the global 'array.order' configuration variable." + ) + warnings.warn(msg, ZarrRuntimeWarning, stacklevel=2) + + +def _default_zarr_format() -> ZarrFormat: + """Return the default zarr_format.""" + return cast("ZarrFormat", int(zarr_config.get("default_zarr_format", 3))) + + +def _subject(name: str, axis: int | None) -> str: + """`name` as the subject of an error message, prefixed by the dimension `axis`.""" + return name[0].upper() + name[1:] if axis is None else f"Dimension {axis}: {name}" + + +def _parse_positive_int(value: object, name: str, axis: int | None) -> int: + """`value` as an `int` of at least 1. A `bool` is read as the `int` it equals; any + other type, a NumPy integer or a float (even an integral one: stored documents with + integral floats are read by `zarr.core.metadata.repair`), is rejected.""" + subject = _subject(name, axis) + if not isinstance(value, int): + raise TypeError(f"{subject} must be an int, got {value!r}") + if value < 1: + raise ValueError(f"{subject} must be >= 1, got {value!r}") + return int(value) + + +def parse_chunk_edge(size: object, axis: int | None = None) -> int: + """Check that `size` is a chunk edge length: an `int` of at least 1 (a `bool` is + read as the `int` it equals). + + This is the one rule for chunk edge lengths in metadata: bare chunk sizes, explicit + edges and run-length encoded sizes. `axis`, when given, is named in the error. + """ + return _parse_positive_int(size, "chunk edge length", axis) + + +def parse_chunk_shape(data: object) -> tuple[int, ...]: + """Check a regular chunk shape: an iterable, other than a string or a mapping, of one + chunk edge length per axis (see `parse_chunk_edge`).""" + match data: + case str() | Mapping(): + pass + case Iterable(): + return tuple(parse_chunk_edge(size, axis) for axis, size in enumerate(data)) + raise TypeError(f"A chunk shape must be an iterable of chunk edge lengths, got {data!r}") + + +def expand_rle(data: Sequence[object], axis: int | None = None) -> list[int]: + """Expand a mixed array of bare integers and RLE pairs, the edges of dimension + `axis` (named in errors, when given). + + Per the rectilinear chunk grid spec, each element can be: + - a bare integer (an explicit edge length) + - a two-element array ``[value, count]`` (run-length encoded) + """ + result: list[int] = [] + for item in data: + if isinstance(item, list): + if len(item) != 2: + subject = _subject("RLE entries", axis) + raise ValueError(f"{subject} must be an integer or [size, count], got {item}") + size, count = item + repeat = _parse_positive_int(count, "RLE repeat count", axis) + result.extend([parse_chunk_edge(size, axis)] * repeat) + else: + result.append(parse_chunk_edge(item, axis)) + return result + + +def compress_rle(sizes: Sequence[int]) -> list[int | list[int]]: + """Compress chunk sizes to mixed RLE format per the rectilinear spec. + + Runs of length > 1 are emitted as ``[value, count]`` pairs; runs of + length 1 are emitted as bare integers:: + + [10, 10, 10, 5] -> [[10, 3], 5] + """ + if not sizes: + return [] + result: list[int | list[int]] = [] + current = sizes[0] + count = 1 + for s in sizes[1:]: + if s == current: + count += 1 + else: + result.append([current, count] if count > 1 else current) + current = s + count = 1 + result.append([current, count] if count > 1 else current) + return result + + +def validate_rectilinear_kind(kind: str | None) -> None: + """Validate the ``kind`` field of a rectilinear chunk grid configuration. + + The rectilinear spec requires ``kind: "inline"``. + """ + if kind is None: + raise ValueError( + "Rectilinear chunk grid configuration requires a 'kind' field. " + "Only 'inline' is currently supported." + ) + if kind != "inline": + raise ValueError( + f"Unsupported rectilinear chunk grid kind: {kind!r}. " + "Only 'inline' is currently supported." + ) + + +def validate_rectilinear_edges( + chunk_shapes: Sequence[int | Sequence[int]], array_shape: Sequence[int] +) -> None: + """Validate that rectilinear chunk edges cover the array extent per dimension. + + Bare-int dimensions (regular step) always cover any extent, so they are + skipped. Explicit edge lists must sum to at least the array extent. + """ + for i, (dim_spec, extent) in enumerate(zip(chunk_shapes, array_shape, strict=True)): + if isinstance(dim_spec, int): + continue + edge_sum = sum(dim_spec) + if edge_sum < extent: + raise ValueError( + f"Rectilinear chunk edges for dimension {i} sum to {edge_sum} " + f"but array shape extent is {extent} (edge sum must be >= extent)" + ) diff --git a/src/zarr/core/group.py b/src/zarr/core/group.py index 2932aeb6a7..b16ee41058 100644 --- a/src/zarr/core/group.py +++ b/src/zarr/core/group.py @@ -1,3950 +1,3950 @@ -from __future__ import annotations - -import asyncio -import logging -import unicodedata -import warnings -from collections import defaultdict -from dataclasses import asdict, dataclass, field, replace -from itertools import accumulate -from typing import TYPE_CHECKING, Final, Literal, assert_never, cast, overload - -import numpy as np - -import zarr.api.asynchronous as async_api -from zarr.abc.metadata import Metadata -from zarr.abc.store import Store, set_or_delete -from zarr.core._info import GroupInfo -from zarr.core._json import buffer_to_json_object, json_to_buffer -from zarr.core.array import ( - DEFAULT_FILL_VALUE, - Array, - AsyncArray, - CompressorLike, - CompressorsLike, - FiltersLike, - SerializerLike, - ShardsLike, - _parse_deprecated_compressor, - create_array, -) -from zarr.core.array_spec import parse_array_config -from zarr.core.attributes import Attributes -from zarr.core.buffer import default_buffer_prototype -from zarr.core.common import ( - JSON, - ZARR_JSON, - ZARRAY_JSON, - ZATTRS_JSON, - ZGROUP_JSON, - ZMETADATA_V2_JSON, - ChunksLike, - DimensionNamesLike, - NodeType, - ShapeLike, - ZarrFormat, - parse_shapelike, - validate_node_path, -) -from zarr.core.config import config -from zarr.core.dtype import parse_data_type -from zarr.core.json_parse import parse_field -from zarr.core.metadata import ArrayV2Metadata, ArrayV3Metadata -from zarr.core.metadata.io import ( - encode_documents, - save_metadata, - save_new_metadata, - store_documents, -) -from zarr.core.metadata.v3 import AllowedExtraField, check_storable, parse_extra_fields -from zarr.core.sync import SyncMixin, sync -from zarr.errors import ( - ArrayNotFoundError, - ContainsArrayError, - ContainsGroupError, - GroupNotFoundError, - MetadataValidationError, - NodeTypeValidationError, - ZarrUserWarning, -) -from zarr.storage import StoreLike, StorePath -from zarr.storage._common import make_store_path -from zarr.storage._utils import _join_paths, _normalize_path_keys, normalize_path - -if TYPE_CHECKING: - from collections.abc import ( - AsyncGenerator, - AsyncIterator, - Coroutine, - Generator, - Iterable, - Iterator, - Mapping, - ) - from typing import Any - - from zarr.core.array_spec import ArrayConfigLike - from zarr.core.buffer import Buffer, BufferPrototype - from zarr.core.chunk_key_encodings import ChunkKeyEncodingLike - from zarr.core.common import MemoryOrder - from zarr.core.dtype import ZDTypeLike - from zarr.types import AnyArray, AnyAsyncArray, ArrayV2, ArrayV3, AsyncArrayV2, AsyncArrayV3 - -logger = logging.getLogger("zarr.group") - - -def parse_zarr_format(data: Any) -> ZarrFormat: - """Parse the zarr_format field from metadata.""" - return cast("ZarrFormat", parse_field(data, Literal[2, 3], "zarr_format")) - - -def parse_node_type(data: Any) -> NodeType: - """Parse the node_type field from metadata.""" - return cast( - "Literal['array', 'group']", - parse_field(data, Literal["array", "group"], "node_type", error=MetadataValidationError), - ) - - -# todo: convert None to empty dict -def parse_attributes(data: Any) -> dict[str, Any]: - """Parse the attributes field from metadata.""" - if data is None: - return {} - elif isinstance(data, dict) and all(isinstance(k, str) for k in data): - return dict(data) - msg = f"Expected dict with string keys. Got {type(data)} instead." - raise TypeError(msg) - - -@overload -def _parse_async_node(node: AsyncArrayV3) -> ArrayV3: ... - - -@overload -def _parse_async_node(node: AsyncArrayV2) -> ArrayV2: ... - - -@overload -def _parse_async_node(node: AsyncGroup) -> Group: ... - - -def _parse_async_node( - node: AnyAsyncArray | AsyncGroup, -) -> AnyArray | Group: - """Wrap an AsyncArray in an Array, or an AsyncGroup in a Group.""" - if isinstance(node, AsyncArray): - return Array(node) - elif isinstance(node, AsyncGroup): - return Group(node) - else: - raise TypeError(f"Unknown node type, got {type(node)}") - - -@dataclass(frozen=True) -class ConsolidatedMetadata: - """ - Consolidated Metadata for this Group. - - This stores the metadata of child nodes below this group. Any child groups - will have their consolidated metadata set appropriately. - """ - - metadata: dict[str, ArrayV2Metadata | ArrayV3Metadata | GroupMetadata] - kind: Literal["inline"] = "inline" - must_understand: Literal[False] = False - - def to_dict(self) -> dict[str, JSON]: - """The consolidated metadata document. An array read from a stored document that - had to be repaired is written as that document was stored, so every reader of - the consolidated metadata reads it as repaired again (see `mark_repaired`).""" - return { - "kind": self.kind, - "must_understand": self.must_understand, - "metadata": { - k: v.to_dict() - if isinstance(v, GroupMetadata) or v._stored_document is None - else dict(v._stored_document) - for k, v in sorted( - self.flattened_metadata.items(), - key=lambda item: ( - item[0].count("/"), - unicodedata.normalize("NFKC", item[0]).casefold(), - ), - ) - }, - } - - @classmethod - def from_dict(cls, data: dict[str, JSON], *, path: str | None = None) -> ConsolidatedMetadata: - """Read consolidated metadata, naming each member by its path under the group at - `path` (or relative to that group, when `path` is not given) in warnings.""" - data = dict(data) - - kind = data.get("kind") - if kind != "inline": - raise ValueError(f"Consolidated metadata kind='{kind}' is not supported.") - - raw_metadata = data.get("metadata") - if not isinstance(raw_metadata, dict): - raise TypeError(f"Unexpected type for 'metadata': {type(raw_metadata)}") - - metadata: dict[str, ArrayV2Metadata | ArrayV3Metadata | GroupMetadata] = {} - if raw_metadata: - for k, v in raw_metadata.items(): - member = k if path is None else _join_paths([path, k]) - if not isinstance(v, dict): - raise TypeError( - f"Invalid value for metadata items. key='{k}', type='{type(v).__name__}'" - ) - - # zarr_format is present in v2 and v3. - zarr_format = parse_zarr_format(v["zarr_format"]) - - if zarr_format == 3: - node_type = parse_node_type(v.get("node_type", None)) - if node_type == "group": - metadata[k] = GroupMetadata.from_dict(v, path=member) - elif node_type == "array": - metadata[k] = ArrayV3Metadata.from_dict(v, path=member) - else: - assert_never(node_type) - elif zarr_format == 2: - if "shape" in v: - metadata[k] = ArrayV2Metadata.from_dict(v, path=member) - else: - metadata[k] = GroupMetadata.from_dict(v, path=member) - else: - assert_never(zarr_format) - - cls._flat_to_nested(metadata) - - return cls(metadata=metadata) - - @staticmethod - def _flat_to_nested( - metadata: dict[str, ArrayV2Metadata | ArrayV3Metadata | GroupMetadata], - ) -> None: - """ - Convert a flat metadata representation to a nested one. - - Notes - ----- - Flat metadata is used when persisting the consolidated metadata. The keys - include the full path, not just the node name. The key prefixes can be - used to determine which nodes are children of which other nodes. - - Nested metadata is used in-memory. The outermost level will only have the - *immediate* children of the Group. All nested child groups will be stored - under the consolidated metadata of their immediate parent. - """ - # We have a flat mapping from {k: v} where the keys include the *full* - # path segment: - # { - # "/a/b": { group_metadata }, - # "/a/b/array-0": { array_metadata }, - # "/a/b/array-1": { array_metadata }, - # } - # - # We want to reorganize the metadata such that each Group contains the - # array metadata of its immediate children. - # In the example, the group at `/a/b` will have consolidated metadata - # for its children `array-0` and `array-1`. - - # Group keys by their parent path. This must not rely on same-parent keys - # being adjacent: the persisted key order is arbitrary, so accumulate - # instead of using itertools.groupby, which only groups consecutive runs. - grouped: dict[str, list[str]] = defaultdict(list) - for k in sorted(metadata, key=lambda k: k.count("/")): - grouped[k.rsplit("/", 1)[0]].append(k) - - # we go top down and directly manipulate metadata. - for key, children_keys in grouped.items(): - # key is a key like "a", "a/b", "a/b/c" - # The basic idea is to find the immediate parent (so "", "a", or "a/b") - # and update that node's consolidated metadata to include the metadata - # in children_keys - *prefixes, name = key.split("/") - parent = metadata - - while prefixes: - # e.g. a/b/c has a parent "a/b". Walk through to get - # metadata["a"]["b"] - part = prefixes.pop(0) - # we can assume that parent[part] here is a group - # otherwise we wouldn't have a node with this `part` prefix. - # We can also assume that the parent node will have consolidated metadata, - # because we're walking top to bottom. - parent = parent[part].consolidated_metadata.metadata # type: ignore[union-attr] - - node = parent[name] - children_keys = list(children_keys) - - if isinstance(node, ArrayV2Metadata | ArrayV3Metadata): - # These are already present, either thanks to being an array in the - # root, or by being collected as a child in the else clause - continue - children: dict[str, ArrayV2Metadata | ArrayV3Metadata | GroupMetadata] = {} - # We pop from metadata, since we're *moving* this under group. - # While doing this, normalize leaf groups to carry empty consolidated metadata. - for child_key in children_keys: - if child_key == key: - continue - child = metadata.pop(child_key) - if isinstance(child, GroupMetadata) and child.consolidated_metadata is None: - child = replace(child, consolidated_metadata=ConsolidatedMetadata(metadata={})) - children[child_key.split("/")[-1]] = child - - parent[name] = replace( - node, consolidated_metadata=ConsolidatedMetadata(metadata=children) - ) - - @property - def flattened_metadata(self) -> dict[str, ArrayV2Metadata | ArrayV3Metadata | GroupMetadata]: - """ - Return the flattened representation of Consolidated Metadata. - - The returned dictionary will have a key for each child node in the hierarchy - under this group. Under the default (nested) representation available through - ``self.metadata``, the dictionary only contains keys for immediate children. - - The keys of the dictionary will include the full path to a child node from - the current group, where segments are joined by ``/``. - - Examples - -------- - ```python - from zarr.core.group import ConsolidatedMetadata, GroupMetadata - cm = ConsolidatedMetadata( - metadata={ - "group-0": GroupMetadata( - consolidated_metadata=ConsolidatedMetadata( - { - "group-0-0": GroupMetadata(), - } - ) - ), - "group-1": GroupMetadata(), - } - ) - # {'group-0': GroupMetadata(attributes={}, zarr_format=3, consolidated_metadata=None, node_type='group', extra_fields={}), - # 'group-0/group-0-0': GroupMetadata(attributes={}, zarr_format=3, consolidated_metadata=None, node_type='group', extra_fields={}), - # 'group-1': GroupMetadata(attributes={}, zarr_format=3, consolidated_metadata=None, node_type='group', extra_fields={})} - ``` - """ - metadata = {} - - def flatten( - key: str, group: GroupMetadata | ArrayV2Metadata | ArrayV3Metadata - ) -> dict[str, ArrayV2Metadata | ArrayV3Metadata | GroupMetadata]: - children: dict[str, ArrayV2Metadata | ArrayV3Metadata | GroupMetadata] = {} - if isinstance(group, ArrayV2Metadata | ArrayV3Metadata): - children[key] = group - else: - if group.consolidated_metadata and group.consolidated_metadata.metadata is not None: - children[key] = replace( - group, consolidated_metadata=ConsolidatedMetadata(metadata={}) - ) - for name, val in group.consolidated_metadata.metadata.items(): - full_key = f"{key}/{name}" - if isinstance(val, GroupMetadata): - children.update(flatten(full_key, val)) - else: - children[full_key] = val - else: - children[key] = replace(group, consolidated_metadata=None) - return children - - for k, v in self.metadata.items(): - metadata.update(flatten(k, v)) - - return metadata - - -GROUP_METADATA_KEYS: Final[set[str]] = { - "zarr_format", - "node_type", - "attributes", - "consolidated_metadata", -} -""" -The names of the fields of the group metadata document. `consolidated_metadata` is not in the -Zarr V3 spec; Zarr-Python writes it. -""" - - -@dataclass(frozen=True) -class GroupMetadata(Metadata): - """ - Metadata for a Group. - """ - - attributes: dict[str, Any] = field(default_factory=dict) - zarr_format: ZarrFormat = 3 - consolidated_metadata: ConsolidatedMetadata | None = None - node_type: Literal["group"] = field(default="group", init=False) - extra_fields: dict[str, AllowedExtraField] = field(default_factory=dict) - - def to_buffer_dict(self, prototype: BufferPrototype) -> dict[str, Buffer]: - if self.consolidated_metadata is not None: - for path, member in self.consolidated_metadata.flattened_metadata.items(): - # A member read from a document that had to be repaired is stored as it - # was stored (see `ConsolidatedMetadata.to_dict`). - if isinstance(member, ArrayV3Metadata) and member._stored_document is None: - try: - check_storable(member) - except ValueError as e: - e.add_note(f"Array {path!r} in the consolidated metadata.") - raise - indent = config.get("json_indent") - if self.zarr_format == 3: - return {ZARR_JSON: json_to_buffer(self.to_dict(), prototype=prototype, indent=indent)} - else: - items = { - ZGROUP_JSON: json_to_buffer( - {"zarr_format": self.zarr_format}, prototype=prototype, indent=indent - ), - ZATTRS_JSON: json_to_buffer(self.attributes, prototype=prototype, indent=indent), - } - if self.consolidated_metadata: - d = { - ZGROUP_JSON: {"zarr_format": self.zarr_format}, - ZATTRS_JSON: self.attributes, - } - consolidated_metadata = self.consolidated_metadata.to_dict()["metadata"] - if not isinstance(consolidated_metadata, dict): - raise TypeError( - "Expected consolidated metadata to serialize to a dict, " - f"got {type(consolidated_metadata).__name__}." - ) - for k, v in consolidated_metadata.items(): - attrs = v.pop("attributes", {}) - d[f"{k}/{ZATTRS_JSON}"] = attrs - if "shape" in v: - # it's an array - d[f"{k}/{ZARRAY_JSON}"] = v - else: - d[f"{k}/{ZGROUP_JSON}"] = { - "zarr_format": self.zarr_format, - "consolidated_metadata": { - "metadata": {}, - "must_understand": False, - "kind": "inline", - }, - } - - # The consolidated metadata blob is written compactly (no indent). - items[ZMETADATA_V2_JSON] = json_to_buffer( - {"metadata": d, "zarr_consolidated_format": 1}, prototype=prototype - ) - - return items - - def __init__( - self, - attributes: dict[str, Any] | None = None, - zarr_format: ZarrFormat = 3, - consolidated_metadata: ConsolidatedMetadata | None = None, - extra_fields: Mapping[str, AllowedExtraField] | None = None, - ) -> None: - attributes_parsed = parse_attributes(attributes) - zarr_format_parsed = parse_zarr_format(zarr_format) - if zarr_format_parsed == 2 and extra_fields: - raise ValueError( - "Invalid extra fields. Zarr format 2 group metadata does not support extra fields." - ) - extra_fields_parsed = parse_extra_fields( - extra_fields, reserved_keys=GROUP_METADATA_KEYS, node_type="group" - ) - - object.__setattr__(self, "attributes", attributes_parsed) - object.__setattr__(self, "zarr_format", zarr_format_parsed) - object.__setattr__(self, "consolidated_metadata", consolidated_metadata) - object.__setattr__(self, "extra_fields", extra_fields_parsed) - - @classmethod - def from_dict(cls, data: dict[str, Any], *, path: str | None = None) -> GroupMetadata: - """Read a stored group document; `path` names the group in warnings about the - consolidated metadata it holds.""" - data = dict(data) - node_type = data.pop("node_type", None) - if node_type not in ("group", None): - raise NodeTypeValidationError( - f"Invalid value for 'node_type'. Expected 'group' or None. Got {node_type!r}." - ) - consolidated_metadata = data.pop("consolidated_metadata", None) - if consolidated_metadata: - data["consolidated_metadata"] = ConsolidatedMetadata.from_dict( - consolidated_metadata, path=path - ) - - zarr_format = data.get("zarr_format") - if zarr_format == 3: - # Zarr v3 allows an extra key only if it is an object with "must_understand": false. - extra_fields = parse_extra_fields( - {k: v for k, v in data.items() if k not in GROUP_METADATA_KEYS}, - reserved_keys=GROUP_METADATA_KEYS, - node_type="group", - ) - else: - # Zarr v2 allowed arbitrary keys; they are dropped rather than kept. - extra_fields = {} - data = {k: v for k, v in data.items() if k in GROUP_METADATA_KEYS} - return cls(**data, extra_fields=extra_fields) - - def to_dict(self) -> dict[str, Any]: - result = asdict(replace(self, consolidated_metadata=None)) - result.update(result.pop("extra_fields")) - if self.consolidated_metadata is not None: - result["consolidated_metadata"] = self.consolidated_metadata.to_dict() - else: - # Leave consolidated metadata unset if it's None - result.pop("consolidated_metadata") - return result - - -@dataclass(frozen=True) -class ImplicitGroupMarker(GroupMetadata): - """ - Marker for an implicit group. Instances of this class are only used in the context of group - creation as a placeholder to represent groups that should only be created if they do not - already exist in storage - """ - - -@dataclass(frozen=True) -class AsyncGroup: - """ - Asynchronous Group object. - """ - - metadata: GroupMetadata - store_path: StorePath - - # TODO: make this correct and work - # TODO: ensure that this can be bound properly to subclass of AsyncGroup - - @classmethod - async def from_store( - cls, - store: StoreLike, - *, - attributes: dict[str, Any] | None = None, - overwrite: bool = False, - zarr_format: ZarrFormat = 3, - ) -> AsyncGroup: - store_path = await make_store_path(store) - attributes = attributes or {} - group = cls( - metadata=GroupMetadata(attributes=attributes, zarr_format=zarr_format), - store_path=store_path, - ) - await save_new_metadata( - store_path, group.metadata, overwrite=overwrite, ensure_parents=True - ) - return group - - @classmethod - async def open( - cls, - store: StoreLike, - zarr_format: ZarrFormat | None = 3, - use_consolidated: bool | str | None = None, - ) -> AsyncGroup: - """Open a new AsyncGroup - - Parameters - ---------- - store : StoreLike - zarr_format : {2, 3}, optional - use_consolidated : bool or str, default None - Whether to use consolidated metadata. - - By default, consolidated metadata is used if it's present in the - store (in the ``zarr.json`` for Zarr format 3 and in the ``.zmetadata`` file - for Zarr format 2) and the Store supports it. - - To explicitly require consolidated metadata, set ``use_consolidated=True``. - In this case, if the Store doesn't support consolidation or consolidated metadata is - not found, a ``ValueError`` exception is raised. - - To explicitly *not* use consolidated metadata, set ``use_consolidated=False``, - which will fall back to using the regular, non consolidated metadata. - - Zarr format 2 allowed configuring the key storing the consolidated metadata - (``.zmetadata`` by default). Specify the custom key as ``use_consolidated`` - to load consolidated metadata from a non-default key. - """ - store_path = await make_store_path(store) - if not store_path.store.supports_consolidated_metadata: - # Fail if consolidated metadata was requested but the Store doesn't support it - if use_consolidated: - store_name = type(store_path.store).__name__ - raise ValueError( - f"The Zarr store in use ({store_name}) doesn't support consolidated metadata." - ) - - # if use_consolidated was None (optional), the Store dictates it doesn't want consolidation - use_consolidated = False - - consolidated_key = ZMETADATA_V2_JSON - - if (zarr_format == 2 or zarr_format is None) and isinstance(use_consolidated, str): - consolidated_key = use_consolidated - - if zarr_format == 2: - paths = [store_path / ZGROUP_JSON, store_path / ZATTRS_JSON] - if use_consolidated or use_consolidated is None: - paths.append(store_path / consolidated_key) - - zgroup_bytes, zattrs_bytes, *rest = await asyncio.gather( - *[path.get() for path in paths] - ) - if zgroup_bytes is None: - raise FileNotFoundError(store_path) - - if use_consolidated or use_consolidated is None: - maybe_consolidated_metadata_bytes = rest[0] - - else: - maybe_consolidated_metadata_bytes = None - - elif zarr_format == 3: - zarr_json_bytes = await (store_path / ZARR_JSON).get() - if zarr_json_bytes is None: - raise FileNotFoundError(store_path) - elif zarr_format is None: - ( - zarr_json_bytes, - zgroup_bytes, - zattrs_bytes, - maybe_consolidated_metadata_bytes, - ) = await asyncio.gather( - (store_path / ZARR_JSON).get(), - (store_path / ZGROUP_JSON).get(), - (store_path / ZATTRS_JSON).get(), - (store_path / str(consolidated_key)).get(), - ) - if zarr_json_bytes is not None and zgroup_bytes is not None: - # warn and favor v3 - msg = f"Both zarr.json (Zarr format 3) and .zgroup (Zarr format 2) metadata objects exist at {store_path}. Zarr format 3 will be used." - warnings.warn(msg, category=ZarrUserWarning, stacklevel=1) - if zarr_json_bytes is None and zgroup_bytes is None: - raise FileNotFoundError( - f"could not find zarr.json or .zgroup objects in {store_path}" - ) - # set zarr_format based on which keys were found - if zarr_json_bytes is not None: - zarr_format = 3 - else: - zarr_format = 2 - else: - msg = f"Invalid value for 'zarr_format'. Expected 2, 3, or None. Got '{zarr_format}'." # type: ignore[unreachable] - raise MetadataValidationError(msg) - - if zarr_format == 2: - if zgroup_bytes is None: - raise FileNotFoundError(store_path) - - if use_consolidated and maybe_consolidated_metadata_bytes is None: - # the user requested consolidated metadata, but it was missing - raise ValueError(consolidated_key) - - elif use_consolidated is False: - # the user explicitly opted out of consolidated_metadata. - # Discard anything we might have read. - maybe_consolidated_metadata_bytes = None - - return cls._from_bytes_v2( - store_path, zgroup_bytes, zattrs_bytes, maybe_consolidated_metadata_bytes - ) - else: - # V3 groups are comprised of a zarr.json object - if zarr_json_bytes is None: - raise FileNotFoundError(store_path) - if not isinstance(use_consolidated, bool | None): - raise TypeError("use_consolidated must be a bool or None for Zarr format 3.") - - return cls._from_bytes_v3( - store_path, - zarr_json_bytes, - use_consolidated=use_consolidated, - ) - - @classmethod - def _from_bytes_v2( - cls, - store_path: StorePath, - zgroup_bytes: Buffer, - zattrs_bytes: Buffer | None, - consolidated_metadata_bytes: Buffer | None, - ) -> AsyncGroup: - # V2 groups are comprised of a .zgroup and .zattrs objects - zgroup = buffer_to_json_object(zgroup_bytes) - zattrs = buffer_to_json_object(zattrs_bytes) if zattrs_bytes is not None else {} - group_metadata: dict[str, Any] = {**zgroup, "attributes": zattrs} - - if consolidated_metadata_bytes is not None: - v2_consolidated_doc = buffer_to_json_object(consolidated_metadata_bytes) - v2_consolidated_metadata = cast("dict[str, Any]", v2_consolidated_doc["metadata"]) - # We already read zattrs and zgroup. Should we ignore these? - v2_consolidated_metadata.pop(".zattrs", None) - v2_consolidated_metadata.pop(".zgroup", None) - - consolidated_metadata: defaultdict[str, dict[str, Any]] = defaultdict(dict) - - # keys like air/.zarray, air/.zattrs - for k, v in v2_consolidated_metadata.items(): - path, kind = k.rsplit("/.", 1) - - if kind == "zarray": - consolidated_metadata[path].update(v) - elif kind == "zattrs": - consolidated_metadata[path]["attributes"] = v - elif kind == "zgroup": - consolidated_metadata[path].update(v) - else: - raise ValueError(f"Invalid file type '{kind}' at path '{path}") - - group_metadata["consolidated_metadata"] = { - "metadata": dict(consolidated_metadata), - "kind": "inline", - "must_understand": False, - } - - return cls.from_dict(store_path, group_metadata) - - @classmethod - def _from_bytes_v3( - cls, - store_path: StorePath, - zarr_json_bytes: Buffer, - use_consolidated: bool | None, - ) -> AsyncGroup: - group_metadata = buffer_to_json_object(zarr_json_bytes) - if use_consolidated and group_metadata.get("consolidated_metadata") is None: - msg = f"Consolidated metadata requested with 'use_consolidated=True' but not found in '{store_path.path}'." - raise ValueError(msg) - - elif use_consolidated is False: - # Drop consolidated metadata if it's there. - group_metadata.pop("consolidated_metadata", None) - - return cls.from_dict(store_path, group_metadata) - - @classmethod - def from_dict( - cls, - store_path: StorePath, - data: dict[str, Any], - ) -> AsyncGroup: - node_type = data.pop("node_type", None) - if node_type == "array": - msg = f"An array already exists in store {store_path.store} at path {store_path.path}." - raise ContainsArrayError(msg) - elif node_type not in ("group", None): - msg = f"Node type in metadata ({node_type}) is not 'group'" - raise GroupNotFoundError(msg) - return cls( - metadata=GroupMetadata.from_dict(data, path=str(store_path)), - store_path=store_path, - ) - - async def setitem(self, key: str, value: Any) -> None: - """ - Fastpath for creating a new array - New arrays will be created with default array settings for the array type. - - Parameters - ---------- - key : str - Array name - value : array-like - Array data - """ - path = self.store_path / key - await async_api.save_array( - store=path, arr=value, zarr_format=self.metadata.zarr_format, overwrite=True - ) - - async def getitem( - self, - key: str, - ) -> AnyAsyncArray | AsyncGroup: - """ - Get a subarray or subgroup from the group. - - Parameters - ---------- - key : str - Array or group name - - Returns - ------- - AsyncArray or AsyncGroup - """ - store_path = self.store_path / key - logger.debug("key=%s, store_path=%s", key, store_path) - - # Consolidated metadata lets us avoid some I/O operations so try that first. - if self.metadata.consolidated_metadata is not None: - return self._getitem_consolidated(store_path, key, prefix=self.name) - try: - return await get_node( - store=store_path.store, path=store_path.path, zarr_format=self.metadata.zarr_format - ) - except FileNotFoundError as e: - raise KeyError(key) from e - - def _getitem_consolidated( - self, store_path: StorePath, key: str, prefix: str - ) -> AnyAsyncArray | AsyncGroup: - # getitem, in the special case where we have consolidated metadata. - # Note that this is a regular def (non async) function. - # This shouldn't do any additional I/O. - - # we support nested getitems like group/subgroup/array - indexers = normalize_path(key).split("/") - indexers.reverse() - metadata: ArrayV2Metadata | ArrayV3Metadata | GroupMetadata = self.metadata - - while indexers: - indexer = indexers.pop() - if isinstance(metadata, ArrayV2Metadata | ArrayV3Metadata): - # we've indexed into an array with group["array/subarray"]. Invalid. - raise KeyError(key) - if metadata.consolidated_metadata is None: - # we've indexed into a group without consolidated metadata. - # This isn't normal; typically, consolidated metadata - # will include explicit markers for when there are no child - # nodes as metadata={}. - # We have some freedom in exactly how we interpret this case. - # For now, we treat None as the same as {}, i.e. we don't - # have any children. - raise KeyError(key) - try: - metadata = metadata.consolidated_metadata.metadata[indexer] - except KeyError as e: - # The Group Metadata has consolidated metadata, but the key - # isn't present. We trust this to mean that the key isn't in - # the hierarchy, and *don't* fall back to checking the store. - msg = f"'{key}' not found in consolidated metadata." - raise KeyError(msg) from e - - # update store_path to ensure that AsyncArray/Group.name is correct - if prefix != "/": - key = "/".join([prefix.lstrip("/"), key]) - store_path = StorePath(store=store_path.store, path=key) - - if isinstance(metadata, GroupMetadata): - return AsyncGroup(metadata=metadata, store_path=store_path) - else: - return AsyncArray(metadata=metadata, store_path=store_path) - - async def delitem(self, key: str) -> None: - """Delete a group member. - - Parameters - ---------- - key : str - Array or group name - """ - store_path = self.store_path / key - consolidated = self.metadata.consolidated_metadata - if consolidated is None: - await store_path.delete_dir() - return - # Encode the group metadata without the member before deleting it: metadata that - # cannot be stored then fails with the store and this group untouched. What is - # stored is encoded after the deletion, from the metadata as it then is, so - # concurrent deletions each store the deletions made before them. - members = {name: node for name, node in consolidated.metadata.items() if name != key} - encode_documents( - self.store_path, - replace(self.metadata, consolidated_metadata=replace(consolidated, metadata=members)), - ) - await store_path.delete_dir() - # In place, so every handle sharing this consolidated metadata (a parent's or a - # subgroup's) sees the deletion. - consolidated.metadata.pop(key, None) - await store_documents(self.store_path, encode_documents(self.store_path, self.metadata)) - - async def get[DefaultT]( - self, key: str, default: DefaultT | None = None - ) -> AnyAsyncArray | AsyncGroup | DefaultT | None: - """Obtain a group member, returning default if not found. - - Parameters - ---------- - key : str - Group member name. - default : object - Default value to return if key is not found (default: None). - - Returns - ------- - object - Group member (AsyncArray or AsyncGroup) or default if not found. - """ - try: - return await self.getitem(key) - except KeyError: - return default - - async def get_array(self, path: str) -> AnyAsyncArray: - """Obtain an array member of this group, raising if it is absent or not an array. - - Parameters - ---------- - path : str - Path of the array relative to this group. May contain `/` to reference - a member of a subgroup, e.g. `subgroup/subarray`. - - Returns - ------- - AsyncArray - The array at the given path. - - Raises - ------ - ArrayNotFoundError - If no node exists at the given path. - ContainsGroupError - If the node at the given path is a group rather than an array. - """ - store_path = self.store_path / path - try: - node = await self.getitem(path) - except KeyError as e: - msg = f"No array found in store {store_path.store!r} at path {store_path.path!r}" - raise ArrayNotFoundError(msg) from e - if isinstance(node, AsyncGroup): - msg = f"A group exists in store {store_path.store!r} at path {store_path.path!r}." - raise ContainsGroupError(msg) - return node - - async def get_group(self, path: str) -> AsyncGroup: - """Obtain a group member of this group, raising if it is absent or not a group. - - Parameters - ---------- - path : str - Path of the group relative to this group. May contain `/` to reference - a member of a subgroup, e.g. `subgroup/subsubgroup`. - - Returns - ------- - AsyncGroup - The group at the given path. - - Raises - ------ - GroupNotFoundError - If no node exists at the given path. - ContainsArrayError - If the node at the given path is an array rather than a group. - """ - store_path = self.store_path / path - try: - node = await self.getitem(path) - except KeyError as e: - msg = f"No group found in store {store_path.store!r} at path {store_path.path!r}" - raise GroupNotFoundError(msg) from e - if isinstance(node, AsyncArray): - msg = f"An array exists in store {store_path.store!r} at path {store_path.path!r}." - raise ContainsArrayError(msg) - return node - - async def _save_metadata(self, ensure_parents: bool = False) -> None: - await save_metadata(self.store_path, self.metadata, ensure_parents=ensure_parents) - - @property - def path(self) -> str: - """Storage path.""" - return self.store_path.path - - @property - def name(self) -> str: - """Group name following h5py convention.""" - if self.path: - # follow h5py convention: add leading slash - name = self.path - if name[0] != "/": - name = f"/{name}" - return name - return "/" - - @property - def basename(self) -> str: - """Final component of name.""" - return self.name.split("/")[-1] - - @property - def attrs(self) -> dict[str, Any]: - return self.metadata.attributes - - @property - def info(self) -> Any: - """ - Return a visual representation of the statically known information about a group. - - Note that this doesn't include dynamic information, like the number of child - Groups or Arrays. - - Returns - ------- - GroupInfo - - Related - ------- - [zarr.AsyncGroup.info_complete][] - All information about a group, including dynamic information - """ - - if self.metadata.consolidated_metadata: - members = list(self.metadata.consolidated_metadata.flattened_metadata.values()) - else: - members = None - return self._info(members=members) - - async def info_complete(self) -> Any: - """ - Return all the information for a group. - - This includes dynamic information like the number - of child Groups or Arrays. If this group doesn't contain consolidated - metadata then this will need to read from the backing Store. - - Returns - ------- - GroupInfo - - Related - ------- - [zarr.AsyncGroup.info][] - """ - members = [x[1].metadata async for x in self.members(max_depth=None)] - return self._info(members=members) - - def _info( - self, members: list[ArrayV2Metadata | ArrayV3Metadata | GroupMetadata] | None = None - ) -> Any: - kwargs = {} - if members is not None: - kwargs["_count_members"] = len(members) - count_arrays = 0 - count_groups = 0 - for member in members: - if isinstance(member, GroupMetadata): - count_groups += 1 - else: - count_arrays += 1 - kwargs["_count_arrays"] = count_arrays - kwargs["_count_groups"] = count_groups - - return GroupInfo( - _name=self.store_path.path, - _read_only=self.read_only, - _store_type=type(self.store_path.store).__name__, - _zarr_format=self.metadata.zarr_format, - # maybe do a typeddict - **kwargs, # type: ignore[arg-type] - ) - - @property - def store(self) -> Store: - return self.store_path.store - - @property - def read_only(self) -> bool: - # Backwards compatibility for 2.x - return self.store_path.read_only - - @property - def synchronizer(self) -> None: - # Backwards compatibility for 2.x - # Not implemented in 3.x yet. - return None - - async def create_group( - self, - name: str, - *, - overwrite: bool = False, - attributes: dict[str, Any] | None = None, - ) -> AsyncGroup: - """Create a sub-group. - - Parameters - ---------- - name : str - Group name. - overwrite : bool, optional - If True, do not raise an error if the group already exists. - attributes : dict, optional - Group attributes. - - Returns - ------- - g : AsyncGroup - """ - attributes = attributes or {} - return await type(self).from_store( - self.store_path / name, - attributes=attributes, - overwrite=overwrite, - zarr_format=self.metadata.zarr_format, - ) - - async def require_group(self, name: str, overwrite: bool = False) -> AsyncGroup: - """Obtain a sub-group, creating one if it doesn't exist. - - Parameters - ---------- - name : str - Group name. - overwrite : bool, optional - Overwrite any existing group with given `name` if present. - - Returns - ------- - g : AsyncGroup - """ - if overwrite: - # TODO: check that overwrite=True errors if an array exists where the group is being created - grp = await self.create_group(name, overwrite=True) - else: - try: - item: AsyncGroup | AnyAsyncArray = await self.getitem(name) - if not isinstance(item, AsyncGroup): - raise TypeError( - f"Incompatible object ({item.__class__.__name__}) already exists" - ) - grp = item - except KeyError: - grp = await self.create_group(name) - return grp - - async def require_groups(self, *names: str) -> tuple[AsyncGroup, ...]: - """Convenience method to require multiple groups in a single call. - - Parameters - ---------- - *names : str - Group names. - - Returns - ------- - Tuple[AsyncGroup, ...] - """ - if not names: - return () - return tuple(await asyncio.gather(*(self.require_group(name) for name in names))) - - async def create_array( - self, - name: str, - *, - shape: ShapeLike | None = None, - dtype: ZDTypeLike | None = None, - data: np.ndarray[Any, np.dtype[Any]] | None = None, - chunks: ChunksLike | Literal["auto"] = "auto", - shards: ShardsLike | None = None, - filters: FiltersLike = "auto", - compressors: CompressorsLike = "auto", - compressor: CompressorLike = "auto", - serializer: SerializerLike = "auto", - fill_value: Any | None = DEFAULT_FILL_VALUE, - order: MemoryOrder | None = None, - attributes: dict[str, JSON] | None = None, - chunk_key_encoding: ChunkKeyEncodingLike | None = None, - dimension_names: DimensionNamesLike = None, - storage_options: dict[str, Any] | None = None, - overwrite: bool = False, - config: ArrayConfigLike | None = None, - write_data: bool = True, - ) -> AnyAsyncArray: - """Create an array within this group. - - This method lightly wraps [zarr.core.array.create_array][]. - - Parameters - ---------- - name : str - The name of the array relative to the group. If `path` is `None`, the array will be located - at the root of the store. - shape : ShapeLike, optional - Shape of the array. Must be `None` if `data` is provided. - dtype : ZDTypeLike | None - Data type of the array. Must be `None` if `data` is provided. - data : np.ndarray, optional - Array-like data to use for initializing the array. If this parameter is provided, the - `shape` and `dtype` parameters must be `None`. - chunks : tuple[int, ...] | Sequence[Sequence[int]] | Literal["auto"], default="auto" - Chunk shape of the array. - If chunks is "auto", a chunk shape is guessed based on the shape of the array and the dtype. - A nested list of per-dimension edge sizes creates a rectilinear grid. - Rectilinear chunk grids are experimental and must be explicitly enabled - with `zarr.config.set({'array.rectilinear_chunks': True})` while the - feature is stabilizing. - shards : tuple[int, ...], optional - Shard shape of the array. The default value of `None` results in no sharding at all. - filters : Iterable[Codec] | Literal["auto"], optional - Iterable of filters to apply to each chunk of the array, in order, before serializing that - chunk to bytes. - - For Zarr format 3, a "filter" is a codec that takes an array and returns an array, - and these values must be instances of [`zarr.abc.codec.ArrayArrayCodec`][], or a - dict representations of [`zarr.abc.codec.ArrayArrayCodec`][]. - - For Zarr format 2, a "filter" can be any numcodecs codec; you should ensure that the - order of your filters is consistent with the behavior of each filter. - - The default value of `"auto"` instructs Zarr to use a default based on the data - type of the array and the Zarr format specified. For all data types in Zarr V3, and most - data types in Zarr V2, the default filters are empty. The only cases where default filters - are not empty is when the Zarr format is 2, and the data type is a variable-length data type like - [`zarr.dtype.VariableLengthUTF8`][] or [`zarr.dtype.VariableLengthBytes`][]. In these cases, - the default filters contains a single element which is a codec specific to that particular data type. - - To create an array with no filters, provide an empty iterable or the value `None`. - compressors : Iterable[Codec], optional - List of compressors to apply to the array. Compressors are applied in order, and after any - filters are applied (if any are specified) and the data is serialized into bytes. - - For Zarr format 3, a "compressor" is a codec that takes a bytestream, and - returns another bytestream. Multiple compressors may be provided for Zarr format 3. - If no `compressors` are provided, a default set of compressors will be used. - These defaults can be changed by modifying the value of `array.v3_default_compressors` - in [`zarr.config`][zarr.config]. - Use `None` to omit default compressors. - - For Zarr format 2, a "compressor" can be any numcodecs codec. Only a single compressor may - be provided for Zarr format 2. - If no `compressor` is provided, a default compressor will be used. - This default can be changed by modifying the value of `array.v2_default_compressor` - in [`zarr.config`][zarr.config]. - Use `None` to omit the default compressor. - compressor : Codec, optional - Deprecated in favor of `compressors`. - serializer : dict[str, JSON] | ArrayBytesCodec, optional - Array-to-bytes codec to use for encoding the array data. - Zarr format 3 only. Zarr format 2 arrays use implicit array-to-bytes conversion. - If no `serializer` is provided, a default serializer will be used. - These defaults can be changed by modifying the value of `array.v3_default_serializer` - in [`zarr.config`][zarr.config]. - fill_value : Any, optional - Fill value for the array. - order : {"C", "F"}, optional - The memory order of the array (default is "C"). - For Zarr format 2, this parameter sets the memory order of the array. - For Zarr format 3, this parameter is deprecated, because memory order - is a runtime parameter for Zarr format 3 arrays. The recommended way to specify the memory - order for Zarr format 3 arrays is via the `config` parameter, e.g. `{'config': 'C'}`. - If no `order` is provided, a default order will be used. - This default can be changed by modifying the value of `array.order` in [`zarr.config`][zarr.config]. - attributes : dict, optional - Attributes for the array. - chunk_key_encoding : ChunkKeyEncodingLike, optional - A specification of how the chunk keys are represented in storage. - For Zarr format 3, the default is `{"name": "default", "separator": "/"}`. - For Zarr format 2, the default is `{"name": "v2", "separator": "."}`. - dimension_names : Iterable[str], optional - The names of the dimensions (default is None). - Zarr format 3 only. Zarr format 2 arrays should not use this parameter. - storage_options : dict, optional - If using an fsspec URL to create the store, these will be passed to the backend implementation. - Ignored otherwise. - overwrite : bool, default False - Whether to overwrite an array with the same name in the store, if one exists. - If `True`, any existing keys under that path are deleted first. - config : ArrayConfigLike, optional - Runtime configuration for the array. - write_data : bool - If a pre-existing array-like object was provided to this function via the `data` parameter - then `write_data` determines whether the values in that array-like object should be - written to the Zarr array created by this function. If `write_data` is `False`, then the - array will be left empty. - - Returns - ------- - AsyncArray - - """ - compressors = _parse_deprecated_compressor( - compressor, compressors, zarr_format=self.metadata.zarr_format - ) - return await create_array( - store=self.store_path, - name=name, - shape=shape, - dtype=dtype, - data=data, - chunks=chunks, - shards=shards, - filters=filters, - compressors=compressors, - serializer=serializer, - fill_value=fill_value, - order=order, - zarr_format=self.metadata.zarr_format, - attributes=attributes, - chunk_key_encoding=chunk_key_encoding, - dimension_names=dimension_names, - storage_options=storage_options, - overwrite=overwrite, - config=config, - write_data=write_data, - ) - - async def require_array( - self, - name: str, - *, - shape: ShapeLike, - dtype: ZDTypeLike | None = None, - exact: bool = False, - config: ArrayConfigLike | None = None, - **kwargs: Any, - ) -> AnyAsyncArray: - """Obtain an array, creating if it doesn't exist. - - Other `kwargs` are as per [zarr.AsyncGroup.create_array][]. - - Parameters - ---------- - name : str - Array name. - shape : int or tuple of ints - Array shape. - dtype : ZDTypeLike, optional - The data type of the array, given as a string, a NumPy dtype, or a - Zarr data type. - exact : bool, optional - If True, require `dtype` to match exactly. If false, require - `dtype` can be cast from array dtype. - config : ArrayConfigLike or None, default=None - Runtime configuration for the array, whether it is created or already exists. - Keys not specified are taken from the global configuration. - - Returns - ------- - a : AsyncArray - """ - try: - ds = await self.getitem(name) - if not isinstance(ds, AsyncArray): - raise TypeError(f"Incompatible object ({ds.__class__.__name__}) already exists") - - shape = parse_shapelike(shape) - if shape != ds.shape: - raise TypeError(f"Incompatible shape ({ds.shape} vs {shape})") - - # `np.dtype(None)` used to resolve to float64 here; keep that default. - dtype = parse_data_type( - "float64" if dtype is None else dtype, - zarr_format=self.metadata.zarr_format, - ).to_native_dtype() - if exact: - if ds.dtype != dtype: - raise TypeError(f"Incompatible dtype ({ds.dtype} vs {dtype})") - else: - if not np.can_cast(ds.dtype, dtype): - raise TypeError(f"Incompatible dtype ({ds.dtype} vs {dtype})") - except KeyError: - return await self.create_array(name, shape=shape, dtype=dtype, config=config, **kwargs) - - # `config` is the runtime configuration of the returned array, not stored metadata, - # so it applies to an existing array too. It is parsed the same way as on the create - # branch: missing keys come from the global configuration, unknown keys raise. - if config is not None: - return ds.with_config(parse_array_config(config)) - return ds - - async def update_attributes(self, new_attributes: dict[str, Any]) -> AsyncGroup: - """Update group attributes. - - Parameters - ---------- - new_attributes : dict - New attributes to set on the group. - - Returns - ------- - self : AsyncGroup - """ - self.metadata.attributes.update(new_attributes) - - # Write new metadata - await self._save_metadata() - - return self - - def __repr__(self) -> str: - return f"" - - async def nmembers( - self, - max_depth: int | None = 0, - ) -> int: - """Count the number of members in this group. - - Parameters - ---------- - max_depth : int, default 0 - The maximum number of levels of the hierarchy to include. By - default, (``max_depth=0``) only immediate children are included. Set - ``max_depth=None`` to include all nodes, and some positive integer - to consider children within that many levels of the root Group. - - Returns - ------- - count : int - """ - # check if we can use consolidated metadata, which requires that we have non-None - # consolidated metadata at all points in the hierarchy. - if self.metadata.consolidated_metadata is not None: - if max_depth is not None and max_depth < 0: - raise ValueError(f"max_depth must be None or >= 0. Got '{max_depth}' instead") - if max_depth is None: - return len(self.metadata.consolidated_metadata.flattened_metadata) - else: - return len( - [ - x - for x in self.metadata.consolidated_metadata.flattened_metadata - if x.count("/") <= max_depth - ] - ) - # TODO: consider using aioitertools.builtins.sum for this - # return await aioitertools.builtins.sum((1 async for _ in self.members()), start=0) - n = 0 - async for _ in self.members(max_depth=max_depth): - n += 1 - return n - - async def members( - self, - max_depth: int | None = 0, - *, - use_consolidated_for_children: bool = True, - ) -> AsyncGenerator[ - tuple[str, AnyAsyncArray | AsyncGroup], - None, - ]: - """ - Returns an AsyncGenerator over the arrays and groups contained in this group. - This method requires that `store_path.store` supports directory listing. - - The results are not guaranteed to be ordered. - - Parameters - ---------- - max_depth : int, default 0 - The maximum number of levels of the hierarchy to include. By - default, (``max_depth=0``) only immediate children are included. Set - ``max_depth=None`` to include all nodes, and some positive integer - to consider children within that many levels of the root Group. - use_consolidated_for_children : bool, default True - Whether to use the consolidated metadata of child groups loaded - from the store. Note that this only affects groups loaded from the - store. If the current Group already has consolidated metadata, it - will always be used. - - Returns - ------- - path: - A string giving the path to the target, relative to the Group ``self``. - value: AsyncArray or AsyncGroup - The AsyncArray or AsyncGroup that is a child of ``self``. - """ - if max_depth is not None and max_depth < 0: - raise ValueError(f"max_depth must be None or >= 0. Got '{max_depth}' instead") - async for item in self._members( - max_depth=max_depth, use_consolidated_for_children=use_consolidated_for_children - ): - yield item - - def _members_consolidated( - self, max_depth: int | None, prefix: str = "" - ) -> Generator[ - tuple[str, AnyAsyncArray | AsyncGroup], - None, - ]: - consolidated_metadata = self.metadata.consolidated_metadata - - do_recursion = max_depth is None or max_depth > 0 - - # we kind of just want the top-level keys. - if consolidated_metadata is not None: - for key in consolidated_metadata.metadata: - obj = self._getitem_consolidated( - self.store_path, key, prefix=self.name - ) # Metadata -> Group/Array - key = f"{prefix}/{key}".lstrip("/") - yield key, obj - - if do_recursion and isinstance(obj, AsyncGroup): - if max_depth is None: - new_depth = None - else: - new_depth = max_depth - 1 - yield from obj._members_consolidated(new_depth, prefix=key) - - async def _members( - self, max_depth: int | None, *, use_consolidated_for_children: bool = True - ) -> AsyncGenerator[tuple[str, AnyAsyncArray | AsyncGroup], None]: - skip_keys: tuple[str, ...] - if self.metadata.zarr_format == 2: - skip_keys = (".zattrs", ".zgroup", ".zarray", ".zmetadata") - elif self.metadata.zarr_format == 3: - skip_keys = ("zarr.json",) - else: - raise ValueError(f"Unknown Zarr format: {self.metadata.zarr_format}") - - if self.metadata.consolidated_metadata is not None: - members = self._members_consolidated(max_depth=max_depth) - for member in members: - yield member - return - - if not self.store_path.store.supports_listing: - msg = ( - f"The store associated with this group ({type(self.store_path.store)}) " - "does not support listing, " - "specifically via the `list_dir` method. " - "This function requires a store that supports listing." - ) - - raise ValueError(msg) - # enforce a concurrency limit by passing a semaphore to all the recursive functions - semaphore = asyncio.Semaphore(config.get("async.concurrency")) - async for member in _iter_members_deep( - self, - max_depth=max_depth, - skip_keys=skip_keys, - semaphore=semaphore, - use_consolidated_for_children=use_consolidated_for_children, - ): - yield member - - async def create_hierarchy( - self, - nodes: dict[str, ArrayV2Metadata | ArrayV3Metadata | GroupMetadata], - *, - overwrite: bool = False, - ) -> AsyncIterator[tuple[str, AsyncGroup | AnyAsyncArray]]: - """ - Create a hierarchy of arrays or groups rooted at this group. - - This function will parse its input to ensure that the hierarchy is complete. Any implicit groups - will be inserted as needed. For example, an input like - ```{'a/b': GroupMetadata}``` will be parsed to - ```{'': GroupMetadata, 'a': GroupMetadata, 'b': Groupmetadata}```. - - Explicitly specifying a root group, e.g. with ``nodes = {'': GroupMetadata()}`` is an error - because this group instance is the root group. - - After input parsing, this function then creates all the nodes in the hierarchy concurrently. - - Arrays and Groups are yielded in the order they are created. This order is not stable and - should not be relied on. - - Parameters - ---------- - nodes : dict[str, GroupMetadata | ArrayV3Metadata | ArrayV2Metadata] - A dictionary defining the hierarchy. The keys are the paths of the nodes in the hierarchy, - relative to the path of the group. The values are instances of ``GroupMetadata`` or ``ArrayMetadata``. Note that - all values must have the same ``zarr_format`` as the parent group -- it is an error to mix zarr versions in the - same hierarchy. - - Leading "/" characters from keys will be removed. - overwrite : bool - Whether to overwrite existing nodes. Defaults to ``False``, in which case an error is - raised instead of overwriting an existing array or group. - - This function will not erase an existing group unless that group is explicitly named in - ``nodes``. If ``nodes`` defines implicit groups, e.g. ``{`'a/b/c': GroupMetadata}``, and a - group already exists at path ``a``, then this function will leave the group at ``a`` as-is. - - Yields - ------ - tuple[str, AsyncArray | AsyncGroup]. - """ - # check that all the nodes have the same zarr_format as Self - prefix = self.path - nodes_parsed = {} - for key, value in nodes.items(): - if value.zarr_format != self.metadata.zarr_format: - msg = ( - "The zarr_format of the nodes must be the same as the parent group. " - f"The node at {key} has zarr_format {value.zarr_format}, but the parent group" - f" has zarr_format {self.metadata.zarr_format}." - ) - raise ValueError(msg) - if normalize_path(key) == "": - msg = ( - "The input defines a root node, but a root node already exists, namely this Group instance." - "It is an error to use this method to create a root node. " - "Remove the root node from the input dict, or use a function like " - "create_rooted_hierarchy to create a rooted hierarchy." - ) - raise ValueError(msg) - else: - nodes_parsed[_join_paths([prefix, key])] = value - - async for key, node in create_hierarchy( - store=self.store, - nodes=nodes_parsed, - overwrite=overwrite, - ): - if prefix == "": - out_key = key - else: - out_key = key.removeprefix(prefix + "/") - yield out_key, node - - async def keys(self) -> AsyncGenerator[str, None]: - """Iterate over member names.""" - async for key, _ in self.members(): - yield key - - async def contains(self, member: str) -> bool: - """Check if a member exists in the group. - - Parameters - ---------- - member : str - Member name. - - Returns - ------- - bool - """ - # TODO: this can be made more efficient. - try: - await self.getitem(member) - except KeyError: - return False - else: - return True - - async def groups(self) -> AsyncGenerator[tuple[str, AsyncGroup], None]: - """Iterate over subgroups.""" - async for name, value in self.members(): - if isinstance(value, AsyncGroup): - yield name, value - - async def group_keys(self) -> AsyncGenerator[str, None]: - """Iterate over group names.""" - async for key, _ in self.groups(): - yield key - - async def group_values(self) -> AsyncGenerator[AsyncGroup, None]: - """Iterate over group values.""" - async for _, group in self.groups(): - yield group - - async def arrays( - self, - ) -> AsyncGenerator[tuple[str, AnyAsyncArray], None]: - """Iterate over arrays.""" - async for key, value in self.members(): - if isinstance(value, AsyncArray): - yield key, value - - async def array_keys(self) -> AsyncGenerator[str, None]: - """Iterate over array names.""" - async for key, _ in self.arrays(): - yield key - - async def array_values( - self, - ) -> AsyncGenerator[AnyAsyncArray, None]: - """Iterate over array values.""" - async for _, array in self.arrays(): - yield array - - async def tree( - self, - expand: bool | None = None, - level: int | None = None, - *, - max_nodes: int = 500, - plain: bool = False, - ) -> Any: - """ - Return a tree-like representation of a hierarchy. - - Parameters - ---------- - expand : bool, optional - This keyword is not yet supported. A NotImplementedError is raised if - it's used. - level : int, optional - The maximum depth below this Group to display in the tree. - max_nodes : int - Maximum number of nodes to display before truncating. Default is 500. - plain : bool, optional - If True, return a plain-text tree without ANSI styling. This is - useful when the output will be consumed by an LLM or written to a - file. Default is False. - - Returns - ------- - TreeRepr - A pretty-printable object displaying the hierarchy. - """ - from zarr.core._tree import group_tree_async - - if expand is not None: - raise NotImplementedError("'expand' is not yet implemented.") - return await group_tree_async(self, max_depth=level, max_nodes=max_nodes, plain=plain) - - def _member_zarr_format(self, zarr_format: ZarrFormat | None) -> ZarrFormat: - """ - The zarr format of a new array in this group, which is the format of the group. - - A group only lists members of its own format, so an array of another format - would be written into the group without becoming a member of it. - """ - if zarr_format is not None and zarr_format != self.metadata.zarr_format: - raise ValueError( - f"Cannot create a zarr_format={zarr_format} array in a " - f"zarr_format={self.metadata.zarr_format} group." - ) - return self.metadata.zarr_format - - async def empty(self, *, name: str, shape: tuple[int, ...], **kwargs: Any) -> AnyAsyncArray: - """Create an empty array with the specified shape in this Group. The contents will - be filled with the array's fill value or zeros if no fill value is provided. - - Parameters - ---------- - name : str - Name of the array. - shape : int or tuple of int - Shape of the empty array. - **kwargs - Keyword arguments passed to [zarr.api.asynchronous.create][]. - - Notes - ----- - The contents of an empty Zarr array are not defined. On attempting to - retrieve data from an empty Zarr array, any values may be returned, - and these are not guaranteed to be stable from one access to the next. - """ - return await async_api.empty( - shape=shape, - store=self.store_path, - path=name, - zarr_format=self._member_zarr_format(kwargs.pop("zarr_format", None)), - **kwargs, - ) - - async def zeros(self, *, name: str, shape: tuple[int, ...], **kwargs: Any) -> AnyAsyncArray: - """Create an array, with zero being used as the default value for uninitialized portions of the array. - - Parameters - ---------- - name : str - Name of the array. - shape : int or tuple of int - Shape of the empty array. - **kwargs - Keyword arguments passed to [zarr.api.asynchronous.create][]. - - Returns - ------- - AsyncArray - The new array. - """ - return await async_api.zeros( - shape=shape, - store=self.store_path, - path=name, - zarr_format=self._member_zarr_format(kwargs.pop("zarr_format", None)), - **kwargs, - ) - - async def ones(self, *, name: str, shape: tuple[int, ...], **kwargs: Any) -> AnyAsyncArray: - """Create an array, with one being used as the default value for uninitialized portions of the array. - - Parameters - ---------- - name : str - Name of the array. - shape : int or tuple of int - Shape of the empty array. - **kwargs - Keyword arguments passed to [zarr.api.asynchronous.create][]. - - Returns - ------- - AsyncArray - The new array. - """ - return await async_api.ones( - shape=shape, - store=self.store_path, - path=name, - zarr_format=self._member_zarr_format(kwargs.pop("zarr_format", None)), - **kwargs, - ) - - async def full( - self, *, name: str, shape: tuple[int, ...], fill_value: Any | None, **kwargs: Any - ) -> AnyAsyncArray: - """Create an array, with "fill_value" being used as the default value for uninitialized portions of the array. - - Parameters - ---------- - name : str - Name of the array. - shape : int or tuple of int - Shape of the empty array. - fill_value : scalar - Value to fill the array with. - **kwargs - Keyword arguments passed to [zarr.api.asynchronous.create][]. - - Returns - ------- - AsyncArray - The new array. - """ - return await async_api.full( - shape=shape, - fill_value=fill_value, - store=self.store_path, - path=name, - zarr_format=self._member_zarr_format(kwargs.pop("zarr_format", None)), - **kwargs, - ) - - async def empty_like( - self, *, name: str, data: async_api.ArrayLike, **kwargs: Any - ) -> AnyAsyncArray: - """Create an empty sub-array like `data`. The contents will be filled with - the array's fill value or zeros if no fill value is provided. - - Parameters - ---------- - name : str - Name of the array. - data : array-like - The array to create an empty array like. - **kwargs - Keyword arguments passed to [zarr.api.asynchronous.create][]. - - Returns - ------- - AsyncArray - The new array. - """ - return await async_api.empty_like( - a=data, - store=self.store_path, - path=name, - zarr_format=self._member_zarr_format(kwargs.pop("zarr_format", None)), - **kwargs, - ) - - async def zeros_like( - self, *, name: str, data: async_api.ArrayLike, **kwargs: Any - ) -> AnyAsyncArray: - """Create a sub-array of zeros like `data`. - - Parameters - ---------- - name : str - Name of the array. - data : array-like - The array to create the new array like. - **kwargs - Keyword arguments passed to [zarr.api.asynchronous.create][]. - - Returns - ------- - AsyncArray - The new array. - """ - return await async_api.zeros_like( - a=data, - store=self.store_path, - path=name, - zarr_format=self._member_zarr_format(kwargs.pop("zarr_format", None)), - **kwargs, - ) - - async def ones_like( - self, *, name: str, data: async_api.ArrayLike, **kwargs: Any - ) -> AnyAsyncArray: - """Create a sub-array of ones like `data`. - - Parameters - ---------- - name : str - Name of the array. - data : array-like - The array to create the new array like. - **kwargs - Keyword arguments passed to [zarr.api.asynchronous.create][]. - - Returns - ------- - AsyncArray - The new array. - """ - return await async_api.ones_like( - a=data, - store=self.store_path, - path=name, - zarr_format=self._member_zarr_format(kwargs.pop("zarr_format", None)), - **kwargs, - ) - - async def full_like( - self, *, name: str, data: async_api.ArrayLike, **kwargs: Any - ) -> AnyAsyncArray: - """Create a sub-array like `data` filled with the `fill_value` of `data` . - - Parameters - ---------- - name : str - Name of the array. - data : array-like - The array to create the new array like. - **kwargs - Keyword arguments passed to [zarr.api.asynchronous.create][]. - - Returns - ------- - AsyncArray - The new array. - """ - return await async_api.full_like( - a=data, - store=self.store_path, - path=name, - zarr_format=self._member_zarr_format(kwargs.pop("zarr_format", None)), - **kwargs, - ) - - async def move(self, source: str, dest: str) -> None: - """Move a sub-group or sub-array from one path to another. - - Notes - ----- - Not implemented - """ - raise NotImplementedError - - -@dataclass(frozen=True) -class Group(SyncMixin): - """ - A Zarr group. - """ - - _async_group: AsyncGroup - - @classmethod - def from_store( - cls, - store: StoreLike, - *, - attributes: dict[str, Any] | None = None, - zarr_format: ZarrFormat = 3, - overwrite: bool = False, - ) -> Group: - """Instantiate a group from an initialized store. - - Parameters - ---------- - store : StoreLike - StoreLike containing the Group. See the - [storage documentation in the user guide][user-guide-store-like] - for a description of all valid StoreLike values. - attributes : dict, optional - A dictionary of JSON-serializable values with user-defined attributes. - zarr_format : {2, 3}, optional - Zarr storage format version. - overwrite : bool, optional - If True, do not raise an error if the group already exists. - - Returns - ------- - Group - Group instantiated from the store. - - Raises - ------ - ContainsArrayError, ContainsGroupError, ContainsArrayAndGroupError - """ - attributes = attributes or {} - obj = sync( - AsyncGroup.from_store( - store, - attributes=attributes, - overwrite=overwrite, - zarr_format=zarr_format, - ), - ) - - return cls(obj) - - @classmethod - def open( - cls, - store: StoreLike, - zarr_format: ZarrFormat | None = 3, - ) -> Group: - """Open a group from an initialized store. - - Parameters - ---------- - store : StoreLike - Store containing the Group. See the - [storage documentation in the user guide][user-guide-store-like] - for a description of all valid StoreLike values. - zarr_format : {2, 3, None}, optional - Zarr storage format version. - - Returns - ------- - Group - Group instantiated from the store. - """ - obj = sync(AsyncGroup.open(store, zarr_format=zarr_format)) - return cls(obj) - - def __getitem__(self, path: str) -> AnyArray | Group: - """Obtain a group member. - - Parameters - ---------- - path : str - Group member name. - - Returns - ------- - Array | Group - Group member (Array or Group) at the specified key - - Examples - -------- - ```python - import zarr - from zarr.core.group import Group - group = Group.from_store(zarr.storage.MemoryStore()) - group.create_array(name="subarray", shape=(10,), chunks=(10,), dtype="float64") - group.create_group(name="subgroup").create_array(name="subarray", shape=(10,), chunks=(10,), dtype="float64") - group["subarray"] - # - group["subgroup"] - # - group["subgroup"]["subarray"] - # - ``` - - """ - obj = self._sync(self._async_group.getitem(path)) - if isinstance(obj, AsyncArray): - return Array(obj) - else: - return Group(obj) - - def get[DefaultT]( - self, path: str, default: DefaultT | None = None - ) -> AnyArray | Group | DefaultT | None: - """Obtain a group member, returning default if not found. - - Parameters - ---------- - path : str - Group member name. - default : object - Default value to return if key is not found (default: None). - - Returns - ------- - object - Group member (Array or Group) or default if not found. - - Examples - -------- - ```python - import zarr - from zarr.core.group import Group - group = Group.from_store(zarr.storage.MemoryStore()) - group.create_array(name="subarray", shape=(10,), chunks=(10,), dtype="float64") - group.create_group(name="subgroup") - group.get("subarray") - # - group.get("subgroup") - # - group.get("nonexistent", None) - # None - ``` - - """ - try: - return self[path] - except KeyError: - return default - - def get_array(self, path: str) -> AnyArray: - """Obtain an array member of this group, raising if it is absent or not an array. - - Parameters - ---------- - path : str - Path of the array relative to this group. May contain `/` to reference - a member of a subgroup, e.g. `subgroup/subarray`. - - Returns - ------- - Array - The array at the given path. - - Raises - ------ - ArrayNotFoundError - If no node exists at the given path. - ContainsGroupError - If the node at the given path is a group rather than an array. - - Examples - -------- - ```python - import zarr - from zarr.core.group import Group - group = Group.from_store(zarr.storage.MemoryStore()) - group.create_array(name="subarray", shape=(10,), chunks=(10,), dtype="float64") - group.get_array("subarray") - # - ``` - """ - return Array(self._sync(self._async_group.get_array(path))) - - def get_group(self, path: str) -> Group: - """Obtain a group member of this group, raising if it is absent or not a group. - - Parameters - ---------- - path : str - Path of the group relative to this group. May contain `/` to reference - a member of a subgroup, e.g. `subgroup/subsubgroup`. - - Returns - ------- - Group - The group at the given path. - - Raises - ------ - GroupNotFoundError - If no node exists at the given path. - ContainsArrayError - If the node at the given path is an array rather than a group. - - Examples - -------- - ```python - import zarr - from zarr.core.group import Group - group = Group.from_store(zarr.storage.MemoryStore()) - group.create_group(name="subgroup") - group.get_group("subgroup") - # - ``` - """ - return Group(self._sync(self._async_group.get_group(path))) - - def __delitem__(self, key: str) -> None: - """Delete a group member. - - Parameters - ---------- - key : str - Group member name. - - Examples - -------- - >>> import zarr - >>> group = Group.from_store(zarr.storage.MemoryStore()) - >>> a = group.create_array(name="subarray", dtype="i1", shape=(10,), chunks=(10,)) - >>> del group["subarray"] - >>> "subarray" in group - False - """ - - self._sync(self._async_group.delitem(key)) - - def __iter__(self) -> Iterator[str]: - """Return an iterator over group member names. - Examples - -------- - >>> import zarr - >>> g1 = zarr.group() - >>> g2 = g1.create_group('foo') - >>> g3 = g1.create_group('bar') - >>> d1 = g1.create_array('baz', dtype="i1", shape=(10,), chunks=(10,)) - >>> d2 = g1.create_array('quux', dtype="i1", shape=(10,), chunks=(10,)) - >>> sorted(g1) - ['bar', 'baz', 'foo', 'quux'] - """ - yield from self.keys() - - def __len__(self) -> int: - """Number of members.""" - return self.nmembers() - - def __setitem__(self, key: str, value: Any) -> None: - """Fastpath for creating a new array. - - New arrays will be created using default settings for the array type. - If you need to create an array with custom settings, use the `create_array` method. - - Parameters - ---------- - key : str - Array name. - value : Any - Array data. - - Examples - -------- - >>> import numpy as np - >>> import zarr - >>> group = zarr.group() - >>> group["foo"] = np.array(zarr.zeros((10,))) - >>> group["foo"] - - """ - self._sync(self._async_group.setitem(key, value)) - - def __repr__(self) -> str: - return f"" - - async def update_attributes_async(self, new_attributes: dict[str, Any]) -> Group: - """Update the attributes of this group. - - Examples - -------- - >>> async def example(): - ... import zarr - ... - ... group = zarr.group() - ... new_group = await group.update_attributes_async({"foo": "bar"}) - ... return new_group.attrs.asdict() - - >>> import asyncio - >>> asyncio.run(example()) - {'foo': 'bar'} - """ - new_metadata = replace(self.metadata, attributes=new_attributes) - - # Write new metadata - to_save = new_metadata.to_buffer_dict(default_buffer_prototype()) - awaitables = [set_or_delete(self.store_path / key, value) for key, value in to_save.items()] - await asyncio.gather(*awaitables) - - async_group = replace(self._async_group, metadata=new_metadata) - return replace(self, _async_group=async_group) - - @property - def store_path(self) -> StorePath: - """Path-like interface for the Store.""" - return self._async_group.store_path - - @property - def metadata(self) -> GroupMetadata: - """Group metadata.""" - return self._async_group.metadata - - @property - def path(self) -> str: - """Storage path.""" - return self._async_group.path - - @property - def name(self) -> str: - """Group name following h5py convention.""" - return self._async_group.name - - @property - def basename(self) -> str: - """Final component of name.""" - return self._async_group.basename - - @property - def attrs(self) -> Attributes: - """Attributes of this Group""" - return Attributes(self) - - @property - def info(self) -> Any: - """ - Return the statically known information for a group. - - Returns - ------- - GroupInfo - - Related - ------- - [zarr.Group.info_complete][] - All information about a group, including dynamic information - like the children members. - """ - return self._async_group.info - - def info_complete(self) -> Any: - """ - Return information for a group. - - If this group doesn't contain consolidated metadata then - this will need to read from the backing Store. - - Returns - ------- - GroupInfo - - Related - ------- - [zarr.Group.info][] - """ - return self._sync(self._async_group.info_complete()) - - @property - def store(self) -> Store: - # Backwards compatibility for 2.x - return self._async_group.store - - @property - def read_only(self) -> bool: - # Backwards compatibility for 2.x - return self._async_group.read_only - - @property - def synchronizer(self) -> None: - # Backwards compatibility for 2.x - # Not implemented in 3.x yet. - return self._async_group.synchronizer - - def update_attributes(self, new_attributes: dict[str, Any]) -> Group: - """Update the attributes of this group. - - Examples - -------- - >>> import zarr - >>> group = zarr.group().update_attributes({"foo": "bar"}) - >>> group.attrs.asdict() - {'foo': 'bar'} - """ - self._sync(self._async_group.update_attributes(new_attributes)) - return self - - def nmembers(self, max_depth: int | None = 0) -> int: - """Count the number of members in this group. - - Parameters - ---------- - max_depth : int, default 0 - The maximum number of levels of the hierarchy to include. By - default, (``max_depth=0``) only immediate children are included. Set - ``max_depth=None`` to include all nodes, and some positive integer - to consider children within that many levels of the root Group. - - Returns - ------- - count : int - """ - - return self._sync(self._async_group.nmembers(max_depth=max_depth)) - - def members( - self, max_depth: int | None = 0, *, use_consolidated_for_children: bool = True - ) -> tuple[tuple[str, AnyArray | Group], ...]: - """ - Returns an AsyncGenerator over the arrays and groups contained in this group. - This method requires that `store_path.store` supports directory listing. - - The results are not guaranteed to be ordered. - - Parameters - ---------- - max_depth : int, default 0 - The maximum number of levels of the hierarchy to include. By - default, (``max_depth=0``) only immediate children are included. Set - ``max_depth=None`` to include all nodes, and some positive integer - to consider children within that many levels of the root Group. - use_consolidated_for_children : bool, default True - Whether to use the consolidated metadata of child groups loaded - from the store. Note that this only affects groups loaded from the - store. If the current Group already has consolidated metadata, it - will always be used. - - Returns - ------- - path: - A string giving the path to the target, relative to the Group ``self``. - value: AsyncArray or AsyncGroup - The AsyncArray or AsyncGroup that is a child of ``self``. - """ - _members = self._sync_iter(self._async_group.members(max_depth=max_depth)) - - return tuple((kv[0], _parse_async_node(kv[1])) for kv in _members) - - def create_hierarchy( - self, - nodes: dict[str, ArrayV2Metadata | ArrayV3Metadata | GroupMetadata], - *, - overwrite: bool = False, - ) -> Iterator[tuple[str, Group | AnyArray]]: - """ - Create a hierarchy of arrays or groups rooted at this group. - - This function will parse its input to ensure that the hierarchy is complete. Any implicit groups - will be inserted as needed. For example, an input like - ```{'a/b': GroupMetadata}``` will be parsed to - ```{'': GroupMetadata, 'a': GroupMetadata, 'b': Groupmetadata}```. - - Explicitly specifying a root group, e.g. with ``nodes = {'': GroupMetadata()}`` is an error - because this group instance is the root group. - - After input parsing, this function then creates all the nodes in the hierarchy concurrently. - - Arrays and Groups are yielded in the order they are created. This order is not stable and - should not be relied on. - - Parameters - ---------- - nodes : dict[str, GroupMetadata | ArrayV3Metadata | ArrayV2Metadata] - A dictionary defining the hierarchy. The keys are the paths of the nodes in the hierarchy, - relative to the path of the group. The values are instances of ``GroupMetadata`` or ``ArrayMetadata``. Note that - all values must have the same ``zarr_format`` as the parent group -- it is an error to mix zarr versions in the - same hierarchy. - - Leading "/" characters from keys will be removed. - overwrite : bool - Whether to overwrite existing nodes. Defaults to ``False``, in which case an error is - raised instead of overwriting an existing array or group. - - This function will not erase an existing group unless that group is explicitly named in - ``nodes``. If ``nodes`` defines implicit groups, e.g. ``{`'a/b/c': GroupMetadata}``, and a - group already exists at path ``a``, then this function will leave the group at ``a`` as-is. - - Yields - ------ - tuple[str, Array | Group]. - - Examples - -------- - >>> import zarr - >>> from zarr.core.group import GroupMetadata - >>> root = zarr.create_group(store={}) - >>> sorted(root.create_hierarchy({'a/b/c': GroupMetadata()})) - [('a', ), - ('a/b', ), - ('a/b/c', )] - """ - for key, node in self._sync_iter( - self._async_group.create_hierarchy(nodes, overwrite=overwrite) - ): - yield (key, _parse_async_node(node)) - - def keys(self) -> Generator[str]: - """Return an iterator over group member names. - - Examples - -------- - >>> import zarr - >>> g1 = zarr.group() - >>> g2 = g1.create_group('foo') - >>> g3 = g1.create_group('bar') - >>> d1 = g1.create_array('baz', dtype="i1", shape=(10,), chunks=(10,)) - >>> d2 = g1.create_array('quux', dtype="i1", shape=(10,), chunks=(10,)) - >>> sorted(g1.keys()) - ['bar', 'baz', 'foo', 'quux'] - """ - yield from self._sync_iter(self._async_group.keys()) - - def __contains__(self, member: str) -> bool: - """Test for group membership. - - Examples - -------- - >>> import zarr - >>> g1 = zarr.group() - >>> g2 = g1.create_group('foo') - >>> d1 = g1.create_array('bar', dtype="i1", shape=(10,), chunks=(10,)) - >>> 'foo' in g1 - True - >>> 'bar' in g1 - True - >>> 'baz' in g1 - False - """ - return self._sync(self._async_group.contains(member)) - - def groups(self) -> Generator[tuple[str, Group], None]: - """Return the sub-groups of this group as a generator of (name, group) pairs. - - Examples - -------- - >>> import zarr - >>> group = zarr.group() - >>> subgroup = group.create_group("subgroup") - >>> list(group.groups()) - [('subgroup', )] - """ - for name, async_group in self._sync_iter(self._async_group.groups()): - yield name, Group(async_group) - - def group_keys(self) -> Generator[str, None]: - """Return an iterator over group member names. - - Examples - -------- - >>> import zarr - >>> group = zarr.group() - >>> subgroup = group.create_group("subgroup") - >>> list(group.group_keys()) - ['subgroup'] - """ - for name, _ in self.groups(): - yield name - - def group_values(self) -> Generator[Group, None]: - """Return an iterator over group members. - - Examples - -------- - >>> import zarr - >>> group = zarr.group() - >>> subgroup = group.create_group("subgroup") - >>> list(group.group_values()) - [] - """ - for _, group in self.groups(): - yield group - - def arrays(self) -> Generator[tuple[str, AnyArray], None]: - """Return the sub-arrays of this group as a generator of (name, array) pairs - - Examples - -------- - >>> import zarr - >>> group = zarr.group() - >>> subarray = group.create_array("subarray", dtype="i1", shape=(10,), chunks=(10,)) - >>> list(group.arrays()) - [('subarray', )] - """ - for name, async_array in self._sync_iter(self._async_group.arrays()): - yield name, Array(async_array) - - def array_keys(self) -> Generator[str, None]: - """Return an iterator over group member names. - - Examples - -------- - >>> import zarr - >>> group = zarr.group() - >>> subarray = group.create_array("subarray", dtype="i1", shape=(10,), chunks=(10,)) - >>> list(group.array_keys()) - ['subarray'] - """ - - for name, _ in self.arrays(): - yield name - - def array_values(self) -> Generator[AnyArray, None]: - """Return an iterator over group members. - - Examples - -------- - >>> import zarr - >>> group = zarr.group() - >>> subarray = group.create_array("subarray", dtype="i1", shape=(10,), chunks=(10,)) - >>> list(group.array_values()) - [] - """ - for _, array in self.arrays(): - yield array - - def tree( - self, - expand: bool | None = None, - level: int | None = None, - *, - max_nodes: int = 500, - plain: bool = False, - ) -> Any: - """ - Return a tree-like representation of a hierarchy. - - Parameters - ---------- - expand : bool, optional - This keyword is not yet supported. A NotImplementedError is raised if - it's used. - level : int, optional - The maximum depth below this Group to display in the tree. - max_nodes : int - Maximum number of nodes to display before truncating. Default is 500. - plain : bool, optional - If True, return a plain-text tree without ANSI styling. This is - useful when the output will be consumed by an LLM or written to a - file. Default is False. - - Returns - ------- - TreeRepr - A pretty-printable object displaying the hierarchy. - """ - return self._sync( - self._async_group.tree(expand=expand, level=level, max_nodes=max_nodes, plain=plain) - ) - - def create_group(self, name: str, **kwargs: Any) -> Group: - """Create a sub-group. - - Parameters - ---------- - name : str - Name of the new subgroup. - - Returns - ------- - Group - - Examples - -------- - >>> import zarr - >>> group = zarr.group() - >>> subgroup = group.create_group("subgroup") - >>> subgroup - - """ - return Group(self._sync(self._async_group.create_group(name, **kwargs))) - - def require_group(self, name: str, **kwargs: Any) -> Group: - """Obtain a sub-group, creating one if it doesn't exist. - - Parameters - ---------- - name : str - Group name. - - Returns - ------- - g : Group - """ - return Group(self._sync(self._async_group.require_group(name, **kwargs))) - - def require_groups(self, *names: str) -> tuple[Group, ...]: - """Convenience method to require multiple groups in a single call. - - Parameters - ---------- - *names : str - Group names. - - Returns - ------- - groups : tuple of Groups - """ - return tuple(map(Group, self._sync(self._async_group.require_groups(*names)))) - - def create( - self, - name: str, - *, - shape: ShapeLike | None = None, - dtype: ZDTypeLike | None = None, - data: np.ndarray[Any, np.dtype[Any]] | None = None, - chunks: ChunksLike | Literal["auto"] = "auto", - shards: ShardsLike | None = None, - filters: FiltersLike = "auto", - compressors: CompressorsLike = "auto", - compressor: CompressorLike = "auto", - serializer: SerializerLike = "auto", - fill_value: Any | None = DEFAULT_FILL_VALUE, - order: MemoryOrder | None = None, - attributes: dict[str, JSON] | None = None, - chunk_key_encoding: ChunkKeyEncodingLike | None = None, - dimension_names: DimensionNamesLike = None, - storage_options: dict[str, Any] | None = None, - overwrite: bool = False, - config: ArrayConfigLike | None = None, - write_data: bool = True, - ) -> AnyArray: - """Create an array within this group. - - This method lightly wraps [`zarr.core.array.create_array`][]. - - Parameters - ---------- - name : str - The name of the array relative to the group. If `path` is `None`, the array will be located - at the root of the store. - shape : ShapeLike, optional - Shape of the array. Must be `None` if `data` is provided. - dtype : ZDTypeLike | None - Data type of the array. Must be `None` if `data` is provided. - data : np.ndarray, optional - Array-like data to use for initializing the array. If this parameter is provided, the - `shape` and `dtype` parameters must be `None`. - chunks : tuple[int, ...] | Sequence[Sequence[int]] | Literal["auto"], default="auto" - Chunk shape of the array. - If chunks is "auto", a chunk shape is guessed based on the shape of the array and the dtype. - A nested list of per-dimension edge sizes creates a rectilinear grid. - Rectilinear chunk grids are experimental and must be explicitly enabled - with `zarr.config.set({'array.rectilinear_chunks': True})` while the - feature is stabilizing. - shards : tuple[int, ...], optional - Shard shape of the array. The default value of `None` results in no sharding at all. - filters : Iterable[Codec] | Literal["auto"], optional - Iterable of filters to apply to each chunk of the array, in order, before serializing that - chunk to bytes. - - For Zarr format 3, a "filter" is a codec that takes an array and returns an array, - and these values must be instances of [`zarr.abc.codec.ArrayArrayCodec`][], or a - dict representations of [`zarr.abc.codec.ArrayArrayCodec`][]. - - For Zarr format 2, a "filter" can be any numcodecs codec; you should ensure that the - order of your filters is consistent with the behavior of each filter. - - The default value of `"auto"` instructs Zarr to use a default based on the data - type of the array and the Zarr format specified. For all data types in Zarr V3, and most - data types in Zarr V2, the default filters are empty. The only cases where default filters - are not empty is when the Zarr format is 2, and the data type is a variable-length data type like - [`zarr.dtype.VariableLengthUTF8`][] or [`zarr.dtype.VariableLengthBytes`][]. In these cases, - the default filters contains a single element which is a codec specific to that particular data type. - - To create an array with no filters, provide an empty iterable or the value `None`. - compressors : Iterable[Codec], optional - List of compressors to apply to the array. Compressors are applied in order, and after any - filters are applied (if any are specified) and the data is serialized into bytes. - - For Zarr format 3, a "compressor" is a codec that takes a bytestream, and - returns another bytestream. Multiple compressors may be provided for Zarr format 3. - If no `compressors` are provided, a default set of compressors will be used. - These defaults can be changed by modifying the value of `array.v3_default_compressors` - in [`zarr.config`][zarr.config]. - Use `None` to omit default compressors. - - For Zarr format 2, a "compressor" can be any numcodecs codec. Only a single compressor may - be provided for Zarr format 2. - If no `compressor` is provided, a default compressor will be used. - This default can be changed by modifying the value of `array.v2_default_compressor` - in [`zarr.config`][zarr.config]. - Use `None` to omit the default compressor. - compressor : Codec, optional - Deprecated in favor of `compressors`. - serializer : dict[str, JSON] | ArrayBytesCodec, optional - Array-to-bytes codec to use for encoding the array data. - Zarr format 3 only. Zarr format 2 arrays use implicit array-to-bytes conversion. - If no `serializer` is provided, a default serializer will be used. - These defaults can be changed by modifying the value of `array.v3_default_serializer` - in [`zarr.config`][zarr.config]. - fill_value : Any, optional - Fill value for the array. - order : {"C", "F"}, optional - The memory order of the array (default is "C"). - For Zarr format 2, this parameter sets the memory order of the array. - For Zarr format 3, this parameter is deprecated, because memory order - is a runtime parameter for Zarr format 3 arrays. The recommended way to specify the memory - order for Zarr format 3 arrays is via the `config` parameter, e.g. `{'config': 'C'}`. - If no `order` is provided, a default order will be used. - This default can be changed by modifying the value of `array.order` in [`zarr.config`][zarr.config]. - attributes : dict, optional - Attributes for the array. - chunk_key_encoding : ChunkKeyEncodingLike, optional - A specification of how the chunk keys are represented in storage. - For Zarr format 3, the default is `{"name": "default", "separator": "/"}`. - For Zarr format 2, the default is `{"name": "v2", "separator": "."}`. - dimension_names : Iterable[str], optional - The names of the dimensions (default is None). - Zarr format 3 only. Zarr format 2 arrays should not use this parameter. - storage_options : dict, optional - If using an fsspec URL to create the store, these will be passed to the backend implementation. - Ignored otherwise. - overwrite : bool, default False - Whether to overwrite an array with the same name in the store, if one exists. - If `True`, any existing keys under that path are deleted first. - config : ArrayConfigLike, optional - Runtime configuration for the array. - write_data : bool - If a pre-existing array-like object was provided to this function via the `data` parameter - then `write_data` determines whether the values in that array-like object should be - written to the Zarr array created by this function. If `write_data` is `False`, then the - array will be left empty. - - Returns - ------- - AsyncArray - """ - return self.create_array( - name, - shape=shape, - dtype=dtype, - data=data, - chunks=chunks, - shards=shards, - filters=filters, - compressors=compressors, - compressor=compressor, - serializer=serializer, - fill_value=fill_value, - order=order, - attributes=attributes, - chunk_key_encoding=chunk_key_encoding, - dimension_names=dimension_names, - storage_options=storage_options, - overwrite=overwrite, - config=config, - write_data=write_data, - ) - - def create_array( - self, - name: str, - *, - shape: ShapeLike | None = None, - dtype: ZDTypeLike | None = None, - data: np.ndarray[Any, np.dtype[Any]] | None = None, - chunks: ChunksLike | Literal["auto"] = "auto", - shards: ShardsLike | None = None, - filters: FiltersLike = "auto", - compressors: CompressorsLike = "auto", - compressor: CompressorLike = "auto", - serializer: SerializerLike = "auto", - fill_value: Any | None = DEFAULT_FILL_VALUE, - order: MemoryOrder | None = None, - attributes: dict[str, JSON] | None = None, - chunk_key_encoding: ChunkKeyEncodingLike | None = None, - dimension_names: DimensionNamesLike = None, - storage_options: dict[str, Any] | None = None, - overwrite: bool = False, - config: ArrayConfigLike | None = None, - write_data: bool = True, - ) -> AnyArray: - """Create an array within this group. - - This method lightly wraps [zarr.core.array.create_array][]. - - Parameters - ---------- - name : str - The name of the array relative to the group. If `path` is `None`, the array will be located - at the root of the store. - shape : ShapeLike, optional - Shape of the array. Must be `None` if `data` is provided. - dtype : ZDTypeLike | None - Data type of the array. Must be `None` if `data` is provided. - data : np.ndarray, optional - Array-like data to use for initializing the array. If this parameter is provided, the - `shape` and `dtype` parameters must be `None`. - chunks : tuple[int, ...] | Sequence[Sequence[int]] | Literal["auto"], default="auto" - Chunk shape of the array. - If chunks is "auto", a chunk shape is guessed based on the shape of the array and the dtype. - A nested list of per-dimension edge sizes creates a rectilinear grid. - Rectilinear chunk grids are experimental and must be explicitly enabled - with `zarr.config.set({'array.rectilinear_chunks': True})` while the - feature is stabilizing. - shards : tuple[int, ...], optional - Shard shape of the array. The default value of `None` results in no sharding at all. - filters : Iterable[Codec] | Literal["auto"], optional - Iterable of filters to apply to each chunk of the array, in order, before serializing that - chunk to bytes. - - For Zarr format 3, a "filter" is a codec that takes an array and returns an array, - and these values must be instances of [`zarr.abc.codec.ArrayArrayCodec`][], or a - dict representations of [`zarr.abc.codec.ArrayArrayCodec`][]. - - For Zarr format 2, a "filter" can be any numcodecs codec; you should ensure that the - order of your filters is consistent with the behavior of each filter. - - The default value of `"auto"` instructs Zarr to use a default based on the data - type of the array and the Zarr format specified. For all data types in Zarr V3, and most - data types in Zarr V2, the default filters are empty. The only cases where default filters - are not empty is when the Zarr format is 2, and the data type is a variable-length data type like - [`zarr.dtype.VariableLengthUTF8`][] or [`zarr.dtype.VariableLengthBytes`][]. In these cases, - the default filters contains a single element which is a codec specific to that particular data type. - - To create an array with no filters, provide an empty iterable or the value `None`. - compressors : Iterable[Codec], optional - List of compressors to apply to the array. Compressors are applied in order, and after any - filters are applied (if any are specified) and the data is serialized into bytes. - - For Zarr format 3, a "compressor" is a codec that takes a bytestream, and - returns another bytestream. Multiple compressors may be provided for Zarr format 3. - If no `compressors` are provided, a default set of compressors will be used. - These defaults can be changed by modifying the value of `array.v3_default_compressors` - in [`zarr.config`][zarr.config]. - Use `None` to omit default compressors. - - For Zarr format 2, a "compressor" can be any numcodecs codec. Only a single compressor may - be provided for Zarr format 2. - If no `compressor` is provided, a default compressor will be used. - This default can be changed by modifying the value of `array.v2_default_compressor` - in [`zarr.config`][zarr.config]. - Use `None` to omit the default compressor. - compressor : Codec, optional - Deprecated in favor of `compressors`. - serializer : dict[str, JSON] | ArrayBytesCodec, optional - Array-to-bytes codec to use for encoding the array data. - Zarr format 3 only. Zarr format 2 arrays use implicit array-to-bytes conversion. - If no `serializer` is provided, a default serializer will be used. - These defaults can be changed by modifying the value of `array.v3_default_serializer` - in [`zarr.config`][zarr.config]. - fill_value : Any, optional - Fill value for the array. - order : {"C", "F"}, optional - The memory order of the array (default is "C"). - For Zarr format 2, this parameter sets the memory order of the array. - For Zarr format 3, this parameter is deprecated, because memory order - is a runtime parameter for Zarr format 3 arrays. The recommended way to specify the memory - order for Zarr format 3 arrays is via the `config` parameter, e.g. `{'config': 'C'}`. - If no `order` is provided, a default order will be used. - This default can be changed by modifying the value of `array.order` in [`zarr.config`][zarr.config]. - attributes : dict, optional - Attributes for the array. - chunk_key_encoding : ChunkKeyEncodingLike, optional - A specification of how the chunk keys are represented in storage. - For Zarr format 3, the default is `{"name": "default", "separator": "/"}`. - For Zarr format 2, the default is `{"name": "v2", "separator": "."}`. - dimension_names : Iterable[str], optional - The names of the dimensions (default is None). - Zarr format 3 only. Zarr format 2 arrays should not use this parameter. - storage_options : dict, optional - If using an fsspec URL to create the store, these will be passed to the backend implementation. - Ignored otherwise. - overwrite : bool, default False - Whether to overwrite an array with the same name in the store, if one exists. - If `True`, any existing keys under that path are deleted first. - config : ArrayConfigLike, optional - Runtime configuration for the array. - write_data : bool - If a pre-existing array-like object was provided to this function via the `data` parameter - then `write_data` determines whether the values in that array-like object should be - written to the Zarr array created by this function. If `write_data` is `False`, then the - array will be left empty. - - Returns - ------- - AsyncArray - """ - compressors = _parse_deprecated_compressor( - compressor, compressors, zarr_format=self.metadata.zarr_format - ) - return Array( - self._sync( - self._async_group.create_array( - name=name, - shape=shape, - dtype=dtype, - data=data, - chunks=chunks, - shards=shards, - fill_value=fill_value, - attributes=attributes, - chunk_key_encoding=chunk_key_encoding, - compressors=compressors, - serializer=serializer, - dimension_names=dimension_names, - order=order, - filters=filters, - overwrite=overwrite, - storage_options=storage_options, - config=config, - write_data=write_data, - ) - ) - ) - - def require_array( - self, - name: str, - *, - shape: ShapeLike, - config: ArrayConfigLike | None = None, - **kwargs: Any, - ) -> AnyArray: - """Obtain an array, creating if it doesn't exist. - - Other `kwargs` are as per [zarr.Group.create_array][]. - - Parameters - ---------- - name : str - Array name. - config : ArrayConfigLike or None, default=None - Runtime configuration for the array, whether it is created or already exists. - Keys not specified are taken from the global configuration. - **kwargs : - See [zarr.Group.create_array][]. - - Returns - ------- - a : Array - """ - return Array( - self._sync(self._async_group.require_array(name, shape=shape, config=config, **kwargs)) - ) - - def empty(self, *, name: str, shape: tuple[int, ...], **kwargs: Any) -> AnyArray: - """Create an empty array with the specified shape in this Group. The contents will be filled with - the array's fill value or zeros if no fill value is provided. - - Parameters - ---------- - name : str - Name of the array. - shape : int or tuple of int - Shape of the empty array. - **kwargs - Keyword arguments passed to [zarr.api.asynchronous.create][]. - - Notes - ----- - The contents of an empty Zarr array are not defined. On attempting to - retrieve data from an empty Zarr array, any values may be returned, - and these are not guaranteed to be stable from one access to the next. - """ - return Array(self._sync(self._async_group.empty(name=name, shape=shape, **kwargs))) - - def zeros(self, *, name: str, shape: tuple[int, ...], **kwargs: Any) -> AnyArray: - """Create an array, with zero being used as the default value for uninitialized portions of the array. - - Parameters - ---------- - name : str - Name of the array. - shape : int or tuple of int - Shape of the empty array. - **kwargs - Keyword arguments passed to [zarr.api.asynchronous.create][]. - - Returns - ------- - Array - The new array. - """ - return Array(self._sync(self._async_group.zeros(name=name, shape=shape, **kwargs))) - - def ones(self, *, name: str, shape: tuple[int, ...], **kwargs: Any) -> AnyArray: - """Create an array, with one being used as the default value for uninitialized portions of the array. - - Parameters - ---------- - name : str - Name of the array. - shape : int or tuple of int - Shape of the empty array. - **kwargs - Keyword arguments passed to [zarr.api.asynchronous.create][]. - - Returns - ------- - Array - The new array. - """ - return Array(self._sync(self._async_group.ones(name=name, shape=shape, **kwargs))) - - def full( - self, *, name: str, shape: tuple[int, ...], fill_value: Any | None, **kwargs: Any - ) -> AnyArray: - """Create an array, with "fill_value" being used as the default value for uninitialized portions of the array. - - Parameters - ---------- - name : str - Name of the array. - shape : int or tuple of int - Shape of the empty array. - fill_value : scalar - Value to fill the array with. - **kwargs - Keyword arguments passed to [zarr.api.asynchronous.create][]. - - Returns - ------- - Array - The new array. - """ - return Array( - self._sync( - self._async_group.full(name=name, shape=shape, fill_value=fill_value, **kwargs) - ) - ) - - def empty_like(self, *, name: str, data: async_api.ArrayLike, **kwargs: Any) -> AnyArray: - """Create an empty sub-array like `data`. The contents will be filled - with the array's fill value or zeros if no fill value is provided. - - Parameters - ---------- - name : str - Name of the array. - data : array-like - The array to create an empty array like. - **kwargs - Keyword arguments passed to [zarr.api.asynchronous.create][]. - - Returns - ------- - Array - The new array. - - Notes - ----- - The contents of an empty Zarr array are not defined. On attempting to - retrieve data from an empty Zarr array, any values may be returned, - and these are not guaranteed to be stable from one access to the next. - """ - return Array(self._sync(self._async_group.empty_like(name=name, data=data, **kwargs))) - - def zeros_like(self, *, name: str, data: async_api.ArrayLike, **kwargs: Any) -> AnyArray: - """Create a sub-array of zeros like `data`. - - Parameters - ---------- - name : str - Name of the array. - data : array-like - The array to create the new array like. - **kwargs - Keyword arguments passed to [zarr.api.asynchronous.create][]. - - Returns - ------- - Array - The new array. - """ - - return Array(self._sync(self._async_group.zeros_like(name=name, data=data, **kwargs))) - - def ones_like(self, *, name: str, data: async_api.ArrayLike, **kwargs: Any) -> AnyArray: - """Create a sub-array of ones like `data`. - - Parameters - ---------- - name : str - Name of the array. - data : array-like - The array to create the new array like. - **kwargs - Keyword arguments passed to [zarr.api.asynchronous.create][]. - - Returns - ------- - Array - The new array. - """ - return Array(self._sync(self._async_group.ones_like(name=name, data=data, **kwargs))) - - def full_like(self, *, name: str, data: async_api.ArrayLike, **kwargs: Any) -> AnyArray: - """Create a sub-array like `data` filled with the `fill_value` of `data` . - - Parameters - ---------- - name : str - Name of the array. - data : array-like - The array to create the new array like. - **kwargs - Keyword arguments passed to [zarr.api.asynchronous.create][]. - - Returns - ------- - Array - The new array. - """ - return Array(self._sync(self._async_group.full_like(name=name, data=data, **kwargs))) - - def move(self, source: str, dest: str) -> None: - """Move a sub-group or sub-array from one path to another. - - Notes - ----- - Not implemented - """ - return self._sync(self._async_group.move(source, dest)) - - -async def create_hierarchy( - *, - store: Store, - nodes: dict[str, GroupMetadata | ArrayV2Metadata | ArrayV3Metadata], - overwrite: bool = False, -) -> AsyncIterator[tuple[str, AsyncGroup | AnyAsyncArray]]: - """ - Create a complete zarr hierarchy from a collection of metadata objects. - - This function will parse its input to ensure that the hierarchy is complete. Any implicit groups - will be inserted as needed. For example, an input like - ```{'a/b': GroupMetadata}``` will be parsed to - ```{'': GroupMetadata, 'a': GroupMetadata, 'b': Groupmetadata}``` - - After input parsing, this function then creates all the nodes in the hierarchy concurrently. - - Arrays and Groups are yielded in the order they are created. This order is not stable and - should not be relied on. - - Parameters - ---------- - store : Store - The storage backend to use. - nodes : dict[str, GroupMetadata | ArrayV3Metadata | ArrayV2Metadata] - A dictionary defining the hierarchy. The keys are the paths of the nodes in the hierarchy, - relative to the root of the ``Store``. The root of the store can be specified with the empty - string ``''``. The values are instances of ``GroupMetadata`` or ``ArrayMetadata``. Note that - all values must have the same ``zarr_format`` -- it is an error to mix zarr versions in the - same hierarchy. - - Leading "/" characters from keys will be removed. - overwrite : bool - Whether to overwrite existing nodes. Defaults to ``False``, in which case an error is - raised instead of overwriting an existing array or group. - - This function will not erase an existing group unless that group is explicitly named in - ``nodes``. If ``nodes`` defines implicit groups, e.g. ``{`'a/b/c': GroupMetadata}``, and a - group already exists at path ``a``, then this function will leave the group at ``a`` as-is. - - Yields - ------ - tuple[str, AsyncGroup | AsyncArray] - Yields (path, node) pairs, in the order the nodes were created. - - Examples - -------- - >>> async def example(): - ... from zarr.api.asynchronous import create_hierarchy - ... from zarr.core.group import GroupMetadata - ... from zarr.storage import MemoryStore - ... - ... store = MemoryStore() - ... nodes = {'a': GroupMetadata(attributes={'name': 'leaf'})} - ... return sorted([x async for x in create_hierarchy(store=store, nodes=nodes)]) - - >>> import asyncio - >>> asyncio.run(example()) - [('', ), ('a', )] - """ - - # normalize the keys to be valid paths - nodes_normed_keys = _normalize_path_keys(nodes) - - # ensure that all nodes have the same zarr_format, and add implicit groups as needed - nodes_parsed = _parse_hierarchy_dict(data=nodes_normed_keys) - redundant_implicit_groups = [] - to_delete_keys: list[str] = [] - - # empty hierarchies should be a no-op - if len(nodes_parsed) > 0: - # figure out which zarr format we are using - zarr_format = next(iter(nodes_parsed.values())).zarr_format - - # check which implicit groups will require materialization - implicit_group_keys = tuple( - filter(lambda k: isinstance(nodes_parsed[k], ImplicitGroupMarker), nodes_parsed) - ) - # read potential group metadata for each implicit group - maybe_extant_group_coros = ( - _read_group_metadata(store, k, zarr_format=zarr_format) for k in implicit_group_keys - ) - maybe_extant_groups = await asyncio.gather( - *maybe_extant_group_coros, return_exceptions=True - ) - - for key, value in zip(implicit_group_keys, maybe_extant_groups, strict=True): - if isinstance(value, BaseException): - if isinstance(value, FileNotFoundError): - # this is fine -- there was no group there, so we will create one - pass - else: - raise value - else: - # a loop exists already at ``key``, so we can avoid creating anything there - redundant_implicit_groups.append(key) - - if overwrite: - # we will remove any nodes that collide with arrays and non-implicit groups defined in - # nodes - to_delete_keys = [k for k in nodes_parsed if k not in implicit_group_keys] - else: - # This type is long. - coros: ( - Generator[Coroutine[Any, Any, ArrayV2Metadata | GroupMetadata], None, None] - | Generator[Coroutine[Any, Any, ArrayV3Metadata | GroupMetadata], None, None] - ) - if zarr_format == 2: - coros = (_read_metadata_v2(store=store, path=key) for key in nodes_parsed) - elif zarr_format == 3: - coros = (_read_metadata_v3(store=store, path=key) for key in nodes_parsed) - else: # pragma: no cover - raise ValueError(f"Invalid zarr_format: {zarr_format}") # pragma: no cover - - extant_node_query = dict( - zip( - nodes_parsed.keys(), - await asyncio.gather(*coros, return_exceptions=True), - strict=False, - ) - ) - # iterate over the existing arrays / groups and figure out which of them conflict - # with the arrays / groups we want to create - for key, extant_node in extant_node_query.items(): - proposed_node = nodes_parsed[key] - if isinstance(extant_node, BaseException): - if isinstance(extant_node, FileNotFoundError): - # ignore FileNotFoundError, because they represent nodes we can safely create - pass - else: - # Any other exception is a real error - raise extant_node - else: - # this is a node that already exists, but a node with the same key was specified - # in nodes_parsed. - if isinstance(extant_node, GroupMetadata): - # a group already exists where we want to create a group - if isinstance(proposed_node, ImplicitGroupMarker): - # we have proposed an implicit group, which is OK -- we will just skip - # creating this particular metadata document - redundant_implicit_groups.append(key) - else: - # we have proposed an explicit group, which is an error, given that a - # group already exists. - msg = f"A group exists in store {store!r} at path {key!r}." - raise ContainsGroupError(msg) - elif isinstance(extant_node, ArrayV2Metadata | ArrayV3Metadata): - # we are trying to overwrite an existing array. this is an error. - msg = f"An array exists in store {store!r} at path {key!r}." - raise ContainsArrayError(msg) - - nodes_explicit: dict[str, GroupMetadata | ArrayV2Metadata | ArrayV3Metadata] = {} - - for k, v in nodes_parsed.items(): - if k not in redundant_implicit_groups: - if isinstance(v, ImplicitGroupMarker): - nodes_explicit[k] = GroupMetadata(zarr_format=v.zarr_format) - else: - nodes_explicit[k] = v - - # Build and encode every node before deleting anything: a node that cannot be built - # or whose metadata cannot be stored then fails with the store untouched. - built, documents = _prepare_nodes(store, nodes_explicit) - await asyncio.gather(*(store.delete_dir(key) for key in to_delete_keys)) - async for key, node in _store_nodes(store, nodes_explicit, built, documents): - yield key, node - - -async def create_nodes( - *, - store: Store, - nodes: dict[str, GroupMetadata | ArrayV2Metadata | ArrayV3Metadata], -) -> AsyncIterator[tuple[str, AsyncGroup | AnyAsyncArray]]: - """Create a collection of arrays and / or groups concurrently. - - Note: no attempt is made to validate that these arrays and / or groups collectively form a - valid Zarr hierarchy. It is the responsibility of the caller of this function to ensure that - the ``nodes`` parameter satisfies any correctness constraints. - - Parameters - ---------- - store : Store - The storage backend to use. - nodes : dict[str, GroupMetadata | ArrayV3Metadata | ArrayV2Metadata] - A dictionary defining the hierarchy. The keys are the paths of the nodes - in the hierarchy, and the values are the metadata of the nodes. The - metadata must be either an instance of GroupMetadata, ArrayV3Metadata - or ArrayV2Metadata. - - Yields - ------ - AsyncGroup | AsyncArray - The created nodes in the order they are created. - """ - async for key, node in _store_nodes(store, nodes, *_prepare_nodes(store, nodes)): - yield key, node - - -def _prepare_nodes( - store: Store, nodes: Mapping[str, GroupMetadata | ArrayV2Metadata | ArrayV3Metadata] -) -> tuple[dict[str, AsyncGroup | AnyAsyncArray], dict[str, Buffer]]: - """The array or group each of `nodes` describes, at its path in `store`, and the - metadata documents of `nodes`, by their keys in the store: every node is built and - encoded before anything is stored.""" - for path, metadata in nodes.items(): - validate_node_path(path, metadata.zarr_format) - - built = { - path: _build_node(store=store, path=path, metadata=meta) for path, meta in nodes.items() - } - documents = { - _join_paths([path, key]): value - for path, metadata in nodes.items() - for key, value in encode_documents(StorePath(store, path), metadata).items() - } - return built, documents - - -async def _store_nodes( - store: Store, - nodes: Mapping[str, GroupMetadata | ArrayV2Metadata | ArrayV3Metadata], - built: Mapping[str, AsyncGroup | AnyAsyncArray], - documents: Mapping[str, Buffer], -) -> AsyncIterator[tuple[str, AsyncGroup | AnyAsyncArray]]: - """Store the `documents` encoded from `nodes` and yield the nodes built from them, as - `_prepare_nodes` returns them (see `create_nodes`).""" - # Note: the only way to alter this value is via the config. If that's undesirable for some reason, - # then we should consider adding a keyword argument to this function - semaphore = asyncio.Semaphore(config.get("async.concurrency")) - create_tasks = [ - _set_return_key(store=store, key=key, value=value, semaphore=semaphore) - for key, value in documents.items() - ] - - created_object_keys = [] - - for coro in asyncio.as_completed(create_tasks): - created_key = await coro - # we need this to track which metadata documents were written so that we can yield a - # complete v2 Array / Group class after both .zattrs and the metadata JSON was created. - created_object_keys.append(created_key) - - # get the node name from the object key - if len(created_key.split("/")) == 1: - # this is the root node - meta_out = nodes[""] - node_name = "" - else: - # turn "foo/" into "foo" - node_name = created_key[: created_key.rfind("/")] - meta_out = nodes[node_name] - if meta_out.zarr_format == 3: - yield node_name, built[node_name] - else: - # For zarr v2 - # we only want to yield when both the metadata and attributes are created - # so we track which keys have been created, and wait for both the meta key and - # the attrs key to be created before yielding back the AsyncArray / AsyncGroup - - attrs_done = _join_paths([node_name, ZATTRS_JSON]) in created_object_keys - - if isinstance(meta_out, GroupMetadata): - meta_done = _join_paths([node_name, ZGROUP_JSON]) in created_object_keys - else: - meta_done = _join_paths([node_name, ZARRAY_JSON]) in created_object_keys - - if meta_done and attrs_done: - yield node_name, built[node_name] - - continue - - -def _get_roots( - data: Iterable[str], -) -> tuple[str, ...]: - """ - Return the keys of the root(s) of the hierarchy. A root is a key with the fewest number of - path segments. - """ - if "" in data: - return ("",) - keys_split = sorted((key.split("/") for key in data), key=len) - groups: defaultdict[int, list[str]] = defaultdict(list) - for key_split in keys_split: - groups[len(key_split)].append("/".join(key_split)) - return tuple(groups[min(groups.keys())]) - - -def _parse_hierarchy_dict( - *, - data: Mapping[str, ImplicitGroupMarker | GroupMetadata | ArrayV2Metadata | ArrayV3Metadata], -) -> dict[str, ImplicitGroupMarker | GroupMetadata | ArrayV2Metadata | ArrayV3Metadata]: - """ - Take an input with type Mapping[str, ArrayMetadata | GroupMetadata] and parse it into - a dict of str: node pairs that models a valid, complete Zarr hierarchy. - - If the input represents a complete Zarr hierarchy, i.e. one with no implicit groups, - then return a dict with the exact same data as the input. - - Otherwise, return a dict derived from the input with GroupMetadata inserted as needed to make - the hierarchy complete. - - For example, an input of {'a/b': ArrayMetadata} is incomplete, because it references two - groups (the root group '' and a group at 'a') that are not specified in the input. Applying this function - to that input will result in a return value of - {'': GroupMetadata, 'a': GroupMetadata, 'a/b': ArrayMetadata}, i.e. the implied groups - were added. - - The input is also checked for the following conditions; an error is raised if any are violated: - - - No arrays can contain group or arrays (i.e., all arrays must be leaf nodes). - - All arrays and groups must have the same ``zarr_format`` value. - - This function ensures that the input is transformed into a specification of a complete and valid - Zarr hierarchy. - """ - - # ensure that all nodes have the same zarr format - data_purified = _ensure_consistent_zarr_format(data) - - # ensure that keys are normalized to zarr paths - data_normed_keys = _normalize_path_keys(data_purified) - - # insert an implicit root group if a root was not specified - # but not if an empty dict was provided, because any empty hierarchy has no nodes - if len(data_normed_keys) > 0 and "" not in data_normed_keys: - z_format = next(iter(data_normed_keys.values())).zarr_format - data_normed_keys = data_normed_keys | {"": ImplicitGroupMarker(zarr_format=z_format)} - - out: dict[str, GroupMetadata | ArrayV2Metadata | ArrayV3Metadata] = {**data_normed_keys} - - for k, v in data_normed_keys.items(): - key_split = k.split("/") - - # get every parent path - *subpaths, _ = accumulate(key_split, lambda a, b: _join_paths([a, b])) - - for subpath in subpaths: - # If a component is not already in the output dict, add ImplicitGroupMetadata - if subpath not in out: - out[subpath] = ImplicitGroupMarker(zarr_format=v.zarr_format) - else: - if not isinstance(out[subpath], GroupMetadata | ImplicitGroupMarker): - msg = ( - f"The node at {subpath} contains other nodes, but it is not a Zarr group. " - "This is invalid. Only Zarr groups can contain other nodes." - ) - raise ValueError(msg) - return out - - -def _ensure_consistent_zarr_format( - data: Mapping[str, GroupMetadata | ArrayV2Metadata | ArrayV3Metadata], -) -> Mapping[str, GroupMetadata | ArrayV2Metadata] | Mapping[str, GroupMetadata | ArrayV3Metadata]: - """ - Ensure that all values of the input dict have the same zarr format. If any do not, - then a value error is raised. - """ - observed_zarr_formats: dict[ZarrFormat, list[str]] = {2: [], 3: []} - - for k, v in data.items(): - observed_zarr_formats[v.zarr_format].append(k) - - if len(observed_zarr_formats[2]) > 0 and len(observed_zarr_formats[3]) > 0: - msg = ( - "Got data with both Zarr v2 and Zarr v3 nodes, which is invalid. " - f"The following keys map to Zarr v2 nodes: {observed_zarr_formats.get(2)}. " - f"The following keys map to Zarr v3 nodes: {observed_zarr_formats.get(3)}." - "Ensure that all nodes have the same Zarr format." - ) - raise ValueError(msg) - - return cast( - "Mapping[str, GroupMetadata | ArrayV2Metadata] | Mapping[str, GroupMetadata | ArrayV3Metadata]", - data, - ) - - -async def _getitem_semaphore( - node: AsyncGroup, key: str, semaphore: asyncio.Semaphore | None -) -> AnyAsyncArray | AsyncGroup: - """ - Wrap Group.getitem with an optional semaphore. - - If the semaphore parameter is an - asyncio.Semaphore instance, then the getitem operation is performed inside an async context - manager provided by that semaphore. If the semaphore parameter is None, then getitem is invoked - without a context manager. - """ - if semaphore is not None: - async with semaphore: - return await node.getitem(key) - else: - return await node.getitem(key) - - -async def _iter_members( - node: AsyncGroup, - skip_keys: tuple[str, ...], - semaphore: asyncio.Semaphore | None, -) -> AsyncGenerator[tuple[str, AnyAsyncArray | AsyncGroup], None]: - """ - Iterate over the arrays and groups contained in a group. - - Parameters - ---------- - node : AsyncGroup - The group to traverse. - skip_keys : tuple[str, ...] - A tuple of keys to skip when iterating over the possible members of the group. - semaphore : asyncio.Semaphore | None - An optional semaphore to use for concurrency control. - - Yields - ------ - tuple[str, AnyAsyncArray | AsyncGroup] - """ - - # retrieve keys from storage - keys = [key async for key in node.store.list_dir(node.path)] - keys_filtered = tuple(filter(lambda v: v not in skip_keys, keys)) - - node_tasks = tuple( - asyncio.create_task(_getitem_semaphore(node, key, semaphore), name=key) - for key in keys_filtered - ) - - for fetched_node_coro in asyncio.as_completed(node_tasks): - try: - fetched_node = await fetched_node_coro - except KeyError as e: - # keyerror is raised when `key` names an object (in the object storage sense), - # as opposed to a prefix, in the store under the prefix associated with this group - # in which case `key` cannot be the name of a sub-array or sub-group. - warnings.warn( - f"Object at {e.args[0]} is not recognized as a component of a Zarr hierarchy.", - ZarrUserWarning, - stacklevel=1, - ) - continue - match fetched_node: - case AsyncArray() | AsyncGroup(): - yield fetched_node.basename, fetched_node - case _: - raise ValueError(f"Unexpected type: {type(fetched_node)}") - - -async def _iter_members_deep( - group: AsyncGroup, - *, - max_depth: int | None, - skip_keys: tuple[str, ...], - semaphore: asyncio.Semaphore | None = None, - use_consolidated_for_children: bool = True, -) -> AsyncGenerator[tuple[str, AnyAsyncArray | AsyncGroup], None]: - """ - Iterate over the arrays and groups contained in a group, and optionally the - arrays and groups contained in those groups. - - Parameters - ---------- - group : AsyncGroup - The group to traverse. - max_depth : int | None - The maximum depth of recursion. - skip_keys : tuple[str, ...] - A tuple of keys to skip when iterating over the possible members of the group. - semaphore : asyncio.Semaphore | None - An optional semaphore to use for concurrency control. - use_consolidated_for_children : bool, default True - Whether to use the consolidated metadata of child groups loaded - from the store. Note that this only affects groups loaded from the - store. If the current Group already has consolidated metadata, it - will always be used. - - Yields - ------ - tuple[str, AnyAsyncArray | AsyncGroup] - """ - - to_recurse = {} - do_recursion = max_depth is None or max_depth > 0 - - if max_depth is None: - new_depth = None - else: - new_depth = max_depth - 1 - async for name, node in _iter_members(group, skip_keys=skip_keys, semaphore=semaphore): - is_group = isinstance(node, AsyncGroup) - if ( - is_group - and not use_consolidated_for_children - and node.metadata.consolidated_metadata is not None - ): - node = cast("AsyncGroup", node) - # We've decided not to trust consolidated metadata at this point, because we're - # reconsolidating the metadata, for example. - node = replace(node, metadata=replace(node.metadata, consolidated_metadata=None)) - yield name, node - if is_group and do_recursion: - node = cast("AsyncGroup", node) - to_recurse[name] = _iter_members_deep( - node, max_depth=new_depth, skip_keys=skip_keys, semaphore=semaphore - ) - - for prefix, subgroup_iter in to_recurse.items(): - async for name, node in subgroup_iter: - key = f"{prefix}/{name}".lstrip("/") - yield key, node - - -async def _read_metadata_v3(store: Store, path: str) -> ArrayV3Metadata | GroupMetadata: - """ - Given a store_path, return ArrayV3Metadata or GroupMetadata defined by the metadata - document stored at store_path.path / zarr.json. If no such document is found, raise a - FileNotFoundError. - """ - zarr_json_bytes = await store.get( - _join_paths([path, ZARR_JSON]), prototype=default_buffer_prototype() - ) - if zarr_json_bytes is None: - raise FileNotFoundError(path) - return _build_metadata_v3( - buffer_to_json_object(zarr_json_bytes), path=str(StorePath(store, path)) - ) - - -async def _read_metadata_v2(store: Store, path: str) -> ArrayV2Metadata | GroupMetadata: - """ - Given a store_path, return ArrayV2Metadata or GroupMetadata defined by the metadata - document stored at store_path.path / (.zgroup | .zarray). If no such document is found, - raise a FileNotFoundError. - """ - # TODO: consider first fetching array metadata, and only fetching group metadata when we don't - # find an array - zarray_bytes, zgroup_bytes, zattrs_bytes = await asyncio.gather( - store.get(_join_paths([path, ZARRAY_JSON]), prototype=default_buffer_prototype()), - store.get(_join_paths([path, ZGROUP_JSON]), prototype=default_buffer_prototype()), - store.get(_join_paths([path, ZATTRS_JSON]), prototype=default_buffer_prototype()), - ) - - zattrs: dict[str, JSON] - if zattrs_bytes is None: - zattrs = {} - else: - zattrs = buffer_to_json_object(zattrs_bytes) - - # TODO: decide how to handle finding both array and group metadata. The spec does not seem to - # consider this situation. A practical approach would be to ignore that combination, and only - # return the array metadata. - if zarray_bytes is not None: - zmeta = buffer_to_json_object(zarray_bytes) - else: - if zgroup_bytes is None: - # neither .zarray or .zgroup were found results in KeyError - raise FileNotFoundError(path) - else: - zmeta = buffer_to_json_object(zgroup_bytes) - - return _build_metadata_v2(zmeta, zattrs, path=str(StorePath(store, path))) - - -async def _read_group_metadata_v2(store: Store, path: str) -> GroupMetadata: - """ - Read group metadata or error - """ - meta = await _read_metadata_v2(store=store, path=path) - if not isinstance(meta, GroupMetadata): - raise FileNotFoundError(f"Group metadata was not found in {store} at {path}") - return meta - - -async def _read_group_metadata_v3(store: Store, path: str) -> GroupMetadata: - """ - Read group metadata or error - """ - meta = await _read_metadata_v3(store=store, path=path) - if not isinstance(meta, GroupMetadata): - raise FileNotFoundError(f"Group metadata was not found in {store} at {path}") - return meta - - -async def _read_group_metadata( - store: Store, path: str, *, zarr_format: ZarrFormat -) -> GroupMetadata: - if zarr_format == 2: - return await _read_group_metadata_v2(store=store, path=path) - return await _read_group_metadata_v3(store=store, path=path) - - -def _build_metadata_v3( - zarr_json: dict[str, JSON], *, path: str | None = None -) -> ArrayV3Metadata | GroupMetadata: - """ - Convert a dict representation of Zarr V3 metadata into the corresponding metadata class. - """ - if "node_type" not in zarr_json: - msg = "Required key 'node_type' is missing from the provided metadata document." - raise MetadataValidationError(msg) - match zarr_json: - case {"node_type": "array"}: - return ArrayV3Metadata.from_dict(zarr_json, path=path) - case {"node_type": "group"}: - return GroupMetadata.from_dict(zarr_json, path=path) - case _: # pragma: no cover - raise ValueError( - "invalid value for `node_type` key in metadata document" - ) # pragma: no cover - - -def _build_metadata_v2( - zarr_json: dict[str, JSON], attrs_json: dict[str, JSON], *, path: str | None = None -) -> ArrayV2Metadata | GroupMetadata: - """ - Convert a dict representation of Zarr V2 metadata into the corresponding metadata class. - """ - match zarr_json: - case {"shape": _}: - return ArrayV2Metadata.from_dict(zarr_json | {"attributes": attrs_json}, path=path) - case _: # pragma: no cover - return GroupMetadata.from_dict(zarr_json | {"attributes": attrs_json}, path=path) - - -@overload -def _build_node(*, store: Store, path: str, metadata: ArrayV2Metadata) -> AsyncArrayV2: ... - - -@overload -def _build_node(*, store: Store, path: str, metadata: ArrayV3Metadata) -> AsyncArrayV3: ... - - -@overload -def _build_node(*, store: Store, path: str, metadata: GroupMetadata) -> AsyncGroup: ... - - -def _build_node( - *, store: Store, path: str, metadata: ArrayV3Metadata | ArrayV2Metadata | GroupMetadata -) -> AnyAsyncArray | AsyncGroup: - """ - Take a metadata object and return a node (AsyncArray or AsyncGroup). - """ - store_path = StorePath(store=store, path=path) - match metadata: - case ArrayV2Metadata() | ArrayV3Metadata(): - return AsyncArray(metadata, store_path=store_path) - case GroupMetadata(): - return AsyncGroup(metadata, store_path=store_path) - case _: # pragma: no cover - raise ValueError(f"Unexpected metadata type: {type(metadata)}") # pragma: no cover - - -async def _get_node_v2(store: Store, path: str) -> AsyncArrayV2 | AsyncGroup: - """ - Read a Zarr v2 AsyncArray or AsyncGroup from a path in a Store. - - Parameters - ---------- - store : Store - The store-like object to read from. - path : str - The path to the node to read. - - Returns - ------- - AsyncArray | AsyncGroup - """ - metadata = await _read_metadata_v2(store=store, path=path) - return _build_node(store=store, path=path, metadata=metadata) - - -async def _get_node_v3(store: Store, path: str) -> AsyncArrayV3 | AsyncGroup: - """ - Read a Zarr v3 AsyncArray or AsyncGroup from a path in a Store. - - Parameters - ---------- - store : Store - The store-like object to read from. - path : str - The path to the node to read. - - Returns - ------- - AsyncArray | AsyncGroup - """ - metadata = await _read_metadata_v3(store=store, path=path) - return _build_node(store=store, path=path, metadata=metadata) - - -async def get_node(store: Store, path: str, zarr_format: ZarrFormat) -> AnyAsyncArray | AsyncGroup: - """ - Get an AsyncArray or AsyncGroup from a path in a Store. - - Parameters - ---------- - store : Store - The store-like object to read from. - path : str - The path to the node to read. - zarr_format : {2, 3} - The zarr format of the node to read. - - Returns - ------- - AsyncArray | AsyncGroup - """ - - match zarr_format: - case 2: - return await _get_node_v2(store=store, path=path) - case 3: - return await _get_node_v3(store=store, path=path) - case _: # pragma: no cover - raise ValueError(f"Unexpected zarr format: {zarr_format}") # pragma: no cover - - -async def _set_return_key( - *, store: Store, key: str, value: Buffer, semaphore: asyncio.Semaphore | None = None -) -> str: - """ - Write a value to storage at the given key. The key is returned. - Useful when saving values via routines that return results in execution order, - like asyncio.as_completed, because in this case we need to know which key was saved in order - to yield the right object to the caller. - - Parameters - ---------- - store : Store - The store to save the value to. - key : str - The key to save the value to. - value : Buffer - The value to save. - semaphore : asyncio.Semaphore | None - An optional semaphore to use to limit the number of concurrent writes. - """ - - if semaphore is not None: - async with semaphore: - await store.set(key, value) - else: - await store.set(key, value) - return key - - -async def create_rooted_hierarchy( - *, - store: Store, - nodes: dict[str, GroupMetadata | ArrayV2Metadata | ArrayV3Metadata], - overwrite: bool = False, -) -> AsyncGroup | AnyAsyncArray: - """ - Create an ``AsyncGroup`` or ``AsyncArray`` from a store and a dict of metadata documents. - This function ensures that its input contains a specification of a root node, - calls ``create_hierarchy`` to create nodes, and returns the root node of the hierarchy. - """ - roots = _get_roots(nodes.keys()) - if len(roots) != 1: - msg = ( - "The input does not specify a root node. " - "This function can only create hierarchies that contain a root node, which is " - "defined as a group that is ancestral to all the other arrays and " - "groups in the hierarchy, or a single array." - ) - raise ValueError(msg) - else: - root_key = roots[0] - - nodes_created = [ - x async for x in create_hierarchy(store=store, nodes=nodes, overwrite=overwrite) - ] - return dict(nodes_created)[root_key] +from __future__ import annotations + +import asyncio +import logging +import unicodedata +import warnings +from collections import defaultdict +from dataclasses import asdict, dataclass, field, replace +from itertools import accumulate +from typing import TYPE_CHECKING, Final, Literal, assert_never, cast, overload + +import numpy as np + +import zarr.api.asynchronous as async_api +from zarr.abc.metadata import Metadata +from zarr.abc.store import Store, set_or_delete +from zarr.core._info import GroupInfo +from zarr.core._json import buffer_to_json_object, json_to_buffer +from zarr.core.array import ( + DEFAULT_FILL_VALUE, + Array, + AsyncArray, + CompressorLike, + CompressorsLike, + FiltersLike, + SerializerLike, + ShardsLike, + _parse_deprecated_compressor, + create_array, +) +from zarr.core.array_spec import parse_array_config +from zarr.core.attributes import Attributes +from zarr.core.buffer import default_buffer_prototype +from zarr.core.common import ( + JSON, + ZARR_JSON, + ZARRAY_JSON, + ZATTRS_JSON, + ZGROUP_JSON, + ZMETADATA_V2_JSON, + ChunksLike, + DimensionNamesLike, + NodeType, + ShapeLike, + ZarrFormat, + parse_shapelike, + validate_node_path, +) +from zarr.core.config import config +from zarr.core.dtype import parse_data_type +from zarr.core.json_parse import parse_field +from zarr.core.metadata import ArrayV2Metadata, ArrayV3Metadata +from zarr.core.metadata.io import ( + encode_documents, + save_metadata, + save_new_metadata, + store_documents, +) +from zarr.core.metadata.v3 import AllowedExtraField, check_storable, parse_extra_fields +from zarr.core.sync import SyncMixin, sync +from zarr.errors import ( + ArrayNotFoundError, + ContainsArrayError, + ContainsGroupError, + GroupNotFoundError, + MetadataValidationError, + NodeTypeValidationError, + ZarrUserWarning, +) +from zarr.storage import StoreLike, StorePath +from zarr.storage._common import make_store_path +from zarr.storage._utils import _join_paths, _normalize_path_keys, normalize_path + +if TYPE_CHECKING: + from collections.abc import ( + AsyncGenerator, + AsyncIterator, + Coroutine, + Generator, + Iterable, + Iterator, + Mapping, + ) + from typing import Any + + from zarr.core.array_spec import ArrayConfigLike + from zarr.core.buffer import Buffer, BufferPrototype + from zarr.core.chunk_key_encodings import ChunkKeyEncodingLike + from zarr.core.common import MemoryOrder + from zarr.core.dtype import ZDTypeLike + from zarr.types import AnyArray, AnyAsyncArray, ArrayV2, ArrayV3, AsyncArrayV2, AsyncArrayV3 + +logger = logging.getLogger("zarr.group") + + +def parse_zarr_format(data: Any) -> ZarrFormat: + """Parse the zarr_format field from metadata.""" + return cast("ZarrFormat", parse_field(data, Literal[2, 3], "zarr_format")) + + +def parse_node_type(data: Any) -> NodeType: + """Parse the node_type field from metadata.""" + return cast( + "Literal['array', 'group']", + parse_field(data, Literal["array", "group"], "node_type", error=MetadataValidationError), + ) + + +# todo: convert None to empty dict +def parse_attributes(data: Any) -> dict[str, Any]: + """Parse the attributes field from metadata.""" + if data is None: + return {} + elif isinstance(data, dict) and all(isinstance(k, str) for k in data): + return dict(data) + msg = f"Expected dict with string keys. Got {type(data)} instead." + raise TypeError(msg) + + +@overload +def _parse_async_node(node: AsyncArrayV3) -> ArrayV3: ... + + +@overload +def _parse_async_node(node: AsyncArrayV2) -> ArrayV2: ... + + +@overload +def _parse_async_node(node: AsyncGroup) -> Group: ... + + +def _parse_async_node( + node: AnyAsyncArray | AsyncGroup, +) -> AnyArray | Group: + """Wrap an AsyncArray in an Array, or an AsyncGroup in a Group.""" + if isinstance(node, AsyncArray): + return Array(node) + elif isinstance(node, AsyncGroup): + return Group(node) + else: + raise TypeError(f"Unknown node type, got {type(node)}") + + +@dataclass(frozen=True) +class ConsolidatedMetadata: + """ + Consolidated Metadata for this Group. + + This stores the metadata of child nodes below this group. Any child groups + will have their consolidated metadata set appropriately. + """ + + metadata: dict[str, ArrayV2Metadata | ArrayV3Metadata | GroupMetadata] + kind: Literal["inline"] = "inline" + must_understand: Literal[False] = False + + def to_dict(self) -> dict[str, JSON]: + """The consolidated metadata document. An array read from a stored document that + had to be repaired is written as that document was stored, so every reader of + the consolidated metadata reads it as repaired again (see `mark_repaired`).""" + return { + "kind": self.kind, + "must_understand": self.must_understand, + "metadata": { + k: v.to_dict() + if isinstance(v, GroupMetadata) or v._stored_document is None + else dict(v._stored_document) + for k, v in sorted( + self.flattened_metadata.items(), + key=lambda item: ( + item[0].count("/"), + unicodedata.normalize("NFKC", item[0]).casefold(), + ), + ) + }, + } + + @classmethod + def from_dict(cls, data: dict[str, JSON], *, path: str | None = None) -> ConsolidatedMetadata: + """Read consolidated metadata, naming each member by its path under the group at + `path` (or relative to that group, when `path` is not given) in warnings.""" + data = dict(data) + + kind = data.get("kind") + if kind != "inline": + raise ValueError(f"Consolidated metadata kind='{kind}' is not supported.") + + raw_metadata = data.get("metadata") + if not isinstance(raw_metadata, dict): + raise TypeError(f"Unexpected type for 'metadata': {type(raw_metadata)}") + + metadata: dict[str, ArrayV2Metadata | ArrayV3Metadata | GroupMetadata] = {} + if raw_metadata: + for k, v in raw_metadata.items(): + member = k if path is None else _join_paths([path, k]) + if not isinstance(v, dict): + raise TypeError( + f"Invalid value for metadata items. key='{k}', type='{type(v).__name__}'" + ) + + # zarr_format is present in v2 and v3. + zarr_format = parse_zarr_format(v["zarr_format"]) + + if zarr_format == 3: + node_type = parse_node_type(v.get("node_type", None)) + if node_type == "group": + metadata[k] = GroupMetadata.from_dict(v, path=member) + elif node_type == "array": + metadata[k] = ArrayV3Metadata.from_dict(v, path=member) + else: + assert_never(node_type) + elif zarr_format == 2: + if "shape" in v: + metadata[k] = ArrayV2Metadata.from_dict(v, path=member) + else: + metadata[k] = GroupMetadata.from_dict(v, path=member) + else: + assert_never(zarr_format) + + cls._flat_to_nested(metadata) + + return cls(metadata=metadata) + + @staticmethod + def _flat_to_nested( + metadata: dict[str, ArrayV2Metadata | ArrayV3Metadata | GroupMetadata], + ) -> None: + """ + Convert a flat metadata representation to a nested one. + + Notes + ----- + Flat metadata is used when persisting the consolidated metadata. The keys + include the full path, not just the node name. The key prefixes can be + used to determine which nodes are children of which other nodes. + + Nested metadata is used in-memory. The outermost level will only have the + *immediate* children of the Group. All nested child groups will be stored + under the consolidated metadata of their immediate parent. + """ + # We have a flat mapping from {k: v} where the keys include the *full* + # path segment: + # { + # "/a/b": { group_metadata }, + # "/a/b/array-0": { array_metadata }, + # "/a/b/array-1": { array_metadata }, + # } + # + # We want to reorganize the metadata such that each Group contains the + # array metadata of its immediate children. + # In the example, the group at `/a/b` will have consolidated metadata + # for its children `array-0` and `array-1`. + + # Group keys by their parent path. This must not rely on same-parent keys + # being adjacent: the persisted key order is arbitrary, so accumulate + # instead of using itertools.groupby, which only groups consecutive runs. + grouped: dict[str, list[str]] = defaultdict(list) + for k in sorted(metadata, key=lambda k: k.count("/")): + grouped[k.rsplit("/", 1)[0]].append(k) + + # we go top down and directly manipulate metadata. + for key, children_keys in grouped.items(): + # key is a key like "a", "a/b", "a/b/c" + # The basic idea is to find the immediate parent (so "", "a", or "a/b") + # and update that node's consolidated metadata to include the metadata + # in children_keys + *prefixes, name = key.split("/") + parent = metadata + + while prefixes: + # e.g. a/b/c has a parent "a/b". Walk through to get + # metadata["a"]["b"] + part = prefixes.pop(0) + # we can assume that parent[part] here is a group + # otherwise we wouldn't have a node with this `part` prefix. + # We can also assume that the parent node will have consolidated metadata, + # because we're walking top to bottom. + parent = parent[part].consolidated_metadata.metadata # type: ignore[union-attr] + + node = parent[name] + children_keys = list(children_keys) + + if isinstance(node, ArrayV2Metadata | ArrayV3Metadata): + # These are already present, either thanks to being an array in the + # root, or by being collected as a child in the else clause + continue + children: dict[str, ArrayV2Metadata | ArrayV3Metadata | GroupMetadata] = {} + # We pop from metadata, since we're *moving* this under group. + # While doing this, normalize leaf groups to carry empty consolidated metadata. + for child_key in children_keys: + if child_key == key: + continue + child = metadata.pop(child_key) + if isinstance(child, GroupMetadata) and child.consolidated_metadata is None: + child = replace(child, consolidated_metadata=ConsolidatedMetadata(metadata={})) + children[child_key.split("/")[-1]] = child + + parent[name] = replace( + node, consolidated_metadata=ConsolidatedMetadata(metadata=children) + ) + + @property + def flattened_metadata(self) -> dict[str, ArrayV2Metadata | ArrayV3Metadata | GroupMetadata]: + """ + Return the flattened representation of Consolidated Metadata. + + The returned dictionary will have a key for each child node in the hierarchy + under this group. Under the default (nested) representation available through + ``self.metadata``, the dictionary only contains keys for immediate children. + + The keys of the dictionary will include the full path to a child node from + the current group, where segments are joined by ``/``. + + Examples + -------- + ```python + from zarr.core.group import ConsolidatedMetadata, GroupMetadata + cm = ConsolidatedMetadata( + metadata={ + "group-0": GroupMetadata( + consolidated_metadata=ConsolidatedMetadata( + { + "group-0-0": GroupMetadata(), + } + ) + ), + "group-1": GroupMetadata(), + } + ) + # {'group-0': GroupMetadata(attributes={}, zarr_format=3, consolidated_metadata=None, node_type='group', extra_fields={}), + # 'group-0/group-0-0': GroupMetadata(attributes={}, zarr_format=3, consolidated_metadata=None, node_type='group', extra_fields={}), + # 'group-1': GroupMetadata(attributes={}, zarr_format=3, consolidated_metadata=None, node_type='group', extra_fields={})} + ``` + """ + metadata = {} + + def flatten( + key: str, group: GroupMetadata | ArrayV2Metadata | ArrayV3Metadata + ) -> dict[str, ArrayV2Metadata | ArrayV3Metadata | GroupMetadata]: + children: dict[str, ArrayV2Metadata | ArrayV3Metadata | GroupMetadata] = {} + if isinstance(group, ArrayV2Metadata | ArrayV3Metadata): + children[key] = group + else: + if group.consolidated_metadata and group.consolidated_metadata.metadata is not None: + children[key] = replace( + group, consolidated_metadata=ConsolidatedMetadata(metadata={}) + ) + for name, val in group.consolidated_metadata.metadata.items(): + full_key = f"{key}/{name}" + if isinstance(val, GroupMetadata): + children.update(flatten(full_key, val)) + else: + children[full_key] = val + else: + children[key] = replace(group, consolidated_metadata=None) + return children + + for k, v in self.metadata.items(): + metadata.update(flatten(k, v)) + + return metadata + + +GROUP_METADATA_KEYS: Final[set[str]] = { + "zarr_format", + "node_type", + "attributes", + "consolidated_metadata", +} +""" +The names of the fields of the group metadata document. `consolidated_metadata` is not in the +Zarr V3 spec; Zarr-Python writes it. +""" + + +@dataclass(frozen=True) +class GroupMetadata(Metadata): + """ + Metadata for a Group. + """ + + attributes: dict[str, Any] = field(default_factory=dict) + zarr_format: ZarrFormat = 3 + consolidated_metadata: ConsolidatedMetadata | None = None + node_type: Literal["group"] = field(default="group", init=False) + extra_fields: dict[str, AllowedExtraField] = field(default_factory=dict) + + def to_buffer_dict(self, prototype: BufferPrototype) -> dict[str, Buffer]: + if self.consolidated_metadata is not None: + for path, member in self.consolidated_metadata.flattened_metadata.items(): + # A member read from a document that had to be repaired is stored as it + # was stored (see `ConsolidatedMetadata.to_dict`). + if isinstance(member, ArrayV3Metadata) and member._stored_document is None: + try: + check_storable(member) + except ValueError as e: + e.add_note(f"Array {path!r} in the consolidated metadata.") + raise + indent = config.get("json_indent") + if self.zarr_format == 3: + return {ZARR_JSON: json_to_buffer(self.to_dict(), prototype=prototype, indent=indent)} + else: + items = { + ZGROUP_JSON: json_to_buffer( + {"zarr_format": self.zarr_format}, prototype=prototype, indent=indent + ), + ZATTRS_JSON: json_to_buffer(self.attributes, prototype=prototype, indent=indent), + } + if self.consolidated_metadata: + d = { + ZGROUP_JSON: {"zarr_format": self.zarr_format}, + ZATTRS_JSON: self.attributes, + } + consolidated_metadata = self.consolidated_metadata.to_dict()["metadata"] + if not isinstance(consolidated_metadata, dict): + raise TypeError( + "Expected consolidated metadata to serialize to a dict, " + f"got {type(consolidated_metadata).__name__}." + ) + for k, v in consolidated_metadata.items(): + attrs = v.pop("attributes", {}) + d[f"{k}/{ZATTRS_JSON}"] = attrs + if "shape" in v: + # it's an array + d[f"{k}/{ZARRAY_JSON}"] = v + else: + d[f"{k}/{ZGROUP_JSON}"] = { + "zarr_format": self.zarr_format, + "consolidated_metadata": { + "metadata": {}, + "must_understand": False, + "kind": "inline", + }, + } + + # The consolidated metadata blob is written compactly (no indent). + items[ZMETADATA_V2_JSON] = json_to_buffer( + {"metadata": d, "zarr_consolidated_format": 1}, prototype=prototype + ) + + return items + + def __init__( + self, + attributes: dict[str, Any] | None = None, + zarr_format: ZarrFormat = 3, + consolidated_metadata: ConsolidatedMetadata | None = None, + extra_fields: Mapping[str, AllowedExtraField] | None = None, + ) -> None: + attributes_parsed = parse_attributes(attributes) + zarr_format_parsed = parse_zarr_format(zarr_format) + if zarr_format_parsed == 2 and extra_fields: + raise ValueError( + "Invalid extra fields. Zarr format 2 group metadata does not support extra fields." + ) + extra_fields_parsed = parse_extra_fields( + extra_fields, reserved_keys=GROUP_METADATA_KEYS, node_type="group" + ) + + object.__setattr__(self, "attributes", attributes_parsed) + object.__setattr__(self, "zarr_format", zarr_format_parsed) + object.__setattr__(self, "consolidated_metadata", consolidated_metadata) + object.__setattr__(self, "extra_fields", extra_fields_parsed) + + @classmethod + def from_dict(cls, data: dict[str, Any], *, path: str | None = None) -> GroupMetadata: + """Read a stored group document; `path` names the group in warnings about the + consolidated metadata it holds.""" + data = dict(data) + node_type = data.pop("node_type", None) + if node_type not in ("group", None): + raise NodeTypeValidationError( + f"Invalid value for 'node_type'. Expected 'group' or None. Got {node_type!r}." + ) + consolidated_metadata = data.pop("consolidated_metadata", None) + if consolidated_metadata: + data["consolidated_metadata"] = ConsolidatedMetadata.from_dict( + consolidated_metadata, path=path + ) + + zarr_format = data.get("zarr_format") + if zarr_format == 3: + # Zarr v3 allows an extra key only if it is an object with "must_understand": false. + extra_fields = parse_extra_fields( + {k: v for k, v in data.items() if k not in GROUP_METADATA_KEYS}, + reserved_keys=GROUP_METADATA_KEYS, + node_type="group", + ) + else: + # Zarr v2 allowed arbitrary keys; they are dropped rather than kept. + extra_fields = {} + data = {k: v for k, v in data.items() if k in GROUP_METADATA_KEYS} + return cls(**data, extra_fields=extra_fields) + + def to_dict(self) -> dict[str, Any]: + result = asdict(replace(self, consolidated_metadata=None)) + result.update(result.pop("extra_fields")) + if self.consolidated_metadata is not None: + result["consolidated_metadata"] = self.consolidated_metadata.to_dict() + else: + # Leave consolidated metadata unset if it's None + result.pop("consolidated_metadata") + return result + + +@dataclass(frozen=True) +class ImplicitGroupMarker(GroupMetadata): + """ + Marker for an implicit group. Instances of this class are only used in the context of group + creation as a placeholder to represent groups that should only be created if they do not + already exist in storage + """ + + +@dataclass(frozen=True) +class AsyncGroup: + """ + Asynchronous Group object. + """ + + metadata: GroupMetadata + store_path: StorePath + + # TODO: make this correct and work + # TODO: ensure that this can be bound properly to subclass of AsyncGroup + + @classmethod + async def from_store( + cls, + store: StoreLike, + *, + attributes: dict[str, Any] | None = None, + overwrite: bool = False, + zarr_format: ZarrFormat = 3, + ) -> AsyncGroup: + store_path = await make_store_path(store) + attributes = attributes or {} + group = cls( + metadata=GroupMetadata(attributes=attributes, zarr_format=zarr_format), + store_path=store_path, + ) + await save_new_metadata( + store_path, group.metadata, overwrite=overwrite, ensure_parents=True + ) + return group + + @classmethod + async def open( + cls, + store: StoreLike, + zarr_format: ZarrFormat | None = 3, + use_consolidated: bool | str | None = None, + ) -> AsyncGroup: + """Open a new AsyncGroup + + Parameters + ---------- + store : StoreLike + zarr_format : {2, 3}, optional + use_consolidated : bool or str, default None + Whether to use consolidated metadata. + + By default, consolidated metadata is used if it's present in the + store (in the ``zarr.json`` for Zarr format 3 and in the ``.zmetadata`` file + for Zarr format 2) and the Store supports it. + + To explicitly require consolidated metadata, set ``use_consolidated=True``. + In this case, if the Store doesn't support consolidation or consolidated metadata is + not found, a ``ValueError`` exception is raised. + + To explicitly *not* use consolidated metadata, set ``use_consolidated=False``, + which will fall back to using the regular, non consolidated metadata. + + Zarr format 2 allowed configuring the key storing the consolidated metadata + (``.zmetadata`` by default). Specify the custom key as ``use_consolidated`` + to load consolidated metadata from a non-default key. + """ + store_path = await make_store_path(store) + if not store_path.store.supports_consolidated_metadata: + # Fail if consolidated metadata was requested but the Store doesn't support it + if use_consolidated: + store_name = type(store_path.store).__name__ + raise ValueError( + f"The Zarr store in use ({store_name}) doesn't support consolidated metadata." + ) + + # if use_consolidated was None (optional), the Store dictates it doesn't want consolidation + use_consolidated = False + + consolidated_key = ZMETADATA_V2_JSON + + if (zarr_format == 2 or zarr_format is None) and isinstance(use_consolidated, str): + consolidated_key = use_consolidated + + if zarr_format == 2: + paths = [store_path / ZGROUP_JSON, store_path / ZATTRS_JSON] + if use_consolidated or use_consolidated is None: + paths.append(store_path / consolidated_key) + + zgroup_bytes, zattrs_bytes, *rest = await asyncio.gather( + *[path.get() for path in paths] + ) + if zgroup_bytes is None: + raise FileNotFoundError(store_path) + + if use_consolidated or use_consolidated is None: + maybe_consolidated_metadata_bytes = rest[0] + + else: + maybe_consolidated_metadata_bytes = None + + elif zarr_format == 3: + zarr_json_bytes = await (store_path / ZARR_JSON).get() + if zarr_json_bytes is None: + raise FileNotFoundError(store_path) + elif zarr_format is None: + ( + zarr_json_bytes, + zgroup_bytes, + zattrs_bytes, + maybe_consolidated_metadata_bytes, + ) = await asyncio.gather( + (store_path / ZARR_JSON).get(), + (store_path / ZGROUP_JSON).get(), + (store_path / ZATTRS_JSON).get(), + (store_path / str(consolidated_key)).get(), + ) + if zarr_json_bytes is not None and zgroup_bytes is not None: + # warn and favor v3 + msg = f"Both zarr.json (Zarr format 3) and .zgroup (Zarr format 2) metadata objects exist at {store_path}. Zarr format 3 will be used." + warnings.warn(msg, category=ZarrUserWarning, stacklevel=1) + if zarr_json_bytes is None and zgroup_bytes is None: + raise FileNotFoundError( + f"could not find zarr.json or .zgroup objects in {store_path}" + ) + # set zarr_format based on which keys were found + if zarr_json_bytes is not None: + zarr_format = 3 + else: + zarr_format = 2 + else: + msg = f"Invalid value for 'zarr_format'. Expected 2, 3, or None. Got '{zarr_format}'." # type: ignore[unreachable] + raise MetadataValidationError(msg) + + if zarr_format == 2: + if zgroup_bytes is None: + raise FileNotFoundError(store_path) + + if use_consolidated and maybe_consolidated_metadata_bytes is None: + # the user requested consolidated metadata, but it was missing + raise ValueError(consolidated_key) + + elif use_consolidated is False: + # the user explicitly opted out of consolidated_metadata. + # Discard anything we might have read. + maybe_consolidated_metadata_bytes = None + + return cls._from_bytes_v2( + store_path, zgroup_bytes, zattrs_bytes, maybe_consolidated_metadata_bytes + ) + else: + # V3 groups are comprised of a zarr.json object + if zarr_json_bytes is None: + raise FileNotFoundError(store_path) + if not isinstance(use_consolidated, bool | None): + raise TypeError("use_consolidated must be a bool or None for Zarr format 3.") + + return cls._from_bytes_v3( + store_path, + zarr_json_bytes, + use_consolidated=use_consolidated, + ) + + @classmethod + def _from_bytes_v2( + cls, + store_path: StorePath, + zgroup_bytes: Buffer, + zattrs_bytes: Buffer | None, + consolidated_metadata_bytes: Buffer | None, + ) -> AsyncGroup: + # V2 groups are comprised of a .zgroup and .zattrs objects + zgroup = buffer_to_json_object(zgroup_bytes) + zattrs = buffer_to_json_object(zattrs_bytes) if zattrs_bytes is not None else {} + group_metadata: dict[str, Any] = {**zgroup, "attributes": zattrs} + + if consolidated_metadata_bytes is not None: + v2_consolidated_doc = buffer_to_json_object(consolidated_metadata_bytes) + v2_consolidated_metadata = cast("dict[str, Any]", v2_consolidated_doc["metadata"]) + # We already read zattrs and zgroup. Should we ignore these? + v2_consolidated_metadata.pop(".zattrs", None) + v2_consolidated_metadata.pop(".zgroup", None) + + consolidated_metadata: defaultdict[str, dict[str, Any]] = defaultdict(dict) + + # keys like air/.zarray, air/.zattrs + for k, v in v2_consolidated_metadata.items(): + path, kind = k.rsplit("/.", 1) + + if kind == "zarray": + consolidated_metadata[path].update(v) + elif kind == "zattrs": + consolidated_metadata[path]["attributes"] = v + elif kind == "zgroup": + consolidated_metadata[path].update(v) + else: + raise ValueError(f"Invalid file type '{kind}' at path '{path}") + + group_metadata["consolidated_metadata"] = { + "metadata": dict(consolidated_metadata), + "kind": "inline", + "must_understand": False, + } + + return cls.from_dict(store_path, group_metadata) + + @classmethod + def _from_bytes_v3( + cls, + store_path: StorePath, + zarr_json_bytes: Buffer, + use_consolidated: bool | None, + ) -> AsyncGroup: + group_metadata = buffer_to_json_object(zarr_json_bytes) + if use_consolidated and group_metadata.get("consolidated_metadata") is None: + msg = f"Consolidated metadata requested with 'use_consolidated=True' but not found in '{store_path.path}'." + raise ValueError(msg) + + elif use_consolidated is False: + # Drop consolidated metadata if it's there. + group_metadata.pop("consolidated_metadata", None) + + return cls.from_dict(store_path, group_metadata) + + @classmethod + def from_dict( + cls, + store_path: StorePath, + data: dict[str, Any], + ) -> AsyncGroup: + node_type = data.pop("node_type", None) + if node_type == "array": + msg = f"An array already exists in store {store_path.store} at path {store_path.path}." + raise ContainsArrayError(msg) + elif node_type not in ("group", None): + msg = f"Node type in metadata ({node_type}) is not 'group'" + raise GroupNotFoundError(msg) + return cls( + metadata=GroupMetadata.from_dict(data, path=str(store_path)), + store_path=store_path, + ) + + async def setitem(self, key: str, value: Any) -> None: + """ + Fastpath for creating a new array + New arrays will be created with default array settings for the array type. + + Parameters + ---------- + key : str + Array name + value : array-like + Array data + """ + path = self.store_path / key + await async_api.save_array( + store=path, arr=value, zarr_format=self.metadata.zarr_format, overwrite=True + ) + + async def getitem( + self, + key: str, + ) -> AnyAsyncArray | AsyncGroup: + """ + Get a subarray or subgroup from the group. + + Parameters + ---------- + key : str + Array or group name + + Returns + ------- + AsyncArray or AsyncGroup + """ + store_path = self.store_path / key + logger.debug("key=%s, store_path=%s", key, store_path) + + # Consolidated metadata lets us avoid some I/O operations so try that first. + if self.metadata.consolidated_metadata is not None: + return self._getitem_consolidated(store_path, key, prefix=self.name) + try: + return await get_node( + store=store_path.store, path=store_path.path, zarr_format=self.metadata.zarr_format + ) + except FileNotFoundError as e: + raise KeyError(key) from e + + def _getitem_consolidated( + self, store_path: StorePath, key: str, prefix: str + ) -> AnyAsyncArray | AsyncGroup: + # getitem, in the special case where we have consolidated metadata. + # Note that this is a regular def (non async) function. + # This shouldn't do any additional I/O. + + # we support nested getitems like group/subgroup/array + indexers = normalize_path(key).split("/") + indexers.reverse() + metadata: ArrayV2Metadata | ArrayV3Metadata | GroupMetadata = self.metadata + + while indexers: + indexer = indexers.pop() + if isinstance(metadata, ArrayV2Metadata | ArrayV3Metadata): + # we've indexed into an array with group["array/subarray"]. Invalid. + raise KeyError(key) + if metadata.consolidated_metadata is None: + # we've indexed into a group without consolidated metadata. + # This isn't normal; typically, consolidated metadata + # will include explicit markers for when there are no child + # nodes as metadata={}. + # We have some freedom in exactly how we interpret this case. + # For now, we treat None as the same as {}, i.e. we don't + # have any children. + raise KeyError(key) + try: + metadata = metadata.consolidated_metadata.metadata[indexer] + except KeyError as e: + # The Group Metadata has consolidated metadata, but the key + # isn't present. We trust this to mean that the key isn't in + # the hierarchy, and *don't* fall back to checking the store. + msg = f"'{key}' not found in consolidated metadata." + raise KeyError(msg) from e + + # update store_path to ensure that AsyncArray/Group.name is correct + if prefix != "/": + key = "/".join([prefix.lstrip("/"), key]) + store_path = StorePath(store=store_path.store, path=key) + + if isinstance(metadata, GroupMetadata): + return AsyncGroup(metadata=metadata, store_path=store_path) + else: + return AsyncArray(metadata=metadata, store_path=store_path) + + async def delitem(self, key: str) -> None: + """Delete a group member. + + Parameters + ---------- + key : str + Array or group name + """ + store_path = self.store_path / key + consolidated = self.metadata.consolidated_metadata + if consolidated is None: + await store_path.delete_dir() + return + # Encode the group metadata without the member before deleting it: metadata that + # cannot be stored then fails with the store and this group untouched. What is + # stored is encoded after the deletion, from the metadata as it then is, so + # concurrent deletions each store the deletions made before them. + members = {name: node for name, node in consolidated.metadata.items() if name != key} + encode_documents( + self.store_path, + replace(self.metadata, consolidated_metadata=replace(consolidated, metadata=members)), + ) + await store_path.delete_dir() + # In place, so every handle sharing this consolidated metadata (a parent's or a + # subgroup's) sees the deletion. + consolidated.metadata.pop(key, None) + await store_documents(self.store_path, encode_documents(self.store_path, self.metadata)) + + async def get[DefaultT]( + self, key: str, default: DefaultT | None = None + ) -> AnyAsyncArray | AsyncGroup | DefaultT | None: + """Obtain a group member, returning default if not found. + + Parameters + ---------- + key : str + Group member name. + default : object + Default value to return if key is not found (default: None). + + Returns + ------- + object + Group member (AsyncArray or AsyncGroup) or default if not found. + """ + try: + return await self.getitem(key) + except KeyError: + return default + + async def get_array(self, path: str) -> AnyAsyncArray: + """Obtain an array member of this group, raising if it is absent or not an array. + + Parameters + ---------- + path : str + Path of the array relative to this group. May contain `/` to reference + a member of a subgroup, e.g. `subgroup/subarray`. + + Returns + ------- + AsyncArray + The array at the given path. + + Raises + ------ + ArrayNotFoundError + If no node exists at the given path. + ContainsGroupError + If the node at the given path is a group rather than an array. + """ + store_path = self.store_path / path + try: + node = await self.getitem(path) + except KeyError as e: + msg = f"No array found in store {store_path.store!r} at path {store_path.path!r}" + raise ArrayNotFoundError(msg) from e + if isinstance(node, AsyncGroup): + msg = f"A group exists in store {store_path.store!r} at path {store_path.path!r}." + raise ContainsGroupError(msg) + return node + + async def get_group(self, path: str) -> AsyncGroup: + """Obtain a group member of this group, raising if it is absent or not a group. + + Parameters + ---------- + path : str + Path of the group relative to this group. May contain `/` to reference + a member of a subgroup, e.g. `subgroup/subsubgroup`. + + Returns + ------- + AsyncGroup + The group at the given path. + + Raises + ------ + GroupNotFoundError + If no node exists at the given path. + ContainsArrayError + If the node at the given path is an array rather than a group. + """ + store_path = self.store_path / path + try: + node = await self.getitem(path) + except KeyError as e: + msg = f"No group found in store {store_path.store!r} at path {store_path.path!r}" + raise GroupNotFoundError(msg) from e + if isinstance(node, AsyncArray): + msg = f"An array exists in store {store_path.store!r} at path {store_path.path!r}." + raise ContainsArrayError(msg) + return node + + async def _save_metadata(self, ensure_parents: bool = False) -> None: + await save_metadata(self.store_path, self.metadata, ensure_parents=ensure_parents) + + @property + def path(self) -> str: + """Storage path.""" + return self.store_path.path + + @property + def name(self) -> str: + """Group name following h5py convention.""" + if self.path: + # follow h5py convention: add leading slash + name = self.path + if name[0] != "/": + name = f"/{name}" + return name + return "/" + + @property + def basename(self) -> str: + """Final component of name.""" + return self.name.split("/")[-1] + + @property + def attrs(self) -> dict[str, Any]: + return self.metadata.attributes + + @property + def info(self) -> Any: + """ + Return a visual representation of the statically known information about a group. + + Note that this doesn't include dynamic information, like the number of child + Groups or Arrays. + + Returns + ------- + GroupInfo + + Related + ------- + [zarr.AsyncGroup.info_complete][] + All information about a group, including dynamic information + """ + + if self.metadata.consolidated_metadata: + members = list(self.metadata.consolidated_metadata.flattened_metadata.values()) + else: + members = None + return self._info(members=members) + + async def info_complete(self) -> Any: + """ + Return all the information for a group. + + This includes dynamic information like the number + of child Groups or Arrays. If this group doesn't contain consolidated + metadata then this will need to read from the backing Store. + + Returns + ------- + GroupInfo + + Related + ------- + [zarr.AsyncGroup.info][] + """ + members = [x[1].metadata async for x in self.members(max_depth=None)] + return self._info(members=members) + + def _info( + self, members: list[ArrayV2Metadata | ArrayV3Metadata | GroupMetadata] | None = None + ) -> Any: + kwargs = {} + if members is not None: + kwargs["_count_members"] = len(members) + count_arrays = 0 + count_groups = 0 + for member in members: + if isinstance(member, GroupMetadata): + count_groups += 1 + else: + count_arrays += 1 + kwargs["_count_arrays"] = count_arrays + kwargs["_count_groups"] = count_groups + + return GroupInfo( + _name=self.store_path.path, + _read_only=self.read_only, + _store_type=type(self.store_path.store).__name__, + _zarr_format=self.metadata.zarr_format, + # maybe do a typeddict + **kwargs, # type: ignore[arg-type] + ) + + @property + def store(self) -> Store: + return self.store_path.store + + @property + def read_only(self) -> bool: + # Backwards compatibility for 2.x + return self.store_path.read_only + + @property + def synchronizer(self) -> None: + # Backwards compatibility for 2.x + # Not implemented in 3.x yet. + return None + + async def create_group( + self, + name: str, + *, + overwrite: bool = False, + attributes: dict[str, Any] | None = None, + ) -> AsyncGroup: + """Create a sub-group. + + Parameters + ---------- + name : str + Group name. + overwrite : bool, optional + If True, do not raise an error if the group already exists. + attributes : dict, optional + Group attributes. + + Returns + ------- + g : AsyncGroup + """ + attributes = attributes or {} + return await type(self).from_store( + self.store_path / name, + attributes=attributes, + overwrite=overwrite, + zarr_format=self.metadata.zarr_format, + ) + + async def require_group(self, name: str, overwrite: bool = False) -> AsyncGroup: + """Obtain a sub-group, creating one if it doesn't exist. + + Parameters + ---------- + name : str + Group name. + overwrite : bool, optional + Overwrite any existing group with given `name` if present. + + Returns + ------- + g : AsyncGroup + """ + if overwrite: + # TODO: check that overwrite=True errors if an array exists where the group is being created + grp = await self.create_group(name, overwrite=True) + else: + try: + item: AsyncGroup | AnyAsyncArray = await self.getitem(name) + if not isinstance(item, AsyncGroup): + raise TypeError( + f"Incompatible object ({item.__class__.__name__}) already exists" + ) + grp = item + except KeyError: + grp = await self.create_group(name) + return grp + + async def require_groups(self, *names: str) -> tuple[AsyncGroup, ...]: + """Convenience method to require multiple groups in a single call. + + Parameters + ---------- + *names : str + Group names. + + Returns + ------- + Tuple[AsyncGroup, ...] + """ + if not names: + return () + return tuple(await asyncio.gather(*(self.require_group(name) for name in names))) + + async def create_array( + self, + name: str, + *, + shape: ShapeLike | None = None, + dtype: ZDTypeLike | None = None, + data: np.ndarray[Any, np.dtype[Any]] | None = None, + chunks: ChunksLike | Literal["auto"] = "auto", + shards: ShardsLike | None = None, + filters: FiltersLike = "auto", + compressors: CompressorsLike = "auto", + compressor: CompressorLike = "auto", + serializer: SerializerLike = "auto", + fill_value: Any | None = DEFAULT_FILL_VALUE, + order: MemoryOrder | None = None, + attributes: dict[str, JSON] | None = None, + chunk_key_encoding: ChunkKeyEncodingLike | None = None, + dimension_names: DimensionNamesLike = None, + storage_options: dict[str, Any] | None = None, + overwrite: bool = False, + config: ArrayConfigLike | None = None, + write_data: bool = True, + ) -> AnyAsyncArray: + """Create an array within this group. + + This method lightly wraps [zarr.core.array.create_array][]. + + Parameters + ---------- + name : str + The name of the array relative to the group. If `path` is `None`, the array will be located + at the root of the store. + shape : ShapeLike, optional + Shape of the array. Must be `None` if `data` is provided. + dtype : ZDTypeLike | None + Data type of the array. Must be `None` if `data` is provided. + data : np.ndarray, optional + Array-like data to use for initializing the array. If this parameter is provided, the + `shape` and `dtype` parameters must be `None`. + chunks : tuple[int, ...] | Sequence[Sequence[int]] | Literal["auto"], default="auto" + Chunk shape of the array. + If chunks is "auto", a chunk shape is guessed based on the shape of the array and the dtype. + A nested list of per-dimension edge sizes creates a rectilinear grid. + Rectilinear chunk grids are experimental and must be explicitly enabled + with `zarr.config.set({'array.rectilinear_chunks': True})` while the + feature is stabilizing. + shards : tuple[int, ...], optional + Shard shape of the array. The default value of `None` results in no sharding at all. + filters : Iterable[Codec] | Literal["auto"], optional + Iterable of filters to apply to each chunk of the array, in order, before serializing that + chunk to bytes. + + For Zarr format 3, a "filter" is a codec that takes an array and returns an array, + and these values must be instances of [`zarr.abc.codec.ArrayArrayCodec`][], or a + dict representations of [`zarr.abc.codec.ArrayArrayCodec`][]. + + For Zarr format 2, a "filter" can be any numcodecs codec; you should ensure that the + order of your filters is consistent with the behavior of each filter. + + The default value of `"auto"` instructs Zarr to use a default based on the data + type of the array and the Zarr format specified. For all data types in Zarr V3, and most + data types in Zarr V2, the default filters are empty. The only cases where default filters + are not empty is when the Zarr format is 2, and the data type is a variable-length data type like + [`zarr.dtype.VariableLengthUTF8`][] or [`zarr.dtype.VariableLengthBytes`][]. In these cases, + the default filters contains a single element which is a codec specific to that particular data type. + + To create an array with no filters, provide an empty iterable or the value `None`. + compressors : Iterable[Codec], optional + List of compressors to apply to the array. Compressors are applied in order, and after any + filters are applied (if any are specified) and the data is serialized into bytes. + + For Zarr format 3, a "compressor" is a codec that takes a bytestream, and + returns another bytestream. Multiple compressors may be provided for Zarr format 3. + If no `compressors` are provided, a default set of compressors will be used. + These defaults can be changed by modifying the value of `array.v3_default_compressors` + in [`zarr.config`][zarr.config]. + Use `None` to omit default compressors. + + For Zarr format 2, a "compressor" can be any numcodecs codec. Only a single compressor may + be provided for Zarr format 2. + If no `compressor` is provided, a default compressor will be used. + This default can be changed by modifying the value of `array.v2_default_compressor` + in [`zarr.config`][zarr.config]. + Use `None` to omit the default compressor. + compressor : Codec, optional + Deprecated in favor of `compressors`. + serializer : dict[str, JSON] | ArrayBytesCodec, optional + Array-to-bytes codec to use for encoding the array data. + Zarr format 3 only. Zarr format 2 arrays use implicit array-to-bytes conversion. + If no `serializer` is provided, a default serializer will be used. + These defaults can be changed by modifying the value of `array.v3_default_serializer` + in [`zarr.config`][zarr.config]. + fill_value : Any, optional + Fill value for the array. + order : {"C", "F"}, optional + The memory order of the array (default is "C"). + For Zarr format 2, this parameter sets the memory order of the array. + For Zarr format 3, this parameter is deprecated, because memory order + is a runtime parameter for Zarr format 3 arrays. The recommended way to specify the memory + order for Zarr format 3 arrays is via the `config` parameter, e.g. `{'config': 'C'}`. + If no `order` is provided, a default order will be used. + This default can be changed by modifying the value of `array.order` in [`zarr.config`][zarr.config]. + attributes : dict, optional + Attributes for the array. + chunk_key_encoding : ChunkKeyEncodingLike, optional + A specification of how the chunk keys are represented in storage. + For Zarr format 3, the default is `{"name": "default", "separator": "/"}`. + For Zarr format 2, the default is `{"name": "v2", "separator": "."}`. + dimension_names : Iterable[str], optional + The names of the dimensions (default is None). + Zarr format 3 only. Zarr format 2 arrays should not use this parameter. + storage_options : dict, optional + If using an fsspec URL to create the store, these will be passed to the backend implementation. + Ignored otherwise. + overwrite : bool, default False + Whether to overwrite an array with the same name in the store, if one exists. + If `True`, any existing keys under that path are deleted first. + config : ArrayConfigLike, optional + Runtime configuration for the array. + write_data : bool + If a pre-existing array-like object was provided to this function via the `data` parameter + then `write_data` determines whether the values in that array-like object should be + written to the Zarr array created by this function. If `write_data` is `False`, then the + array will be left empty. + + Returns + ------- + AsyncArray + + """ + compressors = _parse_deprecated_compressor( + compressor, compressors, zarr_format=self.metadata.zarr_format + ) + return await create_array( + store=self.store_path, + name=name, + shape=shape, + dtype=dtype, + data=data, + chunks=chunks, + shards=shards, + filters=filters, + compressors=compressors, + serializer=serializer, + fill_value=fill_value, + order=order, + zarr_format=self.metadata.zarr_format, + attributes=attributes, + chunk_key_encoding=chunk_key_encoding, + dimension_names=dimension_names, + storage_options=storage_options, + overwrite=overwrite, + config=config, + write_data=write_data, + ) + + async def require_array( + self, + name: str, + *, + shape: ShapeLike, + dtype: ZDTypeLike | None = None, + exact: bool = False, + config: ArrayConfigLike | None = None, + **kwargs: Any, + ) -> AnyAsyncArray: + """Obtain an array, creating if it doesn't exist. + + Other `kwargs` are as per [zarr.AsyncGroup.create_array][]. + + Parameters + ---------- + name : str + Array name. + shape : int or tuple of ints + Array shape. + dtype : ZDTypeLike, optional + The data type of the array, given as a string, a NumPy dtype, or a + Zarr data type. + exact : bool, optional + If True, require `dtype` to match exactly. If false, require + `dtype` can be cast from array dtype. + config : ArrayConfigLike or None, default=None + Runtime configuration for the array, whether it is created or already exists. + Keys not specified are taken from the global configuration. + + Returns + ------- + a : AsyncArray + """ + try: + ds = await self.getitem(name) + if not isinstance(ds, AsyncArray): + raise TypeError(f"Incompatible object ({ds.__class__.__name__}) already exists") + + shape = parse_shapelike(shape) + if shape != ds.shape: + raise TypeError(f"Incompatible shape ({ds.shape} vs {shape})") + + # `np.dtype(None)` used to resolve to float64 here; keep that default. + dtype = parse_data_type( + "float64" if dtype is None else dtype, + zarr_format=self.metadata.zarr_format, + ).to_native_dtype() + if exact: + if ds.dtype != dtype: + raise TypeError(f"Incompatible dtype ({ds.dtype} vs {dtype})") + else: + if not np.can_cast(ds.dtype, dtype): + raise TypeError(f"Incompatible dtype ({ds.dtype} vs {dtype})") + except KeyError: + return await self.create_array(name, shape=shape, dtype=dtype, config=config, **kwargs) + + # `config` is the runtime configuration of the returned array, not stored metadata, + # so it applies to an existing array too. It is parsed the same way as on the create + # branch: missing keys come from the global configuration, unknown keys raise. + if config is not None: + return ds.with_config(parse_array_config(config)) + return ds + + async def update_attributes(self, new_attributes: dict[str, Any]) -> AsyncGroup: + """Update group attributes. + + Parameters + ---------- + new_attributes : dict + New attributes to set on the group. + + Returns + ------- + self : AsyncGroup + """ + self.metadata.attributes.update(new_attributes) + + # Write new metadata + await self._save_metadata() + + return self + + def __repr__(self) -> str: + return f"" + + async def nmembers( + self, + max_depth: int | None = 0, + ) -> int: + """Count the number of members in this group. + + Parameters + ---------- + max_depth : int, default 0 + The maximum number of levels of the hierarchy to include. By + default, (``max_depth=0``) only immediate children are included. Set + ``max_depth=None`` to include all nodes, and some positive integer + to consider children within that many levels of the root Group. + + Returns + ------- + count : int + """ + # check if we can use consolidated metadata, which requires that we have non-None + # consolidated metadata at all points in the hierarchy. + if self.metadata.consolidated_metadata is not None: + if max_depth is not None and max_depth < 0: + raise ValueError(f"max_depth must be None or >= 0. Got '{max_depth}' instead") + if max_depth is None: + return len(self.metadata.consolidated_metadata.flattened_metadata) + else: + return len( + [ + x + for x in self.metadata.consolidated_metadata.flattened_metadata + if x.count("/") <= max_depth + ] + ) + # TODO: consider using aioitertools.builtins.sum for this + # return await aioitertools.builtins.sum((1 async for _ in self.members()), start=0) + n = 0 + async for _ in self.members(max_depth=max_depth): + n += 1 + return n + + async def members( + self, + max_depth: int | None = 0, + *, + use_consolidated_for_children: bool = True, + ) -> AsyncGenerator[ + tuple[str, AnyAsyncArray | AsyncGroup], + None, + ]: + """ + Returns an AsyncGenerator over the arrays and groups contained in this group. + This method requires that `store_path.store` supports directory listing. + + The results are not guaranteed to be ordered. + + Parameters + ---------- + max_depth : int, default 0 + The maximum number of levels of the hierarchy to include. By + default, (``max_depth=0``) only immediate children are included. Set + ``max_depth=None`` to include all nodes, and some positive integer + to consider children within that many levels of the root Group. + use_consolidated_for_children : bool, default True + Whether to use the consolidated metadata of child groups loaded + from the store. Note that this only affects groups loaded from the + store. If the current Group already has consolidated metadata, it + will always be used. + + Returns + ------- + path: + A string giving the path to the target, relative to the Group ``self``. + value: AsyncArray or AsyncGroup + The AsyncArray or AsyncGroup that is a child of ``self``. + """ + if max_depth is not None and max_depth < 0: + raise ValueError(f"max_depth must be None or >= 0. Got '{max_depth}' instead") + async for item in self._members( + max_depth=max_depth, use_consolidated_for_children=use_consolidated_for_children + ): + yield item + + def _members_consolidated( + self, max_depth: int | None, prefix: str = "" + ) -> Generator[ + tuple[str, AnyAsyncArray | AsyncGroup], + None, + ]: + consolidated_metadata = self.metadata.consolidated_metadata + + do_recursion = max_depth is None or max_depth > 0 + + # we kind of just want the top-level keys. + if consolidated_metadata is not None: + for key in consolidated_metadata.metadata: + obj = self._getitem_consolidated( + self.store_path, key, prefix=self.name + ) # Metadata -> Group/Array + key = f"{prefix}/{key}".lstrip("/") + yield key, obj + + if do_recursion and isinstance(obj, AsyncGroup): + if max_depth is None: + new_depth = None + else: + new_depth = max_depth - 1 + yield from obj._members_consolidated(new_depth, prefix=key) + + async def _members( + self, max_depth: int | None, *, use_consolidated_for_children: bool = True + ) -> AsyncGenerator[tuple[str, AnyAsyncArray | AsyncGroup], None]: + skip_keys: tuple[str, ...] + if self.metadata.zarr_format == 2: + skip_keys = (".zattrs", ".zgroup", ".zarray", ".zmetadata") + elif self.metadata.zarr_format == 3: + skip_keys = ("zarr.json",) + else: + raise ValueError(f"Unknown Zarr format: {self.metadata.zarr_format}") + + if self.metadata.consolidated_metadata is not None: + members = self._members_consolidated(max_depth=max_depth) + for member in members: + yield member + return + + if not self.store_path.store.supports_listing: + msg = ( + f"The store associated with this group ({type(self.store_path.store)}) " + "does not support listing, " + "specifically via the `list_dir` method. " + "This function requires a store that supports listing." + ) + + raise ValueError(msg) + # enforce a concurrency limit by passing a semaphore to all the recursive functions + semaphore = asyncio.Semaphore(config.get("async.concurrency")) + async for member in _iter_members_deep( + self, + max_depth=max_depth, + skip_keys=skip_keys, + semaphore=semaphore, + use_consolidated_for_children=use_consolidated_for_children, + ): + yield member + + async def create_hierarchy( + self, + nodes: dict[str, ArrayV2Metadata | ArrayV3Metadata | GroupMetadata], + *, + overwrite: bool = False, + ) -> AsyncIterator[tuple[str, AsyncGroup | AnyAsyncArray]]: + """ + Create a hierarchy of arrays or groups rooted at this group. + + This function will parse its input to ensure that the hierarchy is complete. Any implicit groups + will be inserted as needed. For example, an input like + ```{'a/b': GroupMetadata}``` will be parsed to + ```{'': GroupMetadata, 'a': GroupMetadata, 'b': Groupmetadata}```. + + Explicitly specifying a root group, e.g. with ``nodes = {'': GroupMetadata()}`` is an error + because this group instance is the root group. + + After input parsing, this function then creates all the nodes in the hierarchy concurrently. + + Arrays and Groups are yielded in the order they are created. This order is not stable and + should not be relied on. + + Parameters + ---------- + nodes : dict[str, GroupMetadata | ArrayV3Metadata | ArrayV2Metadata] + A dictionary defining the hierarchy. The keys are the paths of the nodes in the hierarchy, + relative to the path of the group. The values are instances of ``GroupMetadata`` or ``ArrayMetadata``. Note that + all values must have the same ``zarr_format`` as the parent group -- it is an error to mix zarr versions in the + same hierarchy. + + Leading "/" characters from keys will be removed. + overwrite : bool + Whether to overwrite existing nodes. Defaults to ``False``, in which case an error is + raised instead of overwriting an existing array or group. + + This function will not erase an existing group unless that group is explicitly named in + ``nodes``. If ``nodes`` defines implicit groups, e.g. ``{`'a/b/c': GroupMetadata}``, and a + group already exists at path ``a``, then this function will leave the group at ``a`` as-is. + + Yields + ------ + tuple[str, AsyncArray | AsyncGroup]. + """ + # check that all the nodes have the same zarr_format as Self + prefix = self.path + nodes_parsed = {} + for key, value in nodes.items(): + if value.zarr_format != self.metadata.zarr_format: + msg = ( + "The zarr_format of the nodes must be the same as the parent group. " + f"The node at {key} has zarr_format {value.zarr_format}, but the parent group" + f" has zarr_format {self.metadata.zarr_format}." + ) + raise ValueError(msg) + if normalize_path(key) == "": + msg = ( + "The input defines a root node, but a root node already exists, namely this Group instance." + "It is an error to use this method to create a root node. " + "Remove the root node from the input dict, or use a function like " + "create_rooted_hierarchy to create a rooted hierarchy." + ) + raise ValueError(msg) + else: + nodes_parsed[_join_paths([prefix, key])] = value + + async for key, node in create_hierarchy( + store=self.store, + nodes=nodes_parsed, + overwrite=overwrite, + ): + if prefix == "": + out_key = key + else: + out_key = key.removeprefix(prefix + "/") + yield out_key, node + + async def keys(self) -> AsyncGenerator[str, None]: + """Iterate over member names.""" + async for key, _ in self.members(): + yield key + + async def contains(self, member: str) -> bool: + """Check if a member exists in the group. + + Parameters + ---------- + member : str + Member name. + + Returns + ------- + bool + """ + # TODO: this can be made more efficient. + try: + await self.getitem(member) + except KeyError: + return False + else: + return True + + async def groups(self) -> AsyncGenerator[tuple[str, AsyncGroup], None]: + """Iterate over subgroups.""" + async for name, value in self.members(): + if isinstance(value, AsyncGroup): + yield name, value + + async def group_keys(self) -> AsyncGenerator[str, None]: + """Iterate over group names.""" + async for key, _ in self.groups(): + yield key + + async def group_values(self) -> AsyncGenerator[AsyncGroup, None]: + """Iterate over group values.""" + async for _, group in self.groups(): + yield group + + async def arrays( + self, + ) -> AsyncGenerator[tuple[str, AnyAsyncArray], None]: + """Iterate over arrays.""" + async for key, value in self.members(): + if isinstance(value, AsyncArray): + yield key, value + + async def array_keys(self) -> AsyncGenerator[str, None]: + """Iterate over array names.""" + async for key, _ in self.arrays(): + yield key + + async def array_values( + self, + ) -> AsyncGenerator[AnyAsyncArray, None]: + """Iterate over array values.""" + async for _, array in self.arrays(): + yield array + + async def tree( + self, + expand: bool | None = None, + level: int | None = None, + *, + max_nodes: int = 500, + plain: bool = False, + ) -> Any: + """ + Return a tree-like representation of a hierarchy. + + Parameters + ---------- + expand : bool, optional + This keyword is not yet supported. A NotImplementedError is raised if + it's used. + level : int, optional + The maximum depth below this Group to display in the tree. + max_nodes : int + Maximum number of nodes to display before truncating. Default is 500. + plain : bool, optional + If True, return a plain-text tree without ANSI styling. This is + useful when the output will be consumed by an LLM or written to a + file. Default is False. + + Returns + ------- + TreeRepr + A pretty-printable object displaying the hierarchy. + """ + from zarr.core._tree import group_tree_async + + if expand is not None: + raise NotImplementedError("'expand' is not yet implemented.") + return await group_tree_async(self, max_depth=level, max_nodes=max_nodes, plain=plain) + + def _member_zarr_format(self, zarr_format: ZarrFormat | None) -> ZarrFormat: + """ + The zarr format of a new array in this group, which is the format of the group. + + A group only lists members of its own format, so an array of another format + would be written into the group without becoming a member of it. + """ + if zarr_format is not None and zarr_format != self.metadata.zarr_format: + raise ValueError( + f"Cannot create a zarr_format={zarr_format} array in a " + f"zarr_format={self.metadata.zarr_format} group." + ) + return self.metadata.zarr_format + + async def empty(self, *, name: str, shape: tuple[int, ...], **kwargs: Any) -> AnyAsyncArray: + """Create an empty array with the specified shape in this Group. The contents will + be filled with the array's fill value or zeros if no fill value is provided. + + Parameters + ---------- + name : str + Name of the array. + shape : int or tuple of int + Shape of the empty array. + **kwargs + Keyword arguments passed to [zarr.api.asynchronous.create][]. + + Notes + ----- + The contents of an empty Zarr array are not defined. On attempting to + retrieve data from an empty Zarr array, any values may be returned, + and these are not guaranteed to be stable from one access to the next. + """ + return await async_api.empty( + shape=shape, + store=self.store_path, + path=name, + zarr_format=self._member_zarr_format(kwargs.pop("zarr_format", None)), + **kwargs, + ) + + async def zeros(self, *, name: str, shape: tuple[int, ...], **kwargs: Any) -> AnyAsyncArray: + """Create an array, with zero being used as the default value for uninitialized portions of the array. + + Parameters + ---------- + name : str + Name of the array. + shape : int or tuple of int + Shape of the empty array. + **kwargs + Keyword arguments passed to [zarr.api.asynchronous.create][]. + + Returns + ------- + AsyncArray + The new array. + """ + return await async_api.zeros( + shape=shape, + store=self.store_path, + path=name, + zarr_format=self._member_zarr_format(kwargs.pop("zarr_format", None)), + **kwargs, + ) + + async def ones(self, *, name: str, shape: tuple[int, ...], **kwargs: Any) -> AnyAsyncArray: + """Create an array, with one being used as the default value for uninitialized portions of the array. + + Parameters + ---------- + name : str + Name of the array. + shape : int or tuple of int + Shape of the empty array. + **kwargs + Keyword arguments passed to [zarr.api.asynchronous.create][]. + + Returns + ------- + AsyncArray + The new array. + """ + return await async_api.ones( + shape=shape, + store=self.store_path, + path=name, + zarr_format=self._member_zarr_format(kwargs.pop("zarr_format", None)), + **kwargs, + ) + + async def full( + self, *, name: str, shape: tuple[int, ...], fill_value: Any | None, **kwargs: Any + ) -> AnyAsyncArray: + """Create an array, with "fill_value" being used as the default value for uninitialized portions of the array. + + Parameters + ---------- + name : str + Name of the array. + shape : int or tuple of int + Shape of the empty array. + fill_value : scalar + Value to fill the array with. + **kwargs + Keyword arguments passed to [zarr.api.asynchronous.create][]. + + Returns + ------- + AsyncArray + The new array. + """ + return await async_api.full( + shape=shape, + fill_value=fill_value, + store=self.store_path, + path=name, + zarr_format=self._member_zarr_format(kwargs.pop("zarr_format", None)), + **kwargs, + ) + + async def empty_like( + self, *, name: str, data: async_api.ArrayLike, **kwargs: Any + ) -> AnyAsyncArray: + """Create an empty sub-array like `data`. The contents will be filled with + the array's fill value or zeros if no fill value is provided. + + Parameters + ---------- + name : str + Name of the array. + data : array-like + The array to create an empty array like. + **kwargs + Keyword arguments passed to [zarr.api.asynchronous.create][]. + + Returns + ------- + AsyncArray + The new array. + """ + return await async_api.empty_like( + a=data, + store=self.store_path, + path=name, + zarr_format=self._member_zarr_format(kwargs.pop("zarr_format", None)), + **kwargs, + ) + + async def zeros_like( + self, *, name: str, data: async_api.ArrayLike, **kwargs: Any + ) -> AnyAsyncArray: + """Create a sub-array of zeros like `data`. + + Parameters + ---------- + name : str + Name of the array. + data : array-like + The array to create the new array like. + **kwargs + Keyword arguments passed to [zarr.api.asynchronous.create][]. + + Returns + ------- + AsyncArray + The new array. + """ + return await async_api.zeros_like( + a=data, + store=self.store_path, + path=name, + zarr_format=self._member_zarr_format(kwargs.pop("zarr_format", None)), + **kwargs, + ) + + async def ones_like( + self, *, name: str, data: async_api.ArrayLike, **kwargs: Any + ) -> AnyAsyncArray: + """Create a sub-array of ones like `data`. + + Parameters + ---------- + name : str + Name of the array. + data : array-like + The array to create the new array like. + **kwargs + Keyword arguments passed to [zarr.api.asynchronous.create][]. + + Returns + ------- + AsyncArray + The new array. + """ + return await async_api.ones_like( + a=data, + store=self.store_path, + path=name, + zarr_format=self._member_zarr_format(kwargs.pop("zarr_format", None)), + **kwargs, + ) + + async def full_like( + self, *, name: str, data: async_api.ArrayLike, **kwargs: Any + ) -> AnyAsyncArray: + """Create a sub-array like `data` filled with the `fill_value` of `data` . + + Parameters + ---------- + name : str + Name of the array. + data : array-like + The array to create the new array like. + **kwargs + Keyword arguments passed to [zarr.api.asynchronous.create][]. + + Returns + ------- + AsyncArray + The new array. + """ + return await async_api.full_like( + a=data, + store=self.store_path, + path=name, + zarr_format=self._member_zarr_format(kwargs.pop("zarr_format", None)), + **kwargs, + ) + + async def move(self, source: str, dest: str) -> None: + """Move a sub-group or sub-array from one path to another. + + Notes + ----- + Not implemented + """ + raise NotImplementedError + + +@dataclass(frozen=True) +class Group(SyncMixin): + """ + A Zarr group. + """ + + _async_group: AsyncGroup + + @classmethod + def from_store( + cls, + store: StoreLike, + *, + attributes: dict[str, Any] | None = None, + zarr_format: ZarrFormat = 3, + overwrite: bool = False, + ) -> Group: + """Instantiate a group from an initialized store. + + Parameters + ---------- + store : StoreLike + StoreLike containing the Group. See the + [storage documentation in the user guide][user-guide-store-like] + for a description of all valid StoreLike values. + attributes : dict, optional + A dictionary of JSON-serializable values with user-defined attributes. + zarr_format : {2, 3}, optional + Zarr storage format version. + overwrite : bool, optional + If True, do not raise an error if the group already exists. + + Returns + ------- + Group + Group instantiated from the store. + + Raises + ------ + ContainsArrayError, ContainsGroupError, ContainsArrayAndGroupError + """ + attributes = attributes or {} + obj = sync( + AsyncGroup.from_store( + store, + attributes=attributes, + overwrite=overwrite, + zarr_format=zarr_format, + ), + ) + + return cls(obj) + + @classmethod + def open( + cls, + store: StoreLike, + zarr_format: ZarrFormat | None = 3, + ) -> Group: + """Open a group from an initialized store. + + Parameters + ---------- + store : StoreLike + Store containing the Group. See the + [storage documentation in the user guide][user-guide-store-like] + for a description of all valid StoreLike values. + zarr_format : {2, 3, None}, optional + Zarr storage format version. + + Returns + ------- + Group + Group instantiated from the store. + """ + obj = sync(AsyncGroup.open(store, zarr_format=zarr_format)) + return cls(obj) + + def __getitem__(self, path: str) -> AnyArray | Group: + """Obtain a group member. + + Parameters + ---------- + path : str + Group member name. + + Returns + ------- + Array | Group + Group member (Array or Group) at the specified key + + Examples + -------- + ```python + import zarr + from zarr.core.group import Group + group = Group.from_store(zarr.storage.MemoryStore()) + group.create_array(name="subarray", shape=(10,), chunks=(10,), dtype="float64") + group.create_group(name="subgroup").create_array(name="subarray", shape=(10,), chunks=(10,), dtype="float64") + group["subarray"] + # + group["subgroup"] + # + group["subgroup"]["subarray"] + # + ``` + + """ + obj = self._sync(self._async_group.getitem(path)) + if isinstance(obj, AsyncArray): + return Array(obj) + else: + return Group(obj) + + def get[DefaultT]( + self, path: str, default: DefaultT | None = None + ) -> AnyArray | Group | DefaultT | None: + """Obtain a group member, returning default if not found. + + Parameters + ---------- + path : str + Group member name. + default : object + Default value to return if key is not found (default: None). + + Returns + ------- + object + Group member (Array or Group) or default if not found. + + Examples + -------- + ```python + import zarr + from zarr.core.group import Group + group = Group.from_store(zarr.storage.MemoryStore()) + group.create_array(name="subarray", shape=(10,), chunks=(10,), dtype="float64") + group.create_group(name="subgroup") + group.get("subarray") + # + group.get("subgroup") + # + group.get("nonexistent", None) + # None + ``` + + """ + try: + return self[path] + except KeyError: + return default + + def get_array(self, path: str) -> AnyArray: + """Obtain an array member of this group, raising if it is absent or not an array. + + Parameters + ---------- + path : str + Path of the array relative to this group. May contain `/` to reference + a member of a subgroup, e.g. `subgroup/subarray`. + + Returns + ------- + Array + The array at the given path. + + Raises + ------ + ArrayNotFoundError + If no node exists at the given path. + ContainsGroupError + If the node at the given path is a group rather than an array. + + Examples + -------- + ```python + import zarr + from zarr.core.group import Group + group = Group.from_store(zarr.storage.MemoryStore()) + group.create_array(name="subarray", shape=(10,), chunks=(10,), dtype="float64") + group.get_array("subarray") + # + ``` + """ + return Array(self._sync(self._async_group.get_array(path))) + + def get_group(self, path: str) -> Group: + """Obtain a group member of this group, raising if it is absent or not a group. + + Parameters + ---------- + path : str + Path of the group relative to this group. May contain `/` to reference + a member of a subgroup, e.g. `subgroup/subsubgroup`. + + Returns + ------- + Group + The group at the given path. + + Raises + ------ + GroupNotFoundError + If no node exists at the given path. + ContainsArrayError + If the node at the given path is an array rather than a group. + + Examples + -------- + ```python + import zarr + from zarr.core.group import Group + group = Group.from_store(zarr.storage.MemoryStore()) + group.create_group(name="subgroup") + group.get_group("subgroup") + # + ``` + """ + return Group(self._sync(self._async_group.get_group(path))) + + def __delitem__(self, key: str) -> None: + """Delete a group member. + + Parameters + ---------- + key : str + Group member name. + + Examples + -------- + >>> import zarr + >>> group = Group.from_store(zarr.storage.MemoryStore()) + >>> a = group.create_array(name="subarray", dtype="i1", shape=(10,), chunks=(10,)) + >>> del group["subarray"] + >>> "subarray" in group + False + """ + + self._sync(self._async_group.delitem(key)) + + def __iter__(self) -> Iterator[str]: + """Return an iterator over group member names. + Examples + -------- + >>> import zarr + >>> g1 = zarr.group() + >>> g2 = g1.create_group('foo') + >>> g3 = g1.create_group('bar') + >>> d1 = g1.create_array('baz', dtype="i1", shape=(10,), chunks=(10,)) + >>> d2 = g1.create_array('quux', dtype="i1", shape=(10,), chunks=(10,)) + >>> sorted(g1) + ['bar', 'baz', 'foo', 'quux'] + """ + yield from self.keys() + + def __len__(self) -> int: + """Number of members.""" + return self.nmembers() + + def __setitem__(self, key: str, value: Any) -> None: + """Fastpath for creating a new array. + + New arrays will be created using default settings for the array type. + If you need to create an array with custom settings, use the `create_array` method. + + Parameters + ---------- + key : str + Array name. + value : Any + Array data. + + Examples + -------- + >>> import numpy as np + >>> import zarr + >>> group = zarr.group() + >>> group["foo"] = np.array(zarr.zeros((10,))) + >>> group["foo"] + + """ + self._sync(self._async_group.setitem(key, value)) + + def __repr__(self) -> str: + return f"" + + async def update_attributes_async(self, new_attributes: dict[str, Any]) -> Group: + """Update the attributes of this group. + + Examples + -------- + >>> async def example(): + ... import zarr + ... + ... group = zarr.group() + ... new_group = await group.update_attributes_async({"foo": "bar"}) + ... return new_group.attrs.asdict() + + >>> import asyncio + >>> asyncio.run(example()) + {'foo': 'bar'} + """ + new_metadata = replace(self.metadata, attributes=new_attributes) + + # Write new metadata + to_save = new_metadata.to_buffer_dict(default_buffer_prototype()) + awaitables = [set_or_delete(self.store_path / key, value) for key, value in to_save.items()] + await asyncio.gather(*awaitables) + + async_group = replace(self._async_group, metadata=new_metadata) + return replace(self, _async_group=async_group) + + @property + def store_path(self) -> StorePath: + """Path-like interface for the Store.""" + return self._async_group.store_path + + @property + def metadata(self) -> GroupMetadata: + """Group metadata.""" + return self._async_group.metadata + + @property + def path(self) -> str: + """Storage path.""" + return self._async_group.path + + @property + def name(self) -> str: + """Group name following h5py convention.""" + return self._async_group.name + + @property + def basename(self) -> str: + """Final component of name.""" + return self._async_group.basename + + @property + def attrs(self) -> Attributes: + """Attributes of this Group""" + return Attributes(self) + + @property + def info(self) -> Any: + """ + Return the statically known information for a group. + + Returns + ------- + GroupInfo + + Related + ------- + [zarr.Group.info_complete][] + All information about a group, including dynamic information + like the children members. + """ + return self._async_group.info + + def info_complete(self) -> Any: + """ + Return information for a group. + + If this group doesn't contain consolidated metadata then + this will need to read from the backing Store. + + Returns + ------- + GroupInfo + + Related + ------- + [zarr.Group.info][] + """ + return self._sync(self._async_group.info_complete()) + + @property + def store(self) -> Store: + # Backwards compatibility for 2.x + return self._async_group.store + + @property + def read_only(self) -> bool: + # Backwards compatibility for 2.x + return self._async_group.read_only + + @property + def synchronizer(self) -> None: + # Backwards compatibility for 2.x + # Not implemented in 3.x yet. + return self._async_group.synchronizer + + def update_attributes(self, new_attributes: dict[str, Any]) -> Group: + """Update the attributes of this group. + + Examples + -------- + >>> import zarr + >>> group = zarr.group().update_attributes({"foo": "bar"}) + >>> group.attrs.asdict() + {'foo': 'bar'} + """ + self._sync(self._async_group.update_attributes(new_attributes)) + return self + + def nmembers(self, max_depth: int | None = 0) -> int: + """Count the number of members in this group. + + Parameters + ---------- + max_depth : int, default 0 + The maximum number of levels of the hierarchy to include. By + default, (``max_depth=0``) only immediate children are included. Set + ``max_depth=None`` to include all nodes, and some positive integer + to consider children within that many levels of the root Group. + + Returns + ------- + count : int + """ + + return self._sync(self._async_group.nmembers(max_depth=max_depth)) + + def members( + self, max_depth: int | None = 0, *, use_consolidated_for_children: bool = True + ) -> tuple[tuple[str, AnyArray | Group], ...]: + """ + Returns an AsyncGenerator over the arrays and groups contained in this group. + This method requires that `store_path.store` supports directory listing. + + The results are not guaranteed to be ordered. + + Parameters + ---------- + max_depth : int, default 0 + The maximum number of levels of the hierarchy to include. By + default, (``max_depth=0``) only immediate children are included. Set + ``max_depth=None`` to include all nodes, and some positive integer + to consider children within that many levels of the root Group. + use_consolidated_for_children : bool, default True + Whether to use the consolidated metadata of child groups loaded + from the store. Note that this only affects groups loaded from the + store. If the current Group already has consolidated metadata, it + will always be used. + + Returns + ------- + path: + A string giving the path to the target, relative to the Group ``self``. + value: AsyncArray or AsyncGroup + The AsyncArray or AsyncGroup that is a child of ``self``. + """ + _members = self._sync_iter(self._async_group.members(max_depth=max_depth)) + + return tuple((kv[0], _parse_async_node(kv[1])) for kv in _members) + + def create_hierarchy( + self, + nodes: dict[str, ArrayV2Metadata | ArrayV3Metadata | GroupMetadata], + *, + overwrite: bool = False, + ) -> Iterator[tuple[str, Group | AnyArray]]: + """ + Create a hierarchy of arrays or groups rooted at this group. + + This function will parse its input to ensure that the hierarchy is complete. Any implicit groups + will be inserted as needed. For example, an input like + ```{'a/b': GroupMetadata}``` will be parsed to + ```{'': GroupMetadata, 'a': GroupMetadata, 'b': Groupmetadata}```. + + Explicitly specifying a root group, e.g. with ``nodes = {'': GroupMetadata()}`` is an error + because this group instance is the root group. + + After input parsing, this function then creates all the nodes in the hierarchy concurrently. + + Arrays and Groups are yielded in the order they are created. This order is not stable and + should not be relied on. + + Parameters + ---------- + nodes : dict[str, GroupMetadata | ArrayV3Metadata | ArrayV2Metadata] + A dictionary defining the hierarchy. The keys are the paths of the nodes in the hierarchy, + relative to the path of the group. The values are instances of ``GroupMetadata`` or ``ArrayMetadata``. Note that + all values must have the same ``zarr_format`` as the parent group -- it is an error to mix zarr versions in the + same hierarchy. + + Leading "/" characters from keys will be removed. + overwrite : bool + Whether to overwrite existing nodes. Defaults to ``False``, in which case an error is + raised instead of overwriting an existing array or group. + + This function will not erase an existing group unless that group is explicitly named in + ``nodes``. If ``nodes`` defines implicit groups, e.g. ``{`'a/b/c': GroupMetadata}``, and a + group already exists at path ``a``, then this function will leave the group at ``a`` as-is. + + Yields + ------ + tuple[str, Array | Group]. + + Examples + -------- + >>> import zarr + >>> from zarr.core.group import GroupMetadata + >>> root = zarr.create_group(store={}) + >>> sorted(root.create_hierarchy({'a/b/c': GroupMetadata()})) + [('a', ), + ('a/b', ), + ('a/b/c', )] + """ + for key, node in self._sync_iter( + self._async_group.create_hierarchy(nodes, overwrite=overwrite) + ): + yield (key, _parse_async_node(node)) + + def keys(self) -> Generator[str]: + """Return an iterator over group member names. + + Examples + -------- + >>> import zarr + >>> g1 = zarr.group() + >>> g2 = g1.create_group('foo') + >>> g3 = g1.create_group('bar') + >>> d1 = g1.create_array('baz', dtype="i1", shape=(10,), chunks=(10,)) + >>> d2 = g1.create_array('quux', dtype="i1", shape=(10,), chunks=(10,)) + >>> sorted(g1.keys()) + ['bar', 'baz', 'foo', 'quux'] + """ + yield from self._sync_iter(self._async_group.keys()) + + def __contains__(self, member: str) -> bool: + """Test for group membership. + + Examples + -------- + >>> import zarr + >>> g1 = zarr.group() + >>> g2 = g1.create_group('foo') + >>> d1 = g1.create_array('bar', dtype="i1", shape=(10,), chunks=(10,)) + >>> 'foo' in g1 + True + >>> 'bar' in g1 + True + >>> 'baz' in g1 + False + """ + return self._sync(self._async_group.contains(member)) + + def groups(self) -> Generator[tuple[str, Group], None]: + """Return the sub-groups of this group as a generator of (name, group) pairs. + + Examples + -------- + >>> import zarr + >>> group = zarr.group() + >>> subgroup = group.create_group("subgroup") + >>> list(group.groups()) + [('subgroup', )] + """ + for name, async_group in self._sync_iter(self._async_group.groups()): + yield name, Group(async_group) + + def group_keys(self) -> Generator[str, None]: + """Return an iterator over group member names. + + Examples + -------- + >>> import zarr + >>> group = zarr.group() + >>> subgroup = group.create_group("subgroup") + >>> list(group.group_keys()) + ['subgroup'] + """ + for name, _ in self.groups(): + yield name + + def group_values(self) -> Generator[Group, None]: + """Return an iterator over group members. + + Examples + -------- + >>> import zarr + >>> group = zarr.group() + >>> subgroup = group.create_group("subgroup") + >>> list(group.group_values()) + [] + """ + for _, group in self.groups(): + yield group + + def arrays(self) -> Generator[tuple[str, AnyArray], None]: + """Return the sub-arrays of this group as a generator of (name, array) pairs + + Examples + -------- + >>> import zarr + >>> group = zarr.group() + >>> subarray = group.create_array("subarray", dtype="i1", shape=(10,), chunks=(10,)) + >>> list(group.arrays()) + [('subarray', )] + """ + for name, async_array in self._sync_iter(self._async_group.arrays()): + yield name, Array(async_array) + + def array_keys(self) -> Generator[str, None]: + """Return an iterator over group member names. + + Examples + -------- + >>> import zarr + >>> group = zarr.group() + >>> subarray = group.create_array("subarray", dtype="i1", shape=(10,), chunks=(10,)) + >>> list(group.array_keys()) + ['subarray'] + """ + + for name, _ in self.arrays(): + yield name + + def array_values(self) -> Generator[AnyArray, None]: + """Return an iterator over group members. + + Examples + -------- + >>> import zarr + >>> group = zarr.group() + >>> subarray = group.create_array("subarray", dtype="i1", shape=(10,), chunks=(10,)) + >>> list(group.array_values()) + [] + """ + for _, array in self.arrays(): + yield array + + def tree( + self, + expand: bool | None = None, + level: int | None = None, + *, + max_nodes: int = 500, + plain: bool = False, + ) -> Any: + """ + Return a tree-like representation of a hierarchy. + + Parameters + ---------- + expand : bool, optional + This keyword is not yet supported. A NotImplementedError is raised if + it's used. + level : int, optional + The maximum depth below this Group to display in the tree. + max_nodes : int + Maximum number of nodes to display before truncating. Default is 500. + plain : bool, optional + If True, return a plain-text tree without ANSI styling. This is + useful when the output will be consumed by an LLM or written to a + file. Default is False. + + Returns + ------- + TreeRepr + A pretty-printable object displaying the hierarchy. + """ + return self._sync( + self._async_group.tree(expand=expand, level=level, max_nodes=max_nodes, plain=plain) + ) + + def create_group(self, name: str, **kwargs: Any) -> Group: + """Create a sub-group. + + Parameters + ---------- + name : str + Name of the new subgroup. + + Returns + ------- + Group + + Examples + -------- + >>> import zarr + >>> group = zarr.group() + >>> subgroup = group.create_group("subgroup") + >>> subgroup + + """ + return Group(self._sync(self._async_group.create_group(name, **kwargs))) + + def require_group(self, name: str, **kwargs: Any) -> Group: + """Obtain a sub-group, creating one if it doesn't exist. + + Parameters + ---------- + name : str + Group name. + + Returns + ------- + g : Group + """ + return Group(self._sync(self._async_group.require_group(name, **kwargs))) + + def require_groups(self, *names: str) -> tuple[Group, ...]: + """Convenience method to require multiple groups in a single call. + + Parameters + ---------- + *names : str + Group names. + + Returns + ------- + groups : tuple of Groups + """ + return tuple(map(Group, self._sync(self._async_group.require_groups(*names)))) + + def create( + self, + name: str, + *, + shape: ShapeLike | None = None, + dtype: ZDTypeLike | None = None, + data: np.ndarray[Any, np.dtype[Any]] | None = None, + chunks: ChunksLike | Literal["auto"] = "auto", + shards: ShardsLike | None = None, + filters: FiltersLike = "auto", + compressors: CompressorsLike = "auto", + compressor: CompressorLike = "auto", + serializer: SerializerLike = "auto", + fill_value: Any | None = DEFAULT_FILL_VALUE, + order: MemoryOrder | None = None, + attributes: dict[str, JSON] | None = None, + chunk_key_encoding: ChunkKeyEncodingLike | None = None, + dimension_names: DimensionNamesLike = None, + storage_options: dict[str, Any] | None = None, + overwrite: bool = False, + config: ArrayConfigLike | None = None, + write_data: bool = True, + ) -> AnyArray: + """Create an array within this group. + + This method lightly wraps [`zarr.core.array.create_array`][]. + + Parameters + ---------- + name : str + The name of the array relative to the group. If `path` is `None`, the array will be located + at the root of the store. + shape : ShapeLike, optional + Shape of the array. Must be `None` if `data` is provided. + dtype : ZDTypeLike | None + Data type of the array. Must be `None` if `data` is provided. + data : np.ndarray, optional + Array-like data to use for initializing the array. If this parameter is provided, the + `shape` and `dtype` parameters must be `None`. + chunks : tuple[int, ...] | Sequence[Sequence[int]] | Literal["auto"], default="auto" + Chunk shape of the array. + If chunks is "auto", a chunk shape is guessed based on the shape of the array and the dtype. + A nested list of per-dimension edge sizes creates a rectilinear grid. + Rectilinear chunk grids are experimental and must be explicitly enabled + with `zarr.config.set({'array.rectilinear_chunks': True})` while the + feature is stabilizing. + shards : tuple[int, ...], optional + Shard shape of the array. The default value of `None` results in no sharding at all. + filters : Iterable[Codec] | Literal["auto"], optional + Iterable of filters to apply to each chunk of the array, in order, before serializing that + chunk to bytes. + + For Zarr format 3, a "filter" is a codec that takes an array and returns an array, + and these values must be instances of [`zarr.abc.codec.ArrayArrayCodec`][], or a + dict representations of [`zarr.abc.codec.ArrayArrayCodec`][]. + + For Zarr format 2, a "filter" can be any numcodecs codec; you should ensure that the + order of your filters is consistent with the behavior of each filter. + + The default value of `"auto"` instructs Zarr to use a default based on the data + type of the array and the Zarr format specified. For all data types in Zarr V3, and most + data types in Zarr V2, the default filters are empty. The only cases where default filters + are not empty is when the Zarr format is 2, and the data type is a variable-length data type like + [`zarr.dtype.VariableLengthUTF8`][] or [`zarr.dtype.VariableLengthBytes`][]. In these cases, + the default filters contains a single element which is a codec specific to that particular data type. + + To create an array with no filters, provide an empty iterable or the value `None`. + compressors : Iterable[Codec], optional + List of compressors to apply to the array. Compressors are applied in order, and after any + filters are applied (if any are specified) and the data is serialized into bytes. + + For Zarr format 3, a "compressor" is a codec that takes a bytestream, and + returns another bytestream. Multiple compressors may be provided for Zarr format 3. + If no `compressors` are provided, a default set of compressors will be used. + These defaults can be changed by modifying the value of `array.v3_default_compressors` + in [`zarr.config`][zarr.config]. + Use `None` to omit default compressors. + + For Zarr format 2, a "compressor" can be any numcodecs codec. Only a single compressor may + be provided for Zarr format 2. + If no `compressor` is provided, a default compressor will be used. + This default can be changed by modifying the value of `array.v2_default_compressor` + in [`zarr.config`][zarr.config]. + Use `None` to omit the default compressor. + compressor : Codec, optional + Deprecated in favor of `compressors`. + serializer : dict[str, JSON] | ArrayBytesCodec, optional + Array-to-bytes codec to use for encoding the array data. + Zarr format 3 only. Zarr format 2 arrays use implicit array-to-bytes conversion. + If no `serializer` is provided, a default serializer will be used. + These defaults can be changed by modifying the value of `array.v3_default_serializer` + in [`zarr.config`][zarr.config]. + fill_value : Any, optional + Fill value for the array. + order : {"C", "F"}, optional + The memory order of the array (default is "C"). + For Zarr format 2, this parameter sets the memory order of the array. + For Zarr format 3, this parameter is deprecated, because memory order + is a runtime parameter for Zarr format 3 arrays. The recommended way to specify the memory + order for Zarr format 3 arrays is via the `config` parameter, e.g. `{'config': 'C'}`. + If no `order` is provided, a default order will be used. + This default can be changed by modifying the value of `array.order` in [`zarr.config`][zarr.config]. + attributes : dict, optional + Attributes for the array. + chunk_key_encoding : ChunkKeyEncodingLike, optional + A specification of how the chunk keys are represented in storage. + For Zarr format 3, the default is `{"name": "default", "separator": "/"}`. + For Zarr format 2, the default is `{"name": "v2", "separator": "."}`. + dimension_names : Iterable[str], optional + The names of the dimensions (default is None). + Zarr format 3 only. Zarr format 2 arrays should not use this parameter. + storage_options : dict, optional + If using an fsspec URL to create the store, these will be passed to the backend implementation. + Ignored otherwise. + overwrite : bool, default False + Whether to overwrite an array with the same name in the store, if one exists. + If `True`, any existing keys under that path are deleted first. + config : ArrayConfigLike, optional + Runtime configuration for the array. + write_data : bool + If a pre-existing array-like object was provided to this function via the `data` parameter + then `write_data` determines whether the values in that array-like object should be + written to the Zarr array created by this function. If `write_data` is `False`, then the + array will be left empty. + + Returns + ------- + AsyncArray + """ + return self.create_array( + name, + shape=shape, + dtype=dtype, + data=data, + chunks=chunks, + shards=shards, + filters=filters, + compressors=compressors, + compressor=compressor, + serializer=serializer, + fill_value=fill_value, + order=order, + attributes=attributes, + chunk_key_encoding=chunk_key_encoding, + dimension_names=dimension_names, + storage_options=storage_options, + overwrite=overwrite, + config=config, + write_data=write_data, + ) + + def create_array( + self, + name: str, + *, + shape: ShapeLike | None = None, + dtype: ZDTypeLike | None = None, + data: np.ndarray[Any, np.dtype[Any]] | None = None, + chunks: ChunksLike | Literal["auto"] = "auto", + shards: ShardsLike | None = None, + filters: FiltersLike = "auto", + compressors: CompressorsLike = "auto", + compressor: CompressorLike = "auto", + serializer: SerializerLike = "auto", + fill_value: Any | None = DEFAULT_FILL_VALUE, + order: MemoryOrder | None = None, + attributes: dict[str, JSON] | None = None, + chunk_key_encoding: ChunkKeyEncodingLike | None = None, + dimension_names: DimensionNamesLike = None, + storage_options: dict[str, Any] | None = None, + overwrite: bool = False, + config: ArrayConfigLike | None = None, + write_data: bool = True, + ) -> AnyArray: + """Create an array within this group. + + This method lightly wraps [zarr.core.array.create_array][]. + + Parameters + ---------- + name : str + The name of the array relative to the group. If `path` is `None`, the array will be located + at the root of the store. + shape : ShapeLike, optional + Shape of the array. Must be `None` if `data` is provided. + dtype : ZDTypeLike | None + Data type of the array. Must be `None` if `data` is provided. + data : np.ndarray, optional + Array-like data to use for initializing the array. If this parameter is provided, the + `shape` and `dtype` parameters must be `None`. + chunks : tuple[int, ...] | Sequence[Sequence[int]] | Literal["auto"], default="auto" + Chunk shape of the array. + If chunks is "auto", a chunk shape is guessed based on the shape of the array and the dtype. + A nested list of per-dimension edge sizes creates a rectilinear grid. + Rectilinear chunk grids are experimental and must be explicitly enabled + with `zarr.config.set({'array.rectilinear_chunks': True})` while the + feature is stabilizing. + shards : tuple[int, ...], optional + Shard shape of the array. The default value of `None` results in no sharding at all. + filters : Iterable[Codec] | Literal["auto"], optional + Iterable of filters to apply to each chunk of the array, in order, before serializing that + chunk to bytes. + + For Zarr format 3, a "filter" is a codec that takes an array and returns an array, + and these values must be instances of [`zarr.abc.codec.ArrayArrayCodec`][], or a + dict representations of [`zarr.abc.codec.ArrayArrayCodec`][]. + + For Zarr format 2, a "filter" can be any numcodecs codec; you should ensure that the + order of your filters is consistent with the behavior of each filter. + + The default value of `"auto"` instructs Zarr to use a default based on the data + type of the array and the Zarr format specified. For all data types in Zarr V3, and most + data types in Zarr V2, the default filters are empty. The only cases where default filters + are not empty is when the Zarr format is 2, and the data type is a variable-length data type like + [`zarr.dtype.VariableLengthUTF8`][] or [`zarr.dtype.VariableLengthBytes`][]. In these cases, + the default filters contains a single element which is a codec specific to that particular data type. + + To create an array with no filters, provide an empty iterable or the value `None`. + compressors : Iterable[Codec], optional + List of compressors to apply to the array. Compressors are applied in order, and after any + filters are applied (if any are specified) and the data is serialized into bytes. + + For Zarr format 3, a "compressor" is a codec that takes a bytestream, and + returns another bytestream. Multiple compressors may be provided for Zarr format 3. + If no `compressors` are provided, a default set of compressors will be used. + These defaults can be changed by modifying the value of `array.v3_default_compressors` + in [`zarr.config`][zarr.config]. + Use `None` to omit default compressors. + + For Zarr format 2, a "compressor" can be any numcodecs codec. Only a single compressor may + be provided for Zarr format 2. + If no `compressor` is provided, a default compressor will be used. + This default can be changed by modifying the value of `array.v2_default_compressor` + in [`zarr.config`][zarr.config]. + Use `None` to omit the default compressor. + compressor : Codec, optional + Deprecated in favor of `compressors`. + serializer : dict[str, JSON] | ArrayBytesCodec, optional + Array-to-bytes codec to use for encoding the array data. + Zarr format 3 only. Zarr format 2 arrays use implicit array-to-bytes conversion. + If no `serializer` is provided, a default serializer will be used. + These defaults can be changed by modifying the value of `array.v3_default_serializer` + in [`zarr.config`][zarr.config]. + fill_value : Any, optional + Fill value for the array. + order : {"C", "F"}, optional + The memory order of the array (default is "C"). + For Zarr format 2, this parameter sets the memory order of the array. + For Zarr format 3, this parameter is deprecated, because memory order + is a runtime parameter for Zarr format 3 arrays. The recommended way to specify the memory + order for Zarr format 3 arrays is via the `config` parameter, e.g. `{'config': 'C'}`. + If no `order` is provided, a default order will be used. + This default can be changed by modifying the value of `array.order` in [`zarr.config`][zarr.config]. + attributes : dict, optional + Attributes for the array. + chunk_key_encoding : ChunkKeyEncodingLike, optional + A specification of how the chunk keys are represented in storage. + For Zarr format 3, the default is `{"name": "default", "separator": "/"}`. + For Zarr format 2, the default is `{"name": "v2", "separator": "."}`. + dimension_names : Iterable[str], optional + The names of the dimensions (default is None). + Zarr format 3 only. Zarr format 2 arrays should not use this parameter. + storage_options : dict, optional + If using an fsspec URL to create the store, these will be passed to the backend implementation. + Ignored otherwise. + overwrite : bool, default False + Whether to overwrite an array with the same name in the store, if one exists. + If `True`, any existing keys under that path are deleted first. + config : ArrayConfigLike, optional + Runtime configuration for the array. + write_data : bool + If a pre-existing array-like object was provided to this function via the `data` parameter + then `write_data` determines whether the values in that array-like object should be + written to the Zarr array created by this function. If `write_data` is `False`, then the + array will be left empty. + + Returns + ------- + AsyncArray + """ + compressors = _parse_deprecated_compressor( + compressor, compressors, zarr_format=self.metadata.zarr_format + ) + return Array( + self._sync( + self._async_group.create_array( + name=name, + shape=shape, + dtype=dtype, + data=data, + chunks=chunks, + shards=shards, + fill_value=fill_value, + attributes=attributes, + chunk_key_encoding=chunk_key_encoding, + compressors=compressors, + serializer=serializer, + dimension_names=dimension_names, + order=order, + filters=filters, + overwrite=overwrite, + storage_options=storage_options, + config=config, + write_data=write_data, + ) + ) + ) + + def require_array( + self, + name: str, + *, + shape: ShapeLike, + config: ArrayConfigLike | None = None, + **kwargs: Any, + ) -> AnyArray: + """Obtain an array, creating if it doesn't exist. + + Other `kwargs` are as per [zarr.Group.create_array][]. + + Parameters + ---------- + name : str + Array name. + config : ArrayConfigLike or None, default=None + Runtime configuration for the array, whether it is created or already exists. + Keys not specified are taken from the global configuration. + **kwargs : + See [zarr.Group.create_array][]. + + Returns + ------- + a : Array + """ + return Array( + self._sync(self._async_group.require_array(name, shape=shape, config=config, **kwargs)) + ) + + def empty(self, *, name: str, shape: tuple[int, ...], **kwargs: Any) -> AnyArray: + """Create an empty array with the specified shape in this Group. The contents will be filled with + the array's fill value or zeros if no fill value is provided. + + Parameters + ---------- + name : str + Name of the array. + shape : int or tuple of int + Shape of the empty array. + **kwargs + Keyword arguments passed to [zarr.api.asynchronous.create][]. + + Notes + ----- + The contents of an empty Zarr array are not defined. On attempting to + retrieve data from an empty Zarr array, any values may be returned, + and these are not guaranteed to be stable from one access to the next. + """ + return Array(self._sync(self._async_group.empty(name=name, shape=shape, **kwargs))) + + def zeros(self, *, name: str, shape: tuple[int, ...], **kwargs: Any) -> AnyArray: + """Create an array, with zero being used as the default value for uninitialized portions of the array. + + Parameters + ---------- + name : str + Name of the array. + shape : int or tuple of int + Shape of the empty array. + **kwargs + Keyword arguments passed to [zarr.api.asynchronous.create][]. + + Returns + ------- + Array + The new array. + """ + return Array(self._sync(self._async_group.zeros(name=name, shape=shape, **kwargs))) + + def ones(self, *, name: str, shape: tuple[int, ...], **kwargs: Any) -> AnyArray: + """Create an array, with one being used as the default value for uninitialized portions of the array. + + Parameters + ---------- + name : str + Name of the array. + shape : int or tuple of int + Shape of the empty array. + **kwargs + Keyword arguments passed to [zarr.api.asynchronous.create][]. + + Returns + ------- + Array + The new array. + """ + return Array(self._sync(self._async_group.ones(name=name, shape=shape, **kwargs))) + + def full( + self, *, name: str, shape: tuple[int, ...], fill_value: Any | None, **kwargs: Any + ) -> AnyArray: + """Create an array, with "fill_value" being used as the default value for uninitialized portions of the array. + + Parameters + ---------- + name : str + Name of the array. + shape : int or tuple of int + Shape of the empty array. + fill_value : scalar + Value to fill the array with. + **kwargs + Keyword arguments passed to [zarr.api.asynchronous.create][]. + + Returns + ------- + Array + The new array. + """ + return Array( + self._sync( + self._async_group.full(name=name, shape=shape, fill_value=fill_value, **kwargs) + ) + ) + + def empty_like(self, *, name: str, data: async_api.ArrayLike, **kwargs: Any) -> AnyArray: + """Create an empty sub-array like `data`. The contents will be filled + with the array's fill value or zeros if no fill value is provided. + + Parameters + ---------- + name : str + Name of the array. + data : array-like + The array to create an empty array like. + **kwargs + Keyword arguments passed to [zarr.api.asynchronous.create][]. + + Returns + ------- + Array + The new array. + + Notes + ----- + The contents of an empty Zarr array are not defined. On attempting to + retrieve data from an empty Zarr array, any values may be returned, + and these are not guaranteed to be stable from one access to the next. + """ + return Array(self._sync(self._async_group.empty_like(name=name, data=data, **kwargs))) + + def zeros_like(self, *, name: str, data: async_api.ArrayLike, **kwargs: Any) -> AnyArray: + """Create a sub-array of zeros like `data`. + + Parameters + ---------- + name : str + Name of the array. + data : array-like + The array to create the new array like. + **kwargs + Keyword arguments passed to [zarr.api.asynchronous.create][]. + + Returns + ------- + Array + The new array. + """ + + return Array(self._sync(self._async_group.zeros_like(name=name, data=data, **kwargs))) + + def ones_like(self, *, name: str, data: async_api.ArrayLike, **kwargs: Any) -> AnyArray: + """Create a sub-array of ones like `data`. + + Parameters + ---------- + name : str + Name of the array. + data : array-like + The array to create the new array like. + **kwargs + Keyword arguments passed to [zarr.api.asynchronous.create][]. + + Returns + ------- + Array + The new array. + """ + return Array(self._sync(self._async_group.ones_like(name=name, data=data, **kwargs))) + + def full_like(self, *, name: str, data: async_api.ArrayLike, **kwargs: Any) -> AnyArray: + """Create a sub-array like `data` filled with the `fill_value` of `data` . + + Parameters + ---------- + name : str + Name of the array. + data : array-like + The array to create the new array like. + **kwargs + Keyword arguments passed to [zarr.api.asynchronous.create][]. + + Returns + ------- + Array + The new array. + """ + return Array(self._sync(self._async_group.full_like(name=name, data=data, **kwargs))) + + def move(self, source: str, dest: str) -> None: + """Move a sub-group or sub-array from one path to another. + + Notes + ----- + Not implemented + """ + return self._sync(self._async_group.move(source, dest)) + + +async def create_hierarchy( + *, + store: Store, + nodes: dict[str, GroupMetadata | ArrayV2Metadata | ArrayV3Metadata], + overwrite: bool = False, +) -> AsyncIterator[tuple[str, AsyncGroup | AnyAsyncArray]]: + """ + Create a complete zarr hierarchy from a collection of metadata objects. + + This function will parse its input to ensure that the hierarchy is complete. Any implicit groups + will be inserted as needed. For example, an input like + ```{'a/b': GroupMetadata}``` will be parsed to + ```{'': GroupMetadata, 'a': GroupMetadata, 'b': Groupmetadata}``` + + After input parsing, this function then creates all the nodes in the hierarchy concurrently. + + Arrays and Groups are yielded in the order they are created. This order is not stable and + should not be relied on. + + Parameters + ---------- + store : Store + The storage backend to use. + nodes : dict[str, GroupMetadata | ArrayV3Metadata | ArrayV2Metadata] + A dictionary defining the hierarchy. The keys are the paths of the nodes in the hierarchy, + relative to the root of the ``Store``. The root of the store can be specified with the empty + string ``''``. The values are instances of ``GroupMetadata`` or ``ArrayMetadata``. Note that + all values must have the same ``zarr_format`` -- it is an error to mix zarr versions in the + same hierarchy. + + Leading "/" characters from keys will be removed. + overwrite : bool + Whether to overwrite existing nodes. Defaults to ``False``, in which case an error is + raised instead of overwriting an existing array or group. + + This function will not erase an existing group unless that group is explicitly named in + ``nodes``. If ``nodes`` defines implicit groups, e.g. ``{`'a/b/c': GroupMetadata}``, and a + group already exists at path ``a``, then this function will leave the group at ``a`` as-is. + + Yields + ------ + tuple[str, AsyncGroup | AsyncArray] + Yields (path, node) pairs, in the order the nodes were created. + + Examples + -------- + >>> async def example(): + ... from zarr.api.asynchronous import create_hierarchy + ... from zarr.core.group import GroupMetadata + ... from zarr.storage import MemoryStore + ... + ... store = MemoryStore() + ... nodes = {'a': GroupMetadata(attributes={'name': 'leaf'})} + ... return sorted([x async for x in create_hierarchy(store=store, nodes=nodes)]) + + >>> import asyncio + >>> asyncio.run(example()) + [('', ), ('a', )] + """ + + # normalize the keys to be valid paths + nodes_normed_keys = _normalize_path_keys(nodes) + + # ensure that all nodes have the same zarr_format, and add implicit groups as needed + nodes_parsed = _parse_hierarchy_dict(data=nodes_normed_keys) + redundant_implicit_groups = [] + to_delete_keys: list[str] = [] + + # empty hierarchies should be a no-op + if len(nodes_parsed) > 0: + # figure out which zarr format we are using + zarr_format = next(iter(nodes_parsed.values())).zarr_format + + # check which implicit groups will require materialization + implicit_group_keys = tuple( + filter(lambda k: isinstance(nodes_parsed[k], ImplicitGroupMarker), nodes_parsed) + ) + # read potential group metadata for each implicit group + maybe_extant_group_coros = ( + _read_group_metadata(store, k, zarr_format=zarr_format) for k in implicit_group_keys + ) + maybe_extant_groups = await asyncio.gather( + *maybe_extant_group_coros, return_exceptions=True + ) + + for key, value in zip(implicit_group_keys, maybe_extant_groups, strict=True): + if isinstance(value, BaseException): + if isinstance(value, FileNotFoundError): + # this is fine -- there was no group there, so we will create one + pass + else: + raise value + else: + # a loop exists already at ``key``, so we can avoid creating anything there + redundant_implicit_groups.append(key) + + if overwrite: + # we will remove any nodes that collide with arrays and non-implicit groups defined in + # nodes + to_delete_keys = [k for k in nodes_parsed if k not in implicit_group_keys] + else: + # This type is long. + coros: ( + Generator[Coroutine[Any, Any, ArrayV2Metadata | GroupMetadata], None, None] + | Generator[Coroutine[Any, Any, ArrayV3Metadata | GroupMetadata], None, None] + ) + if zarr_format == 2: + coros = (_read_metadata_v2(store=store, path=key) for key in nodes_parsed) + elif zarr_format == 3: + coros = (_read_metadata_v3(store=store, path=key) for key in nodes_parsed) + else: # pragma: no cover + raise ValueError(f"Invalid zarr_format: {zarr_format}") # pragma: no cover + + extant_node_query = dict( + zip( + nodes_parsed.keys(), + await asyncio.gather(*coros, return_exceptions=True), + strict=False, + ) + ) + # iterate over the existing arrays / groups and figure out which of them conflict + # with the arrays / groups we want to create + for key, extant_node in extant_node_query.items(): + proposed_node = nodes_parsed[key] + if isinstance(extant_node, BaseException): + if isinstance(extant_node, FileNotFoundError): + # ignore FileNotFoundError, because they represent nodes we can safely create + pass + else: + # Any other exception is a real error + raise extant_node + else: + # this is a node that already exists, but a node with the same key was specified + # in nodes_parsed. + if isinstance(extant_node, GroupMetadata): + # a group already exists where we want to create a group + if isinstance(proposed_node, ImplicitGroupMarker): + # we have proposed an implicit group, which is OK -- we will just skip + # creating this particular metadata document + redundant_implicit_groups.append(key) + else: + # we have proposed an explicit group, which is an error, given that a + # group already exists. + msg = f"A group exists in store {store!r} at path {key!r}." + raise ContainsGroupError(msg) + elif isinstance(extant_node, ArrayV2Metadata | ArrayV3Metadata): + # we are trying to overwrite an existing array. this is an error. + msg = f"An array exists in store {store!r} at path {key!r}." + raise ContainsArrayError(msg) + + nodes_explicit: dict[str, GroupMetadata | ArrayV2Metadata | ArrayV3Metadata] = {} + + for k, v in nodes_parsed.items(): + if k not in redundant_implicit_groups: + if isinstance(v, ImplicitGroupMarker): + nodes_explicit[k] = GroupMetadata(zarr_format=v.zarr_format) + else: + nodes_explicit[k] = v + + # Build and encode every node before deleting anything: a node that cannot be built + # or whose metadata cannot be stored then fails with the store untouched. + built, documents = _prepare_nodes(store, nodes_explicit) + await asyncio.gather(*(store.delete_dir(key) for key in to_delete_keys)) + async for key, node in _store_nodes(store, nodes_explicit, built, documents): + yield key, node + + +async def create_nodes( + *, + store: Store, + nodes: dict[str, GroupMetadata | ArrayV2Metadata | ArrayV3Metadata], +) -> AsyncIterator[tuple[str, AsyncGroup | AnyAsyncArray]]: + """Create a collection of arrays and / or groups concurrently. + + Note: no attempt is made to validate that these arrays and / or groups collectively form a + valid Zarr hierarchy. It is the responsibility of the caller of this function to ensure that + the ``nodes`` parameter satisfies any correctness constraints. + + Parameters + ---------- + store : Store + The storage backend to use. + nodes : dict[str, GroupMetadata | ArrayV3Metadata | ArrayV2Metadata] + A dictionary defining the hierarchy. The keys are the paths of the nodes + in the hierarchy, and the values are the metadata of the nodes. The + metadata must be either an instance of GroupMetadata, ArrayV3Metadata + or ArrayV2Metadata. + + Yields + ------ + AsyncGroup | AsyncArray + The created nodes in the order they are created. + """ + async for key, node in _store_nodes(store, nodes, *_prepare_nodes(store, nodes)): + yield key, node + + +def _prepare_nodes( + store: Store, nodes: Mapping[str, GroupMetadata | ArrayV2Metadata | ArrayV3Metadata] +) -> tuple[dict[str, AsyncGroup | AnyAsyncArray], dict[str, Buffer]]: + """The array or group each of `nodes` describes, at its path in `store`, and the + metadata documents of `nodes`, by their keys in the store: every node is built and + encoded before anything is stored.""" + for path, metadata in nodes.items(): + validate_node_path(path, metadata.zarr_format) + + built = { + path: _build_node(store=store, path=path, metadata=meta) for path, meta in nodes.items() + } + documents = { + _join_paths([path, key]): value + for path, metadata in nodes.items() + for key, value in encode_documents(StorePath(store, path), metadata).items() + } + return built, documents + + +async def _store_nodes( + store: Store, + nodes: Mapping[str, GroupMetadata | ArrayV2Metadata | ArrayV3Metadata], + built: Mapping[str, AsyncGroup | AnyAsyncArray], + documents: Mapping[str, Buffer], +) -> AsyncIterator[tuple[str, AsyncGroup | AnyAsyncArray]]: + """Store the `documents` encoded from `nodes` and yield the nodes built from them, as + `_prepare_nodes` returns them (see `create_nodes`).""" + # Note: the only way to alter this value is via the config. If that's undesirable for some reason, + # then we should consider adding a keyword argument to this function + semaphore = asyncio.Semaphore(config.get("async.concurrency")) + create_tasks = [ + _set_return_key(store=store, key=key, value=value, semaphore=semaphore) + for key, value in documents.items() + ] + + created_object_keys = [] + + for coro in asyncio.as_completed(create_tasks): + created_key = await coro + # we need this to track which metadata documents were written so that we can yield a + # complete v2 Array / Group class after both .zattrs and the metadata JSON was created. + created_object_keys.append(created_key) + + # get the node name from the object key + if len(created_key.split("/")) == 1: + # this is the root node + meta_out = nodes[""] + node_name = "" + else: + # turn "foo/" into "foo" + node_name = created_key[: created_key.rfind("/")] + meta_out = nodes[node_name] + if meta_out.zarr_format == 3: + yield node_name, built[node_name] + else: + # For zarr v2 + # we only want to yield when both the metadata and attributes are created + # so we track which keys have been created, and wait for both the meta key and + # the attrs key to be created before yielding back the AsyncArray / AsyncGroup + + attrs_done = _join_paths([node_name, ZATTRS_JSON]) in created_object_keys + + if isinstance(meta_out, GroupMetadata): + meta_done = _join_paths([node_name, ZGROUP_JSON]) in created_object_keys + else: + meta_done = _join_paths([node_name, ZARRAY_JSON]) in created_object_keys + + if meta_done and attrs_done: + yield node_name, built[node_name] + + continue + + +def _get_roots( + data: Iterable[str], +) -> tuple[str, ...]: + """ + Return the keys of the root(s) of the hierarchy. A root is a key with the fewest number of + path segments. + """ + if "" in data: + return ("",) + keys_split = sorted((key.split("/") for key in data), key=len) + groups: defaultdict[int, list[str]] = defaultdict(list) + for key_split in keys_split: + groups[len(key_split)].append("/".join(key_split)) + return tuple(groups[min(groups.keys())]) + + +def _parse_hierarchy_dict( + *, + data: Mapping[str, ImplicitGroupMarker | GroupMetadata | ArrayV2Metadata | ArrayV3Metadata], +) -> dict[str, ImplicitGroupMarker | GroupMetadata | ArrayV2Metadata | ArrayV3Metadata]: + """ + Take an input with type Mapping[str, ArrayMetadata | GroupMetadata] and parse it into + a dict of str: node pairs that models a valid, complete Zarr hierarchy. + + If the input represents a complete Zarr hierarchy, i.e. one with no implicit groups, + then return a dict with the exact same data as the input. + + Otherwise, return a dict derived from the input with GroupMetadata inserted as needed to make + the hierarchy complete. + + For example, an input of {'a/b': ArrayMetadata} is incomplete, because it references two + groups (the root group '' and a group at 'a') that are not specified in the input. Applying this function + to that input will result in a return value of + {'': GroupMetadata, 'a': GroupMetadata, 'a/b': ArrayMetadata}, i.e. the implied groups + were added. + + The input is also checked for the following conditions; an error is raised if any are violated: + + - No arrays can contain group or arrays (i.e., all arrays must be leaf nodes). + - All arrays and groups must have the same ``zarr_format`` value. + + This function ensures that the input is transformed into a specification of a complete and valid + Zarr hierarchy. + """ + + # ensure that all nodes have the same zarr format + data_purified = _ensure_consistent_zarr_format(data) + + # ensure that keys are normalized to zarr paths + data_normed_keys = _normalize_path_keys(data_purified) + + # insert an implicit root group if a root was not specified + # but not if an empty dict was provided, because any empty hierarchy has no nodes + if len(data_normed_keys) > 0 and "" not in data_normed_keys: + z_format = next(iter(data_normed_keys.values())).zarr_format + data_normed_keys = data_normed_keys | {"": ImplicitGroupMarker(zarr_format=z_format)} + + out: dict[str, GroupMetadata | ArrayV2Metadata | ArrayV3Metadata] = {**data_normed_keys} + + for k, v in data_normed_keys.items(): + key_split = k.split("/") + + # get every parent path + *subpaths, _ = accumulate(key_split, lambda a, b: _join_paths([a, b])) + + for subpath in subpaths: + # If a component is not already in the output dict, add ImplicitGroupMetadata + if subpath not in out: + out[subpath] = ImplicitGroupMarker(zarr_format=v.zarr_format) + else: + if not isinstance(out[subpath], GroupMetadata | ImplicitGroupMarker): + msg = ( + f"The node at {subpath} contains other nodes, but it is not a Zarr group. " + "This is invalid. Only Zarr groups can contain other nodes." + ) + raise ValueError(msg) + return out + + +def _ensure_consistent_zarr_format( + data: Mapping[str, GroupMetadata | ArrayV2Metadata | ArrayV3Metadata], +) -> Mapping[str, GroupMetadata | ArrayV2Metadata] | Mapping[str, GroupMetadata | ArrayV3Metadata]: + """ + Ensure that all values of the input dict have the same zarr format. If any do not, + then a value error is raised. + """ + observed_zarr_formats: dict[ZarrFormat, list[str]] = {2: [], 3: []} + + for k, v in data.items(): + observed_zarr_formats[v.zarr_format].append(k) + + if len(observed_zarr_formats[2]) > 0 and len(observed_zarr_formats[3]) > 0: + msg = ( + "Got data with both Zarr v2 and Zarr v3 nodes, which is invalid. " + f"The following keys map to Zarr v2 nodes: {observed_zarr_formats.get(2)}. " + f"The following keys map to Zarr v3 nodes: {observed_zarr_formats.get(3)}." + "Ensure that all nodes have the same Zarr format." + ) + raise ValueError(msg) + + return cast( + "Mapping[str, GroupMetadata | ArrayV2Metadata] | Mapping[str, GroupMetadata | ArrayV3Metadata]", + data, + ) + + +async def _getitem_semaphore( + node: AsyncGroup, key: str, semaphore: asyncio.Semaphore | None +) -> AnyAsyncArray | AsyncGroup: + """ + Wrap Group.getitem with an optional semaphore. + + If the semaphore parameter is an + asyncio.Semaphore instance, then the getitem operation is performed inside an async context + manager provided by that semaphore. If the semaphore parameter is None, then getitem is invoked + without a context manager. + """ + if semaphore is not None: + async with semaphore: + return await node.getitem(key) + else: + return await node.getitem(key) + + +async def _iter_members( + node: AsyncGroup, + skip_keys: tuple[str, ...], + semaphore: asyncio.Semaphore | None, +) -> AsyncGenerator[tuple[str, AnyAsyncArray | AsyncGroup], None]: + """ + Iterate over the arrays and groups contained in a group. + + Parameters + ---------- + node : AsyncGroup + The group to traverse. + skip_keys : tuple[str, ...] + A tuple of keys to skip when iterating over the possible members of the group. + semaphore : asyncio.Semaphore | None + An optional semaphore to use for concurrency control. + + Yields + ------ + tuple[str, AnyAsyncArray | AsyncGroup] + """ + + # retrieve keys from storage + keys = [key async for key in node.store.list_dir(node.path)] + keys_filtered = tuple(filter(lambda v: v not in skip_keys, keys)) + + node_tasks = tuple( + asyncio.create_task(_getitem_semaphore(node, key, semaphore), name=key) + for key in keys_filtered + ) + + for fetched_node_coro in asyncio.as_completed(node_tasks): + try: + fetched_node = await fetched_node_coro + except KeyError as e: + # keyerror is raised when `key` names an object (in the object storage sense), + # as opposed to a prefix, in the store under the prefix associated with this group + # in which case `key` cannot be the name of a sub-array or sub-group. + warnings.warn( + f"Object at {e.args[0]} is not recognized as a component of a Zarr hierarchy.", + ZarrUserWarning, + stacklevel=1, + ) + continue + match fetched_node: + case AsyncArray() | AsyncGroup(): + yield fetched_node.basename, fetched_node + case _: + raise ValueError(f"Unexpected type: {type(fetched_node)}") + + +async def _iter_members_deep( + group: AsyncGroup, + *, + max_depth: int | None, + skip_keys: tuple[str, ...], + semaphore: asyncio.Semaphore | None = None, + use_consolidated_for_children: bool = True, +) -> AsyncGenerator[tuple[str, AnyAsyncArray | AsyncGroup], None]: + """ + Iterate over the arrays and groups contained in a group, and optionally the + arrays and groups contained in those groups. + + Parameters + ---------- + group : AsyncGroup + The group to traverse. + max_depth : int | None + The maximum depth of recursion. + skip_keys : tuple[str, ...] + A tuple of keys to skip when iterating over the possible members of the group. + semaphore : asyncio.Semaphore | None + An optional semaphore to use for concurrency control. + use_consolidated_for_children : bool, default True + Whether to use the consolidated metadata of child groups loaded + from the store. Note that this only affects groups loaded from the + store. If the current Group already has consolidated metadata, it + will always be used. + + Yields + ------ + tuple[str, AnyAsyncArray | AsyncGroup] + """ + + to_recurse = {} + do_recursion = max_depth is None or max_depth > 0 + + if max_depth is None: + new_depth = None + else: + new_depth = max_depth - 1 + async for name, node in _iter_members(group, skip_keys=skip_keys, semaphore=semaphore): + is_group = isinstance(node, AsyncGroup) + if ( + is_group + and not use_consolidated_for_children + and node.metadata.consolidated_metadata is not None + ): + node = cast("AsyncGroup", node) + # We've decided not to trust consolidated metadata at this point, because we're + # reconsolidating the metadata, for example. + node = replace(node, metadata=replace(node.metadata, consolidated_metadata=None)) + yield name, node + if is_group and do_recursion: + node = cast("AsyncGroup", node) + to_recurse[name] = _iter_members_deep( + node, max_depth=new_depth, skip_keys=skip_keys, semaphore=semaphore + ) + + for prefix, subgroup_iter in to_recurse.items(): + async for name, node in subgroup_iter: + key = f"{prefix}/{name}".lstrip("/") + yield key, node + + +async def _read_metadata_v3(store: Store, path: str) -> ArrayV3Metadata | GroupMetadata: + """ + Given a store_path, return ArrayV3Metadata or GroupMetadata defined by the metadata + document stored at store_path.path / zarr.json. If no such document is found, raise a + FileNotFoundError. + """ + zarr_json_bytes = await store.get( + _join_paths([path, ZARR_JSON]), prototype=default_buffer_prototype() + ) + if zarr_json_bytes is None: + raise FileNotFoundError(path) + return _build_metadata_v3( + buffer_to_json_object(zarr_json_bytes), path=str(StorePath(store, path)) + ) + + +async def _read_metadata_v2(store: Store, path: str) -> ArrayV2Metadata | GroupMetadata: + """ + Given a store_path, return ArrayV2Metadata or GroupMetadata defined by the metadata + document stored at store_path.path / (.zgroup | .zarray). If no such document is found, + raise a FileNotFoundError. + """ + # TODO: consider first fetching array metadata, and only fetching group metadata when we don't + # find an array + zarray_bytes, zgroup_bytes, zattrs_bytes = await asyncio.gather( + store.get(_join_paths([path, ZARRAY_JSON]), prototype=default_buffer_prototype()), + store.get(_join_paths([path, ZGROUP_JSON]), prototype=default_buffer_prototype()), + store.get(_join_paths([path, ZATTRS_JSON]), prototype=default_buffer_prototype()), + ) + + zattrs: dict[str, JSON] + if zattrs_bytes is None: + zattrs = {} + else: + zattrs = buffer_to_json_object(zattrs_bytes) + + # TODO: decide how to handle finding both array and group metadata. The spec does not seem to + # consider this situation. A practical approach would be to ignore that combination, and only + # return the array metadata. + if zarray_bytes is not None: + zmeta = buffer_to_json_object(zarray_bytes) + else: + if zgroup_bytes is None: + # neither .zarray or .zgroup were found results in KeyError + raise FileNotFoundError(path) + else: + zmeta = buffer_to_json_object(zgroup_bytes) + + return _build_metadata_v2(zmeta, zattrs, path=str(StorePath(store, path))) + + +async def _read_group_metadata_v2(store: Store, path: str) -> GroupMetadata: + """ + Read group metadata or error + """ + meta = await _read_metadata_v2(store=store, path=path) + if not isinstance(meta, GroupMetadata): + raise FileNotFoundError(f"Group metadata was not found in {store} at {path}") + return meta + + +async def _read_group_metadata_v3(store: Store, path: str) -> GroupMetadata: + """ + Read group metadata or error + """ + meta = await _read_metadata_v3(store=store, path=path) + if not isinstance(meta, GroupMetadata): + raise FileNotFoundError(f"Group metadata was not found in {store} at {path}") + return meta + + +async def _read_group_metadata( + store: Store, path: str, *, zarr_format: ZarrFormat +) -> GroupMetadata: + if zarr_format == 2: + return await _read_group_metadata_v2(store=store, path=path) + return await _read_group_metadata_v3(store=store, path=path) + + +def _build_metadata_v3( + zarr_json: dict[str, JSON], *, path: str | None = None +) -> ArrayV3Metadata | GroupMetadata: + """ + Convert a dict representation of Zarr V3 metadata into the corresponding metadata class. + """ + if "node_type" not in zarr_json: + msg = "Required key 'node_type' is missing from the provided metadata document." + raise MetadataValidationError(msg) + match zarr_json: + case {"node_type": "array"}: + return ArrayV3Metadata.from_dict(zarr_json, path=path) + case {"node_type": "group"}: + return GroupMetadata.from_dict(zarr_json, path=path) + case _: # pragma: no cover + raise ValueError( + "invalid value for `node_type` key in metadata document" + ) # pragma: no cover + + +def _build_metadata_v2( + zarr_json: dict[str, JSON], attrs_json: dict[str, JSON], *, path: str | None = None +) -> ArrayV2Metadata | GroupMetadata: + """ + Convert a dict representation of Zarr V2 metadata into the corresponding metadata class. + """ + match zarr_json: + case {"shape": _}: + return ArrayV2Metadata.from_dict(zarr_json | {"attributes": attrs_json}, path=path) + case _: # pragma: no cover + return GroupMetadata.from_dict(zarr_json | {"attributes": attrs_json}, path=path) + + +@overload +def _build_node(*, store: Store, path: str, metadata: ArrayV2Metadata) -> AsyncArrayV2: ... + + +@overload +def _build_node(*, store: Store, path: str, metadata: ArrayV3Metadata) -> AsyncArrayV3: ... + + +@overload +def _build_node(*, store: Store, path: str, metadata: GroupMetadata) -> AsyncGroup: ... + + +def _build_node( + *, store: Store, path: str, metadata: ArrayV3Metadata | ArrayV2Metadata | GroupMetadata +) -> AnyAsyncArray | AsyncGroup: + """ + Take a metadata object and return a node (AsyncArray or AsyncGroup). + """ + store_path = StorePath(store=store, path=path) + match metadata: + case ArrayV2Metadata() | ArrayV3Metadata(): + return AsyncArray(metadata, store_path=store_path) + case GroupMetadata(): + return AsyncGroup(metadata, store_path=store_path) + case _: # pragma: no cover + raise ValueError(f"Unexpected metadata type: {type(metadata)}") # pragma: no cover + + +async def _get_node_v2(store: Store, path: str) -> AsyncArrayV2 | AsyncGroup: + """ + Read a Zarr v2 AsyncArray or AsyncGroup from a path in a Store. + + Parameters + ---------- + store : Store + The store-like object to read from. + path : str + The path to the node to read. + + Returns + ------- + AsyncArray | AsyncGroup + """ + metadata = await _read_metadata_v2(store=store, path=path) + return _build_node(store=store, path=path, metadata=metadata) + + +async def _get_node_v3(store: Store, path: str) -> AsyncArrayV3 | AsyncGroup: + """ + Read a Zarr v3 AsyncArray or AsyncGroup from a path in a Store. + + Parameters + ---------- + store : Store + The store-like object to read from. + path : str + The path to the node to read. + + Returns + ------- + AsyncArray | AsyncGroup + """ + metadata = await _read_metadata_v3(store=store, path=path) + return _build_node(store=store, path=path, metadata=metadata) + + +async def get_node(store: Store, path: str, zarr_format: ZarrFormat) -> AnyAsyncArray | AsyncGroup: + """ + Get an AsyncArray or AsyncGroup from a path in a Store. + + Parameters + ---------- + store : Store + The store-like object to read from. + path : str + The path to the node to read. + zarr_format : {2, 3} + The zarr format of the node to read. + + Returns + ------- + AsyncArray | AsyncGroup + """ + + match zarr_format: + case 2: + return await _get_node_v2(store=store, path=path) + case 3: + return await _get_node_v3(store=store, path=path) + case _: # pragma: no cover + raise ValueError(f"Unexpected zarr format: {zarr_format}") # pragma: no cover + + +async def _set_return_key( + *, store: Store, key: str, value: Buffer, semaphore: asyncio.Semaphore | None = None +) -> str: + """ + Write a value to storage at the given key. The key is returned. + Useful when saving values via routines that return results in execution order, + like asyncio.as_completed, because in this case we need to know which key was saved in order + to yield the right object to the caller. + + Parameters + ---------- + store : Store + The store to save the value to. + key : str + The key to save the value to. + value : Buffer + The value to save. + semaphore : asyncio.Semaphore | None + An optional semaphore to use to limit the number of concurrent writes. + """ + + if semaphore is not None: + async with semaphore: + await store.set(key, value) + else: + await store.set(key, value) + return key + + +async def create_rooted_hierarchy( + *, + store: Store, + nodes: dict[str, GroupMetadata | ArrayV2Metadata | ArrayV3Metadata], + overwrite: bool = False, +) -> AsyncGroup | AnyAsyncArray: + """ + Create an ``AsyncGroup`` or ``AsyncArray`` from a store and a dict of metadata documents. + This function ensures that its input contains a specification of a root node, + calls ``create_hierarchy`` to create nodes, and returns the root node of the hierarchy. + """ + roots = _get_roots(nodes.keys()) + if len(roots) != 1: + msg = ( + "The input does not specify a root node. " + "This function can only create hierarchies that contain a root node, which is " + "defined as a group that is ancestral to all the other arrays and " + "groups in the hierarchy, or a single array." + ) + raise ValueError(msg) + else: + root_key = roots[0] + + nodes_created = [ + x async for x in create_hierarchy(store=store, nodes=nodes, overwrite=overwrite) + ] + return dict(nodes_created)[root_key] From 6c13103624a232e5c32d618e24247298c5b4e349 Mon Sep 17 00:00:00 2001 From: Duo-ee <113885053+Duo-ee@users.noreply.github.com> Date: Thu, 22 Oct 2026 11:18:11 +0800 Subject: [PATCH 03/12] fix: validate metadata node paths Reject reserved Zarr v3 node names before metadata writes. --- src/zarr/core/metadata/io.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/zarr/core/metadata/io.py b/src/zarr/core/metadata/io.py index c1cd927b94..462b7551e6 100644 --- a/src/zarr/core/metadata/io.py +++ b/src/zarr/core/metadata/io.py @@ -9,7 +9,7 @@ from zarr.core._json import buffer_to_json_object, json_equal from zarr.core.buffer.core import default_buffer_prototype from zarr.core.buffer.cpu import buffer_prototype as cpu_buffer_prototype -from zarr.core.common import ZARR_JSON, ZARRAY_JSON, ZATTRS_JSON +from zarr.core.common import ZARR_JSON, ZARRAY_JSON, ZATTRS_JSON, validate_node_path from zarr.core.metadata.repair import mark_repaired, repair_array_document from zarr.errors import ArrayNotFoundError, ContainsArrayError from zarr.storage._common import StorePath, ensure_no_existing_node @@ -228,6 +228,7 @@ async def save_new_metadata( ensure_parents : bool, optional Create any missing parent groups, and check no existing parents are arrays. """ + validate_node_path(store_path.path, metadata.zarr_format) to_save = encode_documents(store_path, metadata) if overwrite and store_path.store.supports_deletes: await store_path.delete_dir() From dfa76dec81819960da889d84f838ebbcbba2fc2e Mon Sep 17 00:00:00 2001 From: Duo-ee <113885053+Duo-ee@users.noreply.github.com> Date: Thu, 22 Oct 2026 11:18:22 +0800 Subject: [PATCH 04/12] test: cover reserved Zarr v3 node names Cover array, group, hierarchy, and v2 compatibility cases. --- tests/test_group.py | 41 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/tests/test_group.py b/tests/test_group.py index 6976ef536c..d2b2bd8651 100644 --- a/tests/test_group.py +++ b/tests/test_group.py @@ -171,6 +171,47 @@ def test_group_name_properties( assert branch.basename == branch_name.split("/")[-1] +@pytest.mark.parametrize("node_type", ["array", "group"]) +@pytest.mark.parametrize( + "name", + ["__reserved", "parent/__reserved", "zarr.json", "parent/zarr.json", "...", "parent/..."], +) +async def test_create_v3_rejects_invalid_node_names( + memory_store: MemoryStore, node_type: Literal["array", "group"], name: str +) -> None: + root = await AsyncGroup.from_store(memory_store, zarr_format=3) + keys_before = {key async for key in memory_store.list()} + if node_type == "array": + create_node = root.create_array(name, shape=(1,), dtype="uint8") + else: + create_node = root.create_group(name) + + with pytest.raises(ValueError, match="Invalid Zarr format 3 node name"): + await create_node + + assert {key async for key in memory_store.list()} == keys_before + + +@pytest.mark.parametrize("name", ["__reserved", "zarr.json"]) +async def test_create_v2_allows_v3_reserved_node_names( + memory_store: MemoryStore, name: str +) -> None: + root = await AsyncGroup.from_store(memory_store, zarr_format=2) + child = await root.create_group(name) + assert child.path == name + + +async def test_create_hierarchy_rejects_invalid_v3_node_name_without_writes( + memory_store: MemoryStore, +) -> None: + nodes = {"parent/__reserved": GroupMetadata(zarr_format=3)} + + with pytest.raises(ValueError, match="Invalid Zarr format 3 node name"): + await _collect_aiterator(create_hierarchy(store=memory_store, nodes=nodes)) + + assert {key async for key in memory_store.list()} == set() + + @pytest.mark.parametrize("consolidated_metadata", [True, False]) def test_group_members(store: Store, zarr_format: ZarrFormat, consolidated_metadata: bool) -> None: """ From 58b6e2ccd5cbd5245b69a52c674d507237b80302 Mon Sep 17 00:00:00 2001 From: Duo-ee <113885053+Duo-ee@users.noreply.github.com> Date: Thu, 22 Oct 2026 11:22:28 +0800 Subject: [PATCH 05/12] docs: add Zarr v3 node name bugfix note Document the reserved-node-name validation change. --- changes/1.bugfix.md | 1 + 1 file changed, 1 insertion(+) create mode 100644 changes/1.bugfix.md diff --git a/changes/1.bugfix.md b/changes/1.bugfix.md new file mode 100644 index 0000000000..51ad387334 --- /dev/null +++ b/changes/1.bugfix.md @@ -0,0 +1 @@ +Zarr v3 node creation now rejects reserved node names before writing metadata. This prevents invalid hierarchies that cannot be discovered or reopened. From 4abc75f9ed96dd92fd5bf2577229510e36a05075 Mon Sep 17 00:00:00 2001 From: Duo-ee <113885053+Duo-ee@users.noreply.github.com> Date: Thu, 22 Oct 2026 11:37:57 +0800 Subject: [PATCH 06/12] fix: update mkdocstrings for docs builds Use the compatibility fix in mkdocstrings 1.0.8 so ReadTheDocs builds work with the current pymdownx.highlight API. --- pyproject.toml | 1008 ++++++++++++++++++++++++------------------------ uv.lock | 12 +- 2 files changed, 510 insertions(+), 510 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index c68450474e..66d3b969c9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,504 +1,504 @@ -[build-system] -requires = ["hatchling>=1.29.0", "hatch-vcs", "setuptools-scm>=9"] -build-backend = "hatchling.build" - -# An allowlist, not a blocklist: anything new — a subpackage under `packages/`, a -# config file in the repository root — stays out of the sdist unless it is named -# here. Beyond `src` and `tests`, the entries are what keeps the shipped test -# suite and docs build runnable from an unpacked sdist: `tests/test_docs.py` -# walks `docs/` and `testpaths` collects `docs/user-guide`; the pages under -# `docs/user-guide/examples/` pull their source out of `examples/` via pymdownx -# snippet includes; `mkdocs.yml` and `mkdocs_hooks.py` let `mkdocs build` run -# too. `pyproject.toml`, `README.md`, `LICENSE.txt` and `.gitignore` are added by -# hatchling itself. -[tool.hatch.build.targets.sdist] -include = [ - "/src", - "/tests", - "/docs", - "/examples", - "/mkdocs.yml", - "/mkdocs_hooks.py", - "/Justfile", - "/ci", -] - -[project] -name = "zarr" -description = "An implementation of chunked, compressed, N-dimensional arrays for Python" -readme = { file = "README.md", content-type = "text/markdown" } -authors = [ - { name = "Alistair Miles", email = "alimanfoo@googlemail.com" }, -] -maintainers = [ - { name = "Davis Bennett", email = "davis.v.bennett@gmail.com" }, - { name = "jakirkham" }, - { name = "Josh Moore", email = "josh@openmicroscopy.org" }, - { name = "Joe Hamman", email = "joe@earthmover.io" }, - { name = "Juan Nunez-Iglesias", email = "juan.nunez-iglesias@monash.edu" }, - { name = "Martin Durant", email = "mdurant@anaconda.com" }, - { name = "Norman Rzepka" }, - { name = "Ryan Abernathey" }, - { name = "David Stansby" }, - { name = "Tom Augspurger", email = "tom.w.augspurger@gmail.com" }, - { name = "Deepak Cherian" } -] -requires-python = ">=3.12" -dependencies = [ - 'packaging>=22.0', - 'numpy>=2', - 'numcodecs>=0.16', - 'google-crc32c>=1.5', - 'typing_extensions>=4.14', - 'donfig>=0.8', - 'msgspec>=0.19', -] - -dynamic = [ - "version", -] -classifiers = [ - 'Development Status :: 6 - Mature', - 'Intended Audience :: Developers', - 'Intended Audience :: Information Technology', - 'Intended Audience :: Science/Research', - 'Programming Language :: Python', - 'Topic :: Software Development :: Libraries :: Python Modules', - 'Operating System :: Unix', - 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.12', - 'Programming Language :: Python :: 3.13', - 'Programming Language :: Python :: 3.14', -] -license = "MIT" -license-files = ["LICENSE.txt"] -keywords = ["Python", "compressed", "ndimensional-arrays", "zarr"] - -[project.optional-dependencies] -# User-facing extras (shipped in package metadata) -remote = [ - "fsspec>=2023.10.0", - "obstore>=0.5.1", -] -gpu = [ - "cupy-cuda12x; sys_platform != 'darwin'", -] -cast-value-rs = ["cast-value-rs>=0.4.2"] -cli = ["typer"] -optional = ["universal-pathlib"] - -[project.scripts] -zarr = "zarr._cli.cli:app" - - -[project.urls] -issues = "https://github.com/zarr-developers/zarr-python/issues" -changelog = "https://zarr.readthedocs.io/en/stable/release-notes.html" -Discussions = "https://github.com/zarr-developers/zarr-python/discussions" -documentation = "https://zarr.readthedocs.io/" -homepage = "https://github.com/zarr-developers/zarr-python" - -# Dev *tooling* is pinned to exact versions for reproducible CI: the hatch envs -# (see `tool.hatch.envs.*`) and bare `uv run` resolve these groups fresh from -# PyPI and do NOT consult uv.lock, so an unrelated tooling release can break CI -# without any change on our side (e.g. the pytest 9.1.0 `duplicate -# parametrization` regression). Runtime/integration deps (fsspec, obstore, s3fs, -# botocore, numcodecs, universal-pathlib) are intentionally left floating so the -# `optional` test matrix keeps exercising their latest releases; their floor and -# bleeding edge are covered by the `min_deps` and `upstream` hatch envs. Bump the -# pins deliberately, e.g. via dependabot or `uv lock --upgrade`. -[dependency-groups] -test = [ - "coverage==7.16.2", - "pytest==9.1.1", - "pytest-asyncio==1.4.0", - "pytest-cov==7.1.0", - "pytest-accept==0.3.0", - "numpydoc==1.11.0", - "hypothesis==6.168.3", - "pytest-reportlog==1.0.0", - "pytest-xdist==3.8.0", - "pytest-benchmark==5.3.0", - "pytest-codspeed==5.0.3", - "tomlkit==0.15.1", - "uv==0.12.21", -] -remote-tests = [ - {include-group = "test"}, - "fsspec>=2023.10.0", - "obstore>=0.5.1", - "botocore", - "s3fs>=2023.10.0", - "moto[s3,server]==5.2.3", - "requests==2.34.2", -] -release = [ - "towncrier==26.9.0", -] -docs = [ - # Doc building - "mkdocs-material[imaging]==9.7.7", - "mkdocs==1.6.1", - "mkdocstrings==1.0.6", - "mkdocstrings-python==2.0.9", - "mike==2.2.0", - "mkdocs-redirects==1.2.3", - "markdown-exec[ansi]==1.12.3", - "griffe-inherited-docstrings==1.1.3", - "ruff==0.16.9", - # Changelog generation - {include-group = "release"}, - # Optional dependencies to run examples - "numcodecs[msgpack]", - "s3fs>=2023.10.0", - "astroid==4.3.3", - "pytest==9.1.1", -] -dev = [ - {include-group = "test"}, - {include-group = "remote-tests"}, - {include-group = "docs"}, - "universal-pathlib", - "mypy==2.3.1", -] - -[tool.coverage.report] -exclude_also = [ - 'if TYPE_CHECKING:', -] - -[tool.coverage.run] -omit = [ - "bench/compress_normal.py", -] - -[tool.hatch] -version.source = "vcs" -# Only consider zarr-python's own `v*` tags when deriving the version. Without -# this filter `git describe` matches the most recent tag of any shape, -# including the `zarr_metadata-v*` and `zarr_http_server-v*` tags used to -# release the subpackages under `packages/` — which would make a from-source -# build report e.g. a `0.2.x` version instead of `3.x`. -version.raw-options = { scm = { git = { describe_command = "git describe --dirty --tags --long --match v*" } } } - -[tool.hatch.build] -hooks.vcs.version-file = "src/zarr/_version.py" - -[tool.hatch.envs.dev] -dependency-groups = ["dev"] - -[tool.hatch.envs.test] -dependency-groups = ["test"] - -[tool.hatch.envs.test.env-vars] - -[[tool.hatch.envs.test.matrix]] -python = ["3.12", "3.13", "3.14"] -deps = ["minimal", "optional"] - -[tool.hatch.envs.test.overrides] -matrix.deps.features = [ - {value = "remote", if = ["optional"]}, - {value = "optional", if = ["optional"]}, - {value = "cli", if = ["optional"]}, - {value = "cast-value-rs", if = ["optional"]}, -] -matrix.deps.dependency-groups = [ - {value = "remote-tests", if = ["optional"]}, -] - -[tool.hatch.envs.gputest] -template = "test" -extra-dependencies = [ - "universal_pathlib", - # Needed so tests/test_docs.py is collectable under `pytest -m gpu`; otherwise its - # module-level importorskip("pytest_examples") skips the whole module and the gpu - # docs example is never executed on GPU hardware. - "pytest-examples", -] -features = ["gpu"] - -[[tool.hatch.envs.gputest.matrix]] -python = ["3.12", "3.13"] - -[tool.hatch.envs.upstream] -template = 'test' -python = "3.14" -extra-dependencies = [ - 'packaging @ git+https://github.com/pypa/packaging', - 'numpy', # from scientific-python-nightly-wheels - 'numcodecs @ git+https://github.com/zarr-developers/numcodecs', - 's3fs @ git+https://github.com/fsspec/s3fs', - 'universal_pathlib @ git+https://github.com/fsspec/universal_pathlib', - 'typing_extensions @ git+https://github.com/python/typing_extensions', - 'donfig @ git+https://github.com/pytroll/donfig', - 'obstore @ git+https://github.com/developmentseed/obstore@main#subdirectory=obstore', -] - -[tool.hatch.envs.upstream.env-vars] -PIP_INDEX_URL = "https://pypi.anaconda.org/scientific-python-nightly-wheels/simple/" -PIP_EXTRA_INDEX_URL = "https://pypi.org/simple/" -PIP_PRE = "1" - -[tool.hatch.envs.min_deps] -description = """Test environment for minimum supported dependencies - -See Spec 0000 for details and drop schedule: https://scientific-python.org/specs/spec-0000/ -""" -template = "test" -python = "3.12" -features = ["remote"] -dependency-groups = ["remote-tests"] -extra-dependencies = [ - 'packaging==22.*', - 'numpy==2.0.*', - 'numcodecs==0.16.*', - 'fsspec==2023.10.0', - 's3fs==2023.10.0', - 'universal_pathlib==0.2.0', - 'typing_extensions==4.14.*', - 'donfig==0.8.*', - 'obstore==0.5.*', - 'msgspec==0.19.*', -] - -[tool.hatch.envs.default] -installer = "uv" - -[tool.hatch.envs.doctest] -description = "Test environment for validating executable code blocks in documentation" -features = ['remote'] -dependency-groups = ['remote-tests'] -extra-dependencies = [ - "pytest-examples", -] - -[tool.ruff] -line-length = 100 -force-exclude = true -extend-exclude = [ - ".bzr", - ".direnv", - ".eggs", - ".git", - ".mypy_cache", - ".nox", - ".pants.d", - ".ruff_cache", - ".venv", - "__pypackages__", - "_build", - "buck-out", - "build", - "dist", - "venv", - "docs", - "tests/test_regression/scripts/", # these are scripts that use a different version of python - "src/zarr/v2/", - "tests/v2/", -] - -[tool.ruff.lint] -extend-select = [ - "ANN", # flake8-annotations - "B", # flake8-bugbear - "C4", # flake8-comprehensions - "EXE", # flake8-executable - "FA", # flake8-future-annotations - "FLY", # flynt - "FURB", # refurb - "G", # flake8-logging-format - "I", # isort - "ISC", # flake8-implicit-str-concat - "LOG", # flake8-logging - "PERF", # Perflint - "PIE", # flake8-pie - "PGH", # pygrep-hooks - "PT", # flake8-pytest-style - "PYI", # flake8-pyi - "RET", # flake8-return - "RSE", # flake8-raise - "RUF", - "S101", # flake8-bandit: no assert in runtime code - "SIM", # flake8-simplify - "SLOT", # flake8-slots - "TC", # flake8-type-checking - "TRY", # tryceratops - "UP", # pyupgrade - "W", # pycodestyle warnings -] -ignore = [ - "ANN401", - "PT011", # TODO: apply this rule - "RET505", - "RET506", - "RUF005", - "RUF043", - "SIM108", - "TRY003", - # https://docs.astral.sh/ruff/formatter/#conflicting-lint-rules - "W191", - "E111", - "E114", - "E117", - "D206", - "D300", - "Q000", - "Q001", - "Q002", - "Q003", - "COM812", - "COM819", - "TC006", -] - -[tool.ruff.lint.extend-per-file-ignores] -"tests/**" = ["ANN001", "ANN201", "RUF029", "SIM117", "SIM300"] -# S101 (no assert) is for runtime code only. Tests, examples, and test-support -# packages (store conformance suite, hypothesis strategies) assert on purpose. -# The globs cover the root package and every package under packages/. -"**/tests/**" = ["S101"] -"**/examples/**" = ["S101"] -"**/testing/**" = ["S101"] - -[tool.mypy] -files = ["src", "tests"] -python_version = "3.12" -ignore_missing_imports = true -namespace_packages = false -pretty = true -show_error_code_links = true -show_error_context = true -strict = true -warn_unreachable = true -enable_error_code = ["ignore-without-code", "redundant-expr", "truthy-bool", "truthy-iterable"] - -[[tool.mypy.overrides]] -module = [ - "tests.package_with_entrypoint.*", - "zarr.testing.stateful", - "tests.test_codecs.test_transpose", - "tests.test_config", - "tests.test_store.test_zip", - "tests.test_store.test_local", - "tests.test_store.test_memory", - "tests.test_codecs.test_codecs", - "tests.test_metadata.*", - "tests.test_store.test_core", - "tests.test_store.test_logging", - "tests.test_store.test_object", - "tests.test_store.test_stateful", - "tests.test_store.test_wrapper", -] -strict = false - -# TODO: Move the next modules up to the strict = false section -# and fix the errors -[[tool.mypy.overrides]] -module = [ - "tests.test_store.test_fsspec", - "tests.test_group", - "tests.test_indexing", - "tests.test_properties", - "tests.test_sync", - "tests.test_regression.scripts.*" -] -ignore_errors = true - -[tool.pytest.ini_options] -minversion = "9" -testpaths = ["src", "tests", "docs/user-guide"] -log_cli_level = "INFO" -log_level = "INFO" -# Enables strict_config, strict_markers, strict_xfail, strict_parametrization_ids, and -# any strictness options added in future pytest releases. Note that the equivalent -# `--strict-config`/`--strict-markers` flags were silently ignored when passed via -# addopts before pytest 9.1 (pytest#14442), so this option is the reliable spelling. -strict = true -# Turn deadlocks into loud failures: if a single test exceeds this many seconds, dump -# every thread's traceback and kill the run (exit_on_timeout is new in pytest 9). Sized -# far above the slowest legitimate test (~20s locally; slower under coverage/Windows/ -# nightly stateful-hypothesis runs) so only a genuine hang can trip it. -faulthandler_timeout = 600 -faulthandler_exit_on_timeout = true -asyncio_mode = "auto" -asyncio_default_fixture_loop_scope = "function" -doctest_optionflags = [ - "NORMALIZE_WHITESPACE", - "ELLIPSIS", - "IGNORE_EXCEPTION_DETAIL", -] -addopts = [ - "--benchmark-columns", "min,mean,stddev,outliers,rounds,iterations", - "--benchmark-disable", # benchmark routines run as tests without benchmarking instrumentation - "--durations", "10", - "-ra", - "--doctest-modules", - "--ignore=tests/test_regression/scripts", - "--ignore=src/zarr/_cli", -] -filterwarnings = [ - "error", - "ignore:Unclosed client session =1.29.0", "hatch-vcs", "setuptools-scm>=9"] +build-backend = "hatchling.build" + +# An allowlist, not a blocklist: anything new — a subpackage under `packages/`, a +# config file in the repository root — stays out of the sdist unless it is named +# here. Beyond `src` and `tests`, the entries are what keeps the shipped test +# suite and docs build runnable from an unpacked sdist: `tests/test_docs.py` +# walks `docs/` and `testpaths` collects `docs/user-guide`; the pages under +# `docs/user-guide/examples/` pull their source out of `examples/` via pymdownx +# snippet includes; `mkdocs.yml` and `mkdocs_hooks.py` let `mkdocs build` run +# too. `pyproject.toml`, `README.md`, `LICENSE.txt` and `.gitignore` are added by +# hatchling itself. +[tool.hatch.build.targets.sdist] +include = [ + "/src", + "/tests", + "/docs", + "/examples", + "/mkdocs.yml", + "/mkdocs_hooks.py", + "/Justfile", + "/ci", +] + +[project] +name = "zarr" +description = "An implementation of chunked, compressed, N-dimensional arrays for Python" +readme = { file = "README.md", content-type = "text/markdown" } +authors = [ + { name = "Alistair Miles", email = "alimanfoo@googlemail.com" }, +] +maintainers = [ + { name = "Davis Bennett", email = "davis.v.bennett@gmail.com" }, + { name = "jakirkham" }, + { name = "Josh Moore", email = "josh@openmicroscopy.org" }, + { name = "Joe Hamman", email = "joe@earthmover.io" }, + { name = "Juan Nunez-Iglesias", email = "juan.nunez-iglesias@monash.edu" }, + { name = "Martin Durant", email = "mdurant@anaconda.com" }, + { name = "Norman Rzepka" }, + { name = "Ryan Abernathey" }, + { name = "David Stansby" }, + { name = "Tom Augspurger", email = "tom.w.augspurger@gmail.com" }, + { name = "Deepak Cherian" } +] +requires-python = ">=3.12" +dependencies = [ + 'packaging>=22.0', + 'numpy>=2', + 'numcodecs>=0.16', + 'google-crc32c>=1.5', + 'typing_extensions>=4.14', + 'donfig>=0.8', + 'msgspec>=0.19', +] + +dynamic = [ + "version", +] +classifiers = [ + 'Development Status :: 6 - Mature', + 'Intended Audience :: Developers', + 'Intended Audience :: Information Technology', + 'Intended Audience :: Science/Research', + 'Programming Language :: Python', + 'Topic :: Software Development :: Libraries :: Python Modules', + 'Operating System :: Unix', + 'Programming Language :: Python :: 3', + 'Programming Language :: Python :: 3.12', + 'Programming Language :: Python :: 3.13', + 'Programming Language :: Python :: 3.14', +] +license = "MIT" +license-files = ["LICENSE.txt"] +keywords = ["Python", "compressed", "ndimensional-arrays", "zarr"] + +[project.optional-dependencies] +# User-facing extras (shipped in package metadata) +remote = [ + "fsspec>=2023.10.0", + "obstore>=0.5.1", +] +gpu = [ + "cupy-cuda12x; sys_platform != 'darwin'", +] +cast-value-rs = ["cast-value-rs>=0.4.2"] +cli = ["typer"] +optional = ["universal-pathlib"] + +[project.scripts] +zarr = "zarr._cli.cli:app" + + +[project.urls] +issues = "https://github.com/zarr-developers/zarr-python/issues" +changelog = "https://zarr.readthedocs.io/en/stable/release-notes.html" +Discussions = "https://github.com/zarr-developers/zarr-python/discussions" +documentation = "https://zarr.readthedocs.io/" +homepage = "https://github.com/zarr-developers/zarr-python" + +# Dev *tooling* is pinned to exact versions for reproducible CI: the hatch envs +# (see `tool.hatch.envs.*`) and bare `uv run` resolve these groups fresh from +# PyPI and do NOT consult uv.lock, so an unrelated tooling release can break CI +# without any change on our side (e.g. the pytest 9.1.0 `duplicate +# parametrization` regression). Runtime/integration deps (fsspec, obstore, s3fs, +# botocore, numcodecs, universal-pathlib) are intentionally left floating so the +# `optional` test matrix keeps exercising their latest releases; their floor and +# bleeding edge are covered by the `min_deps` and `upstream` hatch envs. Bump the +# pins deliberately, e.g. via dependabot or `uv lock --upgrade`. +[dependency-groups] +test = [ + "coverage==7.16.2", + "pytest==9.1.1", + "pytest-asyncio==1.4.0", + "pytest-cov==7.1.0", + "pytest-accept==0.3.0", + "numpydoc==1.11.0", + "hypothesis==6.168.3", + "pytest-reportlog==1.0.0", + "pytest-xdist==3.8.0", + "pytest-benchmark==5.3.0", + "pytest-codspeed==5.0.3", + "tomlkit==0.15.1", + "uv==0.12.21", +] +remote-tests = [ + {include-group = "test"}, + "fsspec>=2023.10.0", + "obstore>=0.5.1", + "botocore", + "s3fs>=2023.10.0", + "moto[s3,server]==5.2.3", + "requests==2.34.2", +] +release = [ + "towncrier==26.9.0", +] +docs = [ + # Doc building + "mkdocs-material[imaging]==9.7.7", + "mkdocs==1.6.1", + "mkdocstrings==1.0.8", + "mkdocstrings-python==2.0.9", + "mike==2.2.0", + "mkdocs-redirects==1.2.3", + "markdown-exec[ansi]==1.12.3", + "griffe-inherited-docstrings==1.1.3", + "ruff==0.16.9", + # Changelog generation + {include-group = "release"}, + # Optional dependencies to run examples + "numcodecs[msgpack]", + "s3fs>=2023.10.0", + "astroid==4.3.3", + "pytest==9.1.1", +] +dev = [ + {include-group = "test"}, + {include-group = "remote-tests"}, + {include-group = "docs"}, + "universal-pathlib", + "mypy==2.3.1", +] + +[tool.coverage.report] +exclude_also = [ + 'if TYPE_CHECKING:', +] + +[tool.coverage.run] +omit = [ + "bench/compress_normal.py", +] + +[tool.hatch] +version.source = "vcs" +# Only consider zarr-python's own `v*` tags when deriving the version. Without +# this filter `git describe` matches the most recent tag of any shape, +# including the `zarr_metadata-v*` and `zarr_http_server-v*` tags used to +# release the subpackages under `packages/` — which would make a from-source +# build report e.g. a `0.2.x` version instead of `3.x`. +version.raw-options = { scm = { git = { describe_command = "git describe --dirty --tags --long --match v*" } } } + +[tool.hatch.build] +hooks.vcs.version-file = "src/zarr/_version.py" + +[tool.hatch.envs.dev] +dependency-groups = ["dev"] + +[tool.hatch.envs.test] +dependency-groups = ["test"] + +[tool.hatch.envs.test.env-vars] + +[[tool.hatch.envs.test.matrix]] +python = ["3.12", "3.13", "3.14"] +deps = ["minimal", "optional"] + +[tool.hatch.envs.test.overrides] +matrix.deps.features = [ + {value = "remote", if = ["optional"]}, + {value = "optional", if = ["optional"]}, + {value = "cli", if = ["optional"]}, + {value = "cast-value-rs", if = ["optional"]}, +] +matrix.deps.dependency-groups = [ + {value = "remote-tests", if = ["optional"]}, +] + +[tool.hatch.envs.gputest] +template = "test" +extra-dependencies = [ + "universal_pathlib", + # Needed so tests/test_docs.py is collectable under `pytest -m gpu`; otherwise its + # module-level importorskip("pytest_examples") skips the whole module and the gpu + # docs example is never executed on GPU hardware. + "pytest-examples", +] +features = ["gpu"] + +[[tool.hatch.envs.gputest.matrix]] +python = ["3.12", "3.13"] + +[tool.hatch.envs.upstream] +template = 'test' +python = "3.14" +extra-dependencies = [ + 'packaging @ git+https://github.com/pypa/packaging', + 'numpy', # from scientific-python-nightly-wheels + 'numcodecs @ git+https://github.com/zarr-developers/numcodecs', + 's3fs @ git+https://github.com/fsspec/s3fs', + 'universal_pathlib @ git+https://github.com/fsspec/universal_pathlib', + 'typing_extensions @ git+https://github.com/python/typing_extensions', + 'donfig @ git+https://github.com/pytroll/donfig', + 'obstore @ git+https://github.com/developmentseed/obstore@main#subdirectory=obstore', +] + +[tool.hatch.envs.upstream.env-vars] +PIP_INDEX_URL = "https://pypi.anaconda.org/scientific-python-nightly-wheels/simple/" +PIP_EXTRA_INDEX_URL = "https://pypi.org/simple/" +PIP_PRE = "1" + +[tool.hatch.envs.min_deps] +description = """Test environment for minimum supported dependencies + +See Spec 0000 for details and drop schedule: https://scientific-python.org/specs/spec-0000/ +""" +template = "test" +python = "3.12" +features = ["remote"] +dependency-groups = ["remote-tests"] +extra-dependencies = [ + 'packaging==22.*', + 'numpy==2.0.*', + 'numcodecs==0.16.*', + 'fsspec==2023.10.0', + 's3fs==2023.10.0', + 'universal_pathlib==0.2.0', + 'typing_extensions==4.14.*', + 'donfig==0.8.*', + 'obstore==0.5.*', + 'msgspec==0.19.*', +] + +[tool.hatch.envs.default] +installer = "uv" + +[tool.hatch.envs.doctest] +description = "Test environment for validating executable code blocks in documentation" +features = ['remote'] +dependency-groups = ['remote-tests'] +extra-dependencies = [ + "pytest-examples", +] + +[tool.ruff] +line-length = 100 +force-exclude = true +extend-exclude = [ + ".bzr", + ".direnv", + ".eggs", + ".git", + ".mypy_cache", + ".nox", + ".pants.d", + ".ruff_cache", + ".venv", + "__pypackages__", + "_build", + "buck-out", + "build", + "dist", + "venv", + "docs", + "tests/test_regression/scripts/", # these are scripts that use a different version of python + "src/zarr/v2/", + "tests/v2/", +] + +[tool.ruff.lint] +extend-select = [ + "ANN", # flake8-annotations + "B", # flake8-bugbear + "C4", # flake8-comprehensions + "EXE", # flake8-executable + "FA", # flake8-future-annotations + "FLY", # flynt + "FURB", # refurb + "G", # flake8-logging-format + "I", # isort + "ISC", # flake8-implicit-str-concat + "LOG", # flake8-logging + "PERF", # Perflint + "PIE", # flake8-pie + "PGH", # pygrep-hooks + "PT", # flake8-pytest-style + "PYI", # flake8-pyi + "RET", # flake8-return + "RSE", # flake8-raise + "RUF", + "S101", # flake8-bandit: no assert in runtime code + "SIM", # flake8-simplify + "SLOT", # flake8-slots + "TC", # flake8-type-checking + "TRY", # tryceratops + "UP", # pyupgrade + "W", # pycodestyle warnings +] +ignore = [ + "ANN401", + "PT011", # TODO: apply this rule + "RET505", + "RET506", + "RUF005", + "RUF043", + "SIM108", + "TRY003", + # https://docs.astral.sh/ruff/formatter/#conflicting-lint-rules + "W191", + "E111", + "E114", + "E117", + "D206", + "D300", + "Q000", + "Q001", + "Q002", + "Q003", + "COM812", + "COM819", + "TC006", +] + +[tool.ruff.lint.extend-per-file-ignores] +"tests/**" = ["ANN001", "ANN201", "RUF029", "SIM117", "SIM300"] +# S101 (no assert) is for runtime code only. Tests, examples, and test-support +# packages (store conformance suite, hypothesis strategies) assert on purpose. +# The globs cover the root package and every package under packages/. +"**/tests/**" = ["S101"] +"**/examples/**" = ["S101"] +"**/testing/**" = ["S101"] + +[tool.mypy] +files = ["src", "tests"] +python_version = "3.12" +ignore_missing_imports = true +namespace_packages = false +pretty = true +show_error_code_links = true +show_error_context = true +strict = true +warn_unreachable = true +enable_error_code = ["ignore-without-code", "redundant-expr", "truthy-bool", "truthy-iterable"] + +[[tool.mypy.overrides]] +module = [ + "tests.package_with_entrypoint.*", + "zarr.testing.stateful", + "tests.test_codecs.test_transpose", + "tests.test_config", + "tests.test_store.test_zip", + "tests.test_store.test_local", + "tests.test_store.test_memory", + "tests.test_codecs.test_codecs", + "tests.test_metadata.*", + "tests.test_store.test_core", + "tests.test_store.test_logging", + "tests.test_store.test_object", + "tests.test_store.test_stateful", + "tests.test_store.test_wrapper", +] +strict = false + +# TODO: Move the next modules up to the strict = false section +# and fix the errors +[[tool.mypy.overrides]] +module = [ + "tests.test_store.test_fsspec", + "tests.test_group", + "tests.test_indexing", + "tests.test_properties", + "tests.test_sync", + "tests.test_regression.scripts.*" +] +ignore_errors = true + +[tool.pytest.ini_options] +minversion = "9" +testpaths = ["src", "tests", "docs/user-guide"] +log_cli_level = "INFO" +log_level = "INFO" +# Enables strict_config, strict_markers, strict_xfail, strict_parametrization_ids, and +# any strictness options added in future pytest releases. Note that the equivalent +# `--strict-config`/`--strict-markers` flags were silently ignored when passed via +# addopts before pytest 9.1 (pytest#14442), so this option is the reliable spelling. +strict = true +# Turn deadlocks into loud failures: if a single test exceeds this many seconds, dump +# every thread's traceback and kill the run (exit_on_timeout is new in pytest 9). Sized +# far above the slowest legitimate test (~20s locally; slower under coverage/Windows/ +# nightly stateful-hypothesis runs) so only a genuine hang can trip it. +faulthandler_timeout = 600 +faulthandler_exit_on_timeout = true +asyncio_mode = "auto" +asyncio_default_fixture_loop_scope = "function" +doctest_optionflags = [ + "NORMALIZE_WHITESPACE", + "ELLIPSIS", + "IGNORE_EXCEPTION_DETAIL", +] +addopts = [ + "--benchmark-columns", "min,mean,stddev,outliers,rounds,iterations", + "--benchmark-disable", # benchmark routines run as tests without benchmarking instrumentation + "--durations", "10", + "-ra", + "--doctest-modules", + "--ignore=tests/test_regression/scripts", + "--ignore=src/zarr/_cli", +] +filterwarnings = [ + "error", + "ignore:Unclosed client session = '3.15'", @@ -1649,7 +1649,7 @@ wheels = [ [[package]] name = "mkdocstrings" -version = "1.0.6" +version = "1.0.8" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "jinja2" }, @@ -1659,9 +1659,9 @@ dependencies = [ { name = "mkdocs-autorefs" }, { name = "pymdown-extensions" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/53/71/f85bdf13355073ae15a7375f09879375a830553552e58c1c4b7e0bbc5c8b/mkdocstrings-1.0.6.tar.gz", hash = "sha256:a0b8c2bdd29a6416c80d717aa369bbf7831946bd9f23c2a66db1b1dbe7693dbd", size = 100649, upload-time = "2026-07-11T19:38:05.732Z" } +sdist = { url = "https://files.pythonhosted.org/packages/27/cc/e3ed8b16851f5d664dffcf8f55fc60d0d3f736a3edca21d1508105d1255a/mkdocstrings-1.0.8.tar.gz", hash = "sha256:b50fc326230906425c82ee9ec59b80e5db00892b2b593d4d0b841a0d4acf537b", size = 107444, upload-time = "2026-10-19T15:43:02.157Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/5d/5b/4c1902e8bdd5c4db63284e9d101dece4038d4025d6d88850ffe0a1578980/mkdocstrings-1.0.6-py3-none-any.whl", hash = "sha256:2703708697487d1b6d6d7b412e176fa436edf120c1bf81dc9e126b12d00893c7", size = 35787, upload-time = "2026-07-11T19:38:04.417Z" }, + { url = "https://files.pythonhosted.org/packages/19/5e/4df7a2290e97c60a3ba410084966e5ab84aa89d15c8c98c195acbaefa8f4/mkdocstrings-1.0.8-py3-none-any.whl", hash = "sha256:4b03ed2b863009f1153f0d04aacb22a85b92afc28a0636d85717c92eecca638c", size = 42209, upload-time = "2026-10-19T15:43:00.65Z" }, ] [[package]] @@ -3843,7 +3843,7 @@ dev = [ { name = "mkdocs", specifier = "==1.6.1" }, { name = "mkdocs-material", extras = ["imaging"], specifier = "==9.7.7" }, { name = "mkdocs-redirects", specifier = "==1.2.3" }, - { name = "mkdocstrings", specifier = "==1.0.6" }, + { name = "mkdocstrings", specifier = "==1.0.8" }, { name = "mkdocstrings-python", specifier = "==2.0.9" }, { name = "moto", extras = ["s3", "server"], specifier = "==5.2.3" }, { name = "mypy", specifier = "==2.3.1" }, @@ -3874,7 +3874,7 @@ docs = [ { name = "mkdocs", specifier = "==1.6.1" }, { name = "mkdocs-material", extras = ["imaging"], specifier = "==9.7.7" }, { name = "mkdocs-redirects", specifier = "==1.2.3" }, - { name = "mkdocstrings", specifier = "==1.0.6" }, + { name = "mkdocstrings", specifier = "==1.0.8" }, { name = "mkdocstrings-python", specifier = "==2.0.9" }, { name = "numcodecs", extras = ["msgpack"] }, { name = "pytest", specifier = "==9.1.1" }, From 5928a2481143e854aea6858a6ed3b41539290c18 Mon Sep 17 00:00:00 2001 From: Duo-ee <113885053+Duo-ee@users.noreply.github.com> Date: Thu, 22 Oct 2026 11:38:34 +0800 Subject: [PATCH 07/12] fix: update zarr-indexing docs dependency Use mkdocstrings 1.0.8, which handles the current pymdownx.highlight constructor API. --- packages/zarr-indexing/pyproject.toml | 404 +++++++++++++------------- packages/zarr-indexing/uv.lock | 10 +- 2 files changed, 207 insertions(+), 207 deletions(-) diff --git a/packages/zarr-indexing/pyproject.toml b/packages/zarr-indexing/pyproject.toml index da7bee34aa..78fab04d04 100644 --- a/packages/zarr-indexing/pyproject.toml +++ b/packages/zarr-indexing/pyproject.toml @@ -1,202 +1,202 @@ -[build-system] -requires = ["hatchling>=1.29.0", "hatch-vcs", "setuptools-scm>=9"] -build-backend = "hatchling.build" - -[project] -name = "zarr-indexing" -dynamic = ["version"] -description = "Composable, lazy coordinate transforms for Zarr array indexing." -readme = "README.md" -requires-python = ">=3.12" -license = "MIT" -license-files = ["LICENSE.txt"] -authors = [ - { name = "Davis Bennett", email = "davis.v.bennett@gmail.com" }, -] -classifiers = [ - "Development Status :: 4 - Beta", - "Intended Audience :: Developers", - "Intended Audience :: Information Technology", - "Intended Audience :: Science/Research", - "License :: OSI Approved :: MIT License", - "Operating System :: OS Independent", - "Programming Language :: Python", - "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.12", - "Programming Language :: Python :: 3.13", - "Programming Language :: Python :: 3.14", - "Topic :: Scientific/Engineering", - "Topic :: Software Development :: Libraries :: Python Modules", - "Typing :: Typed", -] -keywords = ["zarr"] -dependencies = [ - "numpy>=2", -] - -[project.optional-dependencies] -# `zarr_indexing.testing` — a Hypothesis state machine and selection strategies -# for projects checking their own array against this package. Nothing else in -# the package imports hypothesis. -testing = ["hypothesis>=6.160.0"] - -[project.urls] -Homepage = "https://github.com/zarr-developers/zarr-python" -Source = "https://github.com/zarr-developers/zarr-python/tree/main/packages/zarr-indexing" -Issues = "https://github.com/zarr-developers/zarr-python/issues" -Changelog = "https://github.com/zarr-developers/zarr-python/blob/main/packages/zarr-indexing/CHANGELOG.md" -Documentation = "https://zarr-indexing.readthedocs.io/" - -[dependency-groups] -# Package tests use the local grid protocol and grid implementations. The -# repository recipes layer this package into the root test environment. -# This test group and the optional testing extra both declare Hypothesis; -# keep their supported minimum versions aligned. -test = ["pytest", "hypothesis>=6.160.0"] -docs = [ - # Pins match the zarr-python docs environment in the repo-root - # pyproject.toml so every site renders with the same toolchain, and the - # shared packages/mkdocs-base.yml sees one set of plugin versions. - # Bump them together with the root. - "mkdocs-material==9.7.7", - "mkdocs==1.6.1", - "mkdocstrings==1.0.6", - "mkdocstrings-python==2.0.8", - "griffe-inherited-docstrings==1.1.3", - # mkdocstrings uses ruff to format rendered signatures - "ruff==0.16.8", -] - -[tool.hatch.version] -source = "vcs" -tag-pattern = '^zarr_indexing-v(?P.+)$' -# `git_describe_command` ensures we get the zarr_indexing tags instead of latest. -# `local_scheme` strips the git commit info so the appending info is just a counter from latest tag. -# test-pypi doesn't accept git commit info in tags, and the count should be enough to distinguish unique runs. -raw-options = { root = "../..", scm = { git = { describe_command = "git describe --dirty --tags --long --match zarr_indexing-v*" } }, local_scheme = "no-local-version" } - -[tool.hatch.build.targets.wheel] -packages = ["src/zarr_indexing"] - -# An allowlist, so nothing that merely happens to sit in the package directory -# — a scratch script, a stray notebook — can ride along in a release. The list -# includes test assets in the sdist: `tests/` carries the vendored ndsel conformance -# corpus, and `tests/test_doc_examples.py` executes `docs/snippets/*.py` and -# `examples/*/*.py`, so those are part of the suite rather than decoration. -# `pyproject.toml`, `README.md` and `LICENSE.txt` are added by hatchling itself. -[tool.hatch.build.targets.sdist] -include = [ - "/src", - "/tests", - "/docs", - "/examples", - "/mkdocs.yml", - "/justfile", - "/CHANGELOG.md", - "/CONTRIBUTING.md", -] -# The shared docs config and header script are symlinks into packages/ in the -# repository; ship the files they point at, since a link out of the sdist -# dangles and pip refuses to unpack it. -exclude = ["/mkdocs-base.yml", "/docs/_static/source-version.js"] - -[tool.hatch.build.targets.sdist.force-include] -"../mkdocs-base.yml" = "mkdocs-base.yml" -"../source-version.js" = "docs/_static/source-version.js" - -[tool.ruff] -extend = "../../pyproject.toml" -target-version = "py312" - -[tool.ruff.lint.per-file-ignores] -# Chunk discovery deliberately catches Exception: a foreign source's -# attributes may fail, so this path provides fallback metadata. Configured here (not as -# noqa comments) because different ruff versions -# have differed on whether these rules fire; RUF100 can remove unused noqa comments. -"src/zarr_indexing/lazy_array.py" = ["BLE001", "S110"] - -[tool.pytest.ini_options] -minversion = "7" -# src is collected for its doctests: every public object's Examples section -# is eligible for --doctest-modules. Passing examples do not verify all prose -# contracts or optional-dependency paths. -testpaths = ["tests", "src/zarr_indexing"] -pythonpath = ["."] -xfail_strict = true -addopts = ["-ra", "--strict-config", "--strict-markers", "--doctest-modules"] -doctest_optionflags = [ - "NORMALIZE_WHITESPACE", - "ELLIPSIS", - "IGNORE_EXCEPTION_DETAIL", -] -filterwarnings = [ - "error", -] - -[tool.pyright] -include = [ - "src", - "docs/snippets", - "tests/test_doc_examples.py", -] -enableExperimentalFeatures = true -typeCheckingMode = "strict" -pythonVersion = "3.12" -# Partially unknown types from NumPy stubs are reported as warnings. CI's -# pyright invocation fails on errors, while retaining these warnings in its output. -reportUnknownVariableType = "warning" -reportUnknownArgumentType = "warning" -reportUnknownMemberType = "warning" -reportUnknownParameterType = "warning" - -[tool.numpydoc_validation] -checks = [ - "GL10", - "SS04", - "PR02", - "PR03", - "PR05", - "PR06", -] - -[tool.towncrier] -# Fragments for this package live alongside the package source, separate -# from the parent zarr-python `changes/` directory, so a PR touching only -# `packages/zarr-indexing/` produces a release note for this package only. -directory = "changes" -filename = "CHANGELOG.md" -package = "zarr_indexing" -underlines = ["", "", ""] -title_format = "## {version} ({project_date})" -issue_format = "[#{issue}](https://github.com/zarr-developers/zarr-python/pull/{issue})" -start_string = "\n" - -# Declaring any type replaces towncrier's built-in set, so all five the -# `changes/README.md` menu offers are restated here. They are the defaults -# verbatim except for `misc`, whose `showcontent` towncrier defaults to false: -# a `misc` entry would render as a bare PR link, which tells a reader nothing. -# A change worth a release note is worth a sentence, whatever its category. -[[tool.towncrier.type]] -directory = "feature" -name = "Features" -showcontent = true - -[[tool.towncrier.type]] -directory = "bugfix" -name = "Bugfixes" -showcontent = true - -[[tool.towncrier.type]] -directory = "doc" -name = "Improved Documentation" -showcontent = true - -[[tool.towncrier.type]] -directory = "removal" -name = "Deprecations and Removals" -showcontent = true - -[[tool.towncrier.type]] -directory = "misc" -name = "Misc" -showcontent = true +[build-system] +requires = ["hatchling>=1.29.0", "hatch-vcs", "setuptools-scm>=9"] +build-backend = "hatchling.build" + +[project] +name = "zarr-indexing" +dynamic = ["version"] +description = "Composable, lazy coordinate transforms for Zarr array indexing." +readme = "README.md" +requires-python = ">=3.12" +license = "MIT" +license-files = ["LICENSE.txt"] +authors = [ + { name = "Davis Bennett", email = "davis.v.bennett@gmail.com" }, +] +classifiers = [ + "Development Status :: 4 - Beta", + "Intended Audience :: Developers", + "Intended Audience :: Information Technology", + "Intended Audience :: Science/Research", + "License :: OSI Approved :: MIT License", + "Operating System :: OS Independent", + "Programming Language :: Python", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", + "Programming Language :: Python :: 3.14", + "Topic :: Scientific/Engineering", + "Topic :: Software Development :: Libraries :: Python Modules", + "Typing :: Typed", +] +keywords = ["zarr"] +dependencies = [ + "numpy>=2", +] + +[project.optional-dependencies] +# `zarr_indexing.testing` — a Hypothesis state machine and selection strategies +# for projects checking their own array against this package. Nothing else in +# the package imports hypothesis. +testing = ["hypothesis>=6.160.0"] + +[project.urls] +Homepage = "https://github.com/zarr-developers/zarr-python" +Source = "https://github.com/zarr-developers/zarr-python/tree/main/packages/zarr-indexing" +Issues = "https://github.com/zarr-developers/zarr-python/issues" +Changelog = "https://github.com/zarr-developers/zarr-python/blob/main/packages/zarr-indexing/CHANGELOG.md" +Documentation = "https://zarr-indexing.readthedocs.io/" + +[dependency-groups] +# Package tests use the local grid protocol and grid implementations. The +# repository recipes layer this package into the root test environment. +# This test group and the optional testing extra both declare Hypothesis; +# keep their supported minimum versions aligned. +test = ["pytest", "hypothesis>=6.160.0"] +docs = [ + # Pins match the zarr-python docs environment in the repo-root + # pyproject.toml so every site renders with the same toolchain, and the + # shared packages/mkdocs-base.yml sees one set of plugin versions. + # Bump them together with the root. + "mkdocs-material==9.7.7", + "mkdocs==1.6.1", + "mkdocstrings==1.0.8", + "mkdocstrings-python==2.0.8", + "griffe-inherited-docstrings==1.1.3", + # mkdocstrings uses ruff to format rendered signatures + "ruff==0.16.8", +] + +[tool.hatch.version] +source = "vcs" +tag-pattern = '^zarr_indexing-v(?P.+)$' +# `git_describe_command` ensures we get the zarr_indexing tags instead of latest. +# `local_scheme` strips the git commit info so the appending info is just a counter from latest tag. +# test-pypi doesn't accept git commit info in tags, and the count should be enough to distinguish unique runs. +raw-options = { root = "../..", scm = { git = { describe_command = "git describe --dirty --tags --long --match zarr_indexing-v*" } }, local_scheme = "no-local-version" } + +[tool.hatch.build.targets.wheel] +packages = ["src/zarr_indexing"] + +# An allowlist, so nothing that merely happens to sit in the package directory +# — a scratch script, a stray notebook — can ride along in a release. The list +# includes test assets in the sdist: `tests/` carries the vendored ndsel conformance +# corpus, and `tests/test_doc_examples.py` executes `docs/snippets/*.py` and +# `examples/*/*.py`, so those are part of the suite rather than decoration. +# `pyproject.toml`, `README.md` and `LICENSE.txt` are added by hatchling itself. +[tool.hatch.build.targets.sdist] +include = [ + "/src", + "/tests", + "/docs", + "/examples", + "/mkdocs.yml", + "/justfile", + "/CHANGELOG.md", + "/CONTRIBUTING.md", +] +# The shared docs config and header script are symlinks into packages/ in the +# repository; ship the files they point at, since a link out of the sdist +# dangles and pip refuses to unpack it. +exclude = ["/mkdocs-base.yml", "/docs/_static/source-version.js"] + +[tool.hatch.build.targets.sdist.force-include] +"../mkdocs-base.yml" = "mkdocs-base.yml" +"../source-version.js" = "docs/_static/source-version.js" + +[tool.ruff] +extend = "../../pyproject.toml" +target-version = "py312" + +[tool.ruff.lint.per-file-ignores] +# Chunk discovery deliberately catches Exception: a foreign source's +# attributes may fail, so this path provides fallback metadata. Configured here (not as +# noqa comments) because different ruff versions +# have differed on whether these rules fire; RUF100 can remove unused noqa comments. +"src/zarr_indexing/lazy_array.py" = ["BLE001", "S110"] + +[tool.pytest.ini_options] +minversion = "7" +# src is collected for its doctests: every public object's Examples section +# is eligible for --doctest-modules. Passing examples do not verify all prose +# contracts or optional-dependency paths. +testpaths = ["tests", "src/zarr_indexing"] +pythonpath = ["."] +xfail_strict = true +addopts = ["-ra", "--strict-config", "--strict-markers", "--doctest-modules"] +doctest_optionflags = [ + "NORMALIZE_WHITESPACE", + "ELLIPSIS", + "IGNORE_EXCEPTION_DETAIL", +] +filterwarnings = [ + "error", +] + +[tool.pyright] +include = [ + "src", + "docs/snippets", + "tests/test_doc_examples.py", +] +enableExperimentalFeatures = true +typeCheckingMode = "strict" +pythonVersion = "3.12" +# Partially unknown types from NumPy stubs are reported as warnings. CI's +# pyright invocation fails on errors, while retaining these warnings in its output. +reportUnknownVariableType = "warning" +reportUnknownArgumentType = "warning" +reportUnknownMemberType = "warning" +reportUnknownParameterType = "warning" + +[tool.numpydoc_validation] +checks = [ + "GL10", + "SS04", + "PR02", + "PR03", + "PR05", + "PR06", +] + +[tool.towncrier] +# Fragments for this package live alongside the package source, separate +# from the parent zarr-python `changes/` directory, so a PR touching only +# `packages/zarr-indexing/` produces a release note for this package only. +directory = "changes" +filename = "CHANGELOG.md" +package = "zarr_indexing" +underlines = ["", "", ""] +title_format = "## {version} ({project_date})" +issue_format = "[#{issue}](https://github.com/zarr-developers/zarr-python/pull/{issue})" +start_string = "\n" + +# Declaring any type replaces towncrier's built-in set, so all five the +# `changes/README.md` menu offers are restated here. They are the defaults +# verbatim except for `misc`, whose `showcontent` towncrier defaults to false: +# a `misc` entry would render as a bare PR link, which tells a reader nothing. +# A change worth a release note is worth a sentence, whatever its category. +[[tool.towncrier.type]] +directory = "feature" +name = "Features" +showcontent = true + +[[tool.towncrier.type]] +directory = "bugfix" +name = "Bugfixes" +showcontent = true + +[[tool.towncrier.type]] +directory = "doc" +name = "Improved Documentation" +showcontent = true + +[[tool.towncrier.type]] +directory = "removal" +name = "Deprecations and Removals" +showcontent = true + +[[tool.towncrier.type]] +directory = "misc" +name = "Misc" +showcontent = true diff --git a/packages/zarr-indexing/uv.lock b/packages/zarr-indexing/uv.lock index 1a4a90e2ba..7847e8e926 100644 --- a/packages/zarr-indexing/uv.lock +++ b/packages/zarr-indexing/uv.lock @@ -1,5 +1,5 @@ version = 1 -revision = 3 +revision = 5 requires-python = ">=3.12" [[package]] @@ -397,7 +397,7 @@ wheels = [ [[package]] name = "mkdocstrings" -version = "1.0.6" +version = "1.0.8" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "jinja2" }, @@ -407,9 +407,9 @@ dependencies = [ { name = "mkdocs-autorefs" }, { name = "pymdown-extensions" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/53/71/f85bdf13355073ae15a7375f09879375a830553552e58c1c4b7e0bbc5c8b/mkdocstrings-1.0.6.tar.gz", hash = "sha256:a0b8c2bdd29a6416c80d717aa369bbf7831946bd9f23c2a66db1b1dbe7693dbd", size = 100649, upload-time = "2026-07-11T19:38:05.732Z" } +sdist = { url = "https://files.pythonhosted.org/packages/27/cc/e3ed8b16851f5d664dffcf8f55fc60d0d3f736a3edca21d1508105d1255a/mkdocstrings-1.0.8.tar.gz", hash = "sha256:b50fc326230906425c82ee9ec59b80e5db00892b2b593d4d0b841a0d4acf537b", size = 107444, upload-time = "2026-10-19T15:43:02.157Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/5d/5b/4c1902e8bdd5c4db63284e9d101dece4038d4025d6d88850ffe0a1578980/mkdocstrings-1.0.6-py3-none-any.whl", hash = "sha256:2703708697487d1b6d6d7b412e176fa436edf120c1bf81dc9e126b12d00893c7", size = 35787, upload-time = "2026-07-11T19:38:04.417Z" }, + { url = "https://files.pythonhosted.org/packages/19/5e/4df7a2290e97c60a3ba410084966e5ab84aa89d15c8c98c195acbaefa8f4/mkdocstrings-1.0.8-py3-none-any.whl", hash = "sha256:4b03ed2b863009f1153f0d04aacb22a85b92afc28a0636d85717c92eecca638c", size = 42209, upload-time = "2026-10-19T15:43:00.65Z" }, ] [[package]] @@ -759,7 +759,7 @@ docs = [ { name = "griffe-inherited-docstrings", specifier = "==1.1.3" }, { name = "mkdocs", specifier = "==1.6.1" }, { name = "mkdocs-material", specifier = "==9.7.7" }, - { name = "mkdocstrings", specifier = "==1.0.6" }, + { name = "mkdocstrings", specifier = "==1.0.8" }, { name = "mkdocstrings-python", specifier = "==2.0.8" }, { name = "ruff", specifier = "==0.16.8" }, ] From a516d4e36f22e70e2be3289ee399740cb3ce761f Mon Sep 17 00:00:00 2001 From: Duo-ee <113885053+Duo-ee@users.noreply.github.com> Date: Thu, 22 Oct 2026 11:38:59 +0800 Subject: [PATCH 08/12] fix: update zarr-metadata docs dependency Use mkdocstrings 1.0.8, which handles the current pymdownx.highlight constructor API. --- packages/zarr-metadata/pyproject.toml | 428 +++++++++++++------------- packages/zarr-metadata/uv.lock | 10 +- 2 files changed, 219 insertions(+), 219 deletions(-) diff --git a/packages/zarr-metadata/pyproject.toml b/packages/zarr-metadata/pyproject.toml index 31d548bf2a..6a400e154c 100644 --- a/packages/zarr-metadata/pyproject.toml +++ b/packages/zarr-metadata/pyproject.toml @@ -1,214 +1,214 @@ -[build-system] -requires = ["hatchling>=1.29.0", "hatch-vcs", "setuptools-scm>=9"] -build-backend = "hatchling.build" - -[project] -name = "zarr-metadata" -dynamic = ["version"] -description = "Spec-defined metadata types, models, and validators for Zarr v2 and v3." -readme = "README.md" -requires-python = ">=3.11" -license = "MIT" -license-files = ["LICENSE.txt"] -authors = [ - { name = "Davis Bennett", email = "davis.v.bennett@gmail.com" }, -] -classifiers = [ - "Development Status :: 4 - Beta", - "Intended Audience :: Developers", - "Intended Audience :: Information Technology", - "Intended Audience :: Science/Research", - "License :: OSI Approved :: MIT License", - "Operating System :: OS Independent", - "Programming Language :: Python", - "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.11", - "Programming Language :: Python :: 3.12", - "Programming Language :: Python :: 3.13", - "Programming Language :: Python :: 3.14", - "Topic :: Scientific/Engineering", - "Topic :: Software Development :: Libraries :: Python Modules", - "Typing :: Typed", -] -keywords = ["zarr"] -dependencies = [ - # >=4.16: first release where `Sentinel` pickles by reference - # (`__reduce__` returns the sentinel's name), so `UNSET` — and any model - # holding it — can cross process boundaries and be deep-copied with its - # singleton identity intact. 4.15 and earlier refuse to pickle sentinels. - "typing_extensions>=4.16", -] - -[project.urls] -Homepage = "https://github.com/zarr-developers/zarr-python" -Source = "https://github.com/zarr-developers/zarr-python/tree/main/packages/zarr-metadata" -Issues = "https://github.com/zarr-developers/zarr-python/issues" -Changelog = "https://github.com/zarr-developers/zarr-python/blob/main/packages/zarr-metadata/CHANGELOG.md" -Documentation = "https://zarr-metadata.readthedocs.io/" - -[dependency-groups] -test = ["pytest", "pydantic>=2.13", "jsonschema", "hypothesis"] -docs = [ - # Pins match the zarr-python docs environment in the repo-root - # pyproject.toml so every site renders with the same toolchain, and the - # shared packages/mkdocs-base.yml sees one set of plugin versions. - # Bump them together with the root. - "mkdocs-material==9.7.7", - "mkdocs==1.6.1", - "mkdocstrings==1.0.6", - "mkdocstrings-python==2.0.8", - "griffe-inherited-docstrings==1.1.3", - # mkdocstrings uses ruff to format rendered signatures - "ruff==0.16.8", -] - -[tool.hatch.version] -source = "vcs" -tag-pattern = '^zarr_metadata-v(?P.+)$' -# `git_describe_command` ensures we get the zarr_metadata tags instead of latest. -# `local_scheme` strips the git commit info so the appending info is just a counter from latest tag. -# test-pypi doesn't accept git commit info in tags, and the count should be enough to distinguish unique runs. -raw-options = { root = "../..", scm = { git = { describe_command = "git describe --dirty --tags --long --match zarr_metadata-v*" } }, local_scheme = "no-local-version" } - -[tool.hatch.build.targets.wheel] -packages = ["src/zarr_metadata"] - -# An allowlist, so nothing that merely happens to sit in the package directory -# — a scratch script, a stray notebook — can ride along in a release. The list -# keeps an sdist self-testing and self-documenting: every fixture this suite -# reads is a JSON file sitting next to the test module that loads it, so -# `/tests` is the whole test dependency, and `/docs` plus `/mkdocs.yml` are a -# self-contained site (mkdocstrings reads `src`; the shared config and header -# script are copied in below) so `just docs-check` runs from an unpacked sdist too. `changes/` -# and `.readthedocs.yaml` are deliberately absent: towncrier fragments are -# repo bookkeeping, and the RTD config addresses paths from the repo root. -# `pyproject.toml`, `README.md` and `LICENSE.txt` are added by hatchling itself. -[tool.hatch.build.targets.sdist] -include = [ - "/src", - "/tests", - "/docs", - "/mkdocs.yml", - "/justfile", - "/CHANGELOG.md", -] -# The shared docs config and header script are symlinks into packages/ in the -# repository; ship the files they point at, since a link out of the sdist -# dangles and pip refuses to unpack it. -exclude = ["/mkdocs-base.yml", "/docs/_static/source-version.js"] - -[tool.hatch.build.targets.sdist.force-include] -"../mkdocs-base.yml" = "mkdocs-base.yml" -"../source-version.js" = "docs/_static/source-version.js" - -[tool.ruff] -extend = "../../pyproject.toml" -target-version = "py311" - -[tool.ruff.lint] -# `Any` defeats the point of a package whose product is precise types, so it -# is banned in annotations here rather than merely discouraged. Use `object` -# for "any value" (the caller must narrow) and the document TypedDicts where -# the shape is known; a genuinely dynamic annotation needs an explicit noqa -# saying why. -extend-select = ["ANN401"] - -[tool.pytest.ini_options] -minversion = "7" -testpaths = ["tests"] -xfail_strict = true -addopts = ["-ra", "--strict-config", "--strict-markers"] -filterwarnings = [ - "error", - # Pydantic validates these public immutable-shape TypedDicts correctly but - # cannot enforce the type checker's ReadOnly mutation restriction. - "ignore:Items? .* using the `ReadOnly` qualifier.*:UserWarning:pydantic._internal._generate_schema", -] - -[tool.numpydoc_validation] -checks = [ - "GL10", - "SS04", - "PR02", - "PR03", - "PR05", - "PR06", -] - -# The pyright version lives in the justfile, which is what CI runs; pinning it -# keeps a pyright release from turning CI red on its own schedule. -[tool.pyright] -include = ["src", "tests"] -# Pyright is the only checker that runs on this package (the root mypy -# config covers the root package alone), and it honours `# type: ignore` -# as a blanket suppression of every diagnostic on the line. So that -# spelling is switched off: a suppression is `# pyright: ignore[rule]`, -# names the rule, and is an error once it stops suppressing anything. -enableTypeIgnoreComments = false -reportUnnecessaryTypeIgnoreComment = "error" -# `tests` is a package imported as `tests.*`, which pytest resolves from the -# rootdir; pyright needs the same root on its search path. -extraPaths = ["."] -enableExperimentalFeatures = true -typeCheckingMode = "strict" -pythonVersion = "3.11" - -# Tests are checked, but not for the inference strictness the sources are held -# to: test bodies are full of untyped JSON literals whose inferred types are of -# no interest, and strict mode reports about ninety of those. What is kept is -# everything a rename or a signature change breaks -- a dangling import, a call -# that no longer type-checks, an argument of the wrong type -- which is the -# reason for checking tests at all. Tests also reach into private modules and -# define functions purely to be refused at registration, both deliberately. -[[tool.pyright.executionEnvironments]] -root = "tests" -reportMissingTypeArgument = false -reportPrivateUsage = false -reportUnknownArgumentType = false -reportUnknownLambdaType = false -reportUnknownMemberType = false -reportUnknownParameterType = false -reportUnknownVariableType = false -reportUnusedFunction = false - -[tool.towncrier] -# Fragments for this package live alongside the package source, separate -# from the parent zarr-python `changes/` directory, so a PR touching only -# `packages/zarr-metadata/` produces a release note for this package only. -directory = "changes" -filename = "CHANGELOG.md" -package = "zarr_metadata" -underlines = ["", "", ""] -title_format = "## {version} ({project_date})" -issue_format = "[#{issue}](https://github.com/zarr-developers/zarr-python/pull/{issue})" -start_string = "\n" - -# Declaring any type replaces towncrier's built-in set, so all five the -# `changes/README.md` menu offers are restated here. They are the defaults -# verbatim except for `misc`, whose `showcontent` towncrier defaults to false: -# a `misc` entry would render as a bare PR link, which tells a reader nothing. -# A change worth a release note is worth a sentence, whatever its category. -[[tool.towncrier.type]] -directory = "feature" -name = "Features" -showcontent = true - -[[tool.towncrier.type]] -directory = "bugfix" -name = "Bugfixes" -showcontent = true - -[[tool.towncrier.type]] -directory = "doc" -name = "Improved Documentation" -showcontent = true - -[[tool.towncrier.type]] -directory = "removal" -name = "Deprecations and Removals" -showcontent = true - -[[tool.towncrier.type]] -directory = "misc" -name = "Misc" -showcontent = true +[build-system] +requires = ["hatchling>=1.29.0", "hatch-vcs", "setuptools-scm>=9"] +build-backend = "hatchling.build" + +[project] +name = "zarr-metadata" +dynamic = ["version"] +description = "Spec-defined metadata types, models, and validators for Zarr v2 and v3." +readme = "README.md" +requires-python = ">=3.11" +license = "MIT" +license-files = ["LICENSE.txt"] +authors = [ + { name = "Davis Bennett", email = "davis.v.bennett@gmail.com" }, +] +classifiers = [ + "Development Status :: 4 - Beta", + "Intended Audience :: Developers", + "Intended Audience :: Information Technology", + "Intended Audience :: Science/Research", + "License :: OSI Approved :: MIT License", + "Operating System :: OS Independent", + "Programming Language :: Python", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", + "Programming Language :: Python :: 3.14", + "Topic :: Scientific/Engineering", + "Topic :: Software Development :: Libraries :: Python Modules", + "Typing :: Typed", +] +keywords = ["zarr"] +dependencies = [ + # >=4.16: first release where `Sentinel` pickles by reference + # (`__reduce__` returns the sentinel's name), so `UNSET` — and any model + # holding it — can cross process boundaries and be deep-copied with its + # singleton identity intact. 4.15 and earlier refuse to pickle sentinels. + "typing_extensions>=4.16", +] + +[project.urls] +Homepage = "https://github.com/zarr-developers/zarr-python" +Source = "https://github.com/zarr-developers/zarr-python/tree/main/packages/zarr-metadata" +Issues = "https://github.com/zarr-developers/zarr-python/issues" +Changelog = "https://github.com/zarr-developers/zarr-python/blob/main/packages/zarr-metadata/CHANGELOG.md" +Documentation = "https://zarr-metadata.readthedocs.io/" + +[dependency-groups] +test = ["pytest", "pydantic>=2.13", "jsonschema", "hypothesis"] +docs = [ + # Pins match the zarr-python docs environment in the repo-root + # pyproject.toml so every site renders with the same toolchain, and the + # shared packages/mkdocs-base.yml sees one set of plugin versions. + # Bump them together with the root. + "mkdocs-material==9.7.7", + "mkdocs==1.6.1", + "mkdocstrings==1.0.8", + "mkdocstrings-python==2.0.8", + "griffe-inherited-docstrings==1.1.3", + # mkdocstrings uses ruff to format rendered signatures + "ruff==0.16.8", +] + +[tool.hatch.version] +source = "vcs" +tag-pattern = '^zarr_metadata-v(?P.+)$' +# `git_describe_command` ensures we get the zarr_metadata tags instead of latest. +# `local_scheme` strips the git commit info so the appending info is just a counter from latest tag. +# test-pypi doesn't accept git commit info in tags, and the count should be enough to distinguish unique runs. +raw-options = { root = "../..", scm = { git = { describe_command = "git describe --dirty --tags --long --match zarr_metadata-v*" } }, local_scheme = "no-local-version" } + +[tool.hatch.build.targets.wheel] +packages = ["src/zarr_metadata"] + +# An allowlist, so nothing that merely happens to sit in the package directory +# — a scratch script, a stray notebook — can ride along in a release. The list +# keeps an sdist self-testing and self-documenting: every fixture this suite +# reads is a JSON file sitting next to the test module that loads it, so +# `/tests` is the whole test dependency, and `/docs` plus `/mkdocs.yml` are a +# self-contained site (mkdocstrings reads `src`; the shared config and header +# script are copied in below) so `just docs-check` runs from an unpacked sdist too. `changes/` +# and `.readthedocs.yaml` are deliberately absent: towncrier fragments are +# repo bookkeeping, and the RTD config addresses paths from the repo root. +# `pyproject.toml`, `README.md` and `LICENSE.txt` are added by hatchling itself. +[tool.hatch.build.targets.sdist] +include = [ + "/src", + "/tests", + "/docs", + "/mkdocs.yml", + "/justfile", + "/CHANGELOG.md", +] +# The shared docs config and header script are symlinks into packages/ in the +# repository; ship the files they point at, since a link out of the sdist +# dangles and pip refuses to unpack it. +exclude = ["/mkdocs-base.yml", "/docs/_static/source-version.js"] + +[tool.hatch.build.targets.sdist.force-include] +"../mkdocs-base.yml" = "mkdocs-base.yml" +"../source-version.js" = "docs/_static/source-version.js" + +[tool.ruff] +extend = "../../pyproject.toml" +target-version = "py311" + +[tool.ruff.lint] +# `Any` defeats the point of a package whose product is precise types, so it +# is banned in annotations here rather than merely discouraged. Use `object` +# for "any value" (the caller must narrow) and the document TypedDicts where +# the shape is known; a genuinely dynamic annotation needs an explicit noqa +# saying why. +extend-select = ["ANN401"] + +[tool.pytest.ini_options] +minversion = "7" +testpaths = ["tests"] +xfail_strict = true +addopts = ["-ra", "--strict-config", "--strict-markers"] +filterwarnings = [ + "error", + # Pydantic validates these public immutable-shape TypedDicts correctly but + # cannot enforce the type checker's ReadOnly mutation restriction. + "ignore:Items? .* using the `ReadOnly` qualifier.*:UserWarning:pydantic._internal._generate_schema", +] + +[tool.numpydoc_validation] +checks = [ + "GL10", + "SS04", + "PR02", + "PR03", + "PR05", + "PR06", +] + +# The pyright version lives in the justfile, which is what CI runs; pinning it +# keeps a pyright release from turning CI red on its own schedule. +[tool.pyright] +include = ["src", "tests"] +# Pyright is the only checker that runs on this package (the root mypy +# config covers the root package alone), and it honours `# type: ignore` +# as a blanket suppression of every diagnostic on the line. So that +# spelling is switched off: a suppression is `# pyright: ignore[rule]`, +# names the rule, and is an error once it stops suppressing anything. +enableTypeIgnoreComments = false +reportUnnecessaryTypeIgnoreComment = "error" +# `tests` is a package imported as `tests.*`, which pytest resolves from the +# rootdir; pyright needs the same root on its search path. +extraPaths = ["."] +enableExperimentalFeatures = true +typeCheckingMode = "strict" +pythonVersion = "3.11" + +# Tests are checked, but not for the inference strictness the sources are held +# to: test bodies are full of untyped JSON literals whose inferred types are of +# no interest, and strict mode reports about ninety of those. What is kept is +# everything a rename or a signature change breaks -- a dangling import, a call +# that no longer type-checks, an argument of the wrong type -- which is the +# reason for checking tests at all. Tests also reach into private modules and +# define functions purely to be refused at registration, both deliberately. +[[tool.pyright.executionEnvironments]] +root = "tests" +reportMissingTypeArgument = false +reportPrivateUsage = false +reportUnknownArgumentType = false +reportUnknownLambdaType = false +reportUnknownMemberType = false +reportUnknownParameterType = false +reportUnknownVariableType = false +reportUnusedFunction = false + +[tool.towncrier] +# Fragments for this package live alongside the package source, separate +# from the parent zarr-python `changes/` directory, so a PR touching only +# `packages/zarr-metadata/` produces a release note for this package only. +directory = "changes" +filename = "CHANGELOG.md" +package = "zarr_metadata" +underlines = ["", "", ""] +title_format = "## {version} ({project_date})" +issue_format = "[#{issue}](https://github.com/zarr-developers/zarr-python/pull/{issue})" +start_string = "\n" + +# Declaring any type replaces towncrier's built-in set, so all five the +# `changes/README.md` menu offers are restated here. They are the defaults +# verbatim except for `misc`, whose `showcontent` towncrier defaults to false: +# a `misc` entry would render as a bare PR link, which tells a reader nothing. +# A change worth a release note is worth a sentence, whatever its category. +[[tool.towncrier.type]] +directory = "feature" +name = "Features" +showcontent = true + +[[tool.towncrier.type]] +directory = "bugfix" +name = "Bugfixes" +showcontent = true + +[[tool.towncrier.type]] +directory = "doc" +name = "Improved Documentation" +showcontent = true + +[[tool.towncrier.type]] +directory = "removal" +name = "Deprecations and Removals" +showcontent = true + +[[tool.towncrier.type]] +directory = "misc" +name = "Misc" +showcontent = true diff --git a/packages/zarr-metadata/uv.lock b/packages/zarr-metadata/uv.lock index 07ad9a0ebe..64bbeb30dc 100644 --- a/packages/zarr-metadata/uv.lock +++ b/packages/zarr-metadata/uv.lock @@ -1,5 +1,5 @@ version = 1 -revision = 3 +revision = 5 requires-python = ">=3.11" [[package]] @@ -571,7 +571,7 @@ wheels = [ [[package]] name = "mkdocstrings" -version = "1.0.6" +version = "1.0.8" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "jinja2" }, @@ -581,9 +581,9 @@ dependencies = [ { name = "mkdocs-autorefs" }, { name = "pymdown-extensions" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/53/71/f85bdf13355073ae15a7375f09879375a830553552e58c1c4b7e0bbc5c8b/mkdocstrings-1.0.6.tar.gz", hash = "sha256:a0b8c2bdd29a6416c80d717aa369bbf7831946bd9f23c2a66db1b1dbe7693dbd", size = 100649, upload-time = "2026-07-11T19:38:05.732Z" } +sdist = { url = "https://files.pythonhosted.org/packages/27/cc/e3ed8b16851f5d664dffcf8f55fc60d0d3f736a3edca21d1508105d1255a/mkdocstrings-1.0.8.tar.gz", hash = "sha256:b50fc326230906425c82ee9ec59b80e5db00892b2b593d4d0b841a0d4acf537b", size = 107444, upload-time = "2026-10-19T15:43:02.157Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/5d/5b/4c1902e8bdd5c4db63284e9d101dece4038d4025d6d88850ffe0a1578980/mkdocstrings-1.0.6-py3-none-any.whl", hash = "sha256:2703708697487d1b6d6d7b412e176fa436edf120c1bf81dc9e126b12d00893c7", size = 35787, upload-time = "2026-07-11T19:38:04.417Z" }, + { url = "https://files.pythonhosted.org/packages/19/5e/4df7a2290e97c60a3ba410084966e5ab84aa89d15c8c98c195acbaefa8f4/mkdocstrings-1.0.8-py3-none-any.whl", hash = "sha256:4b03ed2b863009f1153f0d04aacb22a85b92afc28a0636d85717c92eecca638c", size = 42209, upload-time = "2026-10-19T15:43:00.65Z" }, ] [[package]] @@ -1162,7 +1162,7 @@ docs = [ { name = "griffe-inherited-docstrings", specifier = "==1.1.3" }, { name = "mkdocs", specifier = "==1.6.1" }, { name = "mkdocs-material", specifier = "==9.7.7" }, - { name = "mkdocstrings", specifier = "==1.0.6" }, + { name = "mkdocstrings", specifier = "==1.0.8" }, { name = "mkdocstrings-python", specifier = "==2.0.8" }, { name = "ruff", specifier = "==0.16.8" }, ] From 4a87b069505928bf5090a194182083b9983088a9 Mon Sep 17 00:00:00 2001 From: Duo-ee <113885053+Duo-ee@users.noreply.github.com> Date: Thu, 22 Oct 2026 11:47:42 +0800 Subject: [PATCH 09/12] chore: normalize zarr-metadata metadata Preserve repository line endings after resolving the branch update. --- packages/zarr-metadata/pyproject.toml | 436 +++++++++++++------------- 1 file changed, 218 insertions(+), 218 deletions(-) diff --git a/packages/zarr-metadata/pyproject.toml b/packages/zarr-metadata/pyproject.toml index c33686c3a7..4c6017c269 100644 --- a/packages/zarr-metadata/pyproject.toml +++ b/packages/zarr-metadata/pyproject.toml @@ -1,218 +1,218 @@ -[build-system] -requires = ["hatchling>=1.29.0", "hatch-vcs", "setuptools-scm>=9"] -build-backend = "hatchling.build" - -[project] -name = "zarr-metadata" -dynamic = ["version"] -description = "Spec-defined metadata types, models, and validators for Zarr v2 and v3." -readme = "README.md" -requires-python = ">=3.11" -license = "MIT" -license-files = ["LICENSE.txt"] -authors = [ - { name = "Davis Bennett", email = "davis.v.bennett@gmail.com" }, -] -classifiers = [ - "Development Status :: 4 - Beta", - "Intended Audience :: Developers", - "Intended Audience :: Information Technology", - "Intended Audience :: Science/Research", - "License :: OSI Approved :: MIT License", - "Operating System :: OS Independent", - "Programming Language :: Python", - "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.11", - "Programming Language :: Python :: 3.12", - "Programming Language :: Python :: 3.13", - "Programming Language :: Python :: 3.14", - "Topic :: Scientific/Engineering", - "Topic :: Software Development :: Libraries :: Python Modules", - "Typing :: Typed", -] -keywords = ["zarr"] -dependencies = [ - # >=4.16: first release where `Sentinel` pickles by reference - # (`__reduce__` returns the sentinel's name), so `UNSET` — and any model - # holding it — can cross process boundaries and be deep-copied with its - # singleton identity intact. 4.15 and earlier refuse to pickle sentinels. - "typing_extensions>=4.16", -] - -[project.urls] -Homepage = "https://github.com/zarr-developers/zarr-python" -Source = "https://github.com/zarr-developers/zarr-python/tree/main/packages/zarr-metadata" -Issues = "https://github.com/zarr-developers/zarr-python/issues" -Changelog = "https://github.com/zarr-developers/zarr-python/blob/main/packages/zarr-metadata/CHANGELOG.md" -Documentation = "https://zarr-metadata.readthedocs.io/" - -[dependency-groups] -test = ["pytest", "pydantic>=2.13", "jsonschema", "hypothesis"] -docs = [ - # Pins match the zarr-python docs environment in the repo-root - # pyproject.toml so every site renders with the same toolchain, and the - # shared packages/mkdocs-base.yml sees one set of plugin versions. - # Bump them together with the root. - "mkdocs-material==9.7.7", - "mkdocs==1.6.1", - "mkdocstrings==1.0.8", - # pymdown-extensions 12.3 changed the Highlight extension API; mkdocstrings 1.0.6 - # cannot drive it (Highlight.__init__() missing argument "md"). Match the - # version in the repo-root uv.lock until mkdocstrings is bumped everywhere. - "pymdown-extensions==11.0.1", - "mkdocstrings-python==2.0.8", - "griffe-inherited-docstrings==1.1.3", - # mkdocstrings uses ruff to format rendered signatures - "ruff==0.16.8", -] - -[tool.hatch.version] -source = "vcs" -tag-pattern = '^zarr_metadata-v(?P.+)$' -# `git_describe_command` ensures we get the zarr_metadata tags instead of latest. -# `local_scheme` strips the git commit info so the appending info is just a counter from latest tag. -# test-pypi doesn't accept git commit info in tags, and the count should be enough to distinguish unique runs. -raw-options = { root = "../..", scm = { git = { describe_command = "git describe --dirty --tags --long --match zarr_metadata-v*" } }, local_scheme = "no-local-version" } - -[tool.hatch.build.targets.wheel] -packages = ["src/zarr_metadata"] - -# An allowlist, so nothing that merely happens to sit in the package directory -# — a scratch script, a stray notebook — can ride along in a release. The list -# keeps an sdist self-testing and self-documenting: every fixture this suite -# reads is a JSON file sitting next to the test module that loads it, so -# `/tests` is the whole test dependency, and `/docs` plus `/mkdocs.yml` are a -# self-contained site (mkdocstrings reads `src`; the shared config and header -# script are copied in below) so `just docs-check` runs from an unpacked sdist too. `changes/` -# and `.readthedocs.yaml` are deliberately absent: towncrier fragments are -# repo bookkeeping, and the RTD config addresses paths from the repo root. -# `pyproject.toml`, `README.md` and `LICENSE.txt` are added by hatchling itself. -[tool.hatch.build.targets.sdist] -include = [ - "/src", - "/tests", - "/docs", - "/mkdocs.yml", - "/justfile", - "/CHANGELOG.md", -] -# The shared docs config and header script are symlinks into packages/ in the -# repository; ship the files they point at, since a link out of the sdist -# dangles and pip refuses to unpack it. -exclude = ["/mkdocs-base.yml", "/docs/_static/source-version.js"] - -[tool.hatch.build.targets.sdist.force-include] -"../mkdocs-base.yml" = "mkdocs-base.yml" -"../source-version.js" = "docs/_static/source-version.js" - -[tool.ruff] -extend = "../../pyproject.toml" -target-version = "py311" - -[tool.ruff.lint] -# `Any` defeats the point of a package whose product is precise types, so it -# is banned in annotations here rather than merely discouraged. Use `object` -# for "any value" (the caller must narrow) and the document TypedDicts where -# the shape is known; a genuinely dynamic annotation needs an explicit noqa -# saying why. -extend-select = ["ANN401"] - -[tool.pytest.ini_options] -minversion = "7" -testpaths = ["tests"] -xfail_strict = true -addopts = ["-ra", "--strict-config", "--strict-markers"] -filterwarnings = [ - "error", - # Pydantic validates these public immutable-shape TypedDicts correctly but - # cannot enforce the type checker's ReadOnly mutation restriction. - "ignore:Items? .* using the `ReadOnly` qualifier.*:UserWarning:pydantic._internal._generate_schema", -] - -[tool.numpydoc_validation] -checks = [ - "GL10", - "SS04", - "PR02", - "PR03", - "PR05", - "PR06", -] - -# The pyright version lives in the justfile, which is what CI runs; pinning it -# keeps a pyright release from turning CI red on its own schedule. -[tool.pyright] -include = ["src", "tests"] -# Pyright is the only checker that runs on this package (the root mypy -# config covers the root package alone), and it honours `# type: ignore` -# as a blanket suppression of every diagnostic on the line. So that -# spelling is switched off: a suppression is `# pyright: ignore[rule]`, -# names the rule, and is an error once it stops suppressing anything. -enableTypeIgnoreComments = false -reportUnnecessaryTypeIgnoreComment = "error" -# `tests` is a package imported as `tests.*`, which pytest resolves from the -# rootdir; pyright needs the same root on its search path. -extraPaths = ["."] -enableExperimentalFeatures = true -typeCheckingMode = "strict" -pythonVersion = "3.11" - -# Tests are checked, but not for the inference strictness the sources are held -# to: test bodies are full of untyped JSON literals whose inferred types are of -# no interest, and strict mode reports about ninety of those. What is kept is -# everything a rename or a signature change breaks -- a dangling import, a call -# that no longer type-checks, an argument of the wrong type -- which is the -# reason for checking tests at all. Tests also reach into private modules and -# define functions purely to be refused at registration, both deliberately. -[[tool.pyright.executionEnvironments]] -root = "tests" -reportMissingTypeArgument = false -reportPrivateUsage = false -reportUnknownArgumentType = false -reportUnknownLambdaType = false -reportUnknownMemberType = false -reportUnknownParameterType = false -reportUnknownVariableType = false -reportUnusedFunction = false - -[tool.towncrier] -# Fragments for this package live alongside the package source, separate -# from the parent zarr-python `changes/` directory, so a PR touching only -# `packages/zarr-metadata/` produces a release note for this package only. -directory = "changes" -filename = "CHANGELOG.md" -package = "zarr_metadata" -underlines = ["", "", ""] -title_format = "## {version} ({project_date})" -issue_format = "[#{issue}](https://github.com/zarr-developers/zarr-python/pull/{issue})" -start_string = "\n" - -# Declaring any type replaces towncrier's built-in set, so all five the -# `changes/README.md` menu offers are restated here. They are the defaults -# verbatim except for `misc`, whose `showcontent` towncrier defaults to false: -# a `misc` entry would render as a bare PR link, which tells a reader nothing. -# A change worth a release note is worth a sentence, whatever its category. -[[tool.towncrier.type]] -directory = "feature" -name = "Features" -showcontent = true - -[[tool.towncrier.type]] -directory = "bugfix" -name = "Bugfixes" -showcontent = true - -[[tool.towncrier.type]] -directory = "doc" -name = "Improved Documentation" -showcontent = true - -[[tool.towncrier.type]] -directory = "removal" -name = "Deprecations and Removals" -showcontent = true - -[[tool.towncrier.type]] -directory = "misc" -name = "Misc" -showcontent = true +[build-system] +requires = ["hatchling>=1.29.0", "hatch-vcs", "setuptools-scm>=9"] +build-backend = "hatchling.build" + +[project] +name = "zarr-metadata" +dynamic = ["version"] +description = "Spec-defined metadata types, models, and validators for Zarr v2 and v3." +readme = "README.md" +requires-python = ">=3.11" +license = "MIT" +license-files = ["LICENSE.txt"] +authors = [ + { name = "Davis Bennett", email = "davis.v.bennett@gmail.com" }, +] +classifiers = [ + "Development Status :: 4 - Beta", + "Intended Audience :: Developers", + "Intended Audience :: Information Technology", + "Intended Audience :: Science/Research", + "License :: OSI Approved :: MIT License", + "Operating System :: OS Independent", + "Programming Language :: Python", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", + "Programming Language :: Python :: 3.14", + "Topic :: Scientific/Engineering", + "Topic :: Software Development :: Libraries :: Python Modules", + "Typing :: Typed", +] +keywords = ["zarr"] +dependencies = [ + # >=4.16: first release where `Sentinel` pickles by reference + # (`__reduce__` returns the sentinel's name), so `UNSET` — and any model + # holding it — can cross process boundaries and be deep-copied with its + # singleton identity intact. 4.15 and earlier refuse to pickle sentinels. + "typing_extensions>=4.16", +] + +[project.urls] +Homepage = "https://github.com/zarr-developers/zarr-python" +Source = "https://github.com/zarr-developers/zarr-python/tree/main/packages/zarr-metadata" +Issues = "https://github.com/zarr-developers/zarr-python/issues" +Changelog = "https://github.com/zarr-developers/zarr-python/blob/main/packages/zarr-metadata/CHANGELOG.md" +Documentation = "https://zarr-metadata.readthedocs.io/" + +[dependency-groups] +test = ["pytest", "pydantic>=2.13", "jsonschema", "hypothesis"] +docs = [ + # Pins match the zarr-python docs environment in the repo-root + # pyproject.toml so every site renders with the same toolchain, and the + # shared packages/mkdocs-base.yml sees one set of plugin versions. + # Bump them together with the root. + "mkdocs-material==9.7.7", + "mkdocs==1.6.1", + "mkdocstrings==1.0.8", + # pymdown-extensions 12.3 changed the Highlight extension API; mkdocstrings 1.0.6 + # cannot drive it (Highlight.__init__() missing argument "md"). Match the + # version in the repo-root uv.lock until mkdocstrings is bumped everywhere. + "pymdown-extensions==11.0.1", + "mkdocstrings-python==2.0.8", + "griffe-inherited-docstrings==1.1.3", + # mkdocstrings uses ruff to format rendered signatures + "ruff==0.16.8", +] + +[tool.hatch.version] +source = "vcs" +tag-pattern = '^zarr_metadata-v(?P.+)$' +# `git_describe_command` ensures we get the zarr_metadata tags instead of latest. +# `local_scheme` strips the git commit info so the appending info is just a counter from latest tag. +# test-pypi doesn't accept git commit info in tags, and the count should be enough to distinguish unique runs. +raw-options = { root = "../..", scm = { git = { describe_command = "git describe --dirty --tags --long --match zarr_metadata-v*" } }, local_scheme = "no-local-version" } + +[tool.hatch.build.targets.wheel] +packages = ["src/zarr_metadata"] + +# An allowlist, so nothing that merely happens to sit in the package directory +# — a scratch script, a stray notebook — can ride along in a release. The list +# keeps an sdist self-testing and self-documenting: every fixture this suite +# reads is a JSON file sitting next to the test module that loads it, so +# `/tests` is the whole test dependency, and `/docs` plus `/mkdocs.yml` are a +# self-contained site (mkdocstrings reads `src`; the shared config and header +# script are copied in below) so `just docs-check` runs from an unpacked sdist too. `changes/` +# and `.readthedocs.yaml` are deliberately absent: towncrier fragments are +# repo bookkeeping, and the RTD config addresses paths from the repo root. +# `pyproject.toml`, `README.md` and `LICENSE.txt` are added by hatchling itself. +[tool.hatch.build.targets.sdist] +include = [ + "/src", + "/tests", + "/docs", + "/mkdocs.yml", + "/justfile", + "/CHANGELOG.md", +] +# The shared docs config and header script are symlinks into packages/ in the +# repository; ship the files they point at, since a link out of the sdist +# dangles and pip refuses to unpack it. +exclude = ["/mkdocs-base.yml", "/docs/_static/source-version.js"] + +[tool.hatch.build.targets.sdist.force-include] +"../mkdocs-base.yml" = "mkdocs-base.yml" +"../source-version.js" = "docs/_static/source-version.js" + +[tool.ruff] +extend = "../../pyproject.toml" +target-version = "py311" + +[tool.ruff.lint] +# `Any` defeats the point of a package whose product is precise types, so it +# is banned in annotations here rather than merely discouraged. Use `object` +# for "any value" (the caller must narrow) and the document TypedDicts where +# the shape is known; a genuinely dynamic annotation needs an explicit noqa +# saying why. +extend-select = ["ANN401"] + +[tool.pytest.ini_options] +minversion = "7" +testpaths = ["tests"] +xfail_strict = true +addopts = ["-ra", "--strict-config", "--strict-markers"] +filterwarnings = [ + "error", + # Pydantic validates these public immutable-shape TypedDicts correctly but + # cannot enforce the type checker's ReadOnly mutation restriction. + "ignore:Items? .* using the `ReadOnly` qualifier.*:UserWarning:pydantic._internal._generate_schema", +] + +[tool.numpydoc_validation] +checks = [ + "GL10", + "SS04", + "PR02", + "PR03", + "PR05", + "PR06", +] + +# The pyright version lives in the justfile, which is what CI runs; pinning it +# keeps a pyright release from turning CI red on its own schedule. +[tool.pyright] +include = ["src", "tests"] +# Pyright is the only checker that runs on this package (the root mypy +# config covers the root package alone), and it honours `# type: ignore` +# as a blanket suppression of every diagnostic on the line. So that +# spelling is switched off: a suppression is `# pyright: ignore[rule]`, +# names the rule, and is an error once it stops suppressing anything. +enableTypeIgnoreComments = false +reportUnnecessaryTypeIgnoreComment = "error" +# `tests` is a package imported as `tests.*`, which pytest resolves from the +# rootdir; pyright needs the same root on its search path. +extraPaths = ["."] +enableExperimentalFeatures = true +typeCheckingMode = "strict" +pythonVersion = "3.11" + +# Tests are checked, but not for the inference strictness the sources are held +# to: test bodies are full of untyped JSON literals whose inferred types are of +# no interest, and strict mode reports about ninety of those. What is kept is +# everything a rename or a signature change breaks -- a dangling import, a call +# that no longer type-checks, an argument of the wrong type -- which is the +# reason for checking tests at all. Tests also reach into private modules and +# define functions purely to be refused at registration, both deliberately. +[[tool.pyright.executionEnvironments]] +root = "tests" +reportMissingTypeArgument = false +reportPrivateUsage = false +reportUnknownArgumentType = false +reportUnknownLambdaType = false +reportUnknownMemberType = false +reportUnknownParameterType = false +reportUnknownVariableType = false +reportUnusedFunction = false + +[tool.towncrier] +# Fragments for this package live alongside the package source, separate +# from the parent zarr-python `changes/` directory, so a PR touching only +# `packages/zarr-metadata/` produces a release note for this package only. +directory = "changes" +filename = "CHANGELOG.md" +package = "zarr_metadata" +underlines = ["", "", ""] +title_format = "## {version} ({project_date})" +issue_format = "[#{issue}](https://github.com/zarr-developers/zarr-python/pull/{issue})" +start_string = "\n" + +# Declaring any type replaces towncrier's built-in set, so all five the +# `changes/README.md` menu offers are restated here. They are the defaults +# verbatim except for `misc`, whose `showcontent` towncrier defaults to false: +# a `misc` entry would render as a bare PR link, which tells a reader nothing. +# A change worth a release note is worth a sentence, whatever its category. +[[tool.towncrier.type]] +directory = "feature" +name = "Features" +showcontent = true + +[[tool.towncrier.type]] +directory = "bugfix" +name = "Bugfixes" +showcontent = true + +[[tool.towncrier.type]] +directory = "doc" +name = "Improved Documentation" +showcontent = true + +[[tool.towncrier.type]] +directory = "removal" +name = "Deprecations and Removals" +showcontent = true + +[[tool.towncrier.type]] +directory = "misc" +name = "Misc" +showcontent = true From 0a61f8ce8f0cec05f475bcb3f3f0cf56d9712295 Mon Sep 17 00:00:00 2001 From: Duo-ee <113885053+Duo-ee@users.noreply.github.com> Date: Thu, 22 Oct 2026 11:48:49 +0800 Subject: [PATCH 10/12] chore: normalize root project line endings From 0bc75f3183532dff95cebdb6f01ccc7d3153d34a Mon Sep 17 00:00:00 2001 From: Duo-ee <113885053+Duo-ee@users.noreply.github.com> Date: Thu, 22 Oct 2026 11:49:14 +0800 Subject: [PATCH 11/12] chore: preserve LF line endings in root metadata From c323737fc94d918f6e161f7b45b8125e821864ed Mon Sep 17 00:00:00 2001 From: Duo-ee <113885053+Duo-ee@users.noreply.github.com> Date: Thu, 22 Oct 2026 16:25:19 +0800 Subject: [PATCH 12/12] chore: restore LF endings in project metadata --- packages/zarr-indexing/pyproject.toml | 412 +++++----- pyproject.toml | 1008 ++++++++++++------------- 2 files changed, 710 insertions(+), 710 deletions(-) diff --git a/packages/zarr-indexing/pyproject.toml b/packages/zarr-indexing/pyproject.toml index 9fcdc82d64..7d6e0ea5f1 100644 --- a/packages/zarr-indexing/pyproject.toml +++ b/packages/zarr-indexing/pyproject.toml @@ -1,206 +1,206 @@ -[build-system] -requires = ["hatchling>=1.29.0", "hatch-vcs", "setuptools-scm>=9"] -build-backend = "hatchling.build" - -[project] -name = "zarr-indexing" -dynamic = ["version"] -description = "Composable, lazy coordinate transforms for Zarr array indexing." -readme = "README.md" -requires-python = ">=3.12" -license = "MIT" -license-files = ["LICENSE.txt"] -authors = [ - { name = "Davis Bennett", email = "davis.v.bennett@gmail.com" }, -] -classifiers = [ - "Development Status :: 4 - Beta", - "Intended Audience :: Developers", - "Intended Audience :: Information Technology", - "Intended Audience :: Science/Research", - "License :: OSI Approved :: MIT License", - "Operating System :: OS Independent", - "Programming Language :: Python", - "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.12", - "Programming Language :: Python :: 3.13", - "Programming Language :: Python :: 3.14", - "Topic :: Scientific/Engineering", - "Topic :: Software Development :: Libraries :: Python Modules", - "Typing :: Typed", -] -keywords = ["zarr"] -dependencies = [ - "numpy>=2", -] - -[project.optional-dependencies] -# `zarr_indexing.testing` — a Hypothesis state machine and selection strategies -# for projects checking their own array against this package. Nothing else in -# the package imports hypothesis. -testing = ["hypothesis>=6.160.0"] - -[project.urls] -Homepage = "https://github.com/zarr-developers/zarr-python" -Source = "https://github.com/zarr-developers/zarr-python/tree/main/packages/zarr-indexing" -Issues = "https://github.com/zarr-developers/zarr-python/issues" -Changelog = "https://github.com/zarr-developers/zarr-python/blob/main/packages/zarr-indexing/CHANGELOG.md" -Documentation = "https://zarr-indexing.readthedocs.io/" - -[dependency-groups] -# Package tests use the local grid protocol and grid implementations. The -# repository recipes layer this package into the root test environment. -# This test group and the optional testing extra both declare Hypothesis; -# keep their supported minimum versions aligned. -test = ["pytest", "hypothesis>=6.160.0"] -docs = [ - # Pins match the zarr-python docs environment in the repo-root - # pyproject.toml so every site renders with the same toolchain, and the - # shared packages/mkdocs-base.yml sees one set of plugin versions. - # Bump them together with the root. - "mkdocs-material==9.7.7", - "mkdocs==1.6.1", - "mkdocstrings==1.0.8", - # pymdown-extensions 12.3 changed the Highlight extension API; mkdocstrings 1.0.6 - # cannot drive it (Highlight.__init__() missing argument "md"). Match the - # version in the repo-root uv.lock until mkdocstrings is bumped everywhere. - "pymdown-extensions==11.0.1", - "mkdocstrings-python==2.0.8", - "griffe-inherited-docstrings==1.1.3", - # mkdocstrings uses ruff to format rendered signatures - "ruff==0.16.8", -] - -[tool.hatch.version] -source = "vcs" -tag-pattern = '^zarr_indexing-v(?P.+)$' -# `git_describe_command` ensures we get the zarr_indexing tags instead of latest. -# `local_scheme` strips the git commit info so the appending info is just a counter from latest tag. -# test-pypi doesn't accept git commit info in tags, and the count should be enough to distinguish unique runs. -raw-options = { root = "../..", scm = { git = { describe_command = "git describe --dirty --tags --long --match zarr_indexing-v*" } }, local_scheme = "no-local-version" } - -[tool.hatch.build.targets.wheel] -packages = ["src/zarr_indexing"] - -# An allowlist, so nothing that merely happens to sit in the package directory -# — a scratch script, a stray notebook — can ride along in a release. The list -# includes test assets in the sdist: `tests/` carries the vendored ndsel conformance -# corpus, and `tests/test_doc_examples.py` executes `docs/snippets/*.py` and -# `examples/*/*.py`, so those are part of the suite rather than decoration. -# `pyproject.toml`, `README.md` and `LICENSE.txt` are added by hatchling itself. -[tool.hatch.build.targets.sdist] -include = [ - "/src", - "/tests", - "/docs", - "/examples", - "/mkdocs.yml", - "/justfile", - "/CHANGELOG.md", - "/CONTRIBUTING.md", -] -# The shared docs config and header script are symlinks into packages/ in the -# repository; ship the files they point at, since a link out of the sdist -# dangles and pip refuses to unpack it. -exclude = ["/mkdocs-base.yml", "/docs/_static/source-version.js"] - -[tool.hatch.build.targets.sdist.force-include] -"../mkdocs-base.yml" = "mkdocs-base.yml" -"../source-version.js" = "docs/_static/source-version.js" - -[tool.ruff] -extend = "../../pyproject.toml" -target-version = "py312" - -[tool.ruff.lint.per-file-ignores] -# Chunk discovery deliberately catches Exception: a foreign source's -# attributes may fail, so this path provides fallback metadata. Configured here (not as -# noqa comments) because different ruff versions -# have differed on whether these rules fire; RUF100 can remove unused noqa comments. -"src/zarr_indexing/lazy_array.py" = ["BLE001", "S110"] - -[tool.pytest.ini_options] -minversion = "7" -# src is collected for its doctests: every public object's Examples section -# is eligible for --doctest-modules. Passing examples do not verify all prose -# contracts or optional-dependency paths. -testpaths = ["tests", "src/zarr_indexing"] -pythonpath = ["."] -xfail_strict = true -addopts = ["-ra", "--strict-config", "--strict-markers", "--doctest-modules"] -doctest_optionflags = [ - "NORMALIZE_WHITESPACE", - "ELLIPSIS", - "IGNORE_EXCEPTION_DETAIL", -] -filterwarnings = [ - "error", -] - -[tool.pyright] -include = [ - "src", - "docs/snippets", - "tests/test_doc_examples.py", -] -enableExperimentalFeatures = true -typeCheckingMode = "strict" -pythonVersion = "3.12" -# Partially unknown types from NumPy stubs are reported as warnings. CI's -# pyright invocation fails on errors, while retaining these warnings in its output. -reportUnknownVariableType = "warning" -reportUnknownArgumentType = "warning" -reportUnknownMemberType = "warning" -reportUnknownParameterType = "warning" - -[tool.numpydoc_validation] -checks = [ - "GL10", - "SS04", - "PR02", - "PR03", - "PR05", - "PR06", -] - -[tool.towncrier] -# Fragments for this package live alongside the package source, separate -# from the parent zarr-python `changes/` directory, so a PR touching only -# `packages/zarr-indexing/` produces a release note for this package only. -directory = "changes" -filename = "CHANGELOG.md" -package = "zarr_indexing" -underlines = ["", "", ""] -title_format = "## {version} ({project_date})" -issue_format = "[#{issue}](https://github.com/zarr-developers/zarr-python/pull/{issue})" -start_string = "\n" - -# Declaring any type replaces towncrier's built-in set, so all five the -# `changes/README.md` menu offers are restated here. They are the defaults -# verbatim except for `misc`, whose `showcontent` towncrier defaults to false: -# a `misc` entry would render as a bare PR link, which tells a reader nothing. -# A change worth a release note is worth a sentence, whatever its category. -[[tool.towncrier.type]] -directory = "feature" -name = "Features" -showcontent = true - -[[tool.towncrier.type]] -directory = "bugfix" -name = "Bugfixes" -showcontent = true - -[[tool.towncrier.type]] -directory = "doc" -name = "Improved Documentation" -showcontent = true - -[[tool.towncrier.type]] -directory = "removal" -name = "Deprecations and Removals" -showcontent = true - -[[tool.towncrier.type]] -directory = "misc" -name = "Misc" -showcontent = true +[build-system] +requires = ["hatchling>=1.29.0", "hatch-vcs", "setuptools-scm>=9"] +build-backend = "hatchling.build" + +[project] +name = "zarr-indexing" +dynamic = ["version"] +description = "Composable, lazy coordinate transforms for Zarr array indexing." +readme = "README.md" +requires-python = ">=3.12" +license = "MIT" +license-files = ["LICENSE.txt"] +authors = [ + { name = "Davis Bennett", email = "davis.v.bennett@gmail.com" }, +] +classifiers = [ + "Development Status :: 4 - Beta", + "Intended Audience :: Developers", + "Intended Audience :: Information Technology", + "Intended Audience :: Science/Research", + "License :: OSI Approved :: MIT License", + "Operating System :: OS Independent", + "Programming Language :: Python", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", + "Programming Language :: Python :: 3.14", + "Topic :: Scientific/Engineering", + "Topic :: Software Development :: Libraries :: Python Modules", + "Typing :: Typed", +] +keywords = ["zarr"] +dependencies = [ + "numpy>=2", +] + +[project.optional-dependencies] +# `zarr_indexing.testing` — a Hypothesis state machine and selection strategies +# for projects checking their own array against this package. Nothing else in +# the package imports hypothesis. +testing = ["hypothesis>=6.160.0"] + +[project.urls] +Homepage = "https://github.com/zarr-developers/zarr-python" +Source = "https://github.com/zarr-developers/zarr-python/tree/main/packages/zarr-indexing" +Issues = "https://github.com/zarr-developers/zarr-python/issues" +Changelog = "https://github.com/zarr-developers/zarr-python/blob/main/packages/zarr-indexing/CHANGELOG.md" +Documentation = "https://zarr-indexing.readthedocs.io/" + +[dependency-groups] +# Package tests use the local grid protocol and grid implementations. The +# repository recipes layer this package into the root test environment. +# This test group and the optional testing extra both declare Hypothesis; +# keep their supported minimum versions aligned. +test = ["pytest", "hypothesis>=6.160.0"] +docs = [ + # Pins match the zarr-python docs environment in the repo-root + # pyproject.toml so every site renders with the same toolchain, and the + # shared packages/mkdocs-base.yml sees one set of plugin versions. + # Bump them together with the root. + "mkdocs-material==9.7.7", + "mkdocs==1.6.1", + "mkdocstrings==1.0.8", + # pymdown-extensions 12.3 changed the Highlight extension API; mkdocstrings 1.0.6 + # cannot drive it (Highlight.__init__() missing argument "md"). Match the + # version in the repo-root uv.lock until mkdocstrings is bumped everywhere. + "pymdown-extensions==11.0.1", + "mkdocstrings-python==2.0.8", + "griffe-inherited-docstrings==1.1.3", + # mkdocstrings uses ruff to format rendered signatures + "ruff==0.16.8", +] + +[tool.hatch.version] +source = "vcs" +tag-pattern = '^zarr_indexing-v(?P.+)$' +# `git_describe_command` ensures we get the zarr_indexing tags instead of latest. +# `local_scheme` strips the git commit info so the appending info is just a counter from latest tag. +# test-pypi doesn't accept git commit info in tags, and the count should be enough to distinguish unique runs. +raw-options = { root = "../..", scm = { git = { describe_command = "git describe --dirty --tags --long --match zarr_indexing-v*" } }, local_scheme = "no-local-version" } + +[tool.hatch.build.targets.wheel] +packages = ["src/zarr_indexing"] + +# An allowlist, so nothing that merely happens to sit in the package directory +# — a scratch script, a stray notebook — can ride along in a release. The list +# includes test assets in the sdist: `tests/` carries the vendored ndsel conformance +# corpus, and `tests/test_doc_examples.py` executes `docs/snippets/*.py` and +# `examples/*/*.py`, so those are part of the suite rather than decoration. +# `pyproject.toml`, `README.md` and `LICENSE.txt` are added by hatchling itself. +[tool.hatch.build.targets.sdist] +include = [ + "/src", + "/tests", + "/docs", + "/examples", + "/mkdocs.yml", + "/justfile", + "/CHANGELOG.md", + "/CONTRIBUTING.md", +] +# The shared docs config and header script are symlinks into packages/ in the +# repository; ship the files they point at, since a link out of the sdist +# dangles and pip refuses to unpack it. +exclude = ["/mkdocs-base.yml", "/docs/_static/source-version.js"] + +[tool.hatch.build.targets.sdist.force-include] +"../mkdocs-base.yml" = "mkdocs-base.yml" +"../source-version.js" = "docs/_static/source-version.js" + +[tool.ruff] +extend = "../../pyproject.toml" +target-version = "py312" + +[tool.ruff.lint.per-file-ignores] +# Chunk discovery deliberately catches Exception: a foreign source's +# attributes may fail, so this path provides fallback metadata. Configured here (not as +# noqa comments) because different ruff versions +# have differed on whether these rules fire; RUF100 can remove unused noqa comments. +"src/zarr_indexing/lazy_array.py" = ["BLE001", "S110"] + +[tool.pytest.ini_options] +minversion = "7" +# src is collected for its doctests: every public object's Examples section +# is eligible for --doctest-modules. Passing examples do not verify all prose +# contracts or optional-dependency paths. +testpaths = ["tests", "src/zarr_indexing"] +pythonpath = ["."] +xfail_strict = true +addopts = ["-ra", "--strict-config", "--strict-markers", "--doctest-modules"] +doctest_optionflags = [ + "NORMALIZE_WHITESPACE", + "ELLIPSIS", + "IGNORE_EXCEPTION_DETAIL", +] +filterwarnings = [ + "error", +] + +[tool.pyright] +include = [ + "src", + "docs/snippets", + "tests/test_doc_examples.py", +] +enableExperimentalFeatures = true +typeCheckingMode = "strict" +pythonVersion = "3.12" +# Partially unknown types from NumPy stubs are reported as warnings. CI's +# pyright invocation fails on errors, while retaining these warnings in its output. +reportUnknownVariableType = "warning" +reportUnknownArgumentType = "warning" +reportUnknownMemberType = "warning" +reportUnknownParameterType = "warning" + +[tool.numpydoc_validation] +checks = [ + "GL10", + "SS04", + "PR02", + "PR03", + "PR05", + "PR06", +] + +[tool.towncrier] +# Fragments for this package live alongside the package source, separate +# from the parent zarr-python `changes/` directory, so a PR touching only +# `packages/zarr-indexing/` produces a release note for this package only. +directory = "changes" +filename = "CHANGELOG.md" +package = "zarr_indexing" +underlines = ["", "", ""] +title_format = "## {version} ({project_date})" +issue_format = "[#{issue}](https://github.com/zarr-developers/zarr-python/pull/{issue})" +start_string = "\n" + +# Declaring any type replaces towncrier's built-in set, so all five the +# `changes/README.md` menu offers are restated here. They are the defaults +# verbatim except for `misc`, whose `showcontent` towncrier defaults to false: +# a `misc` entry would render as a bare PR link, which tells a reader nothing. +# A change worth a release note is worth a sentence, whatever its category. +[[tool.towncrier.type]] +directory = "feature" +name = "Features" +showcontent = true + +[[tool.towncrier.type]] +directory = "bugfix" +name = "Bugfixes" +showcontent = true + +[[tool.towncrier.type]] +directory = "doc" +name = "Improved Documentation" +showcontent = true + +[[tool.towncrier.type]] +directory = "removal" +name = "Deprecations and Removals" +showcontent = true + +[[tool.towncrier.type]] +directory = "misc" +name = "Misc" +showcontent = true diff --git a/pyproject.toml b/pyproject.toml index 4b609ac606..e8f3e09a99 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,504 +1,504 @@ -[build-system] -requires = ["hatchling>=1.29.0", "hatch-vcs", "setuptools-scm>=9"] -build-backend = "hatchling.build" - -# An allowlist, not a blocklist: anything new — a subpackage under `packages/`, a -# config file in the repository root — stays out of the sdist unless it is named -# here. Beyond `src` and `tests`, the entries are what keeps the shipped test -# suite and docs build runnable from an unpacked sdist: `tests/test_docs.py` -# walks `docs/` and `testpaths` collects `docs/user-guide`; the pages under -# `docs/user-guide/examples/` pull their source out of `examples/` via pymdownx -# snippet includes; `mkdocs.yml` and `mkdocs_hooks.py` let `mkdocs build` run -# too. `pyproject.toml`, `README.md`, `LICENSE.txt` and `.gitignore` are added by -# hatchling itself. -[tool.hatch.build.targets.sdist] -include = [ - "/src", - "/tests", - "/docs", - "/examples", - "/mkdocs.yml", - "/mkdocs_hooks.py", - "/Justfile", - "/ci", -] - -[project] -name = "zarr" -description = "An implementation of chunked, compressed, N-dimensional arrays for Python" -readme = { file = "README.md", content-type = "text/markdown" } -authors = [ - { name = "Alistair Miles", email = "alimanfoo@googlemail.com" }, -] -maintainers = [ - { name = "Davis Bennett", email = "davis.v.bennett@gmail.com" }, - { name = "jakirkham" }, - { name = "Josh Moore", email = "josh@openmicroscopy.org" }, - { name = "Joe Hamman", email = "joe@earthmover.io" }, - { name = "Juan Nunez-Iglesias", email = "juan.nunez-iglesias@monash.edu" }, - { name = "Martin Durant", email = "mdurant@anaconda.com" }, - { name = "Norman Rzepka" }, - { name = "Ryan Abernathey" }, - { name = "David Stansby" }, - { name = "Tom Augspurger", email = "tom.w.augspurger@gmail.com" }, - { name = "Deepak Cherian" } -] -requires-python = ">=3.12" -dependencies = [ - 'packaging>=22.0', - 'numpy>=2', - 'numcodecs>=0.16', - 'google-crc32c>=1.5', - 'typing_extensions>=4.14', - 'donfig>=0.8', - 'msgspec>=0.19', -] - -dynamic = [ - "version", -] -classifiers = [ - 'Development Status :: 6 - Mature', - 'Intended Audience :: Developers', - 'Intended Audience :: Information Technology', - 'Intended Audience :: Science/Research', - 'Programming Language :: Python', - 'Topic :: Software Development :: Libraries :: Python Modules', - 'Operating System :: Unix', - 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.12', - 'Programming Language :: Python :: 3.13', - 'Programming Language :: Python :: 3.14', -] -license = "MIT" -license-files = ["LICENSE.txt"] -keywords = ["Python", "compressed", "ndimensional-arrays", "zarr"] - -[project.optional-dependencies] -# User-facing extras (shipped in package metadata) -remote = [ - "fsspec>=2023.10.0", - "obstore>=0.5.1", -] -gpu = [ - "cupy-cuda12x; sys_platform != 'darwin'", -] -cast-value-rs = ["cast-value-rs>=0.4.2"] -cli = ["typer"] -optional = ["universal-pathlib"] - -[project.scripts] -zarr = "zarr._cli.cli:app" - - -[project.urls] -issues = "https://github.com/zarr-developers/zarr-python/issues" -changelog = "https://zarr.readthedocs.io/en/stable/release-notes.html" -Discussions = "https://github.com/zarr-developers/zarr-python/discussions" -documentation = "https://zarr.readthedocs.io/" -homepage = "https://github.com/zarr-developers/zarr-python" - -# Dev *tooling* is pinned to exact versions for reproducible CI: the hatch envs -# (see `tool.hatch.envs.*`) and bare `uv run` resolve these groups fresh from -# PyPI and do NOT consult uv.lock, so an unrelated tooling release can break CI -# without any change on our side (e.g. the pytest 9.1.0 `duplicate -# parametrization` regression). Runtime/integration deps (fsspec, obstore, s3fs, -# botocore, numcodecs, universal-pathlib) are intentionally left floating so the -# `optional` test matrix keeps exercising their latest releases; their floor and -# bleeding edge are covered by the `min_deps` and `upstream` hatch envs. Bump the -# pins deliberately, e.g. via dependabot or `uv lock --upgrade`. -[dependency-groups] -test = [ - "coverage==7.16.2", - "pytest==9.1.1", - "pytest-asyncio==1.4.0", - "pytest-cov==7.1.0", - "pytest-accept==0.3.0", - "numpydoc==1.11.0", - "hypothesis==6.168.3", - "pytest-reportlog==1.0.0", - "pytest-xdist==3.8.0", - "pytest-benchmark==5.3.0", - "pytest-codspeed==5.0.3", - "tomlkit==0.15.1", - "uv==0.13.4", -] -remote-tests = [ - {include-group = "test"}, - "fsspec>=2023.10.0", - "obstore>=0.5.1", - "botocore", - "s3fs>=2023.10.0", - "moto[s3,server]==5.2.3", - "requests==2.34.2", -] -release = [ - "towncrier==26.9.0", -] -docs = [ - # Doc building - "mkdocs-material[imaging]==9.7.7", - "mkdocs==1.6.1", - "mkdocstrings==1.0.8", - "mkdocstrings-python==2.0.9", - "mike==2.2.0", - "mkdocs-redirects==1.2.3", - "markdown-exec[ansi]==1.12.3", - "griffe-inherited-docstrings==1.1.3", - "ruff==0.16.9", - # Changelog generation - {include-group = "release"}, - # Optional dependencies to run examples - "numcodecs[msgpack]", - "s3fs>=2023.10.0", - "astroid==4.3.3", - "pytest==9.1.1", -] -dev = [ - {include-group = "test"}, - {include-group = "remote-tests"}, - {include-group = "docs"}, - "universal-pathlib", - "mypy==2.3.1", -] - -[tool.coverage.report] -exclude_also = [ - 'if TYPE_CHECKING:', -] - -[tool.coverage.run] -omit = [ - "bench/compress_normal.py", -] - -[tool.hatch] -version.source = "vcs" -# Only consider zarr-python's own `v*` tags when deriving the version. Without -# this filter `git describe` matches the most recent tag of any shape, -# including the `zarr_metadata-v*` and `zarr_http_server-v*` tags used to -# release the subpackages under `packages/` — which would make a from-source -# build report e.g. a `0.2.x` version instead of `3.x`. -version.raw-options = { scm = { git = { describe_command = "git describe --dirty --tags --long --match v*" } } } - -[tool.hatch.build] -hooks.vcs.version-file = "src/zarr/_version.py" - -[tool.hatch.envs.dev] -dependency-groups = ["dev"] - -[tool.hatch.envs.test] -dependency-groups = ["test"] - -[tool.hatch.envs.test.env-vars] - -[[tool.hatch.envs.test.matrix]] -python = ["3.12", "3.13", "3.14"] -deps = ["minimal", "optional"] - -[tool.hatch.envs.test.overrides] -matrix.deps.features = [ - {value = "remote", if = ["optional"]}, - {value = "optional", if = ["optional"]}, - {value = "cli", if = ["optional"]}, - {value = "cast-value-rs", if = ["optional"]}, -] -matrix.deps.dependency-groups = [ - {value = "remote-tests", if = ["optional"]}, -] - -[tool.hatch.envs.gputest] -template = "test" -extra-dependencies = [ - "universal_pathlib", - # Needed so tests/test_docs.py is collectable under `pytest -m gpu`; otherwise its - # module-level importorskip("pytest_examples") skips the whole module and the gpu - # docs example is never executed on GPU hardware. - "pytest-examples", -] -features = ["gpu"] - -[[tool.hatch.envs.gputest.matrix]] -python = ["3.12", "3.13"] - -[tool.hatch.envs.upstream] -template = 'test' -python = "3.14" -extra-dependencies = [ - 'packaging @ git+https://github.com/pypa/packaging', - 'numpy', # from scientific-python-nightly-wheels - 'numcodecs @ git+https://github.com/zarr-developers/numcodecs', - 's3fs @ git+https://github.com/fsspec/s3fs', - 'universal_pathlib @ git+https://github.com/fsspec/universal_pathlib', - 'typing_extensions @ git+https://github.com/python/typing_extensions', - 'donfig @ git+https://github.com/pytroll/donfig', - 'obstore @ git+https://github.com/developmentseed/obstore@main#subdirectory=obstore', -] - -[tool.hatch.envs.upstream.env-vars] -PIP_INDEX_URL = "https://pypi.anaconda.org/scientific-python-nightly-wheels/simple/" -PIP_EXTRA_INDEX_URL = "https://pypi.org/simple/" -PIP_PRE = "1" - -[tool.hatch.envs.min_deps] -description = """Test environment for minimum supported dependencies - -See Spec 0000 for details and drop schedule: https://scientific-python.org/specs/spec-0000/ -""" -template = "test" -python = "3.12" -features = ["remote"] -dependency-groups = ["remote-tests"] -extra-dependencies = [ - 'packaging==22.*', - 'numpy==2.0.*', - 'numcodecs==0.16.*', - 'fsspec==2023.10.0', - 's3fs==2023.10.0', - 'universal_pathlib==0.2.0', - 'typing_extensions==4.14.*', - 'donfig==0.8.*', - 'obstore==0.5.*', - 'msgspec==0.19.*', -] - -[tool.hatch.envs.default] -installer = "uv" - -[tool.hatch.envs.doctest] -description = "Test environment for validating executable code blocks in documentation" -features = ['remote'] -dependency-groups = ['remote-tests'] -extra-dependencies = [ - "pytest-examples", -] - -[tool.ruff] -line-length = 100 -force-exclude = true -extend-exclude = [ - ".bzr", - ".direnv", - ".eggs", - ".git", - ".mypy_cache", - ".nox", - ".pants.d", - ".ruff_cache", - ".venv", - "__pypackages__", - "_build", - "buck-out", - "build", - "dist", - "venv", - "docs", - "tests/test_regression/scripts/", # these are scripts that use a different version of python - "src/zarr/v2/", - "tests/v2/", -] - -[tool.ruff.lint] -extend-select = [ - "ANN", # flake8-annotations - "B", # flake8-bugbear - "C4", # flake8-comprehensions - "EXE", # flake8-executable - "FA", # flake8-future-annotations - "FLY", # flynt - "FURB", # refurb - "G", # flake8-logging-format - "I", # isort - "ISC", # flake8-implicit-str-concat - "LOG", # flake8-logging - "PERF", # Perflint - "PIE", # flake8-pie - "PGH", # pygrep-hooks - "PT", # flake8-pytest-style - "PYI", # flake8-pyi - "RET", # flake8-return - "RSE", # flake8-raise - "RUF", - "S101", # flake8-bandit: no assert in runtime code - "SIM", # flake8-simplify - "SLOT", # flake8-slots - "TC", # flake8-type-checking - "TRY", # tryceratops - "UP", # pyupgrade - "W", # pycodestyle warnings -] -ignore = [ - "ANN401", - "PT011", # TODO: apply this rule - "RET505", - "RET506", - "RUF005", - "RUF043", - "SIM108", - "TRY003", - # https://docs.astral.sh/ruff/formatter/#conflicting-lint-rules - "W191", - "E111", - "E114", - "E117", - "D206", - "D300", - "Q000", - "Q001", - "Q002", - "Q003", - "COM812", - "COM819", - "TC006", -] - -[tool.ruff.lint.extend-per-file-ignores] -"tests/**" = ["ANN001", "ANN201", "RUF029", "SIM117", "SIM300"] -# S101 (no assert) is for runtime code only. Tests, examples, and test-support -# packages (store conformance suite, hypothesis strategies) assert on purpose. -# The globs cover the root package and every package under packages/. -"**/tests/**" = ["S101"] -"**/examples/**" = ["S101"] -"**/testing/**" = ["S101"] - -[tool.mypy] -files = ["src", "tests"] -python_version = "3.12" -ignore_missing_imports = true -namespace_packages = false -pretty = true -show_error_code_links = true -show_error_context = true -strict = true -warn_unreachable = true -enable_error_code = ["ignore-without-code", "redundant-expr", "truthy-bool", "truthy-iterable"] - -[[tool.mypy.overrides]] -module = [ - "tests.package_with_entrypoint.*", - "zarr.testing.stateful", - "tests.test_codecs.test_transpose", - "tests.test_config", - "tests.test_store.test_zip", - "tests.test_store.test_local", - "tests.test_store.test_memory", - "tests.test_codecs.test_codecs", - "tests.test_metadata.*", - "tests.test_store.test_core", - "tests.test_store.test_logging", - "tests.test_store.test_object", - "tests.test_store.test_stateful", - "tests.test_store.test_wrapper", -] -strict = false - -# TODO: Move the next modules up to the strict = false section -# and fix the errors -[[tool.mypy.overrides]] -module = [ - "tests.test_store.test_fsspec", - "tests.test_group", - "tests.test_indexing", - "tests.test_properties", - "tests.test_sync", - "tests.test_regression.scripts.*" -] -ignore_errors = true - -[tool.pytest.ini_options] -minversion = "9" -testpaths = ["src", "tests", "docs/user-guide"] -log_cli_level = "INFO" -log_level = "INFO" -# Enables strict_config, strict_markers, strict_xfail, strict_parametrization_ids, and -# any strictness options added in future pytest releases. Note that the equivalent -# `--strict-config`/`--strict-markers` flags were silently ignored when passed via -# addopts before pytest 9.1 (pytest#14442), so this option is the reliable spelling. -strict = true -# Turn deadlocks into loud failures: if a single test exceeds this many seconds, dump -# every thread's traceback and kill the run (exit_on_timeout is new in pytest 9). Sized -# far above the slowest legitimate test (~20s locally; slower under coverage/Windows/ -# nightly stateful-hypothesis runs) so only a genuine hang can trip it. -faulthandler_timeout = 600 -faulthandler_exit_on_timeout = true -asyncio_mode = "auto" -asyncio_default_fixture_loop_scope = "function" -doctest_optionflags = [ - "NORMALIZE_WHITESPACE", - "ELLIPSIS", - "IGNORE_EXCEPTION_DETAIL", -] -addopts = [ - "--benchmark-columns", "min,mean,stddev,outliers,rounds,iterations", - "--benchmark-disable", # benchmark routines run as tests without benchmarking instrumentation - "--durations", "10", - "-ra", - "--doctest-modules", - "--ignore=tests/test_regression/scripts", - "--ignore=src/zarr/_cli", -] -filterwarnings = [ - "error", - "ignore:Unclosed client session =1.29.0", "hatch-vcs", "setuptools-scm>=9"] +build-backend = "hatchling.build" + +# An allowlist, not a blocklist: anything new — a subpackage under `packages/`, a +# config file in the repository root — stays out of the sdist unless it is named +# here. Beyond `src` and `tests`, the entries are what keeps the shipped test +# suite and docs build runnable from an unpacked sdist: `tests/test_docs.py` +# walks `docs/` and `testpaths` collects `docs/user-guide`; the pages under +# `docs/user-guide/examples/` pull their source out of `examples/` via pymdownx +# snippet includes; `mkdocs.yml` and `mkdocs_hooks.py` let `mkdocs build` run +# too. `pyproject.toml`, `README.md`, `LICENSE.txt` and `.gitignore` are added by +# hatchling itself. +[tool.hatch.build.targets.sdist] +include = [ + "/src", + "/tests", + "/docs", + "/examples", + "/mkdocs.yml", + "/mkdocs_hooks.py", + "/Justfile", + "/ci", +] + +[project] +name = "zarr" +description = "An implementation of chunked, compressed, N-dimensional arrays for Python" +readme = { file = "README.md", content-type = "text/markdown" } +authors = [ + { name = "Alistair Miles", email = "alimanfoo@googlemail.com" }, +] +maintainers = [ + { name = "Davis Bennett", email = "davis.v.bennett@gmail.com" }, + { name = "jakirkham" }, + { name = "Josh Moore", email = "josh@openmicroscopy.org" }, + { name = "Joe Hamman", email = "joe@earthmover.io" }, + { name = "Juan Nunez-Iglesias", email = "juan.nunez-iglesias@monash.edu" }, + { name = "Martin Durant", email = "mdurant@anaconda.com" }, + { name = "Norman Rzepka" }, + { name = "Ryan Abernathey" }, + { name = "David Stansby" }, + { name = "Tom Augspurger", email = "tom.w.augspurger@gmail.com" }, + { name = "Deepak Cherian" } +] +requires-python = ">=3.12" +dependencies = [ + 'packaging>=22.0', + 'numpy>=2', + 'numcodecs>=0.16', + 'google-crc32c>=1.5', + 'typing_extensions>=4.14', + 'donfig>=0.8', + 'msgspec>=0.19', +] + +dynamic = [ + "version", +] +classifiers = [ + 'Development Status :: 6 - Mature', + 'Intended Audience :: Developers', + 'Intended Audience :: Information Technology', + 'Intended Audience :: Science/Research', + 'Programming Language :: Python', + 'Topic :: Software Development :: Libraries :: Python Modules', + 'Operating System :: Unix', + 'Programming Language :: Python :: 3', + 'Programming Language :: Python :: 3.12', + 'Programming Language :: Python :: 3.13', + 'Programming Language :: Python :: 3.14', +] +license = "MIT" +license-files = ["LICENSE.txt"] +keywords = ["Python", "compressed", "ndimensional-arrays", "zarr"] + +[project.optional-dependencies] +# User-facing extras (shipped in package metadata) +remote = [ + "fsspec>=2023.10.0", + "obstore>=0.5.1", +] +gpu = [ + "cupy-cuda12x; sys_platform != 'darwin'", +] +cast-value-rs = ["cast-value-rs>=0.4.2"] +cli = ["typer"] +optional = ["universal-pathlib"] + +[project.scripts] +zarr = "zarr._cli.cli:app" + + +[project.urls] +issues = "https://github.com/zarr-developers/zarr-python/issues" +changelog = "https://zarr.readthedocs.io/en/stable/release-notes.html" +Discussions = "https://github.com/zarr-developers/zarr-python/discussions" +documentation = "https://zarr.readthedocs.io/" +homepage = "https://github.com/zarr-developers/zarr-python" + +# Dev *tooling* is pinned to exact versions for reproducible CI: the hatch envs +# (see `tool.hatch.envs.*`) and bare `uv run` resolve these groups fresh from +# PyPI and do NOT consult uv.lock, so an unrelated tooling release can break CI +# without any change on our side (e.g. the pytest 9.1.0 `duplicate +# parametrization` regression). Runtime/integration deps (fsspec, obstore, s3fs, +# botocore, numcodecs, universal-pathlib) are intentionally left floating so the +# `optional` test matrix keeps exercising their latest releases; their floor and +# bleeding edge are covered by the `min_deps` and `upstream` hatch envs. Bump the +# pins deliberately, e.g. via dependabot or `uv lock --upgrade`. +[dependency-groups] +test = [ + "coverage==7.16.2", + "pytest==9.1.1", + "pytest-asyncio==1.4.0", + "pytest-cov==7.1.0", + "pytest-accept==0.3.0", + "numpydoc==1.11.0", + "hypothesis==6.168.3", + "pytest-reportlog==1.0.0", + "pytest-xdist==3.8.0", + "pytest-benchmark==5.3.0", + "pytest-codspeed==5.0.3", + "tomlkit==0.15.1", + "uv==0.13.4", +] +remote-tests = [ + {include-group = "test"}, + "fsspec>=2023.10.0", + "obstore>=0.5.1", + "botocore", + "s3fs>=2023.10.0", + "moto[s3,server]==5.2.3", + "requests==2.34.2", +] +release = [ + "towncrier==26.9.0", +] +docs = [ + # Doc building + "mkdocs-material[imaging]==9.7.7", + "mkdocs==1.6.1", + "mkdocstrings==1.0.8", + "mkdocstrings-python==2.0.9", + "mike==2.2.0", + "mkdocs-redirects==1.2.3", + "markdown-exec[ansi]==1.12.3", + "griffe-inherited-docstrings==1.1.3", + "ruff==0.16.9", + # Changelog generation + {include-group = "release"}, + # Optional dependencies to run examples + "numcodecs[msgpack]", + "s3fs>=2023.10.0", + "astroid==4.3.3", + "pytest==9.1.1", +] +dev = [ + {include-group = "test"}, + {include-group = "remote-tests"}, + {include-group = "docs"}, + "universal-pathlib", + "mypy==2.3.1", +] + +[tool.coverage.report] +exclude_also = [ + 'if TYPE_CHECKING:', +] + +[tool.coverage.run] +omit = [ + "bench/compress_normal.py", +] + +[tool.hatch] +version.source = "vcs" +# Only consider zarr-python's own `v*` tags when deriving the version. Without +# this filter `git describe` matches the most recent tag of any shape, +# including the `zarr_metadata-v*` and `zarr_http_server-v*` tags used to +# release the subpackages under `packages/` — which would make a from-source +# build report e.g. a `0.2.x` version instead of `3.x`. +version.raw-options = { scm = { git = { describe_command = "git describe --dirty --tags --long --match v*" } } } + +[tool.hatch.build] +hooks.vcs.version-file = "src/zarr/_version.py" + +[tool.hatch.envs.dev] +dependency-groups = ["dev"] + +[tool.hatch.envs.test] +dependency-groups = ["test"] + +[tool.hatch.envs.test.env-vars] + +[[tool.hatch.envs.test.matrix]] +python = ["3.12", "3.13", "3.14"] +deps = ["minimal", "optional"] + +[tool.hatch.envs.test.overrides] +matrix.deps.features = [ + {value = "remote", if = ["optional"]}, + {value = "optional", if = ["optional"]}, + {value = "cli", if = ["optional"]}, + {value = "cast-value-rs", if = ["optional"]}, +] +matrix.deps.dependency-groups = [ + {value = "remote-tests", if = ["optional"]}, +] + +[tool.hatch.envs.gputest] +template = "test" +extra-dependencies = [ + "universal_pathlib", + # Needed so tests/test_docs.py is collectable under `pytest -m gpu`; otherwise its + # module-level importorskip("pytest_examples") skips the whole module and the gpu + # docs example is never executed on GPU hardware. + "pytest-examples", +] +features = ["gpu"] + +[[tool.hatch.envs.gputest.matrix]] +python = ["3.12", "3.13"] + +[tool.hatch.envs.upstream] +template = 'test' +python = "3.14" +extra-dependencies = [ + 'packaging @ git+https://github.com/pypa/packaging', + 'numpy', # from scientific-python-nightly-wheels + 'numcodecs @ git+https://github.com/zarr-developers/numcodecs', + 's3fs @ git+https://github.com/fsspec/s3fs', + 'universal_pathlib @ git+https://github.com/fsspec/universal_pathlib', + 'typing_extensions @ git+https://github.com/python/typing_extensions', + 'donfig @ git+https://github.com/pytroll/donfig', + 'obstore @ git+https://github.com/developmentseed/obstore@main#subdirectory=obstore', +] + +[tool.hatch.envs.upstream.env-vars] +PIP_INDEX_URL = "https://pypi.anaconda.org/scientific-python-nightly-wheels/simple/" +PIP_EXTRA_INDEX_URL = "https://pypi.org/simple/" +PIP_PRE = "1" + +[tool.hatch.envs.min_deps] +description = """Test environment for minimum supported dependencies + +See Spec 0000 for details and drop schedule: https://scientific-python.org/specs/spec-0000/ +""" +template = "test" +python = "3.12" +features = ["remote"] +dependency-groups = ["remote-tests"] +extra-dependencies = [ + 'packaging==22.*', + 'numpy==2.0.*', + 'numcodecs==0.16.*', + 'fsspec==2023.10.0', + 's3fs==2023.10.0', + 'universal_pathlib==0.2.0', + 'typing_extensions==4.14.*', + 'donfig==0.8.*', + 'obstore==0.5.*', + 'msgspec==0.19.*', +] + +[tool.hatch.envs.default] +installer = "uv" + +[tool.hatch.envs.doctest] +description = "Test environment for validating executable code blocks in documentation" +features = ['remote'] +dependency-groups = ['remote-tests'] +extra-dependencies = [ + "pytest-examples", +] + +[tool.ruff] +line-length = 100 +force-exclude = true +extend-exclude = [ + ".bzr", + ".direnv", + ".eggs", + ".git", + ".mypy_cache", + ".nox", + ".pants.d", + ".ruff_cache", + ".venv", + "__pypackages__", + "_build", + "buck-out", + "build", + "dist", + "venv", + "docs", + "tests/test_regression/scripts/", # these are scripts that use a different version of python + "src/zarr/v2/", + "tests/v2/", +] + +[tool.ruff.lint] +extend-select = [ + "ANN", # flake8-annotations + "B", # flake8-bugbear + "C4", # flake8-comprehensions + "EXE", # flake8-executable + "FA", # flake8-future-annotations + "FLY", # flynt + "FURB", # refurb + "G", # flake8-logging-format + "I", # isort + "ISC", # flake8-implicit-str-concat + "LOG", # flake8-logging + "PERF", # Perflint + "PIE", # flake8-pie + "PGH", # pygrep-hooks + "PT", # flake8-pytest-style + "PYI", # flake8-pyi + "RET", # flake8-return + "RSE", # flake8-raise + "RUF", + "S101", # flake8-bandit: no assert in runtime code + "SIM", # flake8-simplify + "SLOT", # flake8-slots + "TC", # flake8-type-checking + "TRY", # tryceratops + "UP", # pyupgrade + "W", # pycodestyle warnings +] +ignore = [ + "ANN401", + "PT011", # TODO: apply this rule + "RET505", + "RET506", + "RUF005", + "RUF043", + "SIM108", + "TRY003", + # https://docs.astral.sh/ruff/formatter/#conflicting-lint-rules + "W191", + "E111", + "E114", + "E117", + "D206", + "D300", + "Q000", + "Q001", + "Q002", + "Q003", + "COM812", + "COM819", + "TC006", +] + +[tool.ruff.lint.extend-per-file-ignores] +"tests/**" = ["ANN001", "ANN201", "RUF029", "SIM117", "SIM300"] +# S101 (no assert) is for runtime code only. Tests, examples, and test-support +# packages (store conformance suite, hypothesis strategies) assert on purpose. +# The globs cover the root package and every package under packages/. +"**/tests/**" = ["S101"] +"**/examples/**" = ["S101"] +"**/testing/**" = ["S101"] + +[tool.mypy] +files = ["src", "tests"] +python_version = "3.12" +ignore_missing_imports = true +namespace_packages = false +pretty = true +show_error_code_links = true +show_error_context = true +strict = true +warn_unreachable = true +enable_error_code = ["ignore-without-code", "redundant-expr", "truthy-bool", "truthy-iterable"] + +[[tool.mypy.overrides]] +module = [ + "tests.package_with_entrypoint.*", + "zarr.testing.stateful", + "tests.test_codecs.test_transpose", + "tests.test_config", + "tests.test_store.test_zip", + "tests.test_store.test_local", + "tests.test_store.test_memory", + "tests.test_codecs.test_codecs", + "tests.test_metadata.*", + "tests.test_store.test_core", + "tests.test_store.test_logging", + "tests.test_store.test_object", + "tests.test_store.test_stateful", + "tests.test_store.test_wrapper", +] +strict = false + +# TODO: Move the next modules up to the strict = false section +# and fix the errors +[[tool.mypy.overrides]] +module = [ + "tests.test_store.test_fsspec", + "tests.test_group", + "tests.test_indexing", + "tests.test_properties", + "tests.test_sync", + "tests.test_regression.scripts.*" +] +ignore_errors = true + +[tool.pytest.ini_options] +minversion = "9" +testpaths = ["src", "tests", "docs/user-guide"] +log_cli_level = "INFO" +log_level = "INFO" +# Enables strict_config, strict_markers, strict_xfail, strict_parametrization_ids, and +# any strictness options added in future pytest releases. Note that the equivalent +# `--strict-config`/`--strict-markers` flags were silently ignored when passed via +# addopts before pytest 9.1 (pytest#14442), so this option is the reliable spelling. +strict = true +# Turn deadlocks into loud failures: if a single test exceeds this many seconds, dump +# every thread's traceback and kill the run (exit_on_timeout is new in pytest 9). Sized +# far above the slowest legitimate test (~20s locally; slower under coverage/Windows/ +# nightly stateful-hypothesis runs) so only a genuine hang can trip it. +faulthandler_timeout = 600 +faulthandler_exit_on_timeout = true +asyncio_mode = "auto" +asyncio_default_fixture_loop_scope = "function" +doctest_optionflags = [ + "NORMALIZE_WHITESPACE", + "ELLIPSIS", + "IGNORE_EXCEPTION_DETAIL", +] +addopts = [ + "--benchmark-columns", "min,mean,stddev,outliers,rounds,iterations", + "--benchmark-disable", # benchmark routines run as tests without benchmarking instrumentation + "--durations", "10", + "-ra", + "--doctest-modules", + "--ignore=tests/test_regression/scripts", + "--ignore=src/zarr/_cli", +] +filterwarnings = [ + "error", + "ignore:Unclosed client session