diff --git a/changes/4192.feature.md b/changes/4192.feature.md new file mode 100644 index 0000000000..3ea14b30d6 --- /dev/null +++ b/changes/4192.feature.md @@ -0,0 +1,46 @@ +Added core support for URL pipelines (https://github.com/jbms/url-pipeline): +`|`-chained URLs that address zarr data through nested storage layers, e.g. +`s3://bucket/data.zip|zip:|zarr3:`. This includes the parser +(`zarr.storage.parse_pipeline` / `zarr.storage.resolve_pipeline`), the +single-method `zarr.abc.url_pipeline.URLPipelineAdapter` interface, and the +`zarr.url_adapters` entry-point group through which third-party packages +(e.g. Icechunk) register adapters for their own schemes. Adapters for a scheme +are loaded lazily and individually. Builtin adapters (`zip:`, +`zarr2:`/`zarr3:`) will follow separately. + +Behavior notes: + +- The `|` character is now reserved as the pipeline delimiter in every string + store specification, and no percent-escape is decoded; pass a `pathlib.Path` + to address a local file whose name contains `|`. URLs without a `|` (and + without a registered root adapter scheme) are handled exactly as before — + registered adapters cannot intercept zarr's native `file:`/`memory:` + routing, and fsspec chained URLs (`zip::s3://...`) keep flowing to fsspec. +- Inside a pipeline, `memory:` and `file:` roots follow the URL pipeline + spec's spelling rules (`memory:` ≡ `memory:/` ≡ `memory://`; `file:` must + carry an absolute path with at most a `localhost` authority). Percent-escapes + are *not* decoded in `file:` roots, matching `LocalStore`; this is a + documented divergence from RFC 8089. When fsspec is installed, a plain + `memory://name` URL is still routed to fsspec's in-memory filesystem, whereas + a pipeline's `memory:` root is zarr's `ManagedMemoryStore`; the two do not + share data. +- Wrapper adapters can open a local *file* root only in mode `"r"` (via + `PipelineContext.resolve_preceding(mode="r")`); other modes create a + directory at the root and fail. Writing through a container file is not yet + supported. +- A `zarr.url_adapters` entry point whose scheme is already registered (e.g. by + a builtin adapter) is ignored with a `ZarrUserWarning`, as are duplicate + entry-point names; an entry point that fails to import raises + `URLPipelineError` naming it, and stays discoverable for a retry. +- `zarr.create_array` and `Group.from_store` now default `zarr_format` to + `None`, resolving to the configured default (3) as before unless a URL + pipeline selects a format; `Array.open` gains a `zarr_format` parameter + mirroring `Group.open`. `Group.open` and `Array.open` keep their default of + 3, so pass `zarr_format=None` to defer to a pipeline's format. A format + selected by a pipeline that conflicts with an explicit `zarr_format` raises + `ValueError` in every `open*`/`create*`/`save*` entry point. +- Mode `"a"` (open-or-create, the `zarr.open` default) on a *read-only* store + now serves the "open" half instead of raising upfront, for all stores; + unambiguous write modes (`"w"`, `"w-"`, `"r+"`) still raise. +- For root-adapter URLs (e.g. `gh://org/repo`), `storage_options` are handed + to the adapter and are not validated as used by `make_store`. diff --git a/changes/4476.feature.md b/changes/4476.feature.md new file mode 100644 index 0000000000..413ef01a7d --- /dev/null +++ b/changes/4476.feature.md @@ -0,0 +1,11 @@ +Added the builtin `zarr:`, `zarr2:` and `zarr3:` URL pipeline adapters, so a pipeline +can address a Zarr node within a resource and select its format, e.g. +`zarr.open_group("file:///data/archive|zarr2:path/to/group")`. The text after the colon +is a path within the preceding resource; `zarr2:` / `zarr3:` select the Zarr format and +`zarr:` auto-detects it. These schemes are adapter-only, so strings without `|` behave as +before. + +`Group.open`, `Array.open`, `AsyncGroup.open` and `AsyncArray.open` no longer default to +`zarr_format=3` when given a URL pipeline: if `zarr_format` is not passed, the pipeline's +format segment decides (auto-detection when there is none). For every other store the +default is still 3. diff --git a/changes/4477.feature.md b/changes/4477.feature.md new file mode 100644 index 0000000000..01d102d848 --- /dev/null +++ b/changes/4477.feature.md @@ -0,0 +1,6 @@ +Added a builtin, read-only `zip:` URL pipeline adapter for addressing data inside a ZIP +archive, e.g. `zarr.open_group("s3://bucket/data.zip|zip:inner-dir|zarr3:subgroup")`. The +archive can be a local file, an fsspec object, or an entry of another archive +(`outer.zip|zip:inner.zip|zip:`), and is read with async byte-range requests. Writable +modes (`"w"`, `"w-"`, `"r+"`) raise `URLPipelineError` for now; mode `"a"` opens the +archive read-only. diff --git a/docs/api/zarr/abc/index.md b/docs/api/zarr/abc/index.md index 7e15cb2a51..d155f60a5a 100644 --- a/docs/api/zarr/abc/index.md +++ b/docs/api/zarr/abc/index.md @@ -11,3 +11,4 @@ Abstract base classes for extending Zarr-Python. - **[zarr.abc.metadata](./metadata.md)** - Creating metadata classes compatible with the Zarr API - **[zarr.abc.numcodec](./numcodec.md)** - Protocols and classes for modeling codec interface used by numcodecs - **[zarr.abc.store](./store.md)** - ABC for implementing Zarr stores and managing getting and setting bytes in a store +- **[zarr.abc.url_pipeline](./url_pipeline.md)** - ABC for implementing [URL pipeline](https://github.com/jbms/url-pipeline) adapters diff --git a/docs/api/zarr/abc/url_pipeline.md b/docs/api/zarr/abc/url_pipeline.md new file mode 100644 index 0000000000..2606d4d5d4 --- /dev/null +++ b/docs/api/zarr/abc/url_pipeline.md @@ -0,0 +1,5 @@ +--- +title: url_pipeline +--- + +::: zarr.abc.url_pipeline diff --git a/docs/user-guide/extending.md b/docs/user-guide/extending.md index 507afedea7..d5fe5726ca 100644 --- a/docs/user-guide/extending.md +++ b/docs/user-guide/extending.md @@ -85,6 +85,70 @@ implementation. Custom stores can be created by implementing the [`zarr.abc.store.Store`][] interface. See [developing custom stores](storage.md#developing-custom-stores) for more information. +## Custom URL pipeline adapters + +[URL pipelines](storage.md#user-guide-url-pipelines) are `|`-chained URLs such as +`s3://bucket/data.zip|zip:|zarr3:`. Each sub-URL after the first names an *adapter* +that reinterprets everything to its left. A package provides an adapter for a scheme by +subclassing [`zarr.abc.url_pipeline.URLPipelineAdapter`][] and implementing its single +classmethod, which resolves the adapter's segment into an open store and a residual path: + +```python test="true" session="url-adapter" +import dataclasses + +from zarr.abc.url_pipeline import ( + AdapterResolution, + PipelineContext, + PipelineSegment, + URLPipelineAdapter, +) +from zarr.storage import WrapperStore + + +class MyAdapter(URLPipelineAdapter): + @classmethod + async def open_pipeline_segment( + cls, segment: PipelineSegment, context: PipelineContext + ) -> AdapterResolution: + # A *wrapper* adapter opens the resource to its left and wraps it. Join the + # preceding residual path with this segment's own path, and keep every other + # field (e.g. zarr_format) via dataclasses.replace. + preceding = await context.resolve_preceding() + store = WrapperStore(preceding.store) + path = "/".join(part.strip("/") for part in (preceding.path, segment.body) if part) + return dataclasses.replace(preceding, store=store, path=path) +``` + +Wrapper adapters compose: in `memory://x|mypackage.myscheme:a|mypackage.myscheme:b` the +inner adapter sees the outer one's residual path `a` and the pipeline resolves to `a/b`. + +```python test="true" session="url-adapter" +import asyncio + +from zarr.registry import register_url_adapter +from zarr.storage import resolve_pipeline + +register_url_adapter("mypackage.myscheme", MyAdapter) +resolution = asyncio.run(resolve_pipeline("memory://x|mypackage.myscheme:a|mypackage.myscheme:b")) +assert resolution.path == "a/b" +``` + +Register the class under the `zarr.url_adapters` entry-point group, using the URL +scheme as the entry-point name. Nonstandard schemes should be vendor-prefixed +(`vendor.scheme`), per the URL pipeline specification: + +```toml +[project.entry-points."zarr.url_adapters"] +"mypackage.myscheme" = "mypackage.zarr_adapter:MyAdapter" +``` + +Adapters are loaded lazily, only when a pipeline naming their scheme is resolved. An +adapter may also be used as a *root* scheme (e.g. `mypackage.myscheme://org/repo`), in +which case `context.preceding` is empty. The returned store must honor +`context.read_only`, and `open_pipeline_segment` runs on zarr's internal event loop, so +it must not block or call zarr's synchronous API. See +[`zarr.abc.url_pipeline`][] for the full contract. + ## Custom array buffers Zarr-python provides control over where and how arrays are stored in memory through diff --git a/docs/user-guide/storage.md b/docs/user-guide/storage.md index 88ee2f1f32..d8391cc632 100644 --- a/docs/user-guide/storage.md +++ b/docs/user-guide/storage.md @@ -100,9 +100,79 @@ print(group) group = zarr.open_group(UPath('s3://noaa-nwm-retro-v2-zarr-pds', anon=True), mode='r') ``` +- a [URL pipeline](#user-guide-url-pipelines) string containing `|`, such as + `s3://bucket/data.zip|zip:`, which is resolved through registered adapters. + - a [`Store`][zarr.abc.store.Store] or [`StorePath`][zarr.storage.StorePath] - see explicit store creation below. +## URL Pipelines {#user-guide-url-pipelines} + +Zarr supports [URL pipelines](https://github.com/jbms/url-pipeline): `|`-chained URLs +that address zarr data through nested storage layers, read left to right. The first +sub-URL locates a resource with a conventional URL; each subsequent sub-URL names an +*adapter* that reinterprets everything to its left (e.g. +`s3://bucket/data.zip|zip:|zarr3:`). Adapters are provided by packages through the +`zarr.url_adapters` entry-point group — see +[`zarr.abc.url_pipeline`][zarr.abc.url_pipeline] for the adapter interface. URLs +without a `|` (and without a registered root scheme) are handled exactly as before. + +Zarr ships the `zarr:`, `zarr2:` and `zarr3:` format adapters. The text after the +colon is a path within the resource to its left, and `zarr2:` / `zarr3:` select the +Zarr format (`zarr:` auto-detects it). The colon is optional when the path is empty: + +```python exec="true" session="storage-url-pipeline" source="above" result="ansi" +import tempfile + +import zarr + +root = tempfile.mkdtemp() +group = zarr.open_group(f"{root}|zarr2:", mode="w") +group.create_array("data", shape=(4,), dtype="int32") +print(zarr.open_array(f"{root}|zarr2:data", mode="r").metadata.zarr_format) +print(zarr.open_group(f"file:{root}|zarr", mode="r").metadata.zarr_format) +``` + +A `zarr_format` argument that conflicts with a `zarr2:` / `zarr3:` segment raises +`ValueError`. `Group.open` and `Array.open` take their format from the pipeline when +`zarr_format` is not passed. + +The builtin `zip:` adapter reads a ZIP archive addressed by the sub-URLs to its left, +which may be a local file, an fsspec URL (e.g. `s3://bucket/data.zip`), or an entry of +another archive (`outer.zip|zip:inner.zip|zip:`). The text after the colon is a path +within the archive (one leading `/` is ignored). `zip:` is currently read-only: it opens +archives in modes `"r"` and `"a"` (open-or-create serves the "open" half) and raises +`URLPipelineError` for `"w"`, `"w-"` and `"r+"`; write archives with +[`zarr.storage.ZipStore`][]. + +```python exec="true" session="storage-url-pipeline" source="above" result="ansi" +import os + +archive = os.path.join(root, "example.zip") +with zarr.storage.ZipStore(archive, mode="w") as store: + zarr.create_array(store, name="inner/data", shape=(3,), dtype="int32")[:] = [1, 2, 3] + +print(zarr.open_array(f"{archive}|zip:inner|zarr3:data")[:]) +``` + +`storage_options` passed to `zarr.open` apply to the *root* sub-URL (e.g. fsspec +options for `s3://...`); adapters may consume adapter-specific, namespaced keys. +Non-dict forms of `storage_options` are reserved for future per-segment +configuration. + +The `|` character is reserved as the pipeline delimiter in every string store +specification, and no percent-escape is decoded: to address a local file whose +*name* contains `|` (or `#`), pass a `pathlib.Path` instead of a string. +Registered adapters cannot intercept zarr's native `file:` and `memory:` root +schemes, and fsspec's chained-URL syntax (`zip::s3://...`) keeps flowing to +fsspec. + +Inside a pipeline, a `memory:` root is zarr's managed in-memory store (`memory:`, +`memory:/` and `memory://` are equivalent, and `memory:name` selects a named store). +When fsspec is installed, a plain `memory://name` URL *without* a `|` is still routed to +fsspec's in-memory filesystem, which is a different store. A `file:` root must carry an +absolute path; percent-escapes are not decoded, matching the [local store](#local-store). + ## Explicit Store Creation In some cases, it may be helpful to create a store instance directly. Zarr-Python offers diff --git a/mkdocs.yml b/mkdocs.yml index 4d06701a87..58b5d3344a 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -43,6 +43,7 @@ nav: - ' zarr.abc.metadata': api/zarr/abc/metadata.md - ' zarr.abc.numcodec': api/zarr/abc/numcodec.md - ' zarr.abc.store': api/zarr/abc/store.md + - ' zarr.abc.url_pipeline': api/zarr/abc/url_pipeline.md - ' zarr.api': - api/zarr/api/index.md - ' zarr.api.asynchronous': api/zarr/api/asynchronous.md diff --git a/src/zarr/abc/url_pipeline.py b/src/zarr/abc/url_pipeline.py new file mode 100644 index 0000000000..337563685d --- /dev/null +++ b/src/zarr/abc/url_pipeline.py @@ -0,0 +1,266 @@ +""" +Abstract base class and data model for URL pipeline adapters. + +A URL pipeline is a `|`-separated chain of sub-URLs, read outer-to-inner, +as specified by https://github.com/jbms/url-pipeline. The first sub-URL (the +*root*) locates a resource using a conventional URL, and each subsequent +sub-URL names an *adapter* that reinterprets everything to its left: + + s3://bucket/data.zip|zip:path/inside|zarr3: + +Third-party packages provide adapters by subclassing +[`URLPipelineAdapter`][zarr.abc.url_pipeline.URLPipelineAdapter] and +registering the class under the `zarr.url_adapters` entry-point group, +using the URL scheme as the entry-point name. +""" + +from __future__ import annotations + +import enum +from abc import ABC, abstractmethod +from dataclasses import dataclass +from typing import TYPE_CHECKING, Any + +from zarr.errors import URLPipelineError + +if TYPE_CHECKING: + from zarr.abc.store import Store + from zarr.core.common import AccessModeLiteral, ZarrFormat + +__all__ = [ + "AdapterResolution", + "PipelineContext", + "PipelineSegment", + "URLPipelineAdapter", +] + + +class _Unset(enum.Enum): + token = 0 + + +_UNSET = _Unset.token + + +@dataclass(frozen=True) +class PipelineSegment: + """ + One `|`-delimited sub-URL of a URL pipeline. + + Attributes + ---------- + scheme : str + The lowercased URL scheme. Empty string only for a schemeless root + (a bare local path), which is treated as opaque text. + body : str + The text after `scheme:` and before any `?`. Interpretation is + scheme-defined; it is **not** URL-normalized, so case-significant + content (e.g. icechunk snapshot IDs) is preserved. + query : str | None + The raw query string after `?`, or None. Interpretation is + scheme-defined. + raw : str + The exact original sub-URL text, preserved for lossless + reconstruction of the pipeline. + """ + + scheme: str + body: str + query: str | None + raw: str + + def __str__(self) -> str: + return self.raw + + +@dataclass(frozen=True, eq=False) +class AdapterResolution: + """ + The result of resolving a URL pipeline (or a prefix of one). + + Instances are immutable but compare and hash by identity, since a + [`Store`][zarr.abc.store.Store] is not hashable. + + Attributes + ---------- + store : Store + The resolved store. + path : str + Residual path *within* the store that the pipeline addresses + (e.g. `"path/to/node"` for `...|icechunk://tag.v1/path/to/node`). + Empty string when the pipeline addresses the store root. + zarr_format : ZarrFormat | None + Zarr format selected by a format segment (`zarr2:`/`zarr3:`), + or None if unspecified. A wrapper adapter that re-wraps a preceding + resolution must carry every field it does not change forward — + prefer `dataclasses.replace(preceding, store=..., path=...)` over + reconstructing, so fields added later are never silently dropped. + """ + + store: Store + path: str = "" + zarr_format: ZarrFormat | None = None + + +@dataclass(frozen=True, eq=False) +class PipelineContext: + """ + Context handed to a [`URLPipelineAdapter`][zarr.abc.url_pipeline.URLPipelineAdapter] + describing the pipeline to the left of its segment. + + Instances are immutable but compare and hash by identity. + + Attributes + ---------- + preceding : tuple[PipelineSegment, ...] + The parsed sub-URLs to the left of the adapter's segment, outer to + inner. Empty when the adapter's segment is the pipeline root. + mode : AccessModeLiteral | None + The access mode requested by the caller (e.g. `zarr.open(mode=...)`), + or None when unspecified. Adapters for read-only resources should + raise for unambiguous write modes (`"w"`, `"w-"`, `"r+"`) and + open read-only otherwise. `"a"` (the `zarr.open` default) means + open-or-create: read-only adapters serve the "open" half, and any + subsequent write fails at the store level. + storage_options : dict[str, Any] | None + Options passed by the caller. By convention these configure the + *root* sub-URL (e.g. fsspec options); adapters may consume + adapter-specific keys, and should namespace them (e.g. + `myscheme_credentials`) to avoid collisions with other segments' + backends. An adapter that consumes keys should strip them before + resolving the rest of the pipeline, by passing the reduced mapping + to [`resolve_preceding`][zarr.abc.url_pipeline.PipelineContext.resolve_preceding]. + Non-dict forms of the caller-facing `storage_options` argument are + reserved for future per-segment configuration (one mapping per + pipeline segment); this attribute will remain a single mapping — + the one addressed to this adapter's segment. + """ + + preceding: tuple[PipelineSegment, ...] + mode: AccessModeLiteral | None + storage_options: dict[str, Any] | None + + @property + def read_only(self) -> bool: + """ + True when the caller requires a read-only store (`mode == "r"`). + + Adapters must construct their store read-only when this is set + (the resolver enforces it afterwards); when it is False, they may + construct a writable store if the underlying resource supports + writing. + """ + return self.mode == "r" + + @property + def preceding_url(self) -> str: + """ + The pipeline to the left of this segment, reconstructed exactly. + + An adapter that consumes this string instead of calling + [`resolve_preceding`][zarr.abc.url_pipeline.PipelineContext.resolve_preceding] + takes ownership of the *entire* preceding pipeline: it must + validate every preceding segment itself and raise + [`URLPipelineError`][zarr.errors.URLPipelineError] for segments it + does not understand, so that no segment is ever silently ignored. + """ + return "|".join(segment.raw for segment in self.preceding) + + async def resolve_preceding( + self, + *, + mode: AccessModeLiteral | _Unset | None = _UNSET, + storage_options: dict[str, Any] | _Unset | None = _UNSET, + ) -> AdapterResolution: + """ + Resolve the preceding pipeline into a store. + + This is the entry point for *wrapper* adapters (e.g. `zip:`) that + operate on the resource produced by the segments to their left. It + composes with any preceding adapters, because each segment is + resolved by its own adapter. Adapters backed by their own I/O + machinery (e.g. `icechunk:`) may instead consume + [`preceding_url`][zarr.abc.url_pipeline.PipelineContext.preceding_url] + and never materialize the intermediate store — subject to the + ownership contract documented there. + + Parameters + ---------- + mode : AccessModeLiteral | None, optional + Override the mode used to resolve the preceding pipeline. + Wrapper adapters that only read the preceding resource should + pass `mode="r"` so the root is opened read-only and without + create-on-open side effects, regardless of the caller's mode. + When omitted, the caller's mode is used. Note that a local + *file* root (e.g. `data.zip|zip:`) can currently only be + resolved in mode `"r"`: every other mode creates a directory at + the root and fails with `FileExistsError`, because there is no + file-resource primitive yet. Writing *through* a container file + is therefore not supported by wrapper adapters at this time. + storage_options : dict | None, optional + Override the options forwarded to the preceding pipeline. An + adapter that consumed adapter-specific keys should pass the + remaining mapping here (or None when nothing remains), so the + root store never sees keys that were not addressed to it. + When omitted, the caller's options are forwarded unchanged. + """ + from zarr.storage._url_pipeline import _resolve + + if not self.preceding: + raise URLPipelineError( + "this adapter segment is at the pipeline root; " + "there is no preceding sub-URL to resolve" + ) + return await _resolve( + self.preceding, + mode=self.mode if isinstance(mode, _Unset) else mode, + storage_options=( + self.storage_options if isinstance(storage_options, _Unset) else storage_options + ), + ) + + +class URLPipelineAdapter(ABC): + """ + Handler for one URL pipeline scheme. + + Subclasses implement a single classmethod, + [`open_pipeline_segment`][zarr.abc.url_pipeline.URLPipelineAdapter.open_pipeline_segment], + and are registered under the `zarr.url_adapters` entry-point group with + the URL scheme as the entry-point name: + + [project.entry-points."zarr.url_adapters"] + mypackage.myscheme = "mypackage.zarr_adapter:MyAdapter" + + Nonstandard schemes should be vendor-prefixed (`vendor.scheme`) per the + URL pipeline specification. + + An adapter is used in two positions: + + - as an *adapter segment*: `s3://bucket/repo|icechunk://tag.v1` — the + context carries the preceding sub-URLs; + - as a *root scheme*: `gh://org/repo` — `context.preceding` is empty. + """ + + @classmethod + @abstractmethod + async def open_pipeline_segment( + cls, segment: PipelineSegment, context: PipelineContext + ) -> AdapterResolution: + """ + Resolve `segment` (in the context of the pipeline to its left) + into a store and an optional residual path within that store. + + The returned store must already be open and must honor + `context.read_only` (the resolver additionally enforces it by + downgrading — or rejecting — a writable store when the caller + required read-only). + + This coroutine runs on zarr's internal I/O event loop. It must not + block (do I/O through async APIs or a thread executor) and must not + call zarr's synchronous API (`zarr.open`, `Group.open`, or anything + else that uses `zarr.core.sync.sync`) — doing so raises + `SyncError`. To open the preceding pipeline, use + [`resolve_preceding`][zarr.abc.url_pipeline.PipelineContext.resolve_preceding]. + """ + ... diff --git a/src/zarr/api/asynchronous.py b/src/zarr/api/asynchronous.py index 5725b3d13a..4bb3ce621d 100644 --- a/src/zarr/api/asynchronous.py +++ b/src/zarr/api/asynchronous.py @@ -404,6 +404,7 @@ async def open( else: mode = "a" store_path = await make_store_path(store, mode=mode, path=path, storage_options=storage_options) + zarr_format = store_path.resolve_zarr_format(zarr_format) # TODO: the mode check below seems wrong! if "shape" not in kwargs and mode in (*_READ_MODES, "w"): @@ -526,13 +527,14 @@ async def save_array( **kwargs Passed through to [`create`][zarr.api.asynchronous.create], e.g., compressor. """ - if zarr_format is None: - zarr_format = _default_zarr_format() if not isinstance(arr, NDArrayLike): raise TypeError("arr argument must be numpy or other NDArrayLike array") mode = kwargs.pop("mode", "a") store_path = await make_store_path(store, path=path, mode=mode, storage_options=storage_options) + zarr_format = store_path.resolve_zarr_format(zarr_format) + if zarr_format is None: + zarr_format = _default_zarr_format() if np.isscalar(arr): arr = np.array(arr) shape = arr.shape @@ -581,9 +583,6 @@ async def save_group( NumPy arrays with data to save. """ - if zarr_format is None: - zarr_format = _default_zarr_format() - for arg in args: if not isinstance(arg, NDArrayLike): raise TypeError( @@ -602,6 +601,9 @@ async def save_group( get_data_type_from_native_dtype(arr.dtype) store_path = await make_store_path(store, path=path, mode="w", storage_options=storage_options) + zarr_format = store_path.resolve_zarr_format(zarr_format) + if zarr_format is None: + zarr_format = _default_zarr_format() # The group replaces whatever is stored under the path, now that the arguments are known # to be valid. await AsyncGroup.from_store(store_path, zarr_format=zarr_format, overwrite=True) @@ -796,12 +798,12 @@ async def create_group( The new group. """ - if zarr_format is None: - zarr_format = _default_zarr_format() - mode: Literal["a"] = "a" store_path = await make_store_path(store, path=path, mode=mode, storage_options=storage_options) + zarr_format = store_path.resolve_zarr_format(zarr_format) + if zarr_format is None: + zarr_format = _default_zarr_format() return await AsyncGroup.from_store( store=store_path, @@ -892,6 +894,7 @@ async def open_group( ) store_path = await make_store_path(store, mode=mode, storage_options=storage_options, path=path) + zarr_format = store_path.resolve_zarr_format(zarr_format) if attributes is None: attributes = {} @@ -1098,9 +1101,6 @@ async def create( z : array The array. """ - if zarr_format is None: - zarr_format = _default_zarr_format() - _warn_unimplemented_kwargs( { "synchronizer": synchronizer, @@ -1120,6 +1120,9 @@ async def create( mode = "a" overwrite = overwrite or _infer_overwrite(mode) store_path = await make_store_path(store, path=path, mode=mode, storage_options=storage_options) + zarr_format = store_path.resolve_zarr_format(zarr_format) + if zarr_format is None: + zarr_format = _default_zarr_format() config_parsed = parse_array_config(config) @@ -1334,6 +1337,7 @@ async def open_array( mode = kwargs.pop("mode", None) store_path = await make_store_path(store, path=path, mode=mode, storage_options=storage_options) + zarr_format = store_path.resolve_zarr_format(zarr_format) if "write_empty_chunks" in kwargs: _warn_write_empty_chunks_kwarg() diff --git a/src/zarr/api/synchronous.py b/src/zarr/api/synchronous.py index 524f719742..6778f8d907 100644 --- a/src/zarr/api/synchronous.py +++ b/src/zarr/api/synchronous.py @@ -866,7 +866,7 @@ def create_array( serializer: SerializerLike = "auto", fill_value: Any | None = DEFAULT_FILL_VALUE, order: MemoryOrder | None = None, - zarr_format: ZarrFormat | None = 3, + zarr_format: ZarrFormat | None = None, attributes: dict[str, JSON] | None = None, chunk_key_encoding: ChunkKeyEncodingLike | None = None, dimension_names: DimensionNamesLike = None, diff --git a/src/zarr/core/array.py b/src/zarr/core/array.py index 6a53a2c615..c5ffff7654 100644 --- a/src/zarr/core/array.py +++ b/src/zarr/core/array.py @@ -57,6 +57,7 @@ parse_chunk_key_encoding, ) from zarr.core.common import ( + _UNSPECIFIED, JSON, ZARR_JSON, ZARRAY_JSON, @@ -67,6 +68,7 @@ ShapeLike, ZarrFormat, _default_zarr_format, + _UnspecifiedType, _warn_order_kwarg, ceildiv_int, concurrent_map, @@ -154,7 +156,7 @@ _parse_bytes_bytes_codec, get_pipeline_class, ) -from zarr.storage._common import StorePath, make_store_path +from zarr.storage._common import StorePath, _resolve_open_zarr_format, make_store_path from zarr.storage._utils import _relativize_path if TYPE_CHECKING: @@ -466,8 +468,9 @@ async def _create( Deprecated in favor of [`zarr.api.asynchronous.create_array`][]. """ - dtype_parsed = parse_dtype(dtype, zarr_format=zarr_format) store_path = await make_store_path(store) + zarr_format = store_path.resolve_zarr_format(zarr_format) or zarr_format + dtype_parsed = parse_dtype(dtype, zarr_format=zarr_format) shape = parse_shapelike(shape) @@ -778,7 +781,7 @@ def from_dict( async def open( cls, store: StoreLike, - zarr_format: ZarrFormat | None = 3, + zarr_format: ZarrFormat | _UnspecifiedType | None = _UNSPECIFIED, ) -> AnyAsyncArray: """ Async method to open an existing Zarr array from a given store. @@ -790,7 +793,10 @@ async def open( [storage documentation in the user guide][user-guide-store-like] for a description of all valid StoreLike values. zarr_format : ZarrFormat | None, optional - The Zarr format version (default is 3). + The Zarr format version; None means auto-detect from the store + contents. When not passed, a URL pipeline decides (its `zarr2:` / + `zarr3:` segment, or auto-detection for `zarr:` and pipelines without + a format segment), and every other store defaults to 3. Returns ------- @@ -819,6 +825,7 @@ async def example(): ``` """ store_path = await make_store_path(store) + zarr_format = _resolve_open_zarr_format(store, store_path, zarr_format, default=3) metadata_dict = await get_array_metadata(store_path, zarr_format=zarr_format) # TODO: remove this cast when we have better type hints _metadata_dict = cast("ArrayMetadataJSON_V3", metadata_dict) @@ -2053,6 +2060,7 @@ def from_dict( def open( cls, store: StoreLike, + zarr_format: ZarrFormat | _UnspecifiedType | None = _UNSPECIFIED, ) -> Self: """Opens an existing Array from a store. @@ -2062,13 +2070,18 @@ def open( Store containing the Array. See the [storage documentation in the user guide][user-guide-store-like] for a description of all valid StoreLike values. + zarr_format : {2, 3, None}, optional + The zarr format to expect; None means auto-detect from the store + contents. When not passed, a URL pipeline decides (its `zarr2:` / + `zarr3:` segment, or auto-detection for `zarr:` and pipelines without + a format segment), and every other store defaults to 3. Returns ------- Array Array opened from the store. """ - async_array = sync(AsyncArray.open(store)) + async_array = sync(AsyncArray.open(store, zarr_format=zarr_format)) return cls(async_array) @property @@ -4382,6 +4395,7 @@ async def from_array( mode: Literal["a"] = "a" config_parsed = parse_array_config(config) store_path = await make_store_path(store, path=name, mode=mode, storage_options=storage_options) + zarr_format = store_path.resolve_zarr_format(zarr_format) ( chunks, @@ -4707,7 +4721,7 @@ async def create_array( serializer: SerializerLike = "auto", fill_value: Any | None = DEFAULT_FILL_VALUE, order: MemoryOrder | None = None, - zarr_format: ZarrFormat | None = 3, + zarr_format: ZarrFormat | None = None, attributes: dict[str, JSON] | None = None, chunk_key_encoding: ChunkKeyEncodingLike | None = None, dimension_names: DimensionNamesLike = None, @@ -4870,6 +4884,7 @@ async def create_array( store_path = await make_store_path( store, path=name, mode=mode, storage_options=storage_options ) + zarr_format = store_path.resolve_zarr_format(zarr_format) return await init_array( store_path=store_path, shape=shape_parsed, diff --git a/src/zarr/core/common.py b/src/zarr/core/common.py index de009d0b50..e6b13d8ea2 100644 --- a/src/zarr/core/common.py +++ b/src/zarr/core/common.py @@ -277,6 +277,24 @@ def _warn_order_kwarg() -> None: warnings.warn(msg, ZarrRuntimeWarning, stacklevel=2) +class _UnspecifiedType(Enum): + """ + Type of the `_UNSPECIFIED` sentinel. + + Used as a parameter default where "not passed" must be told apart from an + explicit `None` (for example, `zarr_format` in `Group.open`, where `None` + means "auto-detect"). + """ + + UNSPECIFIED = "unspecified" + + def __repr__(self) -> str: + return "" + + +_UNSPECIFIED: Final = _UnspecifiedType.UNSPECIFIED + + def _default_zarr_format() -> ZarrFormat: """Return the default zarr_format.""" return cast("ZarrFormat", int(zarr_config.get("default_zarr_format", 3))) diff --git a/src/zarr/core/group.py b/src/zarr/core/group.py index 7792fa58de..80dfa24aee 100644 --- a/src/zarr/core/group.py +++ b/src/zarr/core/group.py @@ -32,6 +32,7 @@ from zarr.core.attributes import Attributes from zarr.core.buffer import default_buffer_prototype from zarr.core.common import ( + _UNSPECIFIED, JSON, ZARR_JSON, ZARRAY_JSON, @@ -43,6 +44,8 @@ NodeType, ShapeLike, ZarrFormat, + _default_zarr_format, + _UnspecifiedType, parse_shapelike, ) from zarr.core.config import config @@ -67,7 +70,7 @@ ZarrUserWarning, ) from zarr.storage import StoreLike, StorePath -from zarr.storage._common import make_store_path +from zarr.storage._common import _resolve_open_zarr_format, make_store_path from zarr.storage._utils import _join_paths, _normalize_path_keys, normalize_path if TYPE_CHECKING: @@ -530,9 +533,12 @@ async def from_store( *, attributes: dict[str, Any] | None = None, overwrite: bool = False, - zarr_format: ZarrFormat = 3, + zarr_format: ZarrFormat | None = None, ) -> AsyncGroup: store_path = await make_store_path(store) + zarr_format = store_path.resolve_zarr_format(zarr_format) + if zarr_format is None: + zarr_format = _default_zarr_format() attributes = attributes or {} group = cls( metadata=GroupMetadata(attributes=attributes, zarr_format=zarr_format), @@ -547,7 +553,7 @@ async def from_store( async def open( cls, store: StoreLike, - zarr_format: ZarrFormat | None = 3, + zarr_format: ZarrFormat | _UnspecifiedType | None = _UNSPECIFIED, use_consolidated: bool | str | None = None, ) -> AsyncGroup: """Open a new AsyncGroup @@ -555,7 +561,13 @@ async def open( Parameters ---------- store : StoreLike - zarr_format : {2, 3}, optional + zarr_format : {2, 3, None}, optional + The Zarr format to expect; None means auto-detect from the store + contents. When not passed, a URL pipeline decides (its `zarr2:` / + `zarr3:` segment, or auto-detection for `zarr:` and pipelines without + a format segment), and every other store defaults to 3. An explicit + value that conflicts with a pipeline's format segment raises + `ValueError`. use_consolidated : bool or str, default None Whether to use consolidated metadata. @@ -575,6 +587,7 @@ async def open( to load consolidated metadata from a non-default key. """ store_path = await make_store_path(store) + zarr_format = _resolve_open_zarr_format(store, store_path, zarr_format, default=3) if not store_path.store.supports_consolidated_metadata: # Fail if consolidated metadata was requested but the Store doesn't support it if use_consolidated: @@ -1954,7 +1967,7 @@ def from_store( store: StoreLike, *, attributes: dict[str, Any] | None = None, - zarr_format: ZarrFormat = 3, + zarr_format: ZarrFormat | None = None, overwrite: bool = False, ) -> Group: """Instantiate a group from an initialized store. @@ -1967,8 +1980,9 @@ def from_store( for a description of all valid StoreLike values. attributes : dict, optional A dictionary of JSON-serializable values with user-defined attributes. - zarr_format : {2, 3}, optional - Zarr storage format version. + zarr_format : {2, 3, None}, optional + Zarr storage format version. If None, the format selected by a URL + pipeline segment is used, falling back to the configured default. overwrite : bool, optional If True, do not raise an error if the group already exists. @@ -1997,7 +2011,7 @@ def from_store( def open( cls, store: StoreLike, - zarr_format: ZarrFormat | None = 3, + zarr_format: ZarrFormat | _UnspecifiedType | None = _UNSPECIFIED, ) -> Group: """Open a group from an initialized store. @@ -2008,7 +2022,10 @@ def open( [storage documentation in the user guide][user-guide-store-like] for a description of all valid StoreLike values. zarr_format : {2, 3, None}, optional - Zarr storage format version. + Zarr storage format version; None means auto-detect from the store + contents. When not passed, a URL pipeline decides (its `zarr2:` / + `zarr3:` segment, or auto-detection for `zarr:` and pipelines without + a format segment), and every other store defaults to 3. Returns ------- diff --git a/src/zarr/errors.py b/src/zarr/errors.py index 3e445de2e9..d9029a7430 100644 --- a/src/zarr/errors.py +++ b/src/zarr/errors.py @@ -12,6 +12,7 @@ "MetadataValidationError", "NegativeStepError", "NodeTypeValidationError", + "URLPipelineError", "UnknownCodecError", "UnstableSpecificationWarning", "VindexInvalidSelectionError", @@ -101,6 +102,12 @@ class UnknownCodecError(BaseZarrError): """ +class URLPipelineError(BaseZarrError): + """ + Raised when a URL pipeline cannot be parsed or resolved. + """ + + class NodeTypeValidationError(MetadataValidationError): """ Specialized exception when the node_type of the metadata document is incorrect. diff --git a/src/zarr/registry.py b/src/zarr/registry.py index 307c323153..260d5c4ddb 100644 --- a/src/zarr/registry.py +++ b/src/zarr/registry.py @@ -1,5 +1,6 @@ from __future__ import annotations +import threading import warnings from collections import defaultdict from collections.abc import Mapping @@ -8,7 +9,7 @@ from zarr.core.config import BadConfigError, config from zarr.core.dtype import data_type_registry -from zarr.errors import UnknownCodecError, ZarrUserWarning +from zarr.errors import UnknownCodecError, URLPipelineError, ZarrUserWarning if TYPE_CHECKING: from importlib.metadata import EntryPoint @@ -22,6 +23,7 @@ CodecPipeline, ) from zarr.abc.numcodec import Numcodec + from zarr.abc.url_pipeline import URLPipelineAdapter from zarr.core.buffer import Buffer, NDBuffer from zarr.core.chunk_key_encodings import ChunkKeyEncoding from zarr.core.common import JSON, ZarrFormat @@ -33,11 +35,14 @@ "get_codec_class", "get_ndbuffer_class", "get_pipeline_class", + "get_url_adapter", + "list_url_adapter_schemes", "register_buffer", "register_chunk_key_encoding", "register_codec", "register_ndbuffer", "register_pipeline", + "register_url_adapter", ] _ZARR_CODEC_DOCS_URL = "https://zarr.readthedocs.io/en/stable/user-guide/extending/#custom-codecs" @@ -257,6 +262,7 @@ def register(self, cls: type[T], qualname: str | None = None) -> None: _buffer_registry: Registry[Buffer] = Registry() _ndbuffer_registry: Registry[NDBuffer] = Registry() _chunk_key_encoding_registry: Registry[ChunkKeyEncoding] = Registry() +_url_adapter_registry: Registry[URLPipelineAdapter] = Registry() """ The registry module is responsible for managing implementations of codecs, @@ -303,6 +309,20 @@ def _collect_entrypoints() -> list[Registry[Any]]: entry_points.select(group="zarr", name="chunk_key_encoding") ) + # Builtin adapters go first, as lazily loaded entry points, so that they + # are imported only when a pipeline naming them is resolved and so that a + # third-party entry point with the same name is reported as shadowed. + _url_adapter_registry.lazy_load_list.extend( + _builtin_url_adapter_entry_point(scheme) + for scheme in _BUILTIN_URL_ADAPTERS + if scheme not in _url_adapter_registry + and not any( + e.name == scheme and _is_builtin_url_adapter_entry_point(e) + for e in _url_adapter_registry.lazy_load_list + ) + ) + _url_adapter_registry.lazy_load_list.extend(entry_points.select(group="zarr.url_adapters")) + _pipeline_registry.lazy_load_list.extend(entry_points.select(group="zarr.codec_pipeline")) _pipeline_registry.lazy_load_list.extend( entry_points.select(group="zarr", name="codec_pipeline") @@ -319,6 +339,7 @@ def _collect_entrypoints() -> list[Registry[Any]]: _buffer_registry, _ndbuffer_registry, _chunk_key_encoding_registry, + _url_adapter_registry, ] @@ -504,6 +525,152 @@ def get_chunk_key_encoding_class(key: str) -> type[ChunkKeyEncoding]: return _chunk_key_encoding_registry[key] +# URL pipeline adapters shipped with zarr-python, as `scheme -> "module:attr"`. +# They are registered as lazily loaded entry points (see `_collect_entrypoints`), +# so importing zarr never imports them. These are adapter-only schemes: they are +# never dispatched to as a pipeline *root* (see `zarr.storage._url_pipeline`). +_BUILTIN_URL_ADAPTERS: dict[str, str] = { + "zarr": "zarr.storage._url_adapters._format:ZarrAdapter", + "zarr2": "zarr.storage._url_adapters._format:Zarr2Adapter", + "zarr3": "zarr.storage._url_adapters._format:Zarr3Adapter", + "zip": "zarr.storage._url_adapters._zip:ZipAdapter", +} + + +def _builtin_url_adapter_entry_point(scheme: str) -> EntryPoint: + from importlib.metadata import EntryPoint + + return EntryPoint(name=scheme, value=_BUILTIN_URL_ADAPTERS[scheme], group="zarr.url_adapters") + + +def _is_builtin_url_adapter_entry_point(entry_point: EntryPoint) -> bool: + return _BUILTIN_URL_ADAPTERS.get(entry_point.name.lower()) == entry_point.value + + +def register_url_adapter(scheme: str, cls: type[URLPipelineAdapter]) -> None: + """ + Register a [`URLPipelineAdapter`][zarr.abc.url_pipeline.URLPipelineAdapter] + class for a URL scheme. + + Registering a scheme that already has an adapter replaces it and emits a + [`ZarrUserWarning`][zarr.errors.ZarrUserWarning]. + """ + key = scheme.lower() + previous = _url_adapter_registry.get(key) + if previous is not None and previous is not cls: + warnings.warn( + f"URL pipeline adapter for scheme {scheme!r} is being replaced: " + f"{fully_qualified_name(previous)} -> {fully_qualified_name(cls)}", + category=ZarrUserWarning, + stacklevel=2, + ) + _url_adapter_registry.register(cls, key) + + +def list_url_adapter_schemes() -> set[str]: + """ + The set of URL schemes with a registered URL pipeline adapter. + + Includes adapters advertised via not-yet-loaded `zarr.url_adapters` + entry points; consulting this does not import any adapter code. + Schemes are case-insensitive and reported lowercased. + """ + return set(_url_adapter_registry) | { + e.name.lower() for e in _url_adapter_registry.lazy_load_list + } + + +_url_adapter_lock = threading.Lock() + + +def get_url_adapter(scheme: str) -> type[URLPipelineAdapter]: + """ + Get the URL pipeline adapter class registered for `scheme`. + + Loads pending `zarr.url_adapters` entry points for this scheme only, so + resolving one scheme never imports other providers' packages. zarr's + builtin adapters (`zarr:`, `zarr2:`, `zarr3:`, ...) are pending entry points + too, and take precedence over a third-party entry point of the same name. + When the scheme is already registered, or a builtin provides it, a + same-named third-party entry point is discarded with a + [`ZarrUserWarning`][zarr.errors.ZarrUserWarning]; when several third-party + entry points share the name, the first one wins and a warning names the + ones ignored. An explicit `register_url_adapter` call overrides a builtin + silently. + + Raises + ------ + URLPipelineError + If no adapter is registered for the scheme, or if the entry point + providing it fails to import. + """ + key = scheme.lower() + # Take the matching entry points out of the pending list under the lock, + # so concurrent first-time resolutions of different schemes never clobber + # each other's view of the list. The import itself runs *outside* the lock: + # it may take arbitrarily long and may itself resolve URL adapters. + with _url_adapter_lock: + registered = _url_adapter_registry.get(key) + pending = [e for e in _url_adapter_registry.lazy_load_list if e.name.lower() == key] + if pending: + _url_adapter_registry.lazy_load_list[:] = [ + e for e in _url_adapter_registry.lazy_load_list if e.name.lower() != key + ] + # A pending builtin entry point is redundant once the scheme is registered + # explicitly (an intentional override), so only third-party entry points + # are reported as shadowed. + third_party = [e for e in pending if not _is_builtin_url_adapter_entry_point(e)] + if pending and registered is not None: + if third_party: + warnings.warn( + f"URL pipeline adapter for scheme {scheme!r} is already registered " + f"({fully_qualified_name(registered)}); ignoring entry point(s) " + f"{[e.value for e in third_party]} of the same name", + category=ZarrUserWarning, + stacklevel=2, + ) + elif pending: + entry_point, *duplicates = pending + if duplicates and _is_builtin_url_adapter_entry_point(entry_point): + warnings.warn( + f"the builtin URL pipeline adapter for scheme {entry_point.name!r} takes " + f"precedence; ignoring 'zarr.url_adapters' entry point(s) " + f"{[e.value for e in duplicates]} of the same name", + category=ZarrUserWarning, + stacklevel=2, + ) + elif duplicates: + warnings.warn( + f"multiple 'zarr.url_adapters' entry points are named {entry_point.name!r}; " + f"using {entry_point.value!r} and ignoring {[e.value for e in duplicates]}", + category=ZarrUserWarning, + stacklevel=2, + ) + try: + cls = entry_point.load() + except Exception as exc: + # leave the entry points discoverable so a transient import + # failure is not permanent + with _url_adapter_lock: + _url_adapter_registry.lazy_load_list.extend(pending) + raise URLPipelineError( + f"the 'zarr.url_adapters' entry point {entry_point.name!r} " + f"({entry_point.value!r}) could not be loaded: {exc!r}" + ) from exc + with _url_adapter_lock: + if key not in _url_adapter_registry: + _url_adapter_registry.register(cls, qualname=key) + try: + return _url_adapter_registry[key] + except KeyError: + registered_schemes = sorted(list_url_adapter_schemes()) + raise URLPipelineError( + f"no URL pipeline adapter is registered for scheme {scheme!r}. " + f"Registered schemes: {registered_schemes}. Adapters are provided by " + "packages via the 'zarr.url_adapters' entry-point group." + ) from None + + _collect_entrypoints() diff --git a/src/zarr/storage/__init__.py b/src/zarr/storage/__init__.py index f1bd1724af..edc01f1a5b 100644 --- a/src/zarr/storage/__init__.py +++ b/src/zarr/storage/__init__.py @@ -10,6 +10,7 @@ from zarr.storage._logging import LoggingStore from zarr.storage._memory import GpuMemoryStore, ManagedMemoryStore, MemoryStore from zarr.storage._obstore import ObjectStore +from zarr.storage._url_pipeline import parse_pipeline, resolve_pipeline from zarr.storage._wrapper import WrapperStore from zarr.storage._zip import ZipStore @@ -25,6 +26,8 @@ "StorePath", "WrapperStore", "ZipStore", + "parse_pipeline", + "resolve_pipeline", ] diff --git a/src/zarr/storage/_common.py b/src/zarr/storage/_common.py index 8591447449..0e3f34afbd 100644 --- a/src/zarr/storage/_common.py +++ b/src/zarr/storage/_common.py @@ -20,10 +20,17 @@ ZGROUP_JSON, AccessModeLiteral, ZarrFormat, + _UnspecifiedType, +) +from zarr.errors import ( + ContainsArrayAndGroupError, + ContainsArrayError, + ContainsGroupError, + URLPipelineError, ) -from zarr.errors import ContainsArrayAndGroupError, ContainsArrayError, ContainsGroupError from zarr.storage._local import LocalStore from zarr.storage._memory import ManagedMemoryStore, MemoryStore +from zarr.storage._url_pipeline import is_url_pipeline, resolve_pipeline from zarr.storage._utils import UPath, _join_paths, normalize_path, parse_store_url _has_fsspec = importlib.util.find_spec("fsspec") @@ -47,19 +54,57 @@ class StorePath: The store to use. path : str The path within the store. + + Attributes + ---------- + zarr_format : ZarrFormat | None + Zarr format selected by a `zarr2:`/`zarr3:` URL pipeline segment, + or None. Not part of the StorePath's identity (ignored by `__eq__`). """ store: Store path: str + zarr_format: ZarrFormat | None def __init__(self, store: Store, path: str = "") -> None: self.store = store self.path = normalize_path(path) + self.zarr_format = None @property def read_only(self) -> bool: return self.store.read_only + def resolve_zarr_format(self, zarr_format: ZarrFormat | None) -> ZarrFormat | None: + """ + Combine the zarr format selected by a URL pipeline segment (`zarr2:` / + `zarr3:`) with a caller-supplied `zarr_format`. + + Parameters + ---------- + zarr_format : ZarrFormat | None + The format requested by the caller, or None when unspecified. + + Returns + ------- + ZarrFormat | None + The pipeline's format when the caller did not specify one, otherwise the + caller's format. None when neither is set. + + Raises + ------ + ValueError + If the caller's format differs from the one selected by the pipeline. + """ + if self.zarr_format is None: + return zarr_format + if zarr_format is not None and zarr_format != self.zarr_format: + raise ValueError( + f"zarr_format={zarr_format} conflicts with the 'zarr{self.zarr_format}:' " + "segment of the URL pipeline; pass zarr_format=None to use the pipeline's format" + ) + return self.zarr_format + @classmethod async def _create_open_instance(cls, store: Store, path: str) -> Self: """Helper to create and return a StorePath instance.""" @@ -72,7 +117,10 @@ async def open(cls, store: Store, path: str, mode: AccessModeLiteral | None = No Open StorePath based on the provided mode. * If the mode is None, return an opened version of the store with no changes. - * If the mode is 'r+', 'w-', 'w', or 'a' and the store is read-only, raise a ValueError. + * If the mode is 'r+', 'w-', or 'w' and the store is read-only, raise a ValueError. + * If the mode is 'a' (open-or-create) and the store is read-only, serve the + "open" half: the StorePath is opened read-only, and any subsequent write + fails at the store level. * If the mode is 'r' and the store is not read-only, return a copy of the store with read_only set to True. * If the mode is 'w-' and the store is not read-only and the StorePath contains keys, raise a FileExistsError. @@ -98,7 +146,8 @@ async def open(cls, store: Store, path: str, mode: AccessModeLiteral | None = No FileExistsError If the mode is 'w-' and the store path already exists. ValueError - If the mode is not "r" and the store is read-only, or + If the mode is "r+", "w-", or "w" and the store is read-only, or if the + mode is not a valid access mode. """ # fastpath if mode is None @@ -109,8 +158,9 @@ async def open(cls, store: Store, path: str, mode: AccessModeLiteral | None = No raise ValueError(f"Invalid mode: {mode}, expected one of {ANY_ACCESS_MODE}") if store.read_only: - # Don't allow write operations on a read-only store - if mode != "r": + # mode "a" (open-or-create) on a read-only store means open-only; + # unambiguous write modes are rejected outright. + if mode not in ("r", "a"): raise ValueError( f"Store is read-only but mode is {mode!r}. Create a writable store or use 'r' mode." ) @@ -271,7 +321,9 @@ def delete_sync(self) -> None: def __truediv__(self, other: str) -> StorePath: """Combine this store path with another path""" - return self.__class__(self.store, _join_paths([self.path, other])) + result = self.__class__(self.store, _join_paths([self.path, other])) + result.zarr_format = self.zarr_format + return result def __str__(self) -> str: return _join_paths([str(self.store), self.path]) @@ -320,6 +372,9 @@ async def make_store( - `None` = `MemoryStore` object. - `FSMap` = `FsspecStore` object. - `UPath` = `FsspecStore` object, or `LocalStore` for a local `UPath`. + - `str` containing `|` (or whose scheme has a registered root adapter) = the store + resolved by the [URL pipeline][user-guide-url-pipelines]; the pipeline must address + the store root (use `make_store_path` for pipelines that address a path within a store). Parameters ---------- @@ -346,6 +401,20 @@ async def make_store( """ from zarr.storage._fsspec import FsspecStore # circular import + if mode is not None and mode not in ANY_ACCESS_MODE: + raise ValueError(f"Invalid mode: {mode}, expected one of {ANY_ACCESS_MODE}") + _read_only = mode == "r" + + if isinstance(store_like, str) and is_url_pipeline(store_like): + result = await resolve_pipeline(store_like, mode=mode, storage_options=storage_options) + if normalize_path(result.path): + result.store.close() + raise URLPipelineError( + f"the URL pipeline {store_like!r} resolves to a path inside a store; " + "use zarr.open() or make_store_path() instead of make_store()" + ) + return result.store + # Parse URL early so we can reuse the result for both validation and routing parsed = parse_store_url(store_like) if isinstance(store_like, str) else None @@ -358,10 +427,6 @@ async def make_store( "'storage_options' is only used when the store is passed as an FSSpec URI string.", ) - if mode is not None and mode not in ANY_ACCESS_MODE: - raise ValueError(f"Invalid mode: {mode}, expected one of {ANY_ACCESS_MODE}") - _read_only = mode == "r" - if isinstance(store_like, StorePath): # Get underlying store return store_like.store @@ -456,6 +521,9 @@ async def make_store_path( If the StoreLike object is not one of the supported types, or if storage_options is provided but not used. ValueError If path is provided for a store that does not support it. + URLPipelineError + If `store_like` is a [URL pipeline][user-guide-url-pipelines] string that + cannot be resolved. See Also -------- @@ -463,6 +531,13 @@ async def make_store_path( """ path_normalized = normalize_path(path) + if isinstance(store_like, str) and is_url_pipeline(store_like): + result = await resolve_pipeline(store_like, mode=mode, storage_options=storage_options) + combined_path = _join_paths([normalize_path(result.path), path_normalized]) + store_path = await StorePath.open(result.store, path=combined_path, mode=mode) + store_path.zarr_format = result.zarr_format + return store_path + if isinstance(store_like, StorePath): # Already a StorePath if storage_options: @@ -482,6 +557,48 @@ async def make_store_path( return await StorePath.open(store, path=path_normalized, mode=mode) +def _resolve_open_zarr_format( + store_like: StoreLike, + store_path: StorePath, + zarr_format: ZarrFormat | _UnspecifiedType | None, + *, + default: ZarrFormat | None, +) -> ZarrFormat | None: + """ + Resolve the `zarr_format` of an `open` classmethod whose default is unspecified. + + An explicit `zarr_format` (including None, meaning auto-detect) is merged with + the format selected by a URL pipeline as in + [`StorePath.resolve_zarr_format`][zarr.storage.StorePath.resolve_zarr_format]. + When the caller did not pass one, a URL pipeline decides: its format segment + (`zarr2:`/`zarr3:`) if any, otherwise auto-detection. Every other store keeps + `default`, so non-pipeline callers see no change. + + Parameters + ---------- + store_like : StoreLike + The store argument the caller passed. + store_path : StorePath + The store path resolved from `store_like`. + zarr_format : ZarrFormat | None | _UnspecifiedType + The caller's format, or the unspecified sentinel. + default : ZarrFormat | None + The format to use when neither the caller nor a URL pipeline selects one. + + Returns + ------- + ZarrFormat | None + The format to open with, or None to auto-detect. + """ + if not isinstance(zarr_format, _UnspecifiedType): + return store_path.resolve_zarr_format(zarr_format) + if store_path.zarr_format is not None: + return store_path.zarr_format + if isinstance(store_like, str) and is_url_pipeline(store_like): + return None + return default + + async def ensure_no_existing_node( store_path: StorePath, zarr_format: ZarrFormat, diff --git a/src/zarr/storage/_url_adapters/__init__.py b/src/zarr/storage/_url_adapters/__init__.py new file mode 100644 index 0000000000..04f042c1ae --- /dev/null +++ b/src/zarr/storage/_url_adapters/__init__.py @@ -0,0 +1,7 @@ +""" +URL pipeline adapters shipped with zarr-python. + +These are registered as lazily loaded builtin entry points by +`zarr.registry` (see `_BUILTIN_URL_ADAPTERS`), so nothing in this package is +imported until a pipeline naming one of its schemes is resolved. +""" diff --git a/src/zarr/storage/_url_adapters/_format.py b/src/zarr/storage/_url_adapters/_format.py new file mode 100644 index 0000000000..99b44ebff8 --- /dev/null +++ b/src/zarr/storage/_url_adapters/_format.py @@ -0,0 +1,117 @@ +""" +The `zarr:`, `zarr2:` and `zarr3:` URL pipeline adapters. + +Per the URL pipeline specification (`schemes/zarr.md`), these address a Zarr +array or group within the `directory` resource to their left: + + file:///path/to/data.zarr/|zarr3:path/to/node + +The segment body is a path *within* the preceding resource. `zarr2:` and +`zarr3:` select the Zarr format; `zarr:` leaves it to auto-detection. +""" + +from __future__ import annotations + +import dataclasses +from typing import TYPE_CHECKING, ClassVar + +from zarr.abc.url_pipeline import AdapterResolution, URLPipelineAdapter +from zarr.errors import URLPipelineError +from zarr.storage._utils import _join_paths, normalize_path + +if TYPE_CHECKING: + from zarr.abc.url_pipeline import PipelineContext, PipelineSegment + from zarr.core.common import ZarrFormat + +__all__ = ["Zarr2Adapter", "Zarr3Adapter", "ZarrAdapter"] + + +class _ZarrFormatAdapter(URLPipelineAdapter): + """ + Shared implementation of the format adapters. + + Resolves the preceding pipeline with the caller's mode and options, joins + the segment body onto the preceding residual path, and records the Zarr + format. Every other field of the preceding resolution is carried forward + unchanged via `dataclasses.replace`. + """ + + zarr_format: ClassVar[ZarrFormat | None] + + @classmethod + async def open_pipeline_segment( + cls, segment: PipelineSegment, context: PipelineContext + ) -> AdapterResolution: + """ + Resolve a `zarr:`/`zarr2:`/`zarr3:` segment. + + Parameters + ---------- + segment : PipelineSegment + The format segment. Its body is a path within the preceding resource. + context : PipelineContext + The pipeline to the left of the segment. + + Returns + ------- + AdapterResolution + The preceding resolution with the segment path appended and the + Zarr format recorded. + + Raises + ------ + URLPipelineError + If the segment carries a query, has an invalid path, is the pipeline + root, or selects a format that conflicts with an earlier format segment. + """ + if segment.query is not None: + raise URLPipelineError( + f"'{segment.scheme}:' pipeline segments do not accept a query: {segment.raw!r}" + ) + try: + subpath = normalize_path(segment.body) + except ValueError as exc: + raise URLPipelineError( + f"invalid path in pipeline segment {segment.raw!r}: {exc}" + ) from exc + + preceding = await context.resolve_preceding() + + zarr_format = cls.zarr_format + if preceding.zarr_format is not None: + if zarr_format is None: + # `zarr:` (auto-detect) never discards a format pinned earlier + zarr_format = preceding.zarr_format + elif zarr_format != preceding.zarr_format: + preceding.store.close() + raise URLPipelineError( + f"pipeline segment {segment.raw!r} selects Zarr format {zarr_format}, " + f"but an earlier segment selected Zarr format {preceding.zarr_format}" + ) + + path = _join_paths([normalize_path(preceding.path), subpath]) + return dataclasses.replace(preceding, path=path, zarr_format=zarr_format) + + +class ZarrAdapter(_ZarrFormatAdapter): + """ + The `zarr:` adapter: a Zarr node whose format is auto-detected. + """ + + zarr_format = None + + +class Zarr2Adapter(_ZarrFormatAdapter): + """ + The `zarr2:` adapter: a Zarr format 2 node. + """ + + zarr_format = 2 + + +class Zarr3Adapter(_ZarrFormatAdapter): + """ + The `zarr3:` adapter: a Zarr format 3 node. + """ + + zarr_format = 3 diff --git a/src/zarr/storage/_url_adapters/_zip.py b/src/zarr/storage/_url_adapters/_zip.py new file mode 100644 index 0000000000..fab07ddcae --- /dev/null +++ b/src/zarr/storage/_url_adapters/_zip.py @@ -0,0 +1,447 @@ +""" +The read-only `zip:` URL pipeline adapter. + +Per the URL pipeline specification (`schemes/zip.md`), `zip:` addresses an +entry or directory within the ZIP archive that the preceding pipeline +resolves to: + + s3://bucket/archive.zip|zip:path/within/archive|zarr3: + +The archive is read through the preceding store with async byte-range +requests: the central directory is fetched from the end of the archive once, +and each member is fetched (and decompressed) on demand. This works the same +way for local, fsspec, in-memory and nested (`zip:` inside `zip:`) archives, +and never blocks zarr's I/O event loop on file I/O. +""" + +from __future__ import annotations + +import asyncio +import bz2 +import dataclasses +import io +import struct +import zipfile +import zlib +from typing import TYPE_CHECKING + +from zarr.abc.store import ( + ByteRequest, + OffsetByteRequest, + RangeByteRequest, + Store, + SuffixByteRequest, +) +from zarr.abc.url_pipeline import AdapterResolution, URLPipelineAdapter +from zarr.core.buffer import default_buffer_prototype +from zarr.errors import URLPipelineError +from zarr.storage._utils import normalize_path + +if TYPE_CHECKING: + from collections.abc import AsyncIterator, Iterable + + from zarr.abc.url_pipeline import PipelineContext, PipelineSegment + from zarr.core.buffer import Buffer, BufferPrototype + +__all__ = ["ZipAdapter", "ZipReaderStore"] + +# The end-of-central-directory record is 22 bytes plus a comment of at most +# 65535 bytes, so this suffix always contains it. +_EOCD_SEARCH_BYTES = 22 + 0xFFFF +# Smallest range fetched when the central directory extends past the suffix. +_MIN_FETCH_BYTES = 1 << 16 +# Members whose compressed size exceeds this are decompressed in a thread. +_THREAD_DECOMPRESS_BYTES = 1 << 20 + +_LOCAL_HEADER = struct.Struct("<4s2B4HL2L2H") +_LOCAL_HEADER_SIGNATURE = b"PK\x03\x04" +_WRITE_MODES = frozenset({"w", "w-", "r+"}) + + +async def _read_resource_range( + store: Store, key: str, byte_range: ByteRequest | None = None +) -> bytes: + """ + Read bytes of the single-object resource at `key` in `store`. + + This is the one place that relies on a `file`-kind resource being readable + as a store key: an archive that is an entry of another store (e.g. a nested + `zip:`) is the value at `key`; an archive that *is* the preceding root (a + local file, an fsspec object) is read with the empty key, which `LocalStore` + and `FsspecStore` happen to resolve to the root object itself. + """ + value = await store.get(key, prototype=default_buffer_prototype(), byte_range=byte_range) + if value is None: + raise FileNotFoundError(f"{key!r} in {store}" if key else str(store)) + return value.to_bytes() + + +async def _resource_size(store: Store, key: str) -> int: + """The size in bytes of the single-object resource at `key` in `store`.""" + try: + return await store.getsize(key) + except FileNotFoundError as exc: + raise FileNotFoundError(f"{key!r} in {store}" if key else str(store)) from exc + + +class _MissingRangeError(Exception): + def __init__(self, start: int, stop: int) -> None: + super().__init__(start, stop) + self.start = start + self.stop = stop + + +class _SparseFile(io.RawIOBase): + """ + A read-only, seekable view of a file of known size of which only some + byte ranges have been fetched. Reading outside them raises + `_MissingRangeError`, so the caller can fetch the range and retry. + """ + + def __init__(self, size: int, segments: dict[int, bytes]) -> None: + self._size = size + self._segments = segments + self._pos = 0 + + def readable(self) -> bool: + return True + + def seekable(self) -> bool: + return True + + def tell(self) -> int: + return self._pos + + def seek(self, pos: int, whence: int = io.SEEK_SET) -> int: + if whence == io.SEEK_SET: + self._pos = pos + elif whence == io.SEEK_CUR: + self._pos += pos + else: + self._pos = self._size + pos + return self._pos + + def readinto(self, b: bytearray | memoryview) -> int: # type: ignore[override] + n = min(len(b), self._size - self._pos) + if n <= 0: + return 0 + for start, data in self._segments.items(): + if start <= self._pos and self._pos + n <= start + len(data): + offset = self._pos - start + b[:n] = data[offset : offset + n] + self._pos += n + return n + raise _MissingRangeError(self._pos, self._pos + n) + + +def _parse_central_directory(size: int, segments: dict[int, bytes]) -> list[zipfile.ZipInfo]: + with zipfile.ZipFile(_SparseFile(size, segments)) as zf: + return zf.infolist() + + +def _decompress(info: zipfile.ZipInfo, data: bytes) -> bytes: + if info.compress_type == zipfile.ZIP_STORED: + result = data + elif info.compress_type == zipfile.ZIP_DEFLATED: + result = zlib.decompress(data, -15) + elif info.compress_type == zipfile.ZIP_BZIP2: + result = bz2.decompress(data) + else: + raise NotImplementedError( + f"ZIP member {info.filename!r} uses compression method {info.compress_type}, " + "which zip: does not support (supported: stored, deflate, bzip2)" + ) + if zlib.crc32(result) != info.CRC: + raise OSError(f"bad CRC-32 for ZIP member {info.filename!r}") + return result + + +def _slice(data: bytes, byte_range: ByteRequest | None) -> bytes: + if byte_range is None: + return data + if isinstance(byte_range, RangeByteRequest): + return data[byte_range.start : byte_range.end] + if isinstance(byte_range, OffsetByteRequest): + return data[byte_range.offset :] + if isinstance(byte_range, SuffixByteRequest): + return data[max(0, len(data) - byte_range.suffix) :] + raise TypeError(f"Unexpected byte_range, got {byte_range}.") + + +class ZipReaderStore(Store): + """ + A read-only store over a ZIP archive that is itself a resource of another store. + + The archive is the value at `key` in `source` (the empty key addresses a + store rooted at a single object, such as a local file). It is read with + async range requests only. Use `open_archive` to create one; the store owns + `source` and closes it on `close`. + + Parameters + ---------- + source : Store + The store holding the archive. + key : str + The key of the archive in `source`. + size : int + The archive size in bytes. + infos : Iterable[zipfile.ZipInfo] + The archive's central directory entries. + """ + + supports_writes: bool = False + supports_deletes: bool = False + supports_listing: bool = True + + def __init__( + self, source: Store, key: str, size: int, infos: Iterable[zipfile.ZipInfo] + ) -> None: + super().__init__(read_only=True) + self._source = source + self._key = key + self._size = size + # zarr keys are files: directory entries carry no data + self._infos = {info.filename: info for info in infos if not info.is_dir()} + self._data_offsets: dict[str, int] = {} + self._is_open = True + + @classmethod + async def open_archive(cls, source: Store, key: str = "") -> ZipReaderStore: + """ + Read the central directory of the archive at `key` in `source`. + + Parameters + ---------- + source : Store + The store holding the archive. + key : str, optional + The key of the archive in `source`; empty for a store rooted at + the archive itself. + + Returns + ------- + ZipReaderStore + An open, read-only store over the archive. + + Raises + ------ + zipfile.BadZipFile + If the resource is not a ZIP archive. + FileNotFoundError + If there is no resource at `key`. + """ + size = await _resource_size(source, key) + tail = min(size, _EOCD_SEARCH_BYTES) + segments = { + size - tail: await _read_resource_range( + source, key, RangeByteRequest(size - tail, size) + ) + } + while True: + try: + # parsing a large central directory is CPU-bound + infos = await asyncio.to_thread(_parse_central_directory, size, segments) + except _MissingRangeError as missing: + stop = min(size, max(missing.stop, missing.start + _MIN_FETCH_BYTES)) + segments[missing.start] = await _read_resource_range( + source, key, RangeByteRequest(missing.start, stop) + ) + continue + return cls(source, key, size, infos) + + def __eq__(self, other: object) -> bool: + return ( + isinstance(other, type(self)) + and self._source == other._source + and self._key == other._key + ) + + def __str__(self) -> str: + return f"{self._source}|zip:" if not self._key else f"{self._source}/{self._key}|zip:" + + def __repr__(self) -> str: + return f"ZipReaderStore('{self}')" + + def close(self) -> None: + # docstring inherited + super().close() + self._source.close() + + async def _data_offset(self, info: zipfile.ZipInfo) -> int: + """The archive offset of a member's data, after its local header.""" + offset = self._data_offsets.get(info.filename) + if offset is None: + header = await _read_resource_range( + self._source, + self._key, + RangeByteRequest(info.header_offset, info.header_offset + _LOCAL_HEADER.size), + ) + fields = _LOCAL_HEADER.unpack(header) + if fields[0] != _LOCAL_HEADER_SIGNATURE: + raise zipfile.BadZipFile(f"bad local file header for {info.filename!r}") + name_length, extra_length = fields[-2], fields[-1] + offset = info.header_offset + _LOCAL_HEADER.size + name_length + extra_length + self._data_offsets[info.filename] = offset + return offset + + async def get( + self, + key: str, + prototype: BufferPrototype, + byte_range: ByteRequest | None = None, + ) -> Buffer | None: + # docstring inherited + info = self._infos.get(key) + if info is None: + return None + if info.flag_bits & 0x1: + raise NotImplementedError(f"ZIP member {key!r} is encrypted") + start = await self._data_offset(info) + if info.compress_type == zipfile.ZIP_STORED and byte_range is not None: + # uncompressed members support true range reads + length = info.file_size + if isinstance(byte_range, RangeByteRequest): + lo, hi = min(byte_range.start, length), min(byte_range.end, length) + elif isinstance(byte_range, OffsetByteRequest): + lo, hi = min(byte_range.offset, length), length + elif isinstance(byte_range, SuffixByteRequest): + lo, hi = max(0, length - byte_range.suffix), length + else: + raise TypeError(f"Unexpected byte_range, got {byte_range}.") + if hi <= lo: + return prototype.buffer.from_bytes(b"") + data = await _read_resource_range( + self._source, self._key, RangeByteRequest(start + lo, start + hi) + ) + return prototype.buffer.from_bytes(data) + raw = await _read_resource_range( + self._source, self._key, RangeByteRequest(start, start + info.compress_size) + ) + if info.compress_size > _THREAD_DECOMPRESS_BYTES: + data = await asyncio.to_thread(_decompress, info, raw) + else: + data = _decompress(info, raw) + return prototype.buffer.from_bytes(_slice(data, byte_range)) + + async def get_partial_values( + self, + prototype: BufferPrototype, + key_ranges: Iterable[tuple[str, ByteRequest | None]], + ) -> list[Buffer | None]: + # docstring inherited + return list( + await asyncio.gather( + *(self.get(key, prototype, byte_range) for key, byte_range in key_ranges) + ) + ) + + async def getsize(self, key: str) -> int: + # docstring inherited + info = self._infos.get(key) + if info is None: + raise FileNotFoundError(key) + return info.file_size + + async def exists(self, key: str) -> bool: + # docstring inherited + return key in self._infos + + async def set(self, key: str, value: Buffer) -> None: + # docstring inherited + self._check_writable() + + async def delete(self, key: str) -> None: + # docstring inherited + self._check_writable() + + async def list(self) -> AsyncIterator[str]: + # docstring inherited + for key in self._infos: + yield key + + async def list_prefix(self, prefix: str) -> AsyncIterator[str]: + # docstring inherited + for key in self._infos: + if key.startswith(prefix): + yield key + + async def list_dir(self, prefix: str) -> AsyncIterator[str]: + # docstring inherited + prefix = prefix.rstrip("/") + prefix = f"{prefix}/" if prefix else "" + seen: set[str] = set() + for key in self._infos: + if key.startswith(prefix): + child = key[len(prefix) :].split("/", 1)[0] + if child and child not in seen: + seen.add(child) + yield child + + +def _parse_zip_path(segment: PipelineSegment) -> str: + """The path within the archive addressed by a `zip:` segment.""" + if segment.query is not None: + raise URLPipelineError(f"'zip:' pipeline segments do not accept a query: {segment.raw!r}") + body = segment.body + if body.startswith("//"): + raise URLPipelineError( + f"invalid 'zip:' path in {segment.raw!r}: more than one leading '/' is not allowed" + ) + try: + return normalize_path(body.removeprefix("/")) + except ValueError as exc: + raise URLPipelineError(f"invalid path in pipeline segment {segment.raw!r}: {exc}") from exc + + +class ZipAdapter(URLPipelineAdapter): + """ + The `zip:` adapter: an entry or directory within a ZIP archive (read-only). + """ + + @classmethod + async def open_pipeline_segment( + cls, segment: PipelineSegment, context: PipelineContext + ) -> AdapterResolution: + """ + Resolve a `zip:` segment. + + Parameters + ---------- + segment : PipelineSegment + The `zip:` segment. Its body is a path within the archive. + context : PipelineContext + The pipeline to the left of the segment, which must resolve to the archive. + + Returns + ------- + AdapterResolution + A read-only store over the archive, with the segment path as the residual path. + + Raises + ------ + URLPipelineError + If the mode requires writing, the segment is malformed, or the preceding + resource is not a readable ZIP archive. + """ + if context.mode in _WRITE_MODES: + raise URLPipelineError( + f"zip: is read-only until writable ZIP support lands; cannot open " + f"{segment.raw!r} with mode={context.mode!r}. Use mode='r' (or 'a' to open " + "an existing archive), or write the archive with zarr.storage.ZipStore." + ) + path = _parse_zip_path(segment) + preceding = await context.resolve_preceding(mode="r") + try: + store = await ZipReaderStore.open_archive( + preceding.store, normalize_path(preceding.path) + ) + except (OSError, zipfile.BadZipFile, ValueError) as exc: + preceding.store.close() + raise URLPipelineError( + f"could not open the ZIP archive {context.preceding_url!r} for " + f"{segment.raw!r}: {exc}" + ) from exc + except BaseException: + preceding.store.close() + raise + return dataclasses.replace(preceding, store=store, path=path) diff --git a/src/zarr/storage/_url_pipeline.py b/src/zarr/storage/_url_pipeline.py new file mode 100644 index 0000000000..b649244ebc --- /dev/null +++ b/src/zarr/storage/_url_pipeline.py @@ -0,0 +1,337 @@ +""" +Parsing and resolution of URL pipelines (https://github.com/jbms/url-pipeline). + +Importing this module is cheap and has no side effects: third-party adapters +(registered through the `zarr.url_adapters` entry-point group) are loaded +only when a pipeline URL naming their scheme is actually resolved. + +The `|` character is reserved as the pipeline delimiter in every string +store specification: a string containing `|` is always routed through the +pipeline machinery, and no percent-escape is decoded. To address a local +file whose *name* contains `|` (or `#`), pass a `pathlib.Path` instead of a +string. + +As a zarr-python extension to the specification (which requires the root +sub-URL of an absolute pipeline to carry a scheme), the root sub-URL may be +a schemeless local filesystem path, e.g. `data/example.zip|zip:`. Schemeless +roots are treated as opaque text — no query or fragment splitting is applied +to them. Such pipelines are not portable to other URL pipeline +implementations; portable pipelines should spell the root as a `file:` URL. +""" + +from __future__ import annotations + +import dataclasses +import re +from typing import TYPE_CHECKING, Any + +from zarr.abc.url_pipeline import ( + AdapterResolution, + PipelineContext, + PipelineSegment, +) +from zarr.errors import URLPipelineError +from zarr.registry import _BUILTIN_URL_ADAPTERS, get_url_adapter, list_url_adapter_schemes +from zarr.storage._memory import ManagedMemoryStore +from zarr.storage._utils import parse_store_url + +if TYPE_CHECKING: + from zarr.abc.store import Store + from zarr.core.common import AccessModeLiteral + +__all__ = ["is_url_pipeline", "parse_pipeline", "resolve_pipeline"] + +# Adapter scheme per RFC 3986 plus "." to permit vendor-prefixed +# nonstandard schemes (e.g. "earthmover.myscheme"). +_SCHEME_RE = re.compile(r"^[a-zA-Z][a-zA-Z0-9+.\-]*$") + +# Root scheme at the very start of the sub-URL. The negative lookahead +# excludes fsspec's chained-URL syntax (``zip::file://...``), which is not a +# URL pipeline and keeps flowing through the fsspec machinery. +_ROOT_SCHEME_RE = re.compile(r"([a-zA-Z][a-zA-Z0-9+.\-]*):(?!:)") + +# Schemes zarr resolves natively at the pipeline root. These are never +# dispatched to a registered root adapter, so an installed package cannot +# intercept zarr's own local-path and in-memory routing. +_NATIVE_ROOT_SCHEMES = frozenset({"file", "memory"}) + +# Adapter schemes zarr ships (e.g. `zarr3:`). The spec defines these as adapter +# schemes only, so they are never dispatched to as a pipeline root: a string +# without `|` such as `zarr3:foo` keeps its pre-pipeline meaning. +_BUILTIN_ADAPTER_SCHEMES = frozenset(_BUILTIN_URL_ADAPTERS) + +# An absolute Windows drive path (C:\... or C:/...), which counts as an +# absolute path in a `file:` pipeline root. +_WINDOWS_DRIVE_RE = re.compile(r"[A-Za-z]:[/\\]") + + +def _root_scheme(root_sub_url: str) -> str: + """ + Detect the scheme of the root sub-URL, or `""` for an opaque root. + + Scheme extraction happens on the raw string (so nothing `urlparse` + would strip or reject — whitespace, exotic authorities — can desync the + detected scheme from the body). Single-letter candidates are delegated + to `parse_store_url`, which knows Windows drive letters are not schemes. + """ + match = _ROOT_SCHEME_RE.match(root_sub_url) + if match is None: + return "" + scheme = match.group(1) + if len(scheme) == 1: + try: + return parse_store_url(root_sub_url).scheme.lower() + except ValueError: + return scheme.lower() + return scheme.lower() + + +def _split_query(sub_url: str) -> tuple[str, str | None]: + """Split a sub-URL on the first `?`. Fragments are not supported.""" + if "#" in sub_url: + raise URLPipelineError( + f"URL pipeline sub-URLs do not support fragments: {sub_url!r}. " + "Percent-encode '#' as '%23' if it is part of the path." + ) + body, sep, query = sub_url.partition("?") + return body, query if sep else None + + +def parse_pipeline(url: str) -> tuple[PipelineSegment, ...]: + """ + Parse a URL pipeline into its `|`-delimited segments. + + The first segment is the *root* sub-URL; its scheme is detected with the + same rules as ordinary store URLs (Windows drive letters are not + schemes). Subsequent segments are *adapter* sub-URLs of the form + `scheme:body` where the trailing colon is optional when the body is + empty (`zip` is equivalent to `zip:`). + + A schemeless root (a bare local path) is accepted as a zarr-python + extension to the specification and treated as opaque text: no query or + fragment splitting applies, since `?` and `#` are ordinary filename + characters there. See the module docstring. + + Segment text is preserved verbatim (no case or percent-encoding + normalization) except that schemes are lowercased. + """ + parts = url.split("|") + if any(not part for part in parts): + raise URLPipelineError(f"URL pipeline contains an empty sub-URL: {url!r}") + + segments: list[PipelineSegment] = [] + for index, part in enumerate(parts): + if index == 0: + scheme = _root_scheme(part) + if not scheme: + # opaque root (bare local path, fsspec chained URL, ...) + segments.append(PipelineSegment(scheme="", body=part, query=None, raw=part)) + continue + body_and_scheme, query = _split_query(part) + body = body_and_scheme[len(scheme) + 1 :] + segments.append(PipelineSegment(scheme=scheme, body=body, query=query, raw=part)) + else: + body_and_scheme, query = _split_query(part) + scheme, _, body = body_and_scheme.partition(":") + scheme = scheme.lower() + if not _SCHEME_RE.match(scheme): + raise URLPipelineError( + f"invalid adapter scheme {scheme!r} in pipeline segment {part!r}" + ) + segments.append(PipelineSegment(scheme=scheme, body=body, query=query, raw=part)) + return tuple(segments) + + +def _root_routes_to_adapter(scheme: str) -> bool: + """ + Whether a root sub-URL with this scheme is dispatched to a registered + root adapter. Schemes zarr resolves natively (`file:`, `memory:`), the + builtin adapter-only schemes (`zarr:`, `zarr3:`, ...) and opaque roots are + excluded; the registry check inspects entry-point names only — no adapter + code is imported here. + """ + return ( + bool(scheme) + and scheme not in _NATIVE_ROOT_SCHEMES + and scheme not in _BUILTIN_ADAPTER_SCHEMES + and scheme in list_url_adapter_schemes() + ) + + +def is_url_pipeline(url: str) -> bool: + """ + Whether `url` should be routed through the URL pipeline machinery. + + True when the URL contains a `|` separator, or when its scheme has a + registered URL pipeline adapter (a *root adapter* such as `gh:`). + fsspec chained URLs (`zip::file://...`) and zarr's native `file:` / + `memory:` schemes are never routed to a root adapter. + """ + if "|" in url: + return True + return _root_routes_to_adapter(_root_scheme(url)) + + +async def resolve_pipeline( + url: str, + *, + mode: AccessModeLiteral | None = None, + storage_options: dict[str, Any] | None = None, +) -> AdapterResolution: + """ + Resolve a URL pipeline into a store and a residual path. + + Parameters + ---------- + url : str + A URL pipeline, e.g. `"s3://bucket/data.zip|zip:|zarr3:"`. + mode : AccessModeLiteral | None + The caller's access mode. `"r"` requires a read-only store; the + resolver enforces this on whatever the final adapter returns. + storage_options : dict | None + Options forwarded to the root sub-URL's store (and visible to + adapters via the context). Non-dict forms are reserved for future + per-segment configuration (one mapping per pipeline segment). + + Raises + ------ + URLPipelineError + If the URL cannot be parsed, is not a pipeline, names a scheme with + no registered adapter, names an adapter entry point that fails to + import, or has a root sub-URL that cannot be resolved. + TypeError + If `storage_options` are passed to a root that does not accept them + (local paths and `memory:`), as for non-pipeline stores. + OSError + Errors from opening the root resource (e.g. a missing local directory + in mode `"r"`) propagate unchanged, as for non-pipeline stores. + """ + segments = parse_pipeline(url) + if len(segments) == 1 and not _root_routes_to_adapter(segments[0].scheme): + raise URLPipelineError( + f"{url!r} is not a URL pipeline: it has no '|' separator and no " + f"URL pipeline adapter is registered for scheme {segments[0].scheme!r}" + ) + return await _resolve(segments, mode=mode, storage_options=storage_options) + + +async def _resolve( + segments: tuple[PipelineSegment, ...], + *, + mode: AccessModeLiteral | None, + storage_options: dict[str, Any] | None, +) -> AdapterResolution: + if len(segments) == 1 and not _root_routes_to_adapter(segments[0].scheme): + return AdapterResolution( + store=await _resolve_root(segments[0], mode=mode, storage_options=storage_options) + ) + + *preceding, last = segments + try: + adapter_cls = get_url_adapter(last.scheme) + except URLPipelineError as exc: + raise URLPipelineError( + f"{exc} Note: '|' is reserved as the URL pipeline delimiter; to " + "address a local file whose name contains '|', pass a " + "pathlib.Path instead of a string." + ) from None + + context = PipelineContext( + preceding=tuple(preceding), + mode=mode, + storage_options=storage_options, + ) + resolution = await adapter_cls.open_pipeline_segment(last, context) + if mode == "r" and not resolution.store.read_only: + # The caller required read-only; enforce it rather than trusting + # the adapter to have honored context.read_only. + try: + read_only_store = resolution.store.with_read_only(True) + await read_only_store._ensure_open() + except NotImplementedError as exc: + resolution.store.close() + raise URLPipelineError( + f"adapter {last.scheme!r} returned a writable store for mode 'r', " + "and the store does not support read-only conversion via " + ".with_read_only()" + ) from exc + except Exception: + resolution.store.close() + raise + resolution = dataclasses.replace(resolution, store=read_only_store) + return resolution + + +async def _resolve_root( + segment: PipelineSegment, + *, + mode: AccessModeLiteral | None, + storage_options: dict[str, Any] | None, +) -> Store: + """ + Resolve the root sub-URL of a pipeline into a store. + + `memory:` and `file:` roots are resolved here with the URL pipeline + spec's semantics (spelling equivalences, mandatory absolute `file:` + paths); everything else — bare local paths, fsspec URLs — delegates to + the existing `StoreLike` machinery unchanged. + """ + if segment.scheme == "memory": + return _resolve_memory_root(segment, mode=mode, storage_options=storage_options) + from zarr.storage._common import make_store # circular import + + if segment.scheme == "file": + # Per the spec: file://localhost/p and file:///p are equivalent to + # file:/p; other authorities are unsupported; relative paths are + # forbidden; file: URLs carry no query. + if segment.query is not None: + raise URLPipelineError(f"'file:' pipeline roots do not accept a query: {segment.raw!r}") + body = segment.body + if body.startswith("//"): + authority, sep, rest = body[2:].partition("/") + if authority not in ("", "localhost"): + raise URLPipelineError( + f"unsupported authority {authority!r} in 'file:' pipeline " + f"root {segment.raw!r}; only an empty authority or " + "'localhost' is allowed" + ) + body = f"/{rest}" if sep else "" + # a file URL spells a Windows drive path as /C:/...; strip the + # leading slash so the local-path machinery sees the drive + if body.startswith("/") and _WINDOWS_DRIVE_RE.match(body[1:]): + body = body[1:] + if not (body.startswith("/") or _WINDOWS_DRIVE_RE.match(body)): + raise URLPipelineError( + f"'file:' pipeline roots must carry an absolute path: {segment.raw!r}" + ) + return await make_store(f"file:{body}", mode=mode, storage_options=storage_options) + + try: + return await make_store(segment.raw, mode=mode, storage_options=storage_options) + except ValueError as exc: + raise URLPipelineError( + f"could not resolve the pipeline root {segment.raw!r}: {exc}" + ) from exc + + +def _resolve_memory_root( + segment: PipelineSegment, + *, + mode: AccessModeLiteral | None, + storage_options: dict[str, Any] | None, +) -> Store: + """ + Resolve a `memory:` pipeline root per the spec's spelling equivalences: + `memory:` ≡ `memory:/` ≡ `memory://`, and `memory:a` ≡ `memory:/a` ≡ + `memory://a`. The first path component names the managed store; the + remainder is a path within it. + """ + if segment.query is not None: + raise URLPipelineError(f"'memory:' pipeline roots do not accept a query: {segment.raw!r}") + if storage_options: + raise TypeError( + "'storage_options' was provided but unused. " + "'storage_options' is only used when the store is passed as an FSSpec URI string.", + ) + name, _, path = segment.body.lstrip("/").partition("/") + return ManagedMemoryStore(name=name, path=path, read_only=mode == "r") diff --git a/tests/package_with_entrypoint-0.1.dist-info/entry_points.txt b/tests/package_with_entrypoint-0.1.dist-info/entry_points.txt index 7eb0eb7c86..456c725b4e 100644 --- a/tests/package_with_entrypoint-0.1.dist-info/entry_points.txt +++ b/tests/package_with_entrypoint-0.1.dist-info/entry_points.txt @@ -13,4 +13,6 @@ another_ndbuffer = package_with_entrypoint:TestEntrypointGroup.NDBuffer [zarr.codec_pipeline] another_pipeline = package_with_entrypoint:TestEntrypointGroup.Pipeline [zarr.data_type] -new_data_type = package_with_entrypoint:TestDataType \ No newline at end of file +new_data_type = package_with_entrypoint:TestDataType +[zarr.url_adapters] +example-pkg.entrypoint-scheme = package_with_entrypoint:TestEntrypointURLAdapter diff --git a/tests/package_with_entrypoint/__init__.py b/tests/package_with_entrypoint/__init__.py index 23afcf1dc2..cad73d061d 100644 --- a/tests/package_with_entrypoint/__init__.py +++ b/tests/package_with_entrypoint/__init__.py @@ -7,6 +7,7 @@ import zarr.core.buffer from zarr.abc.codec import ArrayBytesCodec, CodecInput, CodecPipeline +from zarr.abc.url_pipeline import AdapterResolution, URLPipelineAdapter from zarr.codecs import BytesCodec from zarr.core.buffer import Buffer, NDBuffer from zarr.core.dtype.npy.bool import Bool @@ -16,6 +17,7 @@ from collections.abc import Iterable from typing import Any, ClassVar, Literal, Self + from zarr.abc.url_pipeline import PipelineContext, PipelineSegment from zarr.core.array_spec import ArraySpec from zarr.core.common import ZarrFormat from zarr.core.dtype.common import DTypeJSON, DTypeSpec_V2 @@ -100,3 +102,16 @@ def to_json(self, zarr_format: ZarrFormat) -> str | DTypeSpec_V2: # type: ignor if zarr_format == 3: return self._zarr_v3_name raise ValueError("zarr_format must be 2 or 3") + + +class TestEntrypointURLAdapter(URLPipelineAdapter): + """URL pipeline adapter discovered via the zarr.url_adapters entry point.""" + + @classmethod + async def open_pipeline_segment( + cls, segment: PipelineSegment, context: PipelineContext + ) -> AdapterResolution: + from zarr.storage import MemoryStore + + store = await MemoryStore.open(read_only=context.read_only) + return AdapterResolution(store=store, path=segment.body) diff --git a/tests/test_api.py b/tests/test_api.py index f874bd45b7..6a2c4f85f8 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -311,9 +311,12 @@ async def test_open_like_creates_array_by_default( assert np.all(Array(new_arr)[:] == ref_arr.fill_value) -async def test_open_like_default_mode_rejects_read_only_store( +async def test_open_like_default_mode_on_read_only_store_raises_not_found( zarr_format: ZarrFormat, ) -> None: + # mode "a" (open-or-create) on a read-only store serves the "open" half: + # since the node does not exist and creating is impossible, the open + # fails with a not-found error rather than rejecting the mode upfront. ref_arr = zarr.create_array( store={}, shape=(11, 12), @@ -322,7 +325,7 @@ async def test_open_like_default_mode_rejects_read_only_store( zarr_format=zarr_format, ) - with pytest.raises(ValueError, match="Store is read-only but mode is 'a'"): + with pytest.raises(ValueError, match="No array found"): await zarr.api.asynchronous.open_like( ref_arr, path="foo", diff --git a/tests/test_url_pipeline/__init__.py b/tests/test_url_pipeline/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/test_url_pipeline/conftest.py b/tests/test_url_pipeline/conftest.py new file mode 100644 index 0000000000..13c717e04e --- /dev/null +++ b/tests/test_url_pipeline/conftest.py @@ -0,0 +1,22 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING + +import pytest + +import zarr.registry + +if TYPE_CHECKING: + from collections.abc import Generator + + +@pytest.fixture +def clean_url_adapter_registry() -> Generator[None, None, None]: + """Snapshot and restore the URL adapter registry around a test.""" + registry = zarr.registry._url_adapter_registry + saved = dict(registry) + saved_lazy = list(registry.lazy_load_list) + yield + registry.clear() + registry.update(saved) + registry.lazy_load_list[:] = saved_lazy diff --git a/tests/test_url_pipeline/test_format_adapters.py b/tests/test_url_pipeline/test_format_adapters.py new file mode 100644 index 0000000000..fa6deef29d --- /dev/null +++ b/tests/test_url_pipeline/test_format_adapters.py @@ -0,0 +1,317 @@ +"""Tests for the builtin `zarr:`, `zarr2:` and `zarr3:` URL pipeline adapters.""" + +from __future__ import annotations + +import dataclasses +import subprocess +import sys +import uuid +from importlib.metadata import EntryPoint +from typing import TYPE_CHECKING + +import numpy as np +import pytest + +import zarr +import zarr.registry +from zarr.abc.url_pipeline import ( + AdapterResolution, + PipelineContext, + PipelineSegment, + URLPipelineAdapter, +) +from zarr.errors import URLPipelineError, ZarrUserWarning +from zarr.registry import get_url_adapter, list_url_adapter_schemes, register_url_adapter +from zarr.storage import LocalStore, ManagedMemoryStore, WrapperStore +from zarr.storage._common import make_store_path +from zarr.storage._url_adapters._format import Zarr2Adapter, Zarr3Adapter, ZarrAdapter +from zarr.storage._url_pipeline import is_url_pipeline, resolve_pipeline +from zarr.storage._utils import _join_paths + +if TYPE_CHECKING: + from collections.abc import Generator + from pathlib import Path + + from zarr.core.common import ZarrFormat + +pytestmark = pytest.mark.usefixtures("clean_url_adapter_registry") + +FORMAT_SEGMENTS: list[tuple[str, ZarrFormat | None]] = [ + ("zarr2:", 2), + ("zarr3:", 3), + ("zarr:", None), +] + + +@pytest.fixture(params=["local", "memory"]) +def root(request: pytest.FixtureRequest, tmp_path: Path) -> Generator[str, None, None]: + """A pipeline root: a local directory, or a fresh named in-memory store.""" + if request.param == "local": + yield str(tmp_path / "root") + return + # a named managed memory store lives only while something references it, + # so hold one for the duration of the test + name = uuid.uuid4().hex + keepalive = ManagedMemoryStore(name=name) + yield f"memory:{name}" + del keepalive + + +class _WrapAdapter(URLPipelineAdapter): + """A wrapper adapter that follows the documented wrapper contract.""" + + @classmethod + async def open_pipeline_segment( + cls, segment: PipelineSegment, context: PipelineContext + ) -> AdapterResolution: + preceding = await context.resolve_preceding() + return dataclasses.replace( + preceding, + store=WrapperStore(preceding.store), + path=_join_paths([preceding.path, segment.body]), + ) + + +def _expected(fmt: ZarrFormat | None) -> ZarrFormat: + """The format a node created through a segment ends up with.""" + return 3 if fmt is None else fmt + + +class TestRegistration: + def test_builtin_schemes_are_listed(self) -> None: + assert {"zarr", "zarr2", "zarr3"} <= list_url_adapter_schemes() + + @pytest.mark.parametrize( + ("scheme", "cls"), [("zarr", ZarrAdapter), ("zarr2", Zarr2Adapter), ("zarr3", Zarr3Adapter)] + ) + def test_builtin_lookup(self, scheme: str, cls: type[URLPipelineAdapter]) -> None: + assert get_url_adapter(scheme) is cls + assert get_url_adapter(scheme.upper()) is cls + + def test_import_zarr_does_not_import_adapters(self) -> None: + # builtins are lazily loaded entry points, like third-party adapters + code = ( + "import sys, zarr, zarr.storage\n" + "assert 'zarr.storage._url_adapters._format' not in sys.modules\n" + ) + subprocess.run([sys.executable, "-c", code], check=True) + + def test_third_party_entry_point_is_shadowed_with_warning(self) -> None: + registry = zarr.registry._url_adapter_registry + registry.pop("zarr3", None) + registry.lazy_load_list[:] = [ + zarr.registry._builtin_url_adapter_entry_point("zarr3"), + *(e for e in registry.lazy_load_list if e.name != "zarr3"), + EntryPoint(name="zarr3", value="other_pkg:Adapter", group="zarr.url_adapters"), + ] + with pytest.warns(ZarrUserWarning, match="builtin URL pipeline adapter.*takes precedence"): + assert get_url_adapter("zarr3") is Zarr3Adapter + assert not any(e.name == "zarr3" for e in registry.lazy_load_list) + + def test_explicit_registration_overrides_builtin_silently(self) -> None: + # an explicit register_url_adapter call is deliberate: the pending + # builtin entry point is dropped without a warning + registry = zarr.registry._url_adapter_registry + registry.pop("zarr3", None) + registry.lazy_load_list[:] = [ + zarr.registry._builtin_url_adapter_entry_point("zarr3"), + *(e for e in registry.lazy_load_list if e.name != "zarr3"), + ] + register_url_adapter("zarr3", ZarrAdapter) + import warnings + + with warnings.catch_warnings(): + warnings.simplefilter("error") + assert get_url_adapter("zarr3") is ZarrAdapter + + def test_collect_entrypoints_does_not_duplicate_builtins(self) -> None: + zarr.registry._collect_entrypoints() + zarr.registry._collect_entrypoints() + pending = [ + e + for e in zarr.registry._url_adapter_registry.lazy_load_list + if zarr.registry._is_builtin_url_adapter_entry_point(e) + ] + assert len(pending) == len({e.name for e in pending}) + + @pytest.mark.parametrize("url", ["zarr:foo", "zarr2:foo", "zarr3://foo", "ZARR3:foo"]) + def test_builtin_schemes_are_not_root_schemes(self, url: str) -> None: + # a string without '|' keeps its pre-pipeline meaning + assert not is_url_pipeline(url) + + +class TestResolution: + @pytest.mark.parametrize(("segment", "fmt"), FORMAT_SEGMENTS) + async def test_sets_format(self, tmp_path: Path, segment: str, fmt: ZarrFormat | None) -> None: + result = await resolve_pipeline(f"{tmp_path}|{segment}") + assert isinstance(result.store, LocalStore) + assert result.path == "" + assert result.zarr_format == fmt + + @pytest.mark.parametrize("scheme", ["zarr", "zarr2", "zarr3"]) + async def test_colon_is_optional(self, tmp_path: Path, scheme: str) -> None: + with_colon = await resolve_pipeline(f"{tmp_path}|{scheme}:") + without = await resolve_pipeline(f"{tmp_path}|{scheme}") + assert with_colon.zarr_format == without.zarr_format + + async def test_body_is_a_path_within_the_preceding_resource(self, tmp_path: Path) -> None: + result = await resolve_pipeline(f"{tmp_path}|zarr3:a/b") + assert result.path == "a/b" + assert result.zarr_format == 3 + + @pytest.mark.parametrize("body", ["/a/b", "a/b/", "/a//b/"]) + async def test_body_is_normalized(self, tmp_path: Path, body: str) -> None: + result = await resolve_pipeline(f"{tmp_path}|zarr3:{body}") + assert result.path == "a/b" + + async def test_body_joins_onto_preceding_residual_path(self) -> None: + # memory:name/sub roots carry their path inside the store; a wrapper + # carries a residual path; both are kept + register_url_adapter("wrap", _WrapAdapter) + result = await resolve_pipeline("memory:fmt-join/sub|wrap:w|zarr2:a/b") + assert result.path == "w/a/b" + assert result.zarr_format == 2 + + async def test_intermediate_segment(self) -> None: + # a format segment may precede another adapter, which carries the + # format and the residual path forward + register_url_adapter("wrap", _WrapAdapter) + result = await resolve_pipeline("memory:fmt-mid|zarr2:a|wrap:b") + assert isinstance(result.store, WrapperStore) + assert result.path == "a/b" + assert result.zarr_format == 2 + + async def test_keeps_other_fields(self) -> None: + # the adapter uses dataclasses.replace, so the preceding store is kept + result = await resolve_pipeline("memory:fmt-keep|zarr3:") + assert isinstance(result.store, ManagedMemoryStore) + + async def test_auto_keeps_an_earlier_format(self, tmp_path: Path) -> None: + result = await resolve_pipeline(f"{tmp_path}|zarr2:a|zarr:b") + assert result.zarr_format == 2 + assert result.path == "a/b" + + async def test_conflicting_format_segments_raise(self, tmp_path: Path) -> None: + with pytest.raises(URLPipelineError, match="earlier segment selected Zarr format 2"): + await resolve_pipeline(f"{tmp_path}|zarr2:|zarr3:") + + async def test_query_is_rejected(self, tmp_path: Path) -> None: + with pytest.raises(URLPipelineError, match="do not accept a query"): + await resolve_pipeline(f"{tmp_path}|zarr3:a?x=1") + + @pytest.mark.parametrize("body", ["..", "a/../b", "./a"]) + async def test_invalid_path_is_rejected(self, tmp_path: Path, body: str) -> None: + with pytest.raises(URLPipelineError, match="invalid path"): + await resolve_pipeline(f"{tmp_path}|zarr3:{body}") + + async def test_mode_is_forwarded(self, tmp_path: Path) -> None: + zarr.create_group(tmp_path) + store_path = await make_store_path(f"{tmp_path}|zarr3:", mode="r") + assert store_path.read_only + assert store_path.zarr_format == 3 + + +class TestEndToEnd: + @pytest.mark.parametrize(("segment", "fmt"), FORMAT_SEGMENTS) + def test_open_group(self, root: str, segment: str, fmt: ZarrFormat | None) -> None: + group = zarr.open_group(f"{root}|{segment}", mode="w") + assert group.metadata.zarr_format == _expected(fmt) + group.create_array("x", shape=(3,), dtype="i4")[:] = [1, 2, 3] + reopened = zarr.open_group(f"{root}|{segment}", mode="r") + assert reopened.metadata.zarr_format == _expected(fmt) + arr = reopened["x"] + assert isinstance(arr, zarr.Array) + np.testing.assert_array_equal(arr[:], [1, 2, 3]) + + @pytest.mark.parametrize(("segment", "fmt"), FORMAT_SEGMENTS) + def test_open(self, root: str, segment: str, fmt: ZarrFormat | None) -> None: + group = zarr.open(f"{root}|{segment}", mode="w") + assert isinstance(group, zarr.Group) + assert group.metadata.zarr_format == _expected(fmt) + arr = zarr.open(f"{root}|{segment}x", mode="w", shape=(2,), dtype="i4") + assert isinstance(arr, zarr.Array) + assert arr.metadata.zarr_format == _expected(fmt) + reopened = zarr.open(f"{root}|{segment}x", mode="r") + assert isinstance(reopened, zarr.Array) + assert reopened.metadata.zarr_format == _expected(fmt) + + @pytest.mark.parametrize(("segment", "fmt"), FORMAT_SEGMENTS) + def test_create_and_open_array(self, root: str, segment: str, fmt: ZarrFormat | None) -> None: + arr = zarr.create_array(f"{root}|{segment}", name="a/b", shape=(4,), dtype="i4") + assert arr.metadata.zarr_format == _expected(fmt) + arr[:] = np.arange(4) + opened = zarr.open_array(f"{root}|{segment}a/b", mode="r") + assert opened.metadata.zarr_format == _expected(fmt) + np.testing.assert_array_equal(opened[:], np.arange(4)) + # the user-supplied path joins onto the segment path + via_path = zarr.open_array(f"{root}|{segment}a", path="b", mode="r") + np.testing.assert_array_equal(via_path[:], np.arange(4)) + + @pytest.mark.parametrize(("segment", "fmt"), FORMAT_SEGMENTS) + def test_group_and_array_open_classmethods( + self, root: str, segment: str, fmt: ZarrFormat | None + ) -> None: + zarr.create_array(f"{root}|{segment}", name="x", shape=(2,), dtype="i4") + group = zarr.Group.open(f"{root}|{segment}") + assert group.metadata.zarr_format == _expected(fmt) + arr = zarr.Array.open(f"{root}|{segment}x") + assert arr.metadata.zarr_format == _expected(fmt) + + @pytest.mark.parametrize("stored", [2, 3]) + def test_auto_detects_existing_format(self, root: str, stored: ZarrFormat) -> None: + zarr.create_array(f"{root}|zarr{stored}:", name="x", shape=(2,), dtype="i4") + assert zarr.open_group(f"{root}|zarr:", mode="r").metadata.zarr_format == stored + assert zarr.open_array(f"{root}|zarr:x", mode="r").metadata.zarr_format == stored + assert zarr.Group.open(f"{root}|zarr:").metadata.zarr_format == stored + assert zarr.Array.open(f"{root}|zarr:x").metadata.zarr_format == stored + + def test_stacked_segment_paths(self, root: str) -> None: + zarr.create_array(f"{root}|zarr3:a/b", name="c", shape=(2,), dtype="i4") + assert zarr.open_array(f"{root}|zarr3:a/b/c", mode="r").shape == (2,) + group = zarr.open_group(f"{root}|zarr3:a", mode="r") + assert "b" in group + + def test_wrong_format_is_not_found(self, root: str) -> None: + zarr.create_group(f"{root}|zarr2:") + with pytest.raises(FileNotFoundError): + zarr.open_group(f"{root}|zarr3:", mode="r") + + @pytest.mark.parametrize( + ("segment", "explicit"), [("zarr2:", 3), ("zarr3:", 2)], ids=["zarr2-vs-3", "zarr3-vs-2"] + ) + def test_explicit_format_conflict(self, root: str, segment: str, explicit: ZarrFormat) -> None: + url = f"{root}|{segment}" + with pytest.raises(ValueError, match="conflicts with"): + zarr.open_group(url, mode="w", zarr_format=explicit) + with pytest.raises(ValueError, match="conflicts with"): + zarr.create_array(url, name="x", shape=(1,), dtype="i4", zarr_format=explicit) + with pytest.raises(ValueError, match="conflicts with"): + zarr.Group.open(url, zarr_format=explicit) + + def test_explicit_format_with_auto_segment(self, root: str) -> None: + # zarr: selects no format, so an explicit format is not a conflict + group = zarr.open_group(f"{root}|zarr:", mode="w", zarr_format=2) + assert group.metadata.zarr_format == 2 + + +ZARR_SPEC_EXAMPLES: list[tuple[str, ZarrFormat | None]] = [ + ("file:///path/to/node.zarr/|zarr3:", 3), + ("file:///path/to/node.zarr/|zarr3", 3), + ("file:///path/to/node.zarr/|zarr2:", 2), + ("file:///path/to/node.zarr/|zarr2", 2), + ("file:///path/to/node.zarr/|zarr:", None), + ("file:///path/to/node.zarr/|zarr", None), +] + + +@pytest.mark.parametrize(("example", "fmt"), ZARR_SPEC_EXAMPLES) +def test_zarr_scheme_spec_examples(tmp_path: Path, example: str, fmt: ZarrFormat | None) -> None: + """Every example in the spec's `schemes/zarr.md` resolves and opens.""" + posix = tmp_path.as_posix() + url_path = posix if posix.startswith("/") else f"/{posix}" + url = example.replace("/path/to", url_path, 1) + stored: ZarrFormat = 3 if fmt is None else fmt + zarr.create_group(tmp_path / "node.zarr", zarr_format=stored) + group = zarr.open_group(url, mode="r") + assert group.metadata.zarr_format == stored + assert zarr.Group.open(url).metadata.zarr_format == stored diff --git a/tests/test_url_pipeline/test_parser.py b/tests/test_url_pipeline/test_parser.py new file mode 100644 index 0000000000..13d7846e85 --- /dev/null +++ b/tests/test_url_pipeline/test_parser.py @@ -0,0 +1,232 @@ +from __future__ import annotations + +import sys + +import pytest +from hypothesis import given +from hypothesis import strategies as st + +from zarr.errors import URLPipelineError +from zarr.storage._url_pipeline import parse_pipeline + + +def test_single_root_url() -> None: + (segment,) = parse_pipeline("s3://bucket/key") + assert segment.scheme == "s3" + assert segment.body == "//bucket/key" + assert segment.query is None + assert segment.raw == "s3://bucket/key" + assert str(segment) == "s3://bucket/key" + + +def test_schemeless_root() -> None: + (segment,) = parse_pipeline("/local/path") + assert segment.scheme == "" + assert segment.body == "/local/path" + assert segment.raw == "/local/path" + + +def test_adapter_chain() -> None: + segments = parse_pipeline("s3://bucket/data.zip|zip:inner/path|zarr3:") + assert [s.scheme for s in segments] == ["s3", "zip", "zarr3"] + assert segments[1].body == "inner/path" + assert segments[2].body == "" + + +def test_trailing_colon_optional() -> None: + with_colon = parse_pipeline("file:/tmp/x.zip|zip:") + without_colon = parse_pipeline("file:/tmp/x.zip|zip") + assert with_colon[1].scheme == without_colon[1].scheme == "zip" + assert with_colon[1].body == without_colon[1].body == "" + + +def test_scheme_case_insensitive() -> None: + segments = parse_pipeline("FILE:/tmp/x.zip|ZIP:Inner/Path") + assert segments[0].scheme == "file" + assert segments[1].scheme == "zip" + # bodies are case-preserved + assert segments[1].body == "Inner/Path" + + +def test_case_preserved_in_raw() -> None: + # e.g. icechunk snapshot IDs are case-significant + segments = parse_pipeline("file:/tmp/repo|icechunk://ABCDEFGH12345678ABCD/x") + assert segments[1].raw == "icechunk://ABCDEFGH12345678ABCD/x" + assert segments[1].body == "//ABCDEFGH12345678ABCD/x" + + +def test_vendor_prefixed_scheme() -> None: + segments = parse_pipeline("file:/data|vendor-1.custom+adapter:sub/path") + assert segments[1].scheme == "vendor-1.custom+adapter" + assert segments[1].body == "sub/path" + + +def test_query_strings() -> None: + segments = parse_pipeline("https://example.com/d.zip?token=abc|zip:x?opt=1") + assert segments[0].query == "token=abc" + assert segments[0].body == "//example.com/d.zip" + assert segments[1].query == "opt=1" + assert segments[1].body == "x" + + +def test_empty_query() -> None: + (segment,) = parse_pipeline("https://example.com/d?") + assert segment.query == "" + + +def test_windows_drive_path_is_not_a_scheme() -> None: + # On Windows parse_store_url treats C:\... as a local path; elsewhere the + # single-letter scheme is preserved but must not crash the parser. + (segment,) = parse_pipeline(r"C:\data\store") + assert segment.raw == r"C:\data\store" + if sys.platform == "win32": + assert segment.scheme == "" + assert segment.body == r"C:\data\store" + else: + assert segment.scheme == "c" + + +@pytest.mark.parametrize("url", ["a||b:", "|zip:", "file:/tmp|", ""]) +def test_empty_sub_url_rejected(url: str) -> None: + with pytest.raises(URLPipelineError, match="empty sub-URL"): + parse_pipeline(url) + + +def test_fragment_rejected() -> None: + with pytest.raises(URLPipelineError, match="fragment"): + parse_pipeline("file:/tmp/x.zip|zip:inner#frag") + + +@pytest.mark.parametrize("segment", ["1zip:", "zi p:x", "zip@:x"]) +def test_invalid_adapter_scheme_rejected(segment: str) -> None: + with pytest.raises(URLPipelineError, match="invalid adapter scheme"): + parse_pipeline(f"file:/tmp/x|{segment}") + + +def test_single_letter_scheme_with_exotic_authority() -> None: + # single-letter candidates are delegated to parse_store_url, which may + # reject an exotic authority; the raw scheme is used as the fallback + segments = parse_pipeline("x://[authority]/path|zip:") + assert segments[0].scheme in ("", "x") + assert segments[0].raw == "x://[authority]/path" + + +def test_fsspec_chained_root_is_opaque() -> None: + # fsspec's ``scheme::`` chaining is not pipeline syntax; such roots are + # schemeless/opaque and flow through the ordinary store machinery + (segment,) = parse_pipeline("zip::file:///tmp/data.zip") + assert segment.scheme == "" + assert segment.body == "zip::file:///tmp/data.zip" + + +def test_whitespace_prefixed_root_is_opaque() -> None: + # urlparse strips leading whitespace when sniffing a scheme; the parser + # must not, or the detected scheme desyncs from the body slice + segments = parse_pipeline("\tfile:/tmp/x|zip:") + assert segments[0].scheme == "" + assert segments[0].body == "\tfile:/tmp/x" + + +def test_schemeless_root_keeps_query_and_fragment_chars() -> None: + # ? and # are ordinary filename characters in a bare local path + segments = parse_pipeline("/tmp/d?v=1|zip:") + assert segments[0].scheme == "" + assert segments[0].body == "/tmp/d?v=1" + assert segments[0].query is None + segments = parse_pipeline("/tmp/d#frag|zip:") + assert segments[0].body == "/tmp/d#frag" + + +def test_round_trip() -> None: + url = "s3://bucket/a.zip?v=2|zip:b/inner.zip|zip:c|zarr3:" + segments = parse_pipeline(url) + assert "|".join(s.raw for s in segments) == url + + +# --- property-based tests ----------------------------------------------- +# +# These exercise the parser's *splitting invariants*, not URL pipeline spec +# validity: parse_pipeline is a permissive segment splitter (bodies and +# queries are opaque text to it), and semantic validation is left to the +# resolver and the adapters. Rather than enumerating registered schemes, +# the strategies sample the full scheme grammar the parser accepts +# (RFC 3986 plus "." for vendor prefixes), so every valid root/adapter +# scheme is reachable. Bodies and queries draw from printable ASCII minus +# the characters the parser itself splits on ("|", "#", and "?" for +# bodies); the generated text is not required to be a well-formed URI. + +_ADAPTER_SCHEME = st.from_regex(r"[a-zA-Z][a-zA-Z0-9+.\-]{0,15}", fullmatch=True) +# two or more characters, so Windows drive-letter handling cannot reclassify +# the root scheme as a local path on one platform but not another +_ROOT_SCHEME = st.from_regex(r"[a-zA-Z]{2}[a-zA-Z0-9+.\-]{0,14}", fullmatch=True) +_BODY = st.text(st.characters(min_codepoint=32, max_codepoint=126, exclude_characters="|#?")) +# a root body starting with ":" would spell fsspec's chained-URL syntax +# ("scheme::..."), which the parser deliberately treats as an opaque root +_ROOT_BODY = _BODY.filter(lambda body: not body.startswith(":")) +_QUERY = st.text(st.characters(min_codepoint=32, max_codepoint=126, exclude_characters="|#")) + + +@st.composite +def pipelines( + draw: st.DrawFn, min_depth: int = 1 +) -> tuple[str, list[tuple[str, str, str | None, str]]]: + """A parseable pipeline URL up to depth 8, with its expected split.""" + depth = draw(st.integers(min_value=min_depth, max_value=8)) + expected = [] + parts = [] + for index in range(depth): + scheme = draw(_ROOT_SCHEME if index == 0 else _ADAPTER_SCHEME) + body = draw(_ROOT_BODY if index == 0 else _BODY) + query = draw(st.none() | _QUERY) + raw = f"{scheme}:{body}" + (f"?{query}" if query is not None else "") + parts.append(raw) + expected.append((scheme.lower(), body, query, raw)) + return "|".join(parts), expected + + +@given(pipelines()) +def test_valid_pipeline_invariants( + case: tuple[str, list[tuple[str, str, str | None, str]]], +) -> None: + url, expected = case + segments = parse_pipeline(url) + # schemes lowercased; bodies and queries preserved verbatim + assert [(s.scheme, s.body, s.query, s.raw) for s in segments] == expected + # lossless round trip + assert "|".join(s.raw for s in segments) == url + + +@given(pipelines(), st.data()) +def test_property_empty_sub_url_rejected( + case: tuple[str, list[tuple[str, str, str | None, str]]], data: st.DataObject +) -> None: + url, _ = case + parts = url.split("|") + parts.insert(data.draw(st.integers(0, len(parts))), "") + with pytest.raises(URLPipelineError, match="empty sub-URL"): + parse_pipeline("|".join(parts)) + + +@given(pipelines(min_depth=2), st.data()) +def test_property_invalid_adapter_scheme_rejected( + case: tuple[str, list[tuple[str, str, str | None, str]]], data: st.DataObject +) -> None: + url, _ = case + parts = url.split("|") + # corrupt one adapter segment's scheme with a leading character the + # scheme grammar forbids + index = data.draw(st.integers(1, len(parts) - 1)) + parts[index] = data.draw(st.sampled_from(["1", " ", "@", "~"])) + parts[index] + with pytest.raises(URLPipelineError, match="invalid adapter scheme"): + parse_pipeline("|".join(parts)) + + +@given(pipelines(), st.data()) +def test_property_fragment_rejected( + case: tuple[str, list[tuple[str, str, str | None, str]]], data: st.DataObject +) -> None: + url, _ = case + parts = url.split("|") + parts[data.draw(st.integers(0, len(parts) - 1))] += "#frag" + with pytest.raises(URLPipelineError, match="fragment"): + parse_pipeline("|".join(parts)) diff --git a/tests/test_url_pipeline/test_resolver.py b/tests/test_url_pipeline/test_resolver.py new file mode 100644 index 0000000000..531fd70e9b --- /dev/null +++ b/tests/test_url_pipeline/test_resolver.py @@ -0,0 +1,910 @@ +from __future__ import annotations + +import dataclasses +import sys +import threading +import warnings +from importlib.metadata import EntryPoint, version +from typing import TYPE_CHECKING, ClassVar + +import numpy as np +import pytest +from packaging.version import parse as parse_version + +import zarr +import zarr.registry +from zarr.abc.store import Store +from zarr.abc.url_pipeline import ( + AdapterResolution, + PipelineContext, + PipelineSegment, + URLPipelineAdapter, +) +from zarr.errors import URLPipelineError, ZarrUserWarning +from zarr.registry import ( + get_url_adapter, + list_url_adapter_schemes, + register_url_adapter, +) +from zarr.storage import ManagedMemoryStore, MemoryStore, WrapperStore +from zarr.storage._common import _has_fsspec, make_store, make_store_path +from zarr.storage._url_pipeline import is_url_pipeline, resolve_pipeline +from zarr.storage._utils import _join_paths + +# fsspec < 2024.12.0 has no AsyncFileSystemWrapper, so a plain memory:// URL +# (a sync filesystem) cannot be opened at all there +_has_async_fsspec_wrapper = _has_fsspec and parse_version(version("fsspec")) >= parse_version( + "2024.12.0" +) + +if TYPE_CHECKING: + from pathlib import Path + + from zarr.core.common import AccessModeLiteral + +pytestmark = pytest.mark.usefixtures("clean_url_adapter_registry") + + +class TracingStore(WrapperStore[Store]): + """Wrapper that records the context it was created from.""" + + context: PipelineContext + segment: PipelineSegment + + +class WrapperAdapter(URLPipelineAdapter): + """ + A wrapper-style adapter: resolves the preceding pipeline into a store. + + Follows the wrapper contract: the preceding resolution's residual path + is carried forward (joined with this segment's own path), and unchanged + fields survive via `dataclasses.replace`. + """ + + @classmethod + async def open_pipeline_segment( + cls, segment: PipelineSegment, context: PipelineContext + ) -> AdapterResolution: + preceding = await context.resolve_preceding() + store = TracingStore(preceding.store) + store.context = context + store.segment = segment + return dataclasses.replace( + preceding, store=store, path=_join_paths([preceding.path, segment.body]) + ) + + +class NativeAdapter(URLPipelineAdapter): + """A native-style adapter: consumes the preceding URL as a string.""" + + seen_urls: ClassVar[list[str]] = [] + + @classmethod + async def open_pipeline_segment( + cls, segment: PipelineSegment, context: PipelineContext + ) -> AdapterResolution: + cls.seen_urls.append(context.preceding_url) + store = await MemoryStore.open(read_only=context.read_only) + return AdapterResolution(store=store, path=segment.body) + + +class RootAdapter(URLPipelineAdapter): + """A root-scheme adapter (no preceding segments), like al://.""" + + @classmethod + async def open_pipeline_segment( + cls, segment: PipelineSegment, context: PipelineContext + ) -> AdapterResolution: + assert context.preceding == () + if context.mode in ("w", "w-", "r+"): + raise ValueError("read-only scheme") + store = await MemoryStore.open(read_only=True) + return AdapterResolution(store=store, path=segment.body.lstrip("/")) + + +class DisobedientAdapter(URLPipelineAdapter): + """An adapter that ignores `context.read_only` (a contract violation).""" + + @classmethod + async def open_pipeline_segment( + cls, segment: PipelineSegment, context: PipelineContext + ) -> AdapterResolution: + return AdapterResolution(store=await MemoryStore.open(read_only=False)) + + +class FormatAdapter(URLPipelineAdapter): + """A dummy format-selecting adapter, like the builtin `zarr2:`.""" + + @classmethod + async def open_pipeline_segment( + cls, segment: PipelineSegment, context: PipelineContext + ) -> AdapterResolution: + preceding = await context.resolve_preceding() + return dataclasses.replace(preceding, zarr_format=2) + + +class TestRegistry: + def test_register_and_get(self) -> None: + register_url_adapter("demo", WrapperAdapter) + assert get_url_adapter("demo") is WrapperAdapter + assert get_url_adapter("DEMO") is WrapperAdapter + assert "demo" in list_url_adapter_schemes() + + def test_unknown_scheme(self) -> None: + with pytest.raises(URLPipelineError, match="no URL pipeline adapter is registered"): + get_url_adapter("nonexistent-scheme") + + def test_reregistering_scheme_warns(self) -> None: + register_url_adapter("demo", WrapperAdapter) + with pytest.warns(ZarrUserWarning, match="is being replaced"): + register_url_adapter("demo", NativeAdapter) + assert get_url_adapter("demo") is NativeAdapter + # re-registering the same class is not a collision + register_url_adapter("demo", NativeAdapter) + + @pytest.mark.usefixtures("set_path") + def test_entrypoint_discovery(self) -> None: + assert "example-pkg.entrypoint-scheme" in list_url_adapter_schemes() + cls = get_url_adapter("example-pkg.entrypoint-scheme") + assert cls.__name__ == "TestEntrypointURLAdapter" + + @pytest.mark.usefixtures("set_path") + async def test_entrypoint_end_to_end(self) -> None: + result = await resolve_pipeline("memory://src|example-pkg.entrypoint-scheme:sub/path") + assert result.path == "sub/path" + + @pytest.mark.usefixtures("set_path") + def test_entrypoint_scheme_lookup_is_case_insensitive(self) -> None: + # entry-point names are matched case-insensitively, like schemes + cls = get_url_adapter("Example-PKG.Entrypoint-Scheme") + assert cls.__name__ == "TestEntrypointURLAdapter" + + @pytest.mark.usefixtures("set_path") + def test_loading_one_scheme_leaves_others_pending(self) -> None: + # resolving one scheme must not import other providers' entry points + registry = zarr.registry._url_adapter_registry + assert any(e.name == "example-pkg.entrypoint-scheme" for e in registry.lazy_load_list) + with pytest.raises(URLPipelineError, match="no URL pipeline adapter"): + get_url_adapter("some-other-scheme") + assert any(e.name == "example-pkg.entrypoint-scheme" for e in registry.lazy_load_list) + assert get_url_adapter("example-pkg.entrypoint-scheme").__name__ == ( + "TestEntrypointURLAdapter" + ) + + +def _fake_entry_point( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, *, scheme: str, module: str, source: str +) -> EntryPoint: + """ + Write `source` to an importable module and return a `zarr.url_adapters` + entry point for `scheme` pointing at its `Adapter` attribute. + """ + (tmp_path / f"{module}.py").write_text(source) + monkeypatch.syspath_prepend(str(tmp_path)) + monkeypatch.delitem(sys.modules, module, raising=False) + return EntryPoint(name=scheme, value=f"{module}:Adapter", group="zarr.url_adapters") + + +_ADAPTER_SOURCE = """ +from zarr.abc.url_pipeline import AdapterResolution, URLPipelineAdapter +from zarr.storage import MemoryStore +{prelude} + +class Adapter(URLPipelineAdapter): + @classmethod + async def open_pipeline_segment(cls, segment, context): + return AdapterResolution(store=await MemoryStore.open(read_only=context.read_only)) +""" + + +class TestEntryPointLoading: + """Lazy loading of `zarr.url_adapters` entry points.""" + + def test_adapter_import_may_resolve_other_adapters( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + # an adapter module that itself resolves a URL adapter at import + # time must not deadlock on the registry lock + pending = zarr.registry._url_adapter_registry.lazy_load_list + pending.append( + _fake_entry_point( + tmp_path, + monkeypatch, + scheme="reentrant", + module="reentrant_adapter", + source=_ADAPTER_SOURCE.format( + prelude=( + "import zarr.registry\n" + "try:\n" + " zarr.registry.get_url_adapter('no-such-scheme-at-import')\n" + "except Exception:\n" + " pass\n" + ) + ), + ) + ) + result: list[object] = [] + thread = threading.Thread(target=lambda: result.append(get_url_adapter("reentrant"))) + thread.start() + thread.join(timeout=10) + assert not thread.is_alive(), "get_url_adapter deadlocked on a re-entrant lookup" + assert getattr(result[0], "__name__", None) == "Adapter" + + def test_concurrent_loads_keep_other_schemes_pending( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + # two threads resolving different schemes at once, one with a slow + # import, must not drop a third, unrelated pending entry point + pending = zarr.registry._url_adapter_registry.lazy_load_list + for scheme, prelude in [ + ("slow", "import time\ntime.sleep(0.3)\n"), + ("quick", ""), + ("untouched", ""), + ]: + pending.append( + _fake_entry_point( + tmp_path, + monkeypatch, + scheme=scheme, + module=f"{scheme}_adapter", + source=_ADAPTER_SOURCE.format(prelude=prelude), + ) + ) + threads = [ + threading.Thread(target=get_url_adapter, args=(scheme,)) for scheme in ("slow", "quick") + ] + for thread in threads: + thread.start() + for thread in threads: + thread.join(timeout=10) + assert not any(thread.is_alive() for thread in threads) + assert "untouched" in list_url_adapter_schemes() + assert [ + e.name for e in pending if not zarr.registry._is_builtin_url_adapter_entry_point(e) + ] == ["untouched"] + assert get_url_adapter("untouched").__name__ == "Adapter" + + def test_import_failure_is_wrapped_and_not_permanent(self) -> None: + pending = zarr.registry._url_adapter_registry.lazy_load_list + pending.append( + EntryPoint(name="broken", value="no_such_module_xyz:Adapter", group="zarr.url_adapters") + ) + with pytest.raises(URLPipelineError, match="could not be loaded"): + get_url_adapter("broken") + # still advertised, so a transient failure can be retried + assert "broken" in list_url_adapter_schemes() + with pytest.raises(URLPipelineError, match="could not be loaded"): + get_url_adapter("broken") + + def test_registered_adapter_shadows_entry_point_with_warning( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + # a builtin registered under a scheme wins over a same-named entry + # point, which is discarded loudly rather than left pending forever + pending = zarr.registry._url_adapter_registry.lazy_load_list + pending.append( + _fake_entry_point( + tmp_path, + monkeypatch, + scheme="shadowed", + module="shadowed_adapter", + source=_ADAPTER_SOURCE.format(prelude="raise RuntimeError('must not import')\n"), + ) + ) + register_url_adapter("shadowed", NativeAdapter) + with pytest.warns(ZarrUserWarning, match="already registered"): + assert get_url_adapter("shadowed") is NativeAdapter + assert not any(e.name == "shadowed" for e in pending) + # a second lookup is silent + with warnings.catch_warnings(): + warnings.simplefilter("error") + assert get_url_adapter("shadowed") is NativeAdapter + + def test_duplicate_entry_points_warn_and_first_wins( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + pending = zarr.registry._url_adapter_registry.lazy_load_list + for module in ("dup_first", "dup_second"): + pending.append( + _fake_entry_point( + tmp_path, + monkeypatch, + scheme="dup", + module=module, + source=_ADAPTER_SOURCE.format(prelude=f"ORIGIN = {module!r}\n"), + ) + ) + with pytest.warns(ZarrUserWarning, match="multiple 'zarr.url_adapters' entry points"): + cls = get_url_adapter("dup") + assert getattr(sys.modules[cls.__module__], "ORIGIN", None) == "dup_first" + assert not any(e.name == "dup" for e in pending) + + +class TestIsURLPipeline: + def test_pipe_routes(self) -> None: + assert is_url_pipeline("memory://x|demo:") + + def test_registered_root_scheme_routes(self) -> None: + register_url_adapter("rooty", RootAdapter) + assert is_url_pipeline("rooty://org/repo") + + @pytest.mark.parametrize("url", ["s3://bucket/key", "/local/path", "memory://x", "C:.zarr"]) + def test_plain_urls_do_not_route(self, url: str) -> None: + assert not is_url_pipeline(url) + + def test_fsspec_chained_urls_do_not_route(self) -> None: + # fsspec's ``scheme::`` chaining is not a URL pipeline: even with a + # same-named root adapter registered, these keep flowing to fsspec. + register_url_adapter("zip", RootAdapter) + assert not is_url_pipeline("zip::file:///tmp/data.zip") + assert not is_url_pipeline("simplecache::s3://bucket/key") + + def test_native_schemes_do_not_route_to_adapters(self) -> None: + # an installed package must not be able to intercept zarr's own + # local-path and in-memory routing by registering file:/memory: + register_url_adapter("file", RootAdapter) + register_url_adapter("memory", RootAdapter) + assert not is_url_pipeline("file:/tmp/x") + assert not is_url_pipeline("memory://x") + + def test_exotic_authority_root_scheme(self) -> None: + # The spec's own root-URL example: urlparse rejects the bracketed + # non-IP authority, so scheme detection must work on the raw string + # rather than raising before routing. + url = "vendor1-2.custom-1+proto.ext://[authority]/path?query/part?x" + assert not is_url_pipeline(url) + register_url_adapter("vendor1-2.custom-1+proto.ext", RootAdapter) + assert is_url_pipeline(url) + + +class TestResolve: + async def test_wrapper_adapter_chain(self, tmp_path: Path) -> None: + register_url_adapter("wrap", WrapperAdapter) + result = await resolve_pipeline(f"{tmp_path}|wrap:inner/path") + assert isinstance(result.store, TracingStore) + assert result.path == "inner/path" + assert result.store.context.preceding_url == str(tmp_path) + assert result.store.segment.scheme == "wrap" + assert result.store.segment.body == "inner/path" + + async def test_nested_wrappers_preserve_residual_paths(self, tmp_path: Path) -> None: + # each wrapper joins the preceding residual path with its own, so + # no segment's path is lost in root|wrap:a|wrap:b + register_url_adapter("wrap", WrapperAdapter) + result = await resolve_pipeline(f"{tmp_path}|wrap:a|wrap:b") + assert result.path == "a/b" + + async def test_native_adapter_gets_preceding_url(self) -> None: + register_url_adapter("native", NativeAdapter) + NativeAdapter.seen_urls.clear() + await resolve_pipeline("s3://bucket/repo|native:") + assert NativeAdapter.seen_urls == ["s3://bucket/repo"] + + async def test_multi_segment_preceding_url(self) -> None: + register_url_adapter("wrap", WrapperAdapter) + register_url_adapter("native", NativeAdapter) + NativeAdapter.seen_urls.clear() + await resolve_pipeline("memory://base|wrap:a|native:x") + assert NativeAdapter.seen_urls == ["memory://base|wrap:a"] + + async def test_root_adapter(self) -> None: + register_url_adapter("rooty", RootAdapter) + result = await resolve_pipeline("rooty://org/repo") + assert result.path == "org/repo" + assert result.store.read_only + + async def test_root_adapter_composes_with_chain(self) -> None: + register_url_adapter("rooty", RootAdapter) + register_url_adapter("native", NativeAdapter) + NativeAdapter.seen_urls.clear() + await resolve_pipeline("rooty://org/repo|native:x") + assert NativeAdapter.seen_urls == ["rooty://org/repo"] + + async def test_read_only_flag(self, tmp_path: Path) -> None: + register_url_adapter("wrap", WrapperAdapter) + result = await resolve_pipeline(f"{tmp_path}|wrap:", mode="r") + assert isinstance(result.store, TracingStore) + assert result.store.context.read_only + assert result.store.context.mode == "r" + result = await resolve_pipeline(f"{tmp_path}|wrap:") + assert isinstance(result.store, TracingStore) + assert not result.store.context.read_only + assert result.store.context.mode is None + + async def test_read_only_is_enforced_on_disobedient_adapters(self) -> None: + # the resolver downgrades a writable store returned under mode "r" + # instead of trusting the adapter to have honored context.read_only + register_url_adapter("bad", DisobedientAdapter) + result = await resolve_pipeline("memory://base|bad:", mode="r") + assert result.store.read_only + store = await make_store("memory://base|bad:", mode="r") + assert store.read_only + + async def test_read_only_enforcement_without_conversion_raises(self) -> None: + # a disobedient adapter whose store cannot be converted read-only + class StubbornStore(MemoryStore): + def with_read_only(self, read_only: bool = False) -> MemoryStore: + raise NotImplementedError + + class StubbornAdapter(URLPipelineAdapter): + @classmethod + async def open_pipeline_segment( + cls, segment: PipelineSegment, context: PipelineContext + ) -> AdapterResolution: + return AdapterResolution(store=await StubbornStore.open(read_only=False)) + + register_url_adapter("stubborn", StubbornAdapter) + with pytest.raises(URLPipelineError, match="does not support read-only conversion"): + await resolve_pipeline("memory://base|stubborn:", mode="r") + + async def test_read_only_enforcement_failure_closes_store(self) -> None: + # the adapter's store must not leak when it cannot be made read-only + closed: list[bool] = [] + + class StubbornStore(MemoryStore): + def with_read_only(self, read_only: bool = False) -> MemoryStore: + raise NotImplementedError + + def close(self) -> None: + closed.append(True) + super().close() + + class StubbornAdapter(URLPipelineAdapter): + @classmethod + async def open_pipeline_segment( + cls, segment: PipelineSegment, context: PipelineContext + ) -> AdapterResolution: + return AdapterResolution(store=await StubbornStore.open(read_only=False)) + + register_url_adapter("stubborn", StubbornAdapter) + with pytest.raises(URLPipelineError): + await resolve_pipeline("memory://base|stubborn:", mode="r") + assert closed == [True] + + async def test_resolve_preceding_mode_override(self, tmp_path: Path) -> None: + # a wrapper that only reads the preceding resource can open it + # read-only regardless of the caller's mode + class ReadOnlyRootWrapper(WrapperAdapter): + @classmethod + async def open_pipeline_segment( + cls, segment: PipelineSegment, context: PipelineContext + ) -> AdapterResolution: + preceding = await context.resolve_preceding(mode="r") + assert preceding.store.read_only + return dataclasses.replace(preceding, path=segment.body) + + register_url_adapter("rowrap", ReadOnlyRootWrapper) + result = await resolve_pipeline(f"{tmp_path}|rowrap:x", mode="w") + assert result.path == "x" + + async def test_wrapper_on_local_file_root_read_only(self, tmp_path: Path) -> None: + # a wrapper that opens the preceding local *file* read-only gets a + # store rooted at that file (the mechanism a read-only zip: adapter + # can build on) + target = tmp_path / "data.bin" + target.write_bytes(b"payload") + register_url_adapter("wrap", WrapperAdapter) + result = await resolve_pipeline(f"{target}|wrap:", mode="r") + assert isinstance(result.store, TracingStore) + assert result.store.read_only + + @pytest.mark.parametrize("mode", [None, "a", "w", "r+", "w-"]) + @pytest.mark.xfail( + strict=True, + raises=FileExistsError, + reason=( + "known limitation: there is no file-resource primitive yet, so resolving a " + "local file root in a writable mode hits LocalStore's create-on-open mkdir" + ), + ) + async def test_wrapper_on_local_file_root_writable_modes( + self, tmp_path: Path, mode: AccessModeLiteral | None + ) -> None: + target = tmp_path / "data.bin" + target.write_bytes(b"payload") + register_url_adapter("wrap", WrapperAdapter) + await resolve_pipeline(f"{target}|wrap:", mode=mode) + + async def test_resolve_preceding_storage_options_override(self, tmp_path: Path) -> None: + # an adapter that consumed its namespaced keys strips them before + # the root store ever sees them + class ConsumingWrapper(WrapperAdapter): + seen: ClassVar[object | None] = None + + @classmethod + async def open_pipeline_segment( + cls, segment: PipelineSegment, context: PipelineContext + ) -> AdapterResolution: + options = dict(context.storage_options or {}) + cls.seen = options.pop("consuming_secret", None) + preceding = await context.resolve_preceding(storage_options=options or None) + return dataclasses.replace(preceding, path=segment.body) + + register_url_adapter("consuming", ConsumingWrapper) + # the local-path root rejects any surviving storage_options, so this + # passing proves the adapter's keys were stripped before resolution + result = await resolve_pipeline( + f"{tmp_path}|consuming:x", storage_options={"consuming_secret": "s3cr3t"} + ) + assert result.path == "x" + assert ConsumingWrapper.seen == "s3cr3t" + + async def test_storage_options_visible_to_adapter(self) -> None: + register_url_adapter("native", NativeAdapter) + + class OptionsProbe(NativeAdapter): + seen_options: dict[str, object] | None = None + + @classmethod + async def open_pipeline_segment( + cls, segment: PipelineSegment, context: PipelineContext + ) -> AdapterResolution: + cls.seen_options = context.storage_options + return await super().open_pipeline_segment(segment, context) + + register_url_adapter("probe", OptionsProbe) + opts = {"anon": True} + await resolve_pipeline("s3://bucket/x|probe:", storage_options=opts) + assert OptionsProbe.seen_options == opts + + async def test_wrapper_at_root_position_raises(self) -> None: + # a wrapper adapter used as the pipeline root has nothing to wrap + register_url_adapter("wrap", WrapperAdapter) + with pytest.raises(URLPipelineError, match="no preceding sub-URL"): + await resolve_pipeline("wrap:whatever") + + async def test_not_a_pipeline_raises(self) -> None: + with pytest.raises(URLPipelineError, match="is not a URL pipeline"): + await resolve_pipeline("s3://bucket/plain") + + async def test_unknown_adapter_scheme_raises(self) -> None: + with pytest.raises(URLPipelineError, match="no URL pipeline adapter is registered"): + await resolve_pipeline("memory://base|no-such-adapter:") + + async def test_unknown_adapter_error_hints_at_reserved_pipe(self, tmp_path: Path) -> None: + # a filename containing '|' routes here; the error points at the fix + with pytest.raises(URLPipelineError, match="pass a\\s+pathlib.Path"): + await resolve_pipeline(f"{tmp_path}/a|b") + + async def test_base_case_value_error_is_wrapped(self) -> None: + # a root that only the fallback scheme detection accepts must not + # leak urlparse's ValueError out of the base case + register_url_adapter("wrap", WrapperAdapter) + with pytest.raises(URLPipelineError, match="could not resolve the pipeline root"): + await resolve_pipeline("vendor.x://[authority]/path|wrap:") + + async def test_zarr_format_flows_from_adapter(self, tmp_path: Path) -> None: + register_url_adapter("fmt2", FormatAdapter) + result = await resolve_pipeline(f"{tmp_path}|fmt2:") + assert result.zarr_format == 2 + # wrapper adapters carry the format through (dataclasses.replace) + register_url_adapter("wrap", WrapperAdapter) + result = await resolve_pipeline(f"{tmp_path}|fmt2:|wrap:") + assert result.zarr_format == 2 + + +class TestSpecRoots: + """The spec's `memory:` / `file:` root semantics inside pipelines.""" + + @pytest.mark.parametrize("root", ["memory:", "memory:/", "memory://"]) + async def test_memory_root_spellings_equivalent(self, root: str) -> None: + register_url_adapter("wrap", WrapperAdapter) + result = await resolve_pipeline(f"{root}|wrap:") + assert isinstance(result.store, TracingStore) + inner = result.store._store + assert isinstance(inner, ManagedMemoryStore) + assert inner._name == "" + assert inner.path == "" + + @pytest.mark.parametrize("root", ["memory:a/b", "memory:/a/b", "memory://a/b"]) + async def test_memory_root_named_spellings_equivalent(self, root: str) -> None: + register_url_adapter("wrap", WrapperAdapter) + result = await resolve_pipeline(f"{root}|wrap:") + assert isinstance(result.store, TracingStore) + inner = result.store._store + assert isinstance(inner, ManagedMemoryStore) + assert inner._name == "a" + assert inner.path == "b" + + async def test_memory_root_shares_data_across_spellings(self) -> None: + register_url_adapter("wrap", WrapperAdapter) + group = zarr.open_group("memory:pipe-share|wrap:", mode="w") + group.create_array("x", shape=(2,), dtype="i4") + reopened = zarr.open_group("memory://pipe-share|wrap:", mode="r") + assert "x" in reopened + + async def test_memory_root_rejects_query(self) -> None: + register_url_adapter("wrap", WrapperAdapter) + with pytest.raises(URLPipelineError, match="do not accept a query"): + await resolve_pipeline("memory:a?opt=1|wrap:") + + async def test_memory_root_rejects_storage_options(self) -> None: + register_url_adapter("wrap", WrapperAdapter) + with pytest.raises(TypeError, match="'storage_options' was provided but unused"): + await resolve_pipeline("memory:a|wrap:", storage_options={"anon": True}) + + async def test_file_root_localhost_authority(self, tmp_path: Path) -> None: + # file://localhost/p and file:///p are equivalent to file:/p + register_url_adapter("wrap", WrapperAdapter) + # spell the path URL-style: on Windows C:/... gains a leading slash + posix = tmp_path.as_posix() + url_path = posix if posix.startswith("/") else f"/{posix}" + zarr.open_group(f"file://localhost{url_path}|wrap:", mode="w") + opened = zarr.open_group(f"file:{tmp_path}|wrap:", mode="r") + assert isinstance(opened, zarr.Group) + zarr.open_group(f"file://{url_path}|wrap:", mode="r") + + async def test_file_root_windows_drive_spellings( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + # Windows drive paths count as absolute, and the URL spelling + # file:///C:/... has its leading slash stripped before the drive + register_url_adapter("wrap", WrapperAdapter) + if sys.platform == "win32": + drive_path = tmp_path.as_posix() # C:/Users/... + else: + # off Windows a drive path is just an odd relative directory; + # chdir into tmp so it is created there + monkeypatch.chdir(tmp_path) + drive_path = "C:/drive/data" + r1 = await resolve_pipeline(f"file:{drive_path}|wrap:") + r2 = await resolve_pipeline(f"file:///{drive_path}|wrap:") + assert isinstance(r1.store, TracingStore) + assert isinstance(r2.store, TracingStore) + + async def test_file_root_rejects_other_authority(self) -> None: + register_url_adapter("wrap", WrapperAdapter) + with pytest.raises(URLPipelineError, match="unsupported authority"): + await resolve_pipeline("file://example.com/tmp/x|wrap:") + + @pytest.mark.parametrize("root", ["file:relative/path", "file://localhost"]) + async def test_file_root_rejects_relative_paths(self, root: str) -> None: + register_url_adapter("wrap", WrapperAdapter) + with pytest.raises(URLPipelineError, match="absolute path"): + await resolve_pipeline(f"{root}|wrap:") + + async def test_file_root_rejects_query(self, tmp_path: Path) -> None: + register_url_adapter("wrap", WrapperAdapter) + with pytest.raises(URLPipelineError, match="do not accept a query"): + await resolve_pipeline(f"file:{tmp_path}?v=1|wrap:") + + @pytest.mark.skipif( + not _has_async_fsspec_wrapper, + reason="plain memory:// routes to fsspec only with fsspec>=2024.12.0", + ) + async def test_memory_root_is_distinct_from_fsspec_memory_url(self) -> None: + # documented divergence: a plain memory:// URL is fsspec's in-memory + # filesystem when fsspec is installed, while a pipeline's memory: root + # is zarr's ManagedMemoryStore; the two do not share data + register_url_adapter("wrap", WrapperAdapter) + plain = await make_store("memory://distinct-store") + piped = await make_store("memory://distinct-store|wrap:") + assert not isinstance(plain, ManagedMemoryStore) + assert isinstance(piped, TracingStore) + assert isinstance(piped._store, ManagedMemoryStore) + + async def test_file_root_does_not_percent_decode(self, tmp_path: Path) -> None: + # documented divergence from RFC 8089: consistent with LocalStore, no + # percent-escape is decoded, so %20 names a literal directory + register_url_adapter("wrap", WrapperAdapter) + await resolve_pipeline(f"file:{tmp_path.as_posix()}/a%20b|wrap:") + assert (tmp_path / "a%20b").is_dir() + assert not (tmp_path / "a b").exists() + + +class TestMakeStoreIntegration: + async def test_make_store_path_combines_paths(self, tmp_path: Path) -> None: + register_url_adapter("wrap", WrapperAdapter) + store_path = await make_store_path(f"{tmp_path}|wrap:residual", path="user/sub") + assert store_path.path == "residual/user/sub" + + async def test_make_store_rejects_residual_path(self, tmp_path: Path) -> None: + register_url_adapter("wrap", WrapperAdapter) + with pytest.raises(URLPipelineError, match="resolves to a path inside a store"): + await make_store(f"{tmp_path}|wrap:residual") + + async def test_make_store_rejects_slash_root_path(self, tmp_path: Path) -> None: + # an adapter returning path="/" addresses the store root; the raw + # value is normalized before the residual-path rejection + class SlashRootAdapter(WrapperAdapter): + @classmethod + async def open_pipeline_segment( + cls, segment: PipelineSegment, context: PipelineContext + ) -> AdapterResolution: + preceding = await context.resolve_preceding() + return dataclasses.replace(preceding, path="/") + + register_url_adapter("slashy", SlashRootAdapter) + store = await make_store(f"{tmp_path}|slashy:") + assert store is not None + + async def test_make_store_closes_store_on_residual_path_error(self) -> None: + closed: list[bool] = [] + + class ClosingStore(MemoryStore): + def close(self) -> None: + closed.append(True) + super().close() + + class LeakProbe(URLPipelineAdapter): + @classmethod + async def open_pipeline_segment( + cls, segment: PipelineSegment, context: PipelineContext + ) -> AdapterResolution: + return AdapterResolution(store=await ClosingStore.open(), path="residual") + + register_url_adapter("leaky", LeakProbe) + with pytest.raises(URLPipelineError, match="resolves to a path inside a store"): + await make_store("memory://base|leaky:") + assert closed == [True] + + async def test_make_store_no_residual_path(self, tmp_path: Path) -> None: + register_url_adapter("wrap", WrapperAdapter) + store = await make_store(f"{tmp_path}|wrap:") + assert isinstance(store, TracingStore) + + async def test_make_store_rejects_invalid_mode_before_resolving(self, tmp_path: Path) -> None: + # an invalid mode is rejected upfront, like every other StoreLike, + # rather than being handed to adapters + register_url_adapter("wrap", WrapperAdapter) + with pytest.raises(ValueError, match="Invalid mode"): + await make_store(f"{tmp_path}|wrap:", mode="invalid") # type: ignore[arg-type] + + async def test_zarr_open_end_to_end(self, tmp_path: Path) -> None: + register_url_adapter("wrap", WrapperAdapter) + group = zarr.open_group(f"{tmp_path}|wrap:", mode="w") + array = group.create_array("x", shape=(4,), dtype="i4") + array[:] = [1, 2, 3, 4] + assert zarr.open_array(f"{tmp_path}|wrap:x")[2] == 3 + + async def test_pipeline_store_read_only_mode(self) -> None: + register_url_adapter("native", NativeAdapter) + store_path = await make_store_path("memory://base|native:", mode="r") + assert store_path.read_only + + async def test_mode_a_downgrades_to_read_only_open(self) -> None: + # mode "a" (the zarr.open default) is open-or-create: a pipeline + # that resolves to a read-only store serves the "open" half instead + # of failing outright. + register_url_adapter("rooty", RootAdapter) + store_path = await make_store_path("rooty://org/repo", mode="a") + assert store_path.read_only + + async def test_mode_a_downgrade_applies_to_plain_stores_too(self) -> None: + # the open-or-create downgrade lives in StorePath.open, so it is + # uniform across pipeline and non-pipeline stores + store = await MemoryStore.open(read_only=True) + store_path = await make_store_path(store, mode="a") + assert store_path.read_only + + async def test_explicit_write_mode_reaches_adapter(self) -> None: + register_url_adapter("rooty", RootAdapter) + with pytest.raises(ValueError, match="read-only scheme"): + await make_store_path("rooty://org/repo", mode="w") + + async def test_storage_options_forwarded_to_root(self) -> None: + # storage_options reach the root sub-URL via resolve_preceding -> + # make_store. A local-path root does not accept storage_options, so + # forwarding them must raise the same TypeError as a non-pipeline open. + register_url_adapter("wrap", WrapperAdapter) + with pytest.raises(TypeError, match="'storage_options' was provided but unused"): + await make_store("/tmp/some/path|wrap:", storage_options={"anon": True}) + + async def test_store_path_truediv_keeps_format(self, tmp_path: Path) -> None: + # StorePath.__truediv__ constructs a new instance; the pipeline- + # selected format must survive the division + register_url_adapter("fmt2", FormatAdapter) + store_path = await make_store_path(f"{tmp_path}|fmt2:") + assert store_path.zarr_format == 2 + assert (store_path / "sub/node").zarr_format == 2 + + async def test_zarr_format_merges_into_open(self, tmp_path: Path) -> None: + register_url_adapter("fmt2", FormatAdapter) + group = zarr.open_group(f"{tmp_path}|fmt2:", mode="w") + assert group.metadata.zarr_format == 2 + + async def test_conflicting_explicit_format_raises(self, tmp_path: Path) -> None: + register_url_adapter("fmt2", FormatAdapter) + with pytest.raises(ValueError, match="conflicts with"): + zarr.open_group(f"{tmp_path}|fmt2:", mode="w", zarr_format=3) + + async def test_matching_explicit_format_ok(self, tmp_path: Path) -> None: + register_url_adapter("fmt2", FormatAdapter) + group = zarr.open_group(f"{tmp_path}|fmt2:", mode="w", zarr_format=2) + assert group.metadata.zarr_format == 2 + + +class TestZarrFormatMergeInCore: + """ + Every make_store_path caller honors the pipeline-selected zarr format, not + only the top-level zarr.api functions. + """ + + @pytest.fixture(autouse=True) + def _fmt2(self) -> None: + register_url_adapter("fmt2", FormatAdapter) + + async def test_create_array(self, tmp_path: Path) -> None: + arr = zarr.create_array(f"{tmp_path}|fmt2:", name="x", shape=(2,), dtype="i4") + assert arr.metadata.zarr_format == 2 + with pytest.raises(ValueError, match="conflicts with"): + zarr.create_array(f"{tmp_path}|fmt2:", name="y", shape=(2,), dtype="i4", zarr_format=3) + + async def test_create_array_from_data(self, tmp_path: Path) -> None: + # the data path of create_array goes through from_array + arr = zarr.create_array(f"{tmp_path}|fmt2:", name="x", data=np.arange(3)) + assert arr.metadata.zarr_format == 2 + with pytest.raises(ValueError, match="conflicts with"): + zarr.create_array(f"{tmp_path}|fmt2:", name="y", data=np.arange(3), zarr_format=3) + + async def test_from_array(self, tmp_path: Path) -> None: + arr = zarr.from_array(f"{tmp_path}|fmt2:", name="x", data=np.arange(3)) + assert arr.metadata.zarr_format == 2 + + async def test_group_from_store(self, tmp_path: Path) -> None: + group = zarr.Group.from_store(f"{tmp_path}|fmt2:") + assert group.metadata.zarr_format == 2 + with pytest.raises(ValueError, match="conflicts with"): + zarr.Group.from_store(f"{tmp_path}/other|fmt2:", zarr_format=3) + + async def test_group_from_store_default_without_pipeline(self, tmp_path: Path) -> None: + # the default is still the configured format (3) for plain stores + group = zarr.Group.from_store(tmp_path) + assert group.metadata.zarr_format == 3 + + async def test_group_open(self, tmp_path: Path) -> None: + zarr.Group.from_store(f"{tmp_path}|fmt2:") + # the unspecified default defers to the pipeline, as does explicit + # None; only an explicit, different format conflicts + assert zarr.Group.open(f"{tmp_path}|fmt2:").metadata.zarr_format == 2 + group = zarr.Group.open(f"{tmp_path}|fmt2:", zarr_format=None) + assert group.metadata.zarr_format == 2 + with pytest.raises(ValueError, match="conflicts with"): + zarr.Group.open(f"{tmp_path}|fmt2:", zarr_format=3) + + async def test_array_open(self, tmp_path: Path) -> None: + zarr.create_array(f"{tmp_path}|fmt2:", name="x", shape=(2,), dtype="i4") + assert zarr.Array.open(f"{tmp_path}/x|fmt2:").metadata.zarr_format == 2 + arr = zarr.Array.open(f"{tmp_path}/x|fmt2:", zarr_format=None) + assert arr.metadata.zarr_format == 2 + with pytest.raises(ValueError, match="conflicts with"): + zarr.Array.open(f"{tmp_path}/x|fmt2:", zarr_format=3) + + async def test_async_open_classmethods(self, tmp_path: Path) -> None: + from zarr.core.array import AsyncArray + from zarr.core.group import AsyncGroup + + zarr.create_array(f"{tmp_path}|fmt2:", name="x", shape=(2,), dtype="i4") + group = await AsyncGroup.open(f"{tmp_path}|fmt2:") + assert group.metadata.zarr_format == 2 + arr = await AsyncArray.open(f"{tmp_path}/x|fmt2:") + assert arr.metadata.zarr_format == 2 + with pytest.raises(ValueError, match="conflicts with"): + await AsyncArray.open(f"{tmp_path}/x|fmt2:", zarr_format=3) + + async def test_open_classmethod_default_without_pipeline(self, tmp_path: Path) -> None: + # non-pipeline callers keep the historical default of 3: a v2 node is + # not found unless the caller asks for 2 or None + zarr.create_array(tmp_path / "x", shape=(2,), dtype="i4", zarr_format=2) + zarr.create_group(tmp_path / "g", zarr_format=2) + with pytest.raises(FileNotFoundError): + zarr.Array.open(tmp_path / "x") + with pytest.raises(FileNotFoundError): + zarr.Group.open(tmp_path / "g") + assert zarr.Array.open(tmp_path / "x", zarr_format=None).metadata.zarr_format == 2 + assert zarr.Group.open(tmp_path / "g", zarr_format=None).metadata.zarr_format == 2 + + async def test_open_like(self, tmp_path: Path) -> None: + # open_like inherits v2 filters/compressor from the reference; the + # pipeline supplies the format, which open_array then creates with + ref = zarr.create_array({}, shape=(2,), dtype="i4", zarr_format=2) + arr = zarr.open_like(ref, store=f"{tmp_path}|fmt2:", path="x") + assert arr.metadata.zarr_format == 2 + + async def test_deprecated_async_array_create(self, tmp_path: Path) -> None: + from zarr.core.array import AsyncArray + + arr = await AsyncArray._create(f"{tmp_path}|fmt2:", shape=(2,), dtype="i4", zarr_format=2) + assert arr.metadata.zarr_format == 2 + with pytest.raises(ValueError, match="conflicts with"): + await AsyncArray._create(f"{tmp_path}/o|fmt2:", shape=(2,), dtype="i4", zarr_format=3) diff --git a/tests/test_url_pipeline/test_spec_examples.py b/tests/test_url_pipeline/test_spec_examples.py new file mode 100644 index 0000000000..5538074bea --- /dev/null +++ b/tests/test_url_pipeline/test_spec_examples.py @@ -0,0 +1,258 @@ +""" +Conformance tests against the URL pipeline specification. + +Every entry in `SPEC_EXAMPLES` is an example URL from the specification +repository (https://github.com/jbms/url-pipeline @ 1a01ce6), extracted +with the same rule as the spec's own linter (`scripts/lint.py`): backticked +examples on bullet lines, skipping spans without a `:` or `|`. The spec +validates each example (and an uppercased-scheme variant) against its ABNF +grammar, so this corpus is grammar-valid by construction. + +The parser must accept every example — including schemes zarr-python has no +adapter for, since parsing is independent of adapter availability — split it +into the expected sub-URL schemes, and preserve the text losslessly. + +To regenerate after a spec update: extract examples per the rule above and +recompute the expected scheme tuples with `parse_pipeline`, reviewing any +changes against the spec diff. +""" + +from __future__ import annotations + +import re + +import pytest + +from zarr.storage._url_pipeline import parse_pipeline + +SPEC_EXAMPLES: list[tuple[str, tuple[str, ...]]] = [ + # README.md + ("s3://bucket/path/to/archive.zip|zip:path/within/zip.zarr/|zarr3:", ("s3", "zip", "zarr3")), + ( + "file:///tmp/dataset.ocdbt/|ocdbt://2025-01-01T01:23:45.678Z/path/within/database", + ("file", "ocdbt"), + ), + ( + "s3+https://example.com/path/to/database.icechunk/|icechunk://tag.v5/path/to/node/|zarr3:", + ("s3+https", "icechunk", "zarr3"), + ), + ( + "vendor1-2.custom-1+proto.ext://[authority]/path?query/part?x", + ("vendor1-2.custom-1+proto.ext",), + ), + ( + "http://example.com/file|vendor1-2.custom+adapter:/path/within/adapter", + ("http", "vendor1-2.custom+adapter"), + ), + ("a.b:?", ("a.b",)), + ("http://somehost/downloads/somefile.zip|zip:", ("http", "zip")), + ("http://example.com/archive.jar|zip:path/to/file.txt", ("http", "zip")), + ("https://host/archive.zip|zip:path/in/outer.zip|zip:path/in/inner", ("https", "zip", "zip")), + ("file:///path/to/archive.zip|zip:path/within/archive", ("file", "zip")), + # avif.md + ("file:/path/to/image.avif|avif:", ("file", "avif")), + ("file:/path/to/image.avif|avif", ("file", "avif")), + # bmp.md + ("file:/path/to/image.bmp|bmp:", ("file", "bmp")), + ("file:/path/to/image.bmp|bmp", ("file", "bmp")), + # byte-range.md + ("file:/path/to/data|byte-range:1000-2000", ("file", "byte-range")), + ("file:/path/to/data|byte-range:0-1", ("file", "byte-range")), + # file.md + ("file:/", ("file",)), + ("file:/a", ("file",)), + ("file:/path/to/file.txt", ("file",)), + ("file://localhost/path/to/file.txt", ("file",)), + ("file://LOCALHOST/path/to/file.txt", ("file",)), + ("file:///path/to/file.txt", ("file",)), + ("file://somehost/sharename/path/to/file.txt", ("file",)), + # gs.md + ("gs://bucket", ("gs",)), + ("gs://bucket/", ("gs",)), + ("gs://bucket/path/within/bucket", ("gs",)), + ("gs://aaa", ("gs",)), + ( + "gs://label1-is-sixty-two-characters-long-xxxxxxxxxxxxxxxxxxxxxxxxxx.label2-is-sixty-two-characters-long-yyyyyyyyyyyyyyyyyyyyyyyyyy.label3-is-sixty-two-characters-long-zzzzzzzzzzzzzzzzzzzzzzzzzz.label4-is-thirty-one-characters", + ("gs",), + ), + # gzip.md + ("file:/path/to/data.gz|gzip:", ("file", "gzip")), + ("file:/path/to/data.gz|gzip", ("file", "gzip")), + # hdf5.md + ("s3://bucket/path/to/file.h5|hdf5:/path/to/dataset", ("s3", "hdf5")), + ("s3://bucket/path/to/file.h5|hdf5:a", ("s3", "hdf5")), + ("file:///path/to/file.h5|hdf5:", ("file", "hdf5")), + ("file:///path/to/file.h5|hdf5", ("file", "hdf5")), + # http.md + ("https://example.com/path/to/resource%20name", ("https",)), + ("https://example.com/path/to/resource?query=value", ("https",)), + ("https://example.com/path/to/?query=value", ("https",)), + ("https://example.com/path/with:colon", ("https",)), + ("https://server.example.com:1234/path/to/array", ("https",)), + ("http://a", ("http",)), + ("http://example.com", ("http",)), + ("http://local%68ost", ("http",)), + ("http://example.com:", ("http",)), + ("http://192.168.10.1:1234", ("http",)), + ("http://[::1]", ("http",)), + ("http://[::ffff:127.0.0.1]", ("http",)), + ("http://example.com/", ("http",)), + # icechunk.md + ("file:///path/to/repo.zarr.icechunk/|icechunk:", ("file", "icechunk")), + ("file:///path/to/repo.zarr.icechunk/|icechunk", ("file", "icechunk")), + ("file:///path/to/repo.zarr.icechunk/|icechunk://branch.main/", ("file", "icechunk")), + ("file:///path/to/repo.zarr.icechunk/|icechunk:path/to/node/", ("file", "icechunk")), + ("file:///path/to/repo.zarr.icechunk/|icechunk:a", ("file", "icechunk")), + ("file:///path/to/repo.zarr.icechunk/|icechunk:/path/to/node/", ("file", "icechunk")), + ("file:///path/to/repo.zarr.icechunk/|icechunk:path/to/node/zarr.json", ("file", "icechunk")), + ("file:///path/to/repo.zarr.icechunk/|icechunk:path/to/node/c/0/0/1", ("file", "icechunk")), + ( + "file:///path/to/repo.zarr.icechunk/|icechunk://branch.mybranch/path/to/node/", + ("file", "icechunk"), + ), + ("file:///path/to/repo.zarr.icechunk/|icechunk://branch.a", ("file", "icechunk")), + ( + "file:///path/to/repo.zarr.icechunk/|icechunk://tag.mytag/path/to/node/", + ("file", "icechunk"), + ), + ("file:///path/to/repo.zarr.icechunk/|icechunk://tag.a", ("file", "icechunk")), + ( + "file:///path/to/repo.zarr.icechunk/|icechunk://FWWFQGAW742XMX0F5MF0/path/to/node/", + ("file", "icechunk"), + ), + ( + "file:///path/to/repo.zarr.icechunk/|icechunk://ABCDEFGHJKMNPQRSTVWX/path/to/node/", + ("file", "icechunk"), + ), + ( + "file:///path/to/repo.zarr.icechunk/|icechunk:|zarr3:path/to/array/", + ("file", "icechunk", "zarr3"), + ), + ( + "file:///path/to/repo.zarr.icechunk/|icechunk://branch.other/|zarr3:path/to/array/", + ("file", "icechunk", "zarr3"), + ), + ( + "file:///path/to/repo.zarr.icechunk/|icechunk://tag.v5/|zarr3:path/to/array/", + ("file", "icechunk", "zarr3"), + ), + ( + "file:///path/to/repo.zarr.icechunk/|icechunk://4N0217AZA4VNPYD0HR0G/|zarr3:path/to/array/", + ("file", "icechunk", "zarr3"), + ), + # jpeg.md + ("file:/path/to/image.jpeg|jpeg:", ("file", "jpeg")), + ("file:/path/to/image.jpeg|jpeg", ("file", "jpeg")), + # json.md + ("file:/path/to/data.json|json:", ("file", "json")), + ("file:/path/to/data.json|json", ("file", "json")), + ("file:/path/to/data.json|json:/path/to/node", ("file", "json")), + ("file:/path/to/data.json|json:/abc~0def", ("file", "json")), + ("file:/path/to/data.json|json:/abc~1def", ("file", "json")), + ("file:/path/to/data.json|json:/", ("file", "json")), + # memory.md + ("memory:", ("memory",)), + ("memory:/", ("memory",)), + ("memory://", ("memory",)), + ("memory:path/to/resource", ("memory",)), + ("memory:a", ("memory",)), + ("memory:another@path+with,lots&of(special;characters)*_!-$'", ("memory",)), + ("memory:/path/to/resource", ("memory",)), + ("memory://path/to/resource", ("memory",)), + # n5.md + ("file:///tmp/data.n5/|n5:path/to/array", ("file", "n5")), + ("file:///tmp/data.n5/|n5:a", ("file", "n5")), + ("file:///tmp/data.n5/|n5:/path/to/array", ("file", "n5")), + ("file:///tmp/data.n5/|n5:", ("file", "n5")), + ("file:///tmp/data.n5/|n5", ("file", "n5")), + # neuroglancer-precomputed.md + ( + "file:///tmp/dataset.precomputed/|neuroglancer-precomputed:", + ("file", "neuroglancer-precomputed"), + ), + ( + "file:///tmp/dataset.precomputed/|neuroglancer-precomputed", + ("file", "neuroglancer-precomputed"), + ), + # ocdbt.md + ("file:///path/to/repo.ocdbt/|ocdbt:", ("file", "ocdbt")), + ("file:///path/to/repo.ocdbt/|ocdbt", ("file", "ocdbt")), + ("file:///path/to/repo.ocdbt/|ocdbt:path/within/repo/", ("file", "ocdbt")), + ("file:///path/to/repo.ocdbt/|ocdbt:path/within/repo", ("file", "ocdbt")), + ("file:///path/to/repo.ocdbt/|ocdbt:a", ("file", "ocdbt")), + ("file:///path/to/repo.ocdbt/|ocdbt:/path/within/repo", ("file", "ocdbt")), + ("file:///path/to/repo.ocdbt/|ocdbt://v123/", ("file", "ocdbt")), + ("file:///path/to/repo.ocdbt/|ocdbt://v1", ("file", "ocdbt")), + ("file:///path/to/repo.ocdbt/|ocdbt://v123/path/within/repo", ("file", "ocdbt")), + ( + "file:///path/to/repo.ocdbt/|ocdbt://2025-01-01T01:23:45.678Z/path/within/database", + ("file", "ocdbt"), + ), + ("file:///path/to/repo.ocdbt/|ocdbt://2025-01-01T01:23:45Z", ("file", "ocdbt")), + ("file:///path/to/repo.ocdbt/|ocdbt://2025-01-01T01:23:45.1Z", ("file", "ocdbt")), + ("file:///path/to/repo.ocdbt/|ocdbt://2025-01-01T01:23:45.123456789Z", ("file", "ocdbt")), + # png.md + ("file:/path/to/image.png|png:", ("file", "png")), + ("file:/path/to/image.png|png", ("file", "png")), + # s3+http.md + ("s3+https://endpoint/path/within/bucket", ("s3+https",)), + ("s3+https://endpoint/bucket/path/within/bucket", ("s3+https",)), + ("s3+https://mybucket.s3.amazonaws.com/path/to/file", ("s3+https",)), + ("s3+https://s3.amazonaws.com/mybucket/path/to/file", ("s3+https",)), + ("s3+http://example.com", ("s3+http",)), + ("s3+http://example.com/", ("s3+http",)), + # s3.md + ("s3://bucket", ("s3",)), + ("s3://bucket/", ("s3",)), + ("s3://bucket/path/within/bucket", ("s3",)), + ("s3://aaa", ("s3",)), + ( + "s3://aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + ("s3",), + ), + # tiff.md + ("file:/path/to/image.tiff|tiff:", ("file", "tiff")), + ("file:/path/to/image.tiff|tiff", ("file", "tiff")), + # webp.md + ("file:/path/to/image.webp|webp:", ("file", "webp")), + ("file:/path/to/image.webp|webp", ("file", "webp")), + # zarr.md + ("file:///path/to/node.zarr/|zarr3:", ("file", "zarr3")), + ("file:///path/to/node.zarr/|zarr3", ("file", "zarr3")), + ("file:///path/to/node.zarr/|zarr2:", ("file", "zarr2")), + ("file:///path/to/node.zarr/|zarr2", ("file", "zarr2")), + ("file:///path/to/node.zarr/|zarr:", ("file", "zarr")), + ("file:///path/to/node.zarr/|zarr", ("file", "zarr")), + # zip.md + ("file:/path/to/archive.zip|zip:path/to/file.txt", ("file", "zip")), + ("file:/path/to/archive.zip|zip", ("file", "zip")), + ("file:/path/to/archive.zip|zip:/path/to/file.txt", ("file", "zip")), + ("file:/path/to/outer.zip|zip:path/to/inner.zip|zip:path/to/file.txt", ("file", "zip", "zip")), + # zstd.md + ("file:/path/to/data.zstd|zstd:", ("file", "zstd")), + ("file:/path/to/data.zstd|zstd", ("file", "zstd")), +] + + +def _uppercase_schemes(example: str) -> str: + """Uppercase every sub-URL scheme, as the spec's linter does.""" + + def repl(match: re.Match[str]) -> str: + return match.group(1) + match.group(2).upper() + + return re.sub(r"((?:^|\|)\s*)([a-zA-Z][a-zA-Z0-9+.-]*)", repl, example) + + +@pytest.mark.parametrize(("url", "expected_schemes"), SPEC_EXAMPLES, ids=lambda v: str(v)) +def test_spec_example_parses(url: str, expected_schemes: tuple[str, ...]) -> None: + segments = parse_pipeline(url) + assert tuple(segment.scheme for segment in segments) == expected_schemes + # lossless round-trip of the original text + assert "|".join(segment.raw for segment in segments) == url + + +@pytest.mark.parametrize(("url", "expected_schemes"), SPEC_EXAMPLES, ids=lambda v: str(v)) +def test_spec_example_schemes_case_insensitive(url: str, expected_schemes: tuple[str, ...]) -> None: + upper = _uppercase_schemes(url) + segments = parse_pipeline(upper) + assert tuple(segment.scheme for segment in segments) == expected_schemes diff --git a/tests/test_url_pipeline/test_zip_adapter.py b/tests/test_url_pipeline/test_zip_adapter.py new file mode 100644 index 0000000000..4c34cc5a01 --- /dev/null +++ b/tests/test_url_pipeline/test_zip_adapter.py @@ -0,0 +1,358 @@ +"""Tests for the builtin, read-only `zip:` URL pipeline adapter.""" + +from __future__ import annotations + +import dataclasses +import io +import zipfile +from typing import TYPE_CHECKING, ClassVar + +import numpy as np +import pytest + +import zarr +from zarr.abc.store import ( + ByteRequest, + OffsetByteRequest, + RangeByteRequest, + Store, + SuffixByteRequest, +) +from zarr.abc.url_pipeline import ( + AdapterResolution, + PipelineContext, + PipelineSegment, + URLPipelineAdapter, +) +from zarr.core.buffer import default_buffer_prototype +from zarr.errors import URLPipelineError +from zarr.registry import get_url_adapter, register_url_adapter +from zarr.storage import LocalStore, WrapperStore, ZipStore +from zarr.storage._common import _has_fsspec +from zarr.storage._url_adapters._zip import ZipAdapter, ZipReaderStore +from zarr.storage._url_pipeline import is_url_pipeline, resolve_pipeline + +if TYPE_CHECKING: + from pathlib import Path + + from zarr.core.common import AccessModeLiteral + +pytestmark = pytest.mark.usefixtures("clean_url_adapter_registry") + + +def _write_archive(path: Path, *, compression: int = zipfile.ZIP_STORED) -> Path: + """An archive with a v3 hierarchy at `inner-dir/subgroup`, written by ZipStore.""" + with ZipStore(path, mode="w", compression=compression) as store: + root = zarr.open_group(store, mode="w", path="inner-dir") + sub = root.create_group("subgroup", attributes={"title": "headline"}) + arr = sub.create_array("x", shape=(6,), chunks=(2,), dtype="i4") + arr[:] = np.arange(6) + return path + + +def _url_path(path: Path) -> str: + """A path spelled for a `file:` URL (a Windows drive gains a leading slash).""" + posix = path.as_posix() + return posix if posix.startswith("/") else f"/{posix}" + + +@pytest.fixture +def archive(tmp_path: Path) -> Path: + return _write_archive(tmp_path / "my-archive.zip") + + +class _CloseSpyStore(WrapperStore[Store]): + closed: ClassVar[list[str]] = [] + + def close(self) -> None: + type(self).closed.append(str(self)) + super().close() + + +class _CloseSpyAdapter(URLPipelineAdapter): + """Wraps the preceding store so tests can observe it being closed.""" + + @classmethod + async def open_pipeline_segment( + cls, segment: PipelineSegment, context: PipelineContext + ) -> AdapterResolution: + preceding = await context.resolve_preceding() + return dataclasses.replace(preceding, store=_CloseSpyStore(preceding.store)) + + +def test_registered_as_builtin() -> None: + assert get_url_adapter("zip") is ZipAdapter + + +@pytest.mark.parametrize("url", ["zip:/tmp/a.zip", "zip:///tmp/a.zip", "zip::file:///tmp/a.zip"]) +def test_zip_is_not_a_root_scheme(url: str) -> None: + # strings without '|' keep their pre-pipeline meaning (fsspec's zip://) + assert not is_url_pipeline(url) + + +class TestRead: + def test_headline_example(self, archive: Path) -> None: + # zarr-developers/zarr-python#2943's headline example, with a local root + group = zarr.open_group(f"{archive}|zip:inner-dir|zarr3:subgroup", mode="r") + assert group.attrs["title"] == "headline" + x = group["x"] + assert isinstance(x, zarr.Array) + np.testing.assert_array_equal(x[:], np.arange(6)) + + @pytest.mark.parametrize( + "template", + [ + "file:{url}|zip:", + "file://{url}|zip:", + "file://localhost{url}|zip", + "{path}|zip:", + ], + ) + def test_root_spellings(self, archive: Path, template: str) -> None: + url = template.format(url=_url_path(archive), path=archive) + group = zarr.open_group(f"{url}|zarr3:inner-dir/subgroup", mode="r") + assert group.attrs["title"] == "headline" + + @pytest.mark.parametrize("body", ["inner-dir/subgroup", "/inner-dir/subgroup", "inner-dir/"]) + def test_leading_slash_is_ignored(self, archive: Path, body: str) -> None: + result = zarr.open_group(f"file:{_url_path(archive)}|zip:{body}", mode="r") + assert isinstance(result, zarr.Group) + + def test_double_leading_slash_is_invalid(self, archive: Path) -> None: + with pytest.raises(URLPipelineError, match="more than one leading '/'"): + zarr.open_group(f"{archive}|zip://inner-dir", mode="r") + + @pytest.mark.parametrize("body", ["..", "a/../b"]) + def test_invalid_path(self, archive: Path, body: str) -> None: + with pytest.raises(URLPipelineError, match="invalid path"): + zarr.open_group(f"{archive}|zip:{body}", mode="r") + + def test_query_is_rejected(self, archive: Path) -> None: + with pytest.raises(URLPipelineError, match="do not accept a query"): + zarr.open_group(f"{archive}|zip:a?x=1", mode="r") + + def test_open_array_through_paths(self, archive: Path) -> None: + arr = zarr.open_array(f"{archive}|zip:inner-dir/subgroup/x", mode="r") + np.testing.assert_array_equal(arr[1:4], [1, 2, 3]) + via_path = zarr.open_array(f"{archive}|zip:inner-dir", path="subgroup/x", mode="r") + np.testing.assert_array_equal(via_path[:], np.arange(6)) + + @pytest.mark.parametrize("mode", [None, "a", "r"]) + def test_default_and_open_modes_are_read_only( + self, archive: Path, mode: AccessModeLiteral | None + ) -> None: + # "a" (the zarr.open default) is open-or-create: zip: serves the open half + url = f"{archive}|zip:inner-dir/subgroup" + node = zarr.open(url) if mode is None else zarr.open(url, mode=mode) + assert isinstance(node, zarr.Group) + assert node.store.read_only + with pytest.raises(ValueError, match="read-only"): + node.create_group("new") + + @pytest.mark.parametrize("mode", ["w", "w-", "r+"]) + def test_write_modes_raise(self, archive: Path, mode: AccessModeLiteral) -> None: + with pytest.raises(URLPipelineError, match="zip: is read-only until"): + zarr.open_group(f"{archive}|zip:", mode=mode) + + @pytest.mark.parametrize( + "compression", [zipfile.ZIP_STORED, zipfile.ZIP_DEFLATED, zipfile.ZIP_BZIP2] + ) + def test_compression_methods(self, tmp_path: Path, compression: int) -> None: + path = _write_archive(tmp_path / "c.zip", compression=compression) + arr = zarr.open_array(f"{path}|zip:inner-dir/subgroup/x", mode="r") + np.testing.assert_array_equal(arr[:], np.arange(6)) + + def test_unsupported_compression(self, tmp_path: Path) -> None: + path = tmp_path / "lzma.zip" + with zipfile.ZipFile(path, "w", compression=zipfile.ZIP_LZMA) as zf: + zf.writestr("zarr.json", b"{}") + group_url = f"{path}|zip:" + with pytest.raises(NotImplementedError, match="compression method"): + zarr.open_group(group_url, mode="r") + + def test_archive_with_directory_entries(self, tmp_path: Path, archive: Path) -> None: + # archives made by `zip -r` carry directory entries; they are not keys + rewritten = tmp_path / "with-dirs.zip" + with zipfile.ZipFile(archive) as src, zipfile.ZipFile(rewritten, "w") as dst: + dst.writestr("inner-dir/", b"") + dst.writestr("inner-dir/subgroup/", b"") + for info in src.infolist(): + dst.writestr(info, src.read(info)) + group = zarr.open_group(f"{rewritten}|zip:inner-dir", mode="r") + assert sorted(group.group_keys()) == ["subgroup"] + + def test_large_central_directory(self, tmp_path: Path) -> None: + # a central directory larger than the initial suffix read is fetched + # in further range requests + path = tmp_path / "many.zip" + with zipfile.ZipFile(path, "w") as zf: + for i in range(3000): + zf.writestr(f"entries/padding-name-to-grow-the-directory-{i:05d}", b"x") + zf.writestr("zarr.json", b'{"zarr_format": 3, "node_type": "group"}') + group = zarr.open_group(f"{path}|zip:", mode="r") + assert isinstance(group, zarr.Group) + + def test_nested_zip(self, tmp_path: Path, archive: Path) -> None: + outer = tmp_path / "outer.zip" + with zipfile.ZipFile(outer, "w") as zf: + zf.write(archive, "nested/inner.zip") + group = zarr.open_group( + f"file:{_url_path(outer)}|zip:nested/inner.zip|zip:inner-dir|zarr3:subgroup", + mode="r", + ) + assert group.attrs["title"] == "headline" + + def test_nested_zip_compressed_outer(self, tmp_path: Path, archive: Path) -> None: + outer = tmp_path / "outer.zip" + with zipfile.ZipFile(outer, "w", compression=zipfile.ZIP_DEFLATED) as zf: + zf.write(archive, "inner.zip") + arr = zarr.open_array(f"{outer}|zip:inner.zip|zip:inner-dir/subgroup/x", mode="r") + np.testing.assert_array_equal(arr[:], np.arange(6)) + + @pytest.mark.skipif(not _has_fsspec, reason="requires fsspec") + def test_fsspec_root(self, archive: Path) -> None: + # fsspec's `local://` protocol: an FsspecStore root, read by range requests + from zarr.storage import FsspecStore + + result = zarr.open_group(f"local://{archive.as_posix()}|zip:inner-dir|zarr3:subgroup") + assert isinstance(result.store, ZipReaderStore) + assert isinstance(result.store._source, FsspecStore) + assert result.attrs["title"] == "headline" + + @pytest.mark.skipif(not _has_fsspec, reason="requires fsspec") + def test_fsspec_memory_filesystem_root(self, archive: Path) -> None: + # an fsspec in-memory filesystem stands in for a remote object store + fsspec = pytest.importorskip("fsspec") + from fsspec.implementations.memory import MemoryFileSystem + + class RemoteLikeFS(MemoryFileSystem): # type: ignore[misc] + protocol = ("zarr-test-remote",) + store: ClassVar[dict[str, object]] = {} + pseudo_dirs: ClassVar[list[str]] = [""] + + @classmethod + def _strip_protocol(cls, path: str) -> str: + return "/" + path.removeprefix("zarr-test-remote://").lstrip("/") + + fsspec.register_implementation("zarr-test-remote", RemoteLikeFS, clobber=True) + RemoteLikeFS().pipe("/bucket/my-archive.zip", archive.read_bytes()) + group = zarr.open_group("zarr-test-remote://bucket/my-archive.zip|zip:inner-dir/subgroup") + assert group.attrs["title"] == "headline" + + def test_memory_root(self, archive: Path) -> None: + from zarr.core.buffer import cpu + from zarr.storage import ManagedMemoryStore + + holder = ManagedMemoryStore(name="zip-memory-root") + zarr.core.sync.sync(holder.set("dir/a.zip", cpu.Buffer.from_bytes(archive.read_bytes()))) + group = zarr.open_group("memory:zip-memory-root/dir/a.zip|zip:inner-dir/subgroup") + assert group.attrs["title"] == "headline" + + +class TestErrors: + def test_not_a_zip_file(self, tmp_path: Path) -> None: + path = tmp_path / "not.zip" + path.write_bytes(b"definitely not a zip archive") + with pytest.raises(URLPipelineError, match="could not open the ZIP archive"): + zarr.open_group(f"{path}|zip:", mode="r") + + def test_missing_archive(self, tmp_path: Path) -> None: + with pytest.raises((URLPipelineError, FileNotFoundError)): + zarr.open_group(f"{tmp_path / 'missing.zip'}|zip:", mode="r") + + def test_directory_root(self, tmp_path: Path) -> None: + with pytest.raises(URLPipelineError, match="could not open the ZIP archive"): + zarr.open_group(f"{tmp_path}|zip:", mode="r") + + def test_preceding_store_closed_on_error(self, tmp_path: Path) -> None: + register_url_adapter("spy", _CloseSpyAdapter) + _CloseSpyStore.closed.clear() + path = tmp_path / "not.zip" + path.write_bytes(b"definitely not a zip archive") + with pytest.raises(URLPipelineError): + zarr.open_group(f"{path}|spy:|zip:", mode="r") + assert len(_CloseSpyStore.closed) == 1 + + async def test_close_closes_source(self, archive: Path) -> None: + register_url_adapter("spy", _CloseSpyAdapter) + _CloseSpyStore.closed.clear() + result = await resolve_pipeline(f"{archive}|spy:|zip:", mode="r") + assert _CloseSpyStore.closed == [] + result.store.close() + assert len(_CloseSpyStore.closed) == 1 + + +class TestZipReaderStore: + @pytest.fixture + async def store(self, tmp_path: Path) -> ZipReaderStore: + path = tmp_path / "s.zip" + with zipfile.ZipFile(path, "w") as zf: + zf.writestr("a/b", b"0123456789") + zf.writestr("a/c/d", b"x") + zf.writestr("e", b"y") + with zipfile.ZipFile(path, "a", compression=zipfile.ZIP_DEFLATED) as zf: + zf.writestr("z", b"0123456789" * 10) + source = await LocalStore.open(path, read_only=True) + return await ZipReaderStore.open_archive(source) + + async def test_get_and_ranges(self, store: ZipReaderStore) -> None: + proto = default_buffer_prototype() + for key in ("a/b", "z"): + full = await store.get(key, proto) + assert full is not None + data = full.to_bytes() + cases: list[tuple[ByteRequest, bytes]] = [ + (RangeByteRequest(2, 5), data[2:5]), + (RangeByteRequest(8, 50), data[8:50]), + (OffsetByteRequest(7), data[7:]), + (SuffixByteRequest(3), data[-3:]), + (RangeByteRequest(200, 300), b""), + ] + for request, expected in cases: + got = await store.get(key, proto, request) + assert got is not None + assert got.to_bytes() == expected + assert await store.get("missing", proto) is None + values = await store.get_partial_values( + proto, [("e", None), ("a/b", RangeByteRequest(0, 1))] + ) + assert [v.to_bytes() if v else None for v in values] == [b"y", b"0"] + + async def test_listing(self, store: ZipReaderStore) -> None: + assert sorted([k async for k in store.list()]) == ["a/b", "a/c/d", "e", "z"] + assert sorted([k async for k in store.list_prefix("a/")]) == ["a/b", "a/c/d"] + assert sorted([k async for k in store.list_dir("")]) == ["a", "e", "z"] + assert sorted([k async for k in store.list_dir("a")]) == ["b", "c"] + assert await store.exists("e") + assert not await store.exists("a") + assert await store.getsize("z") == 100 + with pytest.raises(FileNotFoundError): + await store.getsize("missing") + + async def test_read_only(self, store: ZipReaderStore) -> None: + from zarr.core.buffer import cpu + + assert store.read_only + with pytest.raises(ValueError, match="read-only"): + await store.set("k", cpu.Buffer.from_bytes(b"")) + with pytest.raises(ValueError, match="read-only"): + await store.delete("e") + + async def test_eq_and_str(self, store: ZipReaderStore) -> None: + same = store + assert store == same + assert store != object() + assert str(store).endswith("|zip:") + assert "ZipReaderStore" in repr(store) + + async def test_bad_crc(self, tmp_path: Path) -> None: + buf = io.BytesIO() + with zipfile.ZipFile(buf, "w") as zf: + zf.writestr("k", b"payload") + raw = bytearray(buf.getvalue()) + raw[raw.index(b"payload")] ^= 0xFF + path = tmp_path / "bad.zip" + path.write_bytes(bytes(raw)) + store = await ZipReaderStore.open_archive(await LocalStore.open(path, read_only=True)) + with pytest.raises(OSError, match="bad CRC-32"): + await store.get("k", default_buffer_prototype())