From 103c87d87bd5afb34c2158f6f73ae9cb3901b60b Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Tue, 6 Oct 2026 10:40:54 +0200 Subject: [PATCH 1/4] refactor(url-pipeline): resolve pipelines from a URLPipeline object, never from plain strings Strings passed as a StoreLike keep their pre-existing meaning: a local path containing `|` still names a directory, whatever adapters are installed. A pipeline is opened either through the new `zarr.open_url`, or by passing a `zarr.storage.URLPipeline` (built with `from_url`) to any function that accepts a StoreLike. - `URLPipeline` holds every parsed segment; a trailing `zarr2:`/`zarr3:` format segment is read from it (`zarr_format`, `path`, `store_segments`) instead of through pseudo-adapters, so `AdapterResolution.zarr_format` is gone. - `StorePath` is back to its state on main: no `zarr_format` attribute, and mode "a" on a read-only store raises again. The open-or-create downgrade applies only to pipelines that resolve read-only. - The format selected by a pipeline is merged with the caller's `zarr_format` by `resolve_zarr_format(store, zarr_format)` at each call site, before the store is resolved. `create_array` and `Group.from_store` default to 3 again; `Array.open` keeps its new `zarr_format` parameter. - `is_url_pipeline` and the `pathlib.Path` escape hatch are removed; `resolve_pipeline` accepts a string or a `URLPipeline`, and a single URL is a trivial pipeline. Assisted-by: ClaudeCode:claude-fable-5-1 Co-Authored-By: Claude Fable 5.1 --- changes/4192.feature.md | 69 ++-- docs/api/zarr/functions/open_url.md | 5 + docs/user-guide/extending.md | 5 +- docs/user-guide/storage.md | 64 ++-- mkdocs.yml | 1 + src/zarr/__init__.py | 2 + src/zarr/abc/url_pipeline.py | 16 +- src/zarr/api/asynchronous.py | 71 ++++- src/zarr/api/synchronous.py | 57 +++- src/zarr/core/array.py | 12 +- src/zarr/core/group.py | 14 +- src/zarr/storage/__init__.py | 3 +- src/zarr/storage/_common.py | 116 +++---- src/zarr/storage/_url_pipeline.py | 206 +++++++++--- tests/test_api.py | 7 +- .../test_url_pipeline/test_pipeline_object.py | 115 +++++++ tests/test_url_pipeline/test_resolver.py | 299 ++++++++++-------- 17 files changed, 730 insertions(+), 332 deletions(-) create mode 100644 docs/api/zarr/functions/open_url.md create mode 100644 tests/test_url_pipeline/test_pipeline_object.py diff --git a/changes/4192.feature.md b/changes/4192.feature.md index 3ea14b30d6..91a4748417 100644 --- a/changes/4192.feature.md +++ b/changes/4192.feature.md @@ -1,46 +1,35 @@ Added core support for URL pipelines (https://github.com/jbms/url-pipeline): `|`-chained URLs that address zarr data through nested storage layers, e.g. -`s3://bucket/data.zip|zip:|zarr3:`. This includes the parser -(`zarr.storage.parse_pipeline` / `zarr.storage.resolve_pipeline`), the -single-method `zarr.abc.url_pipeline.URLPipelineAdapter` interface, and the -`zarr.url_adapters` entry-point group through which third-party packages -(e.g. Icechunk) register adapters for their own schemes. Adapters for a scheme -are loaded lazily and individually. Builtin adapters (`zip:`, -`zarr2:`/`zarr3:`) will follow separately. +`s3://bucket/data.zip|zip:|zarr3:`. A pipeline is opened with the new `zarr.open_url` +function, or by passing a parsed `zarr.storage.URLPipeline` (built with +`URLPipeline.from_url`) to any function that accepts a `StoreLike`. Adapters for the +sub-URLs after the first are provided by third-party packages (e.g. Icechunk) through +the `zarr.url_adapters` entry-point group and the single-method +`zarr.abc.url_pipeline.URLPipelineAdapter` interface; they are loaded lazily and +individually. A builtin `zip:` adapter will follow separately. Behavior notes: -- The `|` character is now reserved as the pipeline delimiter in every string - store specification, and no percent-escape is decoded; pass a `pathlib.Path` - to address a local file whose name contains `|`. URLs without a `|` (and - without a registered root adapter scheme) are handled exactly as before — - registered adapters cannot intercept zarr's native `file:`/`memory:` - routing, and fsspec chained URLs (`zip::s3://...`) keep flowing to fsspec. -- Inside a pipeline, `memory:` and `file:` roots follow the URL pipeline - spec's spelling rules (`memory:` ≡ `memory:/` ≡ `memory://`; `file:` must - carry an absolute path with at most a `localhost` authority). Percent-escapes - are *not* decoded in `file:` roots, matching `LocalStore`; this is a - documented divergence from RFC 8089. When fsspec is installed, a plain - `memory://name` URL is still routed to fsspec's in-memory filesystem, whereas - a pipeline's `memory:` root is zarr's `ManagedMemoryStore`; the two do not - share data. +- Plain strings are never interpreted as pipelines: `zarr.open("data|x.zarr")` still + addresses a local directory of that name, whatever adapters are installed. Only + `zarr.open_url` and `URLPipeline.from_url` read a string as a pipeline. +- A trailing `zarr2:`/`zarr3:` segment selects the zarr format of the open or create + call, and its body is the path of the node to open. A `zarr_format` argument that + conflicts with it raises `ValueError`; functions whose `zarr_format` defaults to 3 + (`zarr.create_array`, `Group.from_store`, `Group.open`, `Array.open`) need an explicit + `zarr_format=None` to defer to the pipeline. `Array.open` gains a `zarr_format` + parameter mirroring `Group.open`. +- With the default mode `"a"`, a pipeline that resolves to a read-only resource is + opened for reading instead of raising; other stores behave as before. +- Inside a pipeline, `memory:` and `file:` roots follow the URL pipeline spec's spelling + rules (`memory:` ≡ `memory:/` ≡ `memory://`; `file:` must carry an absolute path with + at most a `localhost` authority). Percent-escapes are not decoded, matching + `LocalStore`. A pipeline's `memory:` root is zarr's `ManagedMemoryStore`, which does + not share data with the fsspec in-memory filesystem a plain `memory://name` string + resolves to. - Wrapper adapters can open a local *file* root only in mode `"r"` (via - `PipelineContext.resolve_preceding(mode="r")`); other modes create a - directory at the root and fail. Writing through a container file is not yet - supported. -- A `zarr.url_adapters` entry point whose scheme is already registered (e.g. by - a builtin adapter) is ignored with a `ZarrUserWarning`, as are duplicate - entry-point names; an entry point that fails to import raises - `URLPipelineError` naming it, and stays discoverable for a retry. -- `zarr.create_array` and `Group.from_store` now default `zarr_format` to - `None`, resolving to the configured default (3) as before unless a URL - pipeline selects a format; `Array.open` gains a `zarr_format` parameter - mirroring `Group.open`. `Group.open` and `Array.open` keep their default of - 3, so pass `zarr_format=None` to defer to a pipeline's format. A format - selected by a pipeline that conflicts with an explicit `zarr_format` raises - `ValueError` in every `open*`/`create*`/`save*` entry point. -- Mode `"a"` (open-or-create, the `zarr.open` default) on a *read-only* store - now serves the "open" half instead of raising upfront, for all stores; - unambiguous write modes (`"w"`, `"w-"`, `"r+"`) still raise. -- For root-adapter URLs (e.g. `gh://org/repo`), `storage_options` are handed - to the adapter and are not validated as used by `make_store`. + `PipelineContext.resolve_preceding(mode="r")`); writing through a container file is + not yet supported. +- A `zarr.url_adapters` entry point whose scheme is already registered is ignored with a + `ZarrUserWarning`, as are duplicate entry-point names; an entry point that fails to + import raises `URLPipelineError` naming it, and stays discoverable for a retry. diff --git a/docs/api/zarr/functions/open_url.md b/docs/api/zarr/functions/open_url.md new file mode 100644 index 0000000000..84774a1710 --- /dev/null +++ b/docs/api/zarr/functions/open_url.md @@ -0,0 +1,5 @@ +--- +title: zarr.open_url +--- + +::: zarr.open_url diff --git a/docs/user-guide/extending.md b/docs/user-guide/extending.md index d5fe5726ca..5138ba3a10 100644 --- a/docs/user-guide/extending.md +++ b/docs/user-guide/extending.md @@ -112,7 +112,7 @@ class MyAdapter(URLPipelineAdapter): ) -> AdapterResolution: # A *wrapper* adapter opens the resource to its left and wraps it. Join the # preceding residual path with this segment's own path, and keep every other - # field (e.g. zarr_format) via dataclasses.replace. + # field via dataclasses.replace. preceding = await context.resolve_preceding() store = WrapperStore(preceding.store) path = "/".join(part.strip("/") for part in (preceding.path, segment.body) if part) @@ -144,7 +144,8 @@ scheme as the entry-point name. Nonstandard schemes should be vendor-prefixed Adapters are loaded lazily, only when a pipeline naming their scheme is resolved. An adapter may also be used as a *root* scheme (e.g. `mypackage.myscheme://org/repo`), in -which case `context.preceding` is empty. The returned store must honor +which case `context.preceding` is empty; such a URL reaches the adapter only through +`zarr.open_url` or a `URLPipeline`, never as a plain string store specification. The returned store must honor `context.read_only`, and `open_pipeline_segment` runs on zarr's internal event loop, so it must not block or call zarr's synchronous API. See [`zarr.abc.url_pipeline`][] for the full contract. diff --git a/docs/user-guide/storage.md b/docs/user-guide/storage.md index 010abc08a4..4d4571c7e3 100644 --- a/docs/user-guide/storage.md +++ b/docs/user-guide/storage.md @@ -100,8 +100,9 @@ print(group) group = zarr.open_group(UPath('s3://noaa-nwm-retro-v2-zarr-pds', anon=True), mode='r') ``` -- a [URL pipeline](#user-guide-url-pipelines) string containing `|`, such as - `s3://bucket/data.zip|zip:`, which is resolved through registered adapters. +- a [`URLPipeline`][zarr.storage.URLPipeline], the parsed form of a + [URL pipeline](#user-guide-url-pipelines) such as `s3://bucket/data.zip|zip:`, which + is resolved through registered adapters. Strings are never interpreted as pipelines. - a [`Store`][zarr.abc.store.Store] or [`StorePath`][zarr.storage.StorePath] - see explicit store creation below. @@ -114,28 +115,49 @@ sub-URL locates a resource with a conventional URL; each subsequent sub-URL name *adapter* that reinterprets everything to its left (e.g. `s3://bucket/data.zip|zip:|zarr3:`). Adapters are provided by packages through the `zarr.url_adapters` entry-point group — see -[`zarr.abc.url_pipeline`][zarr.abc.url_pipeline] for the adapter interface. Builtin -adapters (`zip:`, `zarr2:`/`zarr3:`) are under development and will expand this -section. URLs without a `|` (and without a registered root scheme) are handled -exactly as before. - -`storage_options` passed to `zarr.open` apply to the *root* sub-URL (e.g. fsspec -options for `s3://...`); adapters may consume adapter-specific, namespaced keys. -Non-dict forms of `storage_options` are reserved for future per-segment -configuration. - -The `|` character is reserved as the pipeline delimiter in every string store -specification, and no percent-escape is decoded: to address a local file whose -*name* contains `|` (or `#`), pass a `pathlib.Path` instead of a string. -Registered adapters cannot intercept zarr's native `file:` and `memory:` root -schemes, and fsspec's chained-URL syntax (`zip::s3://...`) keeps flowing to -fsspec. +[`zarr.abc.url_pipeline`][zarr.abc.url_pipeline] for the adapter interface. A builtin +`zip:` adapter is under development and will expand this section. + +A pipeline is opened either with [`zarr.open_url`][], which is `zarr.open` with its +string argument read as a pipeline, or by passing a parsed +[`URLPipeline`][zarr.storage.URLPipeline] to any function that accepts a `StoreLike`: + +```python exec="true" session="storage" source="above" result="ansi" +from zarr.storage import URLPipeline + +pipeline = URLPipeline.from_url("memory://pipeline-demo|zarr2:") +group = zarr.open_group(pipeline, mode="w") +print(group.metadata.zarr_format) +# the same thing, as a one-liner +print(zarr.open_url("memory://pipeline-demo|zarr2:").metadata.zarr_format) +``` + +Only these two entry points read a string as a pipeline. Plain string store +specifications keep their existing meaning everywhere else, so a local path that +happens to contain `|` still names a directory, whatever adapters are installed. + +A trailing `zarr2:` or `zarr3:` segment is a *format segment*. It is not an adapter: +zarr-python reads it from the pipeline itself and uses it as the `zarr_format` of the +open or create call, and its body (`zarr3:path/to/node`) is the path of the node to +open. A `zarr_format` argument that conflicts with the format segment raises +`ValueError`; functions whose `zarr_format` defaults to 3 (such as `zarr.create_array` +and `Group.open`) need an explicit `zarr_format=None` to defer to the pipeline. + +`storage_options` apply to the *root* sub-URL (e.g. fsspec options for `s3://...`); +adapters may consume adapter-specific, namespaced keys. Non-dict forms of +`storage_options` are reserved for future per-segment configuration. No percent-escape +is decoded anywhere in a pipeline. Registered adapters cannot intercept zarr's native +`file:` and `memory:` root schemes. With the default mode `"a"`, a pipeline that +resolves to a read-only resource is opened for reading instead of raising. Inside a pipeline, a `memory:` root is zarr's managed in-memory store (`memory:`, `memory:/` and `memory://` are equivalent, and `memory:name` selects a named store). -When fsspec is installed, a plain `memory://name` URL *without* a `|` is still routed to -fsspec's in-memory filesystem, which is a different store. A `file:` root must carry an -absolute path; percent-escapes are not decoded, matching the [local store](#local-store). +When fsspec is installed, a plain `memory://name` string store specification is still +routed to fsspec's in-memory filesystem, which is a different store. A `file:` root +must carry an absolute path; percent-escapes are not decoded, matching the +[local store](#local-store). As a zarr-python extension, the root may also be a bare +local path (`data/example.zip|zip:`); such pipelines are not portable to other +implementations. ## Explicit Store Creation diff --git a/mkdocs.yml b/mkdocs.yml index 58b5d3344a..c0b7a96fa0 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -81,6 +81,7 @@ nav: - ' zarr.open_consolidated': api/zarr/functions/open_consolidated.md - ' zarr.open_group': api/zarr/functions/open_group.md - ' zarr.open_like': api/zarr/functions/open_like.md + - ' zarr.open_url': api/zarr/functions/open_url.md - ' zarr.print_debug_info': api/zarr/functions/print_debug_info.md - ' zarr.registry': api/zarr/registry.md - ' zarr.save': api/zarr/functions/save.md diff --git a/src/zarr/__init__.py b/src/zarr/__init__.py index f305245f4e..b8ac1fb11e 100644 --- a/src/zarr/__init__.py +++ b/src/zarr/__init__.py @@ -27,6 +27,7 @@ open_consolidated, open_group, open_like, + open_url, save, save_array, save_group, @@ -175,6 +176,7 @@ def set_format(log_format: str) -> None: "open_consolidated", "open_group", "open_like", + "open_url", "print_debug_info", "save", "save_array", diff --git a/src/zarr/abc/url_pipeline.py b/src/zarr/abc/url_pipeline.py index 337563685d..e1cf96db41 100644 --- a/src/zarr/abc/url_pipeline.py +++ b/src/zarr/abc/url_pipeline.py @@ -25,7 +25,7 @@ if TYPE_CHECKING: from zarr.abc.store import Store - from zarr.core.common import AccessModeLiteral, ZarrFormat + from zarr.core.common import AccessModeLiteral __all__ = [ "AdapterResolution", @@ -89,17 +89,17 @@ class AdapterResolution: Residual path *within* the store that the pipeline addresses (e.g. `"path/to/node"` for `...|icechunk://tag.v1/path/to/node`). Empty string when the pipeline addresses the store root. - zarr_format : ZarrFormat | None - Zarr format selected by a format segment (`zarr2:`/`zarr3:`), - or None if unspecified. A wrapper adapter that re-wraps a preceding - resolution must carry every field it does not change forward — - prefer `dataclasses.replace(preceding, store=..., path=...)` over - reconstructing, so fields added later are never silently dropped. + + A wrapper adapter that re-wraps a preceding resolution must carry every + field it does not change forward: prefer + `dataclasses.replace(preceding, store=..., path=...)` over + reconstructing, so fields added later are never silently dropped. + Format segments (`zarr2:`/`zarr3:`) are not adapters; they are consumed + by [`URLPipeline`][zarr.storage.URLPipeline] before resolution. """ store: Store path: str = "" - zarr_format: ZarrFormat | None = None @dataclass(frozen=True, eq=False) diff --git a/src/zarr/api/asynchronous.py b/src/zarr/api/asynchronous.py index 4bb3ce621d..9b110ed63e 100644 --- a/src/zarr/api/asynchronous.py +++ b/src/zarr/api/asynchronous.py @@ -48,7 +48,7 @@ ZarrUserWarning, ) from zarr.storage import StorePath -from zarr.storage._common import make_store_path +from zarr.storage._common import make_store_path, resolve_zarr_format if TYPE_CHECKING: from collections.abc import Iterable @@ -88,6 +88,7 @@ "open_consolidated", "open_group", "open_like", + "open_url", "save", "save_array", "save_group", @@ -403,8 +404,8 @@ async def open( mode = "r" else: mode = "a" + zarr_format = resolve_zarr_format(store, zarr_format) store_path = await make_store_path(store, mode=mode, path=path, storage_options=storage_options) - zarr_format = store_path.resolve_zarr_format(zarr_format) # TODO: the mode check below seems wrong! if "shape" not in kwargs and mode in (*_READ_MODES, "w"): @@ -436,6 +437,60 @@ async def open( return await open_group(store=store_path, zarr_format=zarr_format, mode=mode, **kwargs) +async def open_url( + url: str, + *, + mode: AccessModeLiteral | None = None, + zarr_format: ZarrFormat | None = None, + path: str | None = None, + storage_options: dict[str, Any] | None = None, + **kwargs: Any, +) -> AnyAsyncArray | AsyncGroup: + """Open a group or array addressed by a URL pipeline. + + This is [`open`][zarr.api.asynchronous.open] with its `store` argument + interpreted as a [URL pipeline][user-guide-url-pipelines] string, i.e. + `open(URLPipeline.from_url(url), ...)`. It is the only entry point that + reads a string as a pipeline; `open` and the other `StoreLike`-taking + functions treat strings as they always have. + + Parameters + ---------- + url : str + A URL pipeline such as `"s3://bucket/data.zip|zip:|zarr3:"`. A URL + without `|` is a trivial pipeline consisting of its root alone. + mode : {'r', 'r+', 'a', 'w', 'w-'}, optional + Persistence mode, with the same meaning as for `zarr.open`. + zarr_format : {2, 3, None}, optional + The zarr format to use. None (the default) defers to a trailing + `zarr2:`/`zarr3:` segment of the pipeline; a value that conflicts + with such a segment raises `ValueError`. + path : str or None, optional + The path within the store to open, appended to any node path the + pipeline itself addresses. + storage_options : dict + Options for the pipeline's root sub-URL (e.g. fsspec options). + **kwargs + Additional parameters are passed through to `zarr.open_array` or + `zarr.open_group`. + + Returns + ------- + z : array or group + Return type depends on what exists in the given store. + """ + from zarr.storage import URLPipeline + + return await open( + store=URLPipeline.from_url(url), + mode=mode, + zarr_format=zarr_format, + path=path, + storage_options=storage_options, + **kwargs, + ) + + async def open_consolidated( *args: Any, use_consolidated: Literal[True] = True, **kwargs: Any ) -> AsyncGroup: @@ -531,8 +586,8 @@ async def save_array( raise TypeError("arr argument must be numpy or other NDArrayLike array") mode = kwargs.pop("mode", "a") + zarr_format = resolve_zarr_format(store, zarr_format) store_path = await make_store_path(store, path=path, mode=mode, storage_options=storage_options) - zarr_format = store_path.resolve_zarr_format(zarr_format) if zarr_format is None: zarr_format = _default_zarr_format() if np.isscalar(arr): @@ -600,8 +655,8 @@ async def save_group( for arr in (*args, *kwargs.values()): get_data_type_from_native_dtype(arr.dtype) + zarr_format = resolve_zarr_format(store, zarr_format) store_path = await make_store_path(store, path=path, mode="w", storage_options=storage_options) - zarr_format = store_path.resolve_zarr_format(zarr_format) if zarr_format is None: zarr_format = _default_zarr_format() # The group replaces whatever is stored under the path, now that the arguments are known @@ -800,8 +855,8 @@ async def create_group( mode: Literal["a"] = "a" + zarr_format = resolve_zarr_format(store, zarr_format) store_path = await make_store_path(store, path=path, mode=mode, storage_options=storage_options) - zarr_format = store_path.resolve_zarr_format(zarr_format) if zarr_format is None: zarr_format = _default_zarr_format() @@ -893,8 +948,8 @@ async def open_group( } ) + zarr_format = resolve_zarr_format(store, zarr_format) store_path = await make_store_path(store, mode=mode, storage_options=storage_options, path=path) - zarr_format = store_path.resolve_zarr_format(zarr_format) if attributes is None: attributes = {} @@ -1119,8 +1174,8 @@ async def create( if mode is None: mode = "a" overwrite = overwrite or _infer_overwrite(mode) + zarr_format = resolve_zarr_format(store, zarr_format) store_path = await make_store_path(store, path=path, mode=mode, storage_options=storage_options) - zarr_format = store_path.resolve_zarr_format(zarr_format) if zarr_format is None: zarr_format = _default_zarr_format() @@ -1336,8 +1391,8 @@ async def open_array( """ mode = kwargs.pop("mode", None) + zarr_format = resolve_zarr_format(store, zarr_format) store_path = await make_store_path(store, path=path, mode=mode, storage_options=storage_options) - zarr_format = store_path.resolve_zarr_format(zarr_format) if "write_empty_chunks" in kwargs: _warn_write_empty_chunks_kwarg() diff --git a/src/zarr/api/synchronous.py b/src/zarr/api/synchronous.py index 6778f8d907..a599f6b8e1 100644 --- a/src/zarr/api/synchronous.py +++ b/src/zarr/api/synchronous.py @@ -66,6 +66,7 @@ "open_consolidated", "open_group", "open_like", + "open_url", "save", "save_array", "save_group", @@ -246,6 +247,60 @@ def open( return Group(obj) +def open_url( + url: str, + *, + mode: AccessModeLiteral | None = None, + zarr_format: ZarrFormat | None = None, + path: str | None = None, + storage_options: dict[str, Any] | None = None, + **kwargs: Any, +) -> AnyArray | Group: + """Open a group or array addressed by a URL pipeline. + + This is [`open`][zarr.open] with its `store` argument interpreted as a + [URL pipeline][user-guide-url-pipelines] string, i.e. + `zarr.open(URLPipeline.from_url(url), ...)`. It is the only entry point + that reads a string as a pipeline; `zarr.open` and the other + `StoreLike`-taking functions treat strings as they always have. + + Parameters + ---------- + url : str + A URL pipeline such as `"s3://bucket/data.zip|zip:|zarr3:"`. A URL + without `|` is a trivial pipeline consisting of its root alone. + mode : {'r', 'r+', 'a', 'w', 'w-'}, optional + Persistence mode, with the same meaning as for `zarr.open`. + zarr_format : {2, 3, None}, optional + The zarr format to use. None (the default) defers to a trailing + `zarr2:`/`zarr3:` segment of the pipeline; a value that conflicts + with such a segment raises `ValueError`. + path : str or None, optional + The path within the store to open, appended to any node path the + pipeline itself addresses. + storage_options : dict + Options for the pipeline's root sub-URL (e.g. fsspec options). + **kwargs + Additional parameters are passed through to `zarr.open_array` or + `zarr.open_group`. + + Returns + ------- + z : array or group + Return type depends on what exists in the given store. + """ + from zarr.storage import URLPipeline + + return open( + store=URLPipeline.from_url(url), + mode=mode, + zarr_format=zarr_format, + path=path, + storage_options=storage_options, + **kwargs, + ) + + def open_consolidated(*args: Any, use_consolidated: Literal[True] = True, **kwargs: Any) -> Group: """ Alias for [`open_group`][zarr.api.synchronous.open_group] with `use_consolidated=True`. @@ -866,7 +921,7 @@ def create_array( serializer: SerializerLike = "auto", fill_value: Any | None = DEFAULT_FILL_VALUE, order: MemoryOrder | None = None, - zarr_format: ZarrFormat | None = None, + zarr_format: ZarrFormat | None = 3, attributes: dict[str, JSON] | None = None, chunk_key_encoding: ChunkKeyEncodingLike | None = None, dimension_names: DimensionNamesLike = None, diff --git a/src/zarr/core/array.py b/src/zarr/core/array.py index f32ee7a94b..feedea7528 100644 --- a/src/zarr/core/array.py +++ b/src/zarr/core/array.py @@ -154,7 +154,7 @@ _parse_bytes_bytes_codec, get_pipeline_class, ) -from zarr.storage._common import StorePath, make_store_path +from zarr.storage._common import StorePath, make_store_path, resolve_zarr_format from zarr.storage._utils import _relativize_path if TYPE_CHECKING: @@ -466,8 +466,8 @@ async def _create( Deprecated in favor of [`zarr.api.asynchronous.create_array`][]. """ + zarr_format = resolve_zarr_format(store, zarr_format) store_path = await make_store_path(store) - zarr_format = store_path.resolve_zarr_format(zarr_format) or zarr_format dtype_parsed = parse_dtype(dtype, zarr_format=zarr_format) shape = parse_shapelike(shape) @@ -819,8 +819,8 @@ async def example(): # ``` """ + zarr_format = resolve_zarr_format(store, zarr_format) store_path = await make_store_path(store) - zarr_format = store_path.resolve_zarr_format(zarr_format) metadata_dict = await get_array_metadata(store_path, zarr_format=zarr_format) # TODO: remove this cast when we have better type hints _metadata_dict = cast("ArrayMetadataJSON_V3", metadata_dict) @@ -4387,8 +4387,8 @@ async def from_array( """ mode: Literal["a"] = "a" config_parsed = parse_array_config(config) + zarr_format = resolve_zarr_format(store, zarr_format) store_path = await make_store_path(store, path=name, mode=mode, storage_options=storage_options) - zarr_format = store_path.resolve_zarr_format(zarr_format) ( chunks, @@ -4714,7 +4714,7 @@ async def create_array( serializer: SerializerLike = "auto", fill_value: Any | None = DEFAULT_FILL_VALUE, order: MemoryOrder | None = None, - zarr_format: ZarrFormat | None = None, + zarr_format: ZarrFormat | None = 3, attributes: dict[str, JSON] | None = None, chunk_key_encoding: ChunkKeyEncodingLike | None = None, dimension_names: DimensionNamesLike = None, @@ -4874,10 +4874,10 @@ async def create_array( else: mode: Literal["a"] = "a" + zarr_format = resolve_zarr_format(store, zarr_format) store_path = await make_store_path( store, path=name, mode=mode, storage_options=storage_options ) - zarr_format = store_path.resolve_zarr_format(zarr_format) return await init_array( store_path=store_path, shape=shape_parsed, diff --git a/src/zarr/core/group.py b/src/zarr/core/group.py index b541aff437..01d6e05df9 100644 --- a/src/zarr/core/group.py +++ b/src/zarr/core/group.py @@ -68,7 +68,7 @@ ZarrUserWarning, ) from zarr.storage import StoreLike, StorePath -from zarr.storage._common import make_store_path +from zarr.storage._common import make_store_path, resolve_zarr_format from zarr.storage._utils import _join_paths, _normalize_path_keys, normalize_path if TYPE_CHECKING: @@ -531,10 +531,10 @@ async def from_store( *, attributes: dict[str, Any] | None = None, overwrite: bool = False, - zarr_format: ZarrFormat | None = None, + zarr_format: ZarrFormat | None = 3, ) -> AsyncGroup: + zarr_format = resolve_zarr_format(store, zarr_format) store_path = await make_store_path(store) - zarr_format = store_path.resolve_zarr_format(zarr_format) if zarr_format is None: zarr_format = _default_zarr_format() attributes = attributes or {} @@ -578,8 +578,8 @@ async def open( (``.zmetadata`` by default). Specify the custom key as ``use_consolidated`` to load consolidated metadata from a non-default key. """ + zarr_format = resolve_zarr_format(store, zarr_format) store_path = await make_store_path(store) - zarr_format = store_path.resolve_zarr_format(zarr_format) if not store_path.store.supports_consolidated_metadata: # Fail if consolidated metadata was requested but the Store doesn't support it if use_consolidated: @@ -1959,7 +1959,7 @@ def from_store( store: StoreLike, *, attributes: dict[str, Any] | None = None, - zarr_format: ZarrFormat | None = None, + zarr_format: ZarrFormat | None = 3, overwrite: bool = False, ) -> Group: """Instantiate a group from an initialized store. @@ -1973,8 +1973,8 @@ def from_store( attributes : dict, optional A dictionary of JSON-serializable values with user-defined attributes. zarr_format : {2, 3, None}, optional - Zarr storage format version. If None, the format selected by a URL - pipeline segment is used, falling back to the configured default. + Zarr storage format version. None defers to the format selected by a + `URLPipeline` store, falling back to the configured default. overwrite : bool, optional If True, do not raise an error if the group already exists. diff --git a/src/zarr/storage/__init__.py b/src/zarr/storage/__init__.py index edc01f1a5b..66328d5a32 100644 --- a/src/zarr/storage/__init__.py +++ b/src/zarr/storage/__init__.py @@ -10,7 +10,7 @@ from zarr.storage._logging import LoggingStore from zarr.storage._memory import GpuMemoryStore, ManagedMemoryStore, MemoryStore from zarr.storage._obstore import ObjectStore -from zarr.storage._url_pipeline import parse_pipeline, resolve_pipeline +from zarr.storage._url_pipeline import URLPipeline, parse_pipeline, resolve_pipeline from zarr.storage._wrapper import WrapperStore from zarr.storage._zip import ZipStore @@ -24,6 +24,7 @@ "ObjectStore", "StoreLike", "StorePath", + "URLPipeline", "WrapperStore", "ZipStore", "parse_pipeline", diff --git a/src/zarr/storage/_common.py b/src/zarr/storage/_common.py index f649a7eea6..3886e9d0f8 100644 --- a/src/zarr/storage/_common.py +++ b/src/zarr/storage/_common.py @@ -2,7 +2,7 @@ import importlib.util from pathlib import Path -from typing import TYPE_CHECKING, Any, Literal, Self +from typing import TYPE_CHECKING, Any, Literal, Self, overload from zarr.abc.store import ( ByteRequest, @@ -29,7 +29,7 @@ ) from zarr.storage._local import LocalStore from zarr.storage._memory import ManagedMemoryStore, MemoryStore -from zarr.storage._url_pipeline import is_url_pipeline, resolve_pipeline +from zarr.storage._url_pipeline import URLPipeline, resolve_pipeline from zarr.storage._utils import UPath, _join_paths, normalize_path, parse_store_url _has_fsspec = importlib.util.find_spec("fsspec") @@ -53,57 +53,19 @@ class StorePath: The store to use. path : str The path within the store. - - Attributes - ---------- - zarr_format : ZarrFormat | None - Zarr format selected by a `zarr2:`/`zarr3:` URL pipeline segment, - or None. Not part of the StorePath's identity (ignored by `__eq__`). """ store: Store path: str - zarr_format: ZarrFormat | None def __init__(self, store: Store, path: str = "") -> None: self.store = store self.path = normalize_path(path) - self.zarr_format = None @property def read_only(self) -> bool: return self.store.read_only - def resolve_zarr_format(self, zarr_format: ZarrFormat | None) -> ZarrFormat | None: - """ - Combine the zarr format selected by a URL pipeline segment (`zarr2:` / - `zarr3:`) with a caller-supplied `zarr_format`. - - Parameters - ---------- - zarr_format : ZarrFormat | None - The format requested by the caller, or None when unspecified. - - Returns - ------- - ZarrFormat | None - The pipeline's format when the caller did not specify one, otherwise the - caller's format. None when neither is set. - - Raises - ------ - ValueError - If the caller's format differs from the one selected by the pipeline. - """ - if self.zarr_format is None: - return zarr_format - if zarr_format is not None and zarr_format != self.zarr_format: - raise ValueError( - f"zarr_format={zarr_format} conflicts with the 'zarr{self.zarr_format}:' " - "segment of the URL pipeline; pass zarr_format=None to use the pipeline's format" - ) - return self.zarr_format - @classmethod async def _create_open_instance(cls, store: Store, path: str) -> Self: """Helper to create and return a StorePath instance.""" @@ -116,10 +78,7 @@ async def open(cls, store: Store, path: str, mode: AccessModeLiteral | None = No Open StorePath based on the provided mode. * If the mode is None, return an opened version of the store with no changes. - * If the mode is 'r+', 'w-', or 'w' and the store is read-only, raise a ValueError. - * If the mode is 'a' (open-or-create) and the store is read-only, serve the - "open" half: the StorePath is opened read-only, and any subsequent write - fails at the store level. + * If the mode is 'r+', 'w-', 'w', or 'a' and the store is read-only, raise a ValueError. * If the mode is 'r' and the store is not read-only, return a copy of the store with read_only set to True. * If the mode is 'w-' and the store is not read-only and the StorePath contains keys, raise a FileExistsError. @@ -145,8 +104,7 @@ async def open(cls, store: Store, path: str, mode: AccessModeLiteral | None = No FileExistsError If the mode is 'w-' and the store path already exists. ValueError - If the mode is "r+", "w-", or "w" and the store is read-only, or if the - mode is not a valid access mode. + If the mode is not "r" and the store is read-only, or """ # fastpath if mode is None @@ -157,9 +115,8 @@ async def open(cls, store: Store, path: str, mode: AccessModeLiteral | None = No raise ValueError(f"Invalid mode: {mode}, expected one of {ANY_ACCESS_MODE}") if store.read_only: - # mode "a" (open-or-create) on a read-only store means open-only; - # unambiguous write modes are rejected outright. - if mode not in ("r", "a"): + # Don't allow write operations on a read-only store + if mode != "r": raise ValueError( f"Store is read-only but mode is {mode!r}. Create a writable store or use 'r' mode." ) @@ -320,9 +277,7 @@ def delete_sync(self) -> None: def __truediv__(self, other: str) -> StorePath: """Combine this store path with another path""" - result = self.__class__(self.store, _join_paths([self.path, other])) - result.zarr_format = self.zarr_format - return result + return self.__class__(self.store, _join_paths([self.path, other])) def __str__(self) -> str: return _join_paths([str(self.store), self.path]) @@ -350,7 +305,7 @@ def __eq__(self, other: object) -> bool: return False -type StoreLike = Store | StorePath | FSMap | Path | UPath | str | dict[str, Buffer] +type StoreLike = Store | StorePath | URLPipeline | FSMap | Path | UPath | str | dict[str, Buffer] async def make_store( @@ -371,9 +326,9 @@ async def make_store( - `None` = `MemoryStore` object. - `FSMap` = `FsspecStore` object. - `UPath` = `FsspecStore` object, or `LocalStore` for a local `UPath`. - - `str` containing `|` (or whose scheme has a registered root adapter) = the store - resolved by the [URL pipeline][user-guide-url-pipelines]; the pipeline must address - the store root (use `make_store_path` for pipelines that address a path within a store). + - `URLPipeline` = the store resolved by the [URL pipeline][user-guide-url-pipelines]; + the pipeline must address the store root (use `make_store_path` for pipelines that + address a path within a store). Plain strings are never interpreted as pipelines. Parameters ---------- @@ -404,12 +359,12 @@ async def make_store( raise ValueError(f"Invalid mode: {mode}, expected one of {ANY_ACCESS_MODE}") _read_only = mode == "r" - if isinstance(store_like, str) and is_url_pipeline(store_like): + if isinstance(store_like, URLPipeline): result = await resolve_pipeline(store_like, mode=mode, storage_options=storage_options) if normalize_path(result.path): result.store.close() raise URLPipelineError( - f"the URL pipeline {store_like!r} resolves to a path inside a store; " + f"the URL pipeline {str(store_like)!r} resolves to a path inside a store; " "use zarr.open() or make_store_path() instead of make_store()" ) return result.store @@ -521,8 +476,8 @@ async def make_store_path( ValueError If path is provided for a store that does not support it. URLPipelineError - If `store_like` is a [URL pipeline][user-guide-url-pipelines] string that - cannot be resolved. + If `store_like` is a [URL pipeline][user-guide-url-pipelines] that cannot be + resolved. See Also -------- @@ -530,12 +485,15 @@ async def make_store_path( """ path_normalized = normalize_path(path) - if isinstance(store_like, str) and is_url_pipeline(store_like): + if isinstance(store_like, URLPipeline): result = await resolve_pipeline(store_like, mode=mode, storage_options=storage_options) combined_path = _join_paths([normalize_path(result.path), path_normalized]) - store_path = await StorePath.open(result.store, path=combined_path, mode=mode) - store_path.zarr_format = result.zarr_format - return store_path + if mode == "a" and result.store.read_only: + # "a" (open-or-create, the zarr.open default) on a pipeline that + # resolves to a read-only resource serves the "open" half; any + # later write fails at the store level. + mode = "r" + return await StorePath.open(result.store, path=combined_path, mode=mode) if isinstance(store_like, StorePath): # Already a StorePath @@ -556,6 +514,36 @@ async def make_store_path( return await StorePath.open(store, path=path_normalized, mode=mode) +@overload +def resolve_zarr_format(store_like: StoreLike | None, zarr_format: ZarrFormat) -> ZarrFormat: ... + + +@overload +def resolve_zarr_format( + store_like: StoreLike | None, zarr_format: ZarrFormat | None +) -> ZarrFormat | None: ... + + +def resolve_zarr_format( + store_like: StoreLike | None, zarr_format: ZarrFormat | None +) -> ZarrFormat | None: + """ + Merge the zarr format selected by a `URLPipeline` store specification with + the format a caller asked for. + + For any other `StoreLike`, `zarr_format` is returned unchanged. For a + pipeline, see [`URLPipeline.resolve_zarr_format`][zarr.storage.URLPipeline]. + + Raises + ------ + ValueError + If `zarr_format` is not None and differs from the pipeline's format. + """ + if isinstance(store_like, URLPipeline): + return store_like.resolve_zarr_format(zarr_format) + return zarr_format + + async def ensure_no_existing_node( store_path: StorePath, zarr_format: ZarrFormat, diff --git a/src/zarr/storage/_url_pipeline.py b/src/zarr/storage/_url_pipeline.py index 41a2114daf..ec8c9294eb 100644 --- a/src/zarr/storage/_url_pipeline.py +++ b/src/zarr/storage/_url_pipeline.py @@ -5,11 +5,11 @@ (registered through the `zarr.url_adapters` entry-point group) are loaded only when a pipeline URL naming their scheme is actually resolved. -The `|` character is reserved as the pipeline delimiter in every string -store specification: a string containing `|` is always routed through the -pipeline machinery, and no percent-escape is decoded. To address a local -file whose *name* contains `|` (or `#`), pass a `pathlib.Path` instead of a -string. +A string is never interpreted as a pipeline on its own: the user wraps it +in a [`URLPipeline`][zarr.storage.URLPipeline] (or calls `zarr.open_url`), +and that object is what the `StoreLike` machinery resolves. Plain string +store specifications, including local paths that happen to contain `|`, +keep their pre-existing meaning. No percent-escape is decoded. As a zarr-python extension to the specification (which requires the root sub-URL of an absolute pipeline to carry a scheme), the root sub-URL may be @@ -33,13 +33,18 @@ from zarr.errors import URLPipelineError from zarr.registry import get_url_adapter, list_url_adapter_schemes from zarr.storage._memory import ManagedMemoryStore -from zarr.storage._utils import parse_store_url +from zarr.storage._utils import _join_paths, normalize_path, parse_store_url if TYPE_CHECKING: from zarr.abc.store import Store - from zarr.core.common import AccessModeLiteral + from zarr.core.common import AccessModeLiteral, ZarrFormat -__all__ = ["is_url_pipeline", "parse_pipeline", "resolve_pipeline"] +__all__ = ["URLPipeline", "parse_pipeline", "resolve_pipeline"] + +# Format segments are consumed by zarr-python itself rather than by a +# registered adapter: they select the zarr format of the node the pipeline +# addresses, and their body is a path to that node. +_FORMAT_SCHEMES: dict[str, ZarrFormat] = {"zarr2": 2, "zarr3": 3} # Adapter scheme per RFC 3986 plus "." to permit vendor-prefixed # nonstandard schemes (e.g. "earthmover.myscheme"). @@ -137,6 +142,140 @@ def parse_pipeline(url: str) -> tuple[PipelineSegment, ...]: return tuple(segments) +@dataclasses.dataclass(frozen=True) +class URLPipeline: + """ + A parsed URL pipeline, ready to be passed wherever a `StoreLike` is accepted. + + Construct one with [`from_url`][zarr.storage.URLPipeline.from_url]. Because a + `URLPipeline` is an explicit object, no plain string store specification is + ever interpreted as a pipeline: `zarr.open("data|x.zarr")` still addresses a + local directory of that name, while + `zarr.open(URLPipeline.from_url("s3://bucket/data.zip|zip:|zarr3:"))` (or the + equivalent `zarr.open_url(...)`) resolves the pipeline through registered + adapters. + + A pipeline is a value: instances are immutable, hashable, and compare equal + when they hold the same segments. Nothing is resolved until the pipeline is + opened; resolution needs the access mode and storage options of the open + call. + + Attributes + ---------- + segments : tuple[PipelineSegment, ...] + Every `|`-delimited sub-URL of the pipeline, in order: the root, any + adapter segments, and an optional trailing format segment. + + Raises + ------ + URLPipelineError + If there are no segments, if a format segment (`zarr2:`/`zarr3:`) + appears anywhere but last, or if a format segment carries a query. + """ + + segments: tuple[PipelineSegment, ...] + + def __post_init__(self) -> None: + if not self.segments: + raise URLPipelineError("a URL pipeline needs at least one segment") + for segment in self.segments[1:-1]: + if segment.scheme in _FORMAT_SCHEMES: + raise URLPipelineError( + f"the format segment {segment.raw!r} must be the last segment " + f"of the URL pipeline {str(self)!r}" + ) + format_segment = self._format_segment + if format_segment is not None and format_segment.query is not None: + raise URLPipelineError( + f"'{format_segment.scheme}:' segments do not accept a query: {format_segment.raw!r}" + ) + + @classmethod + def from_url(cls, url: str) -> URLPipeline: + """ + Parse a URL pipeline string. + + A URL without a `|` is a trivial pipeline consisting of its root alone. + + Raises + ------ + URLPipelineError + If the string cannot be parsed or the resulting pipeline is invalid + (see the class docstring). + """ + return cls(parse_pipeline(url)) + + @property + def _format_segment(self) -> PipelineSegment | None: + last = self.segments[-1] + if len(self.segments) > 1 and last.scheme in _FORMAT_SCHEMES: + return last + return None + + @property + def store_segments(self) -> tuple[PipelineSegment, ...]: + """ + The segments that address a store: the root and any adapter segments, + i.e. `segments` without a trailing format segment. + """ + if self._format_segment is None: + return self.segments + return self.segments[:-1] + + @property + def zarr_format(self) -> ZarrFormat | None: + """The format selected by a trailing `zarr2:`/`zarr3:` segment, or None.""" + format_segment = self._format_segment + if format_segment is None: + return None + return _FORMAT_SCHEMES[format_segment.scheme] + + @property + def path(self) -> str: + """ + The node path given as the body of a trailing format segment, + normalized. Empty when there is no format segment or its body is empty. + """ + format_segment = self._format_segment + if format_segment is None: + return "" + return normalize_path(format_segment.body) + + def resolve_zarr_format(self, zarr_format: ZarrFormat | None) -> ZarrFormat | None: + """ + Combine the format selected by this pipeline with a caller-supplied one. + + Parameters + ---------- + zarr_format : ZarrFormat | None + The format requested by the caller, or None when unspecified. + + Returns + ------- + ZarrFormat | None + The pipeline's format when the caller did not specify one, otherwise + the caller's format. None when neither is set. + + Raises + ------ + ValueError + If the caller's format differs from the one selected by the pipeline. + """ + pipeline_format = self.zarr_format + if pipeline_format is None: + return zarr_format + if zarr_format is not None and zarr_format != pipeline_format: + raise ValueError( + f"zarr_format={zarr_format} conflicts with the 'zarr{pipeline_format}:' " + f"segment of the URL pipeline {str(self)!r}; pass zarr_format=None to use " + "the pipeline's format" + ) + return pipeline_format + + def __str__(self) -> str: + return "|".join(segment.raw for segment in self.segments) + + def _root_routes_to_adapter(scheme: str) -> bool: """ Whether a root sub-URL with this scheme is dispatched to a registered @@ -149,22 +288,8 @@ def _root_routes_to_adapter(scheme: str) -> bool: ) -def is_url_pipeline(url: str) -> bool: - """ - Whether `url` should be routed through the URL pipeline machinery. - - True when the URL contains a `|` separator, or when its scheme has a - registered URL pipeline adapter (a *root adapter* such as `gh:`). - fsspec chained URLs (`zip::file://...`) and zarr's native `file:` / - `memory:` schemes are never routed to a root adapter. - """ - if "|" in url: - return True - return _root_routes_to_adapter(_root_scheme(url)) - - async def resolve_pipeline( - url: str, + url: str | URLPipeline, *, mode: AccessModeLiteral | None = None, storage_options: dict[str, Any] | None = None, @@ -172,10 +297,17 @@ async def resolve_pipeline( """ Resolve a URL pipeline into a store and a residual path. + The residual path joins the path returned by the adapters with the node + path of a trailing format segment (`...|zarr3:path/to/node`). The format + itself is not part of the resolution; read it from + [`URLPipeline.zarr_format`][zarr.storage.URLPipeline]. + Parameters ---------- - url : str - A URL pipeline, e.g. `"s3://bucket/data.zip|zip:|zarr3:"`. + url : str | URLPipeline + A URL pipeline, e.g. `"s3://bucket/data.zip|zip:|zarr3:"`, or its + parsed form. A URL without `|` is a trivial pipeline consisting of + its root alone. mode : AccessModeLiteral | None The caller's access mode. `"r"` requires a read-only store; the resolver enforces this on whatever the final adapter returns. @@ -187,8 +319,7 @@ async def resolve_pipeline( Raises ------ URLPipelineError - If the URL cannot be parsed, is not a pipeline, names a scheme with - no registered adapter, names an adapter entry point that fails to + If the URL cannot be parsed, names a scheme with no registered adapter, names an adapter entry point that fails to import, or has a root sub-URL that cannot be resolved. TypeError If `storage_options` are passed to a root that does not accept them @@ -197,13 +328,13 @@ async def resolve_pipeline( Errors from opening the root resource (e.g. a missing local directory in mode `"r"`) propagate unchanged, as for non-pipeline stores. """ - segments = parse_pipeline(url) - if len(segments) == 1 and not _root_routes_to_adapter(segments[0].scheme): - raise URLPipelineError( - f"{url!r} is not a URL pipeline: it has no '|' separator and no " - f"URL pipeline adapter is registered for scheme {segments[0].scheme!r}" + pipeline = URLPipeline.from_url(url) if isinstance(url, str) else url + resolution = await _resolve(pipeline.store_segments, mode=mode, storage_options=storage_options) + if pipeline.path: + resolution = dataclasses.replace( + resolution, path=_join_paths([normalize_path(resolution.path), pipeline.path]) ) - return await _resolve(segments, mode=mode, storage_options=storage_options) + return resolution async def _resolve( @@ -218,14 +349,7 @@ async def _resolve( ) *preceding, last = segments - try: - adapter_cls = get_url_adapter(last.scheme) - except URLPipelineError as exc: - raise URLPipelineError( - f"{exc} Note: '|' is reserved as the URL pipeline delimiter; to " - "address a local file whose name contains '|', pass a " - "pathlib.Path instead of a string." - ) from None + adapter_cls = get_url_adapter(last.scheme) context = PipelineContext( preceding=tuple(preceding), diff --git a/tests/test_api.py b/tests/test_api.py index 6a2c4f85f8..f874bd45b7 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -311,12 +311,9 @@ async def test_open_like_creates_array_by_default( assert np.all(Array(new_arr)[:] == ref_arr.fill_value) -async def test_open_like_default_mode_on_read_only_store_raises_not_found( +async def test_open_like_default_mode_rejects_read_only_store( zarr_format: ZarrFormat, ) -> None: - # mode "a" (open-or-create) on a read-only store serves the "open" half: - # since the node does not exist and creating is impossible, the open - # fails with a not-found error rather than rejecting the mode upfront. ref_arr = zarr.create_array( store={}, shape=(11, 12), @@ -325,7 +322,7 @@ async def test_open_like_default_mode_on_read_only_store_raises_not_found( zarr_format=zarr_format, ) - with pytest.raises(ValueError, match="No array found"): + with pytest.raises(ValueError, match="Store is read-only but mode is 'a'"): await zarr.api.asynchronous.open_like( ref_arr, path="foo", diff --git a/tests/test_url_pipeline/test_pipeline_object.py b/tests/test_url_pipeline/test_pipeline_object.py new file mode 100644 index 0000000000..c21e166810 --- /dev/null +++ b/tests/test_url_pipeline/test_pipeline_object.py @@ -0,0 +1,115 @@ +from __future__ import annotations + +import pytest + +from zarr.errors import URLPipelineError +from zarr.storage import URLPipeline +from zarr.storage._url_pipeline import parse_pipeline + + +@pytest.mark.parametrize( + ("url", "schemes", "store_schemes", "zarr_format", "path"), + [ + ( + "s3://bucket/data.zip|zip:inner|zarr3:", + ("s3", "zip", "zarr3"), + ("s3", "zip"), + 3, + "", + ), + # the trailing colon of a format segment is optional + ("file:///data/node.zarr|zarr2", ("file", "zarr2"), ("file",), 2, ""), + # the body of a format segment is a node path, normalized + ( + "file:///repo|icechunk:|zarr3:/path/to/array/", + ("file", "icechunk", "zarr3"), + ("file", "icechunk"), + 3, + "path/to/array", + ), + # no format segment + ("memory://x|wrap:a", ("memory", "wrap"), ("memory", "wrap"), None, ""), + # a single URL is a trivial pipeline + ("s3://bucket/key", ("s3",), ("s3",), None, ""), + # schemeless local root (zarr-python extension) + ("/local/data.zip|zip:", ("", "zip"), ("", "zip"), None, ""), + ], +) +def test_from_url( + url: str, + schemes: tuple[str, ...], + store_schemes: tuple[str, ...], + zarr_format: int | None, + path: str, +) -> None: + """ + `URLPipeline.from_url` holds every parsed segment of the URL, `str()` + reproduces the URL exactly, and the zarr format, node path and + store-addressing segments are derived from a trailing `zarr2:`/`zarr3:` + segment. + """ + pipeline = URLPipeline.from_url(url) + assert pipeline.segments == parse_pipeline(url) + assert tuple(segment.scheme for segment in pipeline.segments) == schemes + assert tuple(segment.scheme for segment in pipeline.store_segments) == store_schemes + assert pipeline.zarr_format == zarr_format + assert pipeline.path == path + assert str(pipeline) == url + assert URLPipeline(pipeline.segments) == pipeline + + +def test_format_segment_must_be_last() -> None: + """A `zarr2:`/`zarr3:` segment anywhere but the end of the pipeline is rejected.""" + with pytest.raises(URLPipelineError, match="must be the last segment"): + URLPipeline.from_url("file:/data|zarr3:|zip:") + + +def test_format_segment_rejects_query() -> None: + """A `zarr2:`/`zarr3:` segment takes a node path only; a query is an error.""" + with pytest.raises(URLPipelineError, match="do not accept a query"): + URLPipeline.from_url("file:/data|zarr3:x?opt=1") + + +def test_direct_construction_is_validated() -> None: + """Validation lives in the constructor, so a hand-built pipeline is checked too.""" + segments = parse_pipeline("file:/data|zarr3:|zip:") + with pytest.raises(URLPipelineError, match="must be the last segment"): + URLPipeline(segments) + + +def test_empty_pipeline_rejected() -> None: + """A pipeline needs at least a root segment.""" + with pytest.raises(URLPipelineError, match="at least one segment"): + URLPipeline(()) + + +def test_hashable() -> None: + """Pipelines are values: equal pipelines hash equal and can key a dict.""" + a = URLPipeline.from_url("s3://b/d.zip|zip:|zarr3:") + b = URLPipeline.from_url("s3://b/d.zip|zip:|zarr3:") + assert a == b + assert hash(a) == hash(b) + assert {a: 1}[b] == 1 + + +@pytest.mark.parametrize( + ("pipeline_format", "requested", "expected"), + [(None, None, None), (None, 3, 3), (2, None, 2), (2, 2, 2)], +) +def test_resolve_zarr_format( + pipeline_format: int | None, requested: int | None, expected: int | None +) -> None: + """ + `resolve_zarr_format` returns the pipeline's format when the caller did not + specify one, and the caller's format otherwise. + """ + suffix = "" if pipeline_format is None else f"|zarr{pipeline_format}:" + pipeline = URLPipeline.from_url(f"memory://x{suffix}") + assert pipeline.resolve_zarr_format(requested) == expected # type: ignore[arg-type] + + +def test_resolve_zarr_format_conflict_raises() -> None: + """A caller-supplied format that differs from the pipeline's segment is an error.""" + pipeline = URLPipeline.from_url("memory://x|zarr2:") + with pytest.raises(ValueError, match="conflicts with the 'zarr2:' segment"): + pipeline.resolve_zarr_format(3) diff --git a/tests/test_url_pipeline/test_resolver.py b/tests/test_url_pipeline/test_resolver.py index f9941b5101..d290a12dee 100644 --- a/tests/test_url_pipeline/test_resolver.py +++ b/tests/test_url_pipeline/test_resolver.py @@ -1,5 +1,6 @@ from __future__ import annotations +import contextlib import dataclasses import sys import threading @@ -26,9 +27,9 @@ list_url_adapter_schemes, register_url_adapter, ) -from zarr.storage import ManagedMemoryStore, MemoryStore, WrapperStore +from zarr.storage import LocalStore, ManagedMemoryStore, MemoryStore, URLPipeline, WrapperStore from zarr.storage._common import _has_fsspec, make_store, make_store_path -from zarr.storage._url_pipeline import is_url_pipeline, resolve_pipeline +from zarr.storage._url_pipeline import resolve_pipeline from zarr.storage._utils import _join_paths # fsspec < 2024.12.0 has no AsyncFileSystemWrapper, so a plain memory:// URL @@ -112,17 +113,6 @@ async def open_pipeline_segment( return AdapterResolution(store=await MemoryStore.open(read_only=False)) -class FormatAdapter(URLPipelineAdapter): - """A dummy format-selecting adapter, like the builtin `zarr2:`.""" - - @classmethod - async def open_pipeline_segment( - cls, segment: PipelineSegment, context: PipelineContext - ) -> AdapterResolution: - preceding = await context.resolve_preceding() - return dataclasses.replace(preceding, zarr_format=2) - - class TestRegistry: def test_register_and_get(self) -> None: register_url_adapter("demo", WrapperAdapter) @@ -318,41 +308,46 @@ def test_duplicate_entry_points_warn_and_first_wins( assert not any(e.name == "dup" for e in pending) -class TestIsURLPipeline: - def test_pipe_routes(self) -> None: - assert is_url_pipeline("memory://x|demo:") +class TestStringsAreNeverPipelines: + """ + Only a `URLPipeline` object is resolved as a pipeline. A plain string + keeps the meaning it had before URL pipelines existed, whatever adapters + are registered. + """ + + async def test_local_path_containing_pipe_is_a_local_store(self, tmp_path: Path) -> None: + """A string path with `|` and a registered adapter's name is still a local directory.""" + register_url_adapter("zip", WrapperAdapter) + store_path = await make_store_path(f"{tmp_path}/a|zip:x", mode="w") + assert isinstance(store_path.store, LocalStore) + assert store_path.store.root == tmp_path / "a|zip:x" - def test_registered_root_scheme_routes(self) -> None: - register_url_adapter("rooty", RootAdapter) - assert is_url_pipeline("rooty://org/repo") + async def test_registered_root_scheme_string_is_not_routed(self) -> None: + """A URL whose scheme names a registered root adapter is not handed to that adapter.""" + calls: list[PipelineSegment] = [] - @pytest.mark.parametrize("url", ["s3://bucket/key", "/local/path", "memory://x", "C:.zarr"]) - def test_plain_urls_do_not_route(self, url: str) -> None: - assert not is_url_pipeline(url) + class CountingRootAdapter(RootAdapter): + @classmethod + async def open_pipeline_segment( + cls, segment: PipelineSegment, context: PipelineContext + ) -> AdapterResolution: + calls.append(segment) + return await super().open_pipeline_segment(segment, context) - def test_fsspec_chained_urls_do_not_route(self) -> None: - # fsspec's ``scheme::`` chaining is not a URL pipeline: even with a - # same-named root adapter registered, these keep flowing to fsspec. - register_url_adapter("zip", RootAdapter) - assert not is_url_pipeline("zip::file:///tmp/data.zip") - assert not is_url_pipeline("simplecache::s3://bucket/key") + register_url_adapter("rooty", CountingRootAdapter) + # the string falls through to the fsspec route, which may raise for + # an unknown protocol; whatever happens, the adapter is not consulted + with contextlib.suppress(Exception): + await make_store_path("rooty://org/repo") + assert calls == [] def test_native_schemes_do_not_route_to_adapters(self) -> None: - # an installed package must not be able to intercept zarr's own - # local-path and in-memory routing by registering file:/memory: + """An installed package cannot intercept `file:`/`memory:` roots by registering them.""" register_url_adapter("file", RootAdapter) register_url_adapter("memory", RootAdapter) - assert not is_url_pipeline("file:/tmp/x") - assert not is_url_pipeline("memory://x") - - def test_exotic_authority_root_scheme(self) -> None: - # The spec's own root-URL example: urlparse rejects the bracketed - # non-IP authority, so scheme detection must work on the raw string - # rather than raising before routing. - url = "vendor1-2.custom-1+proto.ext://[authority]/path?query/part?x" - assert not is_url_pipeline(url) - register_url_adapter("vendor1-2.custom-1+proto.ext", RootAdapter) - assert is_url_pipeline(url) + pipeline = URLPipeline.from_url("memory://x") + store = zarr.core.sync.sync(make_store(pipeline)) + assert isinstance(store, ManagedMemoryStore) class TestResolve: @@ -415,7 +410,7 @@ async def test_read_only_is_enforced_on_disobedient_adapters(self) -> None: register_url_adapter("bad", DisobedientAdapter) result = await resolve_pipeline("memory://base|bad:", mode="r") assert result.store.read_only - store = await make_store("memory://base|bad:", mode="r") + store = await make_store(URLPipeline.from_url("memory://base|bad:"), mode="r") assert store.read_only async def test_read_only_enforcement_without_conversion_raises(self) -> None: @@ -551,19 +546,29 @@ async def test_wrapper_at_root_position_raises(self) -> None: with pytest.raises(URLPipelineError, match="no preceding sub-URL"): await resolve_pipeline("wrap:whatever") - async def test_not_a_pipeline_raises(self) -> None: - with pytest.raises(URLPipelineError, match="is not a URL pipeline"): - await resolve_pipeline("s3://bucket/plain") + async def test_single_url_is_a_trivial_pipeline(self) -> None: + """A URL without `|` resolves as a pipeline consisting of its root alone.""" + result = await resolve_pipeline("memory://plain") + assert isinstance(result.store, ManagedMemoryStore) + assert result.path == "" + + async def test_accepts_pipeline_object(self, tmp_path: Path) -> None: + """`resolve_pipeline` takes a parsed `URLPipeline` as well as a string.""" + register_url_adapter("wrap", WrapperAdapter) + result = await resolve_pipeline(URLPipeline.from_url(f"{tmp_path}|wrap:a")) + assert isinstance(result.store, TracingStore) + assert result.path == "a" + + async def test_format_segment_path_joins_residual_path(self, tmp_path: Path) -> None: + """The node path of a trailing format segment extends the adapters' residual path.""" + register_url_adapter("wrap", WrapperAdapter) + result = await resolve_pipeline(f"{tmp_path}|wrap:a|zarr3:b") + assert result.path == "a/b" async def test_unknown_adapter_scheme_raises(self) -> None: with pytest.raises(URLPipelineError, match="no URL pipeline adapter is registered"): await resolve_pipeline("memory://base|no-such-adapter:") - async def test_unknown_adapter_error_hints_at_reserved_pipe(self, tmp_path: Path) -> None: - # a filename containing '|' routes here; the error points at the fix - with pytest.raises(URLPipelineError, match="pass a\\s+pathlib.Path"): - await resolve_pipeline(f"{tmp_path}/a|b") - async def test_base_case_value_error_is_wrapped(self) -> None: # a root that only the fallback scheme detection accepts must not # leak urlparse's ValueError out of the base case @@ -571,15 +576,6 @@ async def test_base_case_value_error_is_wrapped(self) -> None: with pytest.raises(URLPipelineError, match="could not resolve the pipeline root"): await resolve_pipeline("vendor.x://[authority]/path|wrap:") - async def test_zarr_format_flows_from_adapter(self, tmp_path: Path) -> None: - register_url_adapter("fmt2", FormatAdapter) - result = await resolve_pipeline(f"{tmp_path}|fmt2:") - assert result.zarr_format == 2 - # wrapper adapters carry the format through (dataclasses.replace) - register_url_adapter("wrap", WrapperAdapter) - result = await resolve_pipeline(f"{tmp_path}|fmt2:|wrap:") - assert result.zarr_format == 2 - class TestSpecRoots: """The spec's `memory:` / `file:` root semantics inside pipelines.""" @@ -606,9 +602,9 @@ async def test_memory_root_named_spellings_equivalent(self, root: str) -> None: async def test_memory_root_shares_data_across_spellings(self) -> None: register_url_adapter("wrap", WrapperAdapter) - group = zarr.open_group("memory:pipe-share|wrap:", mode="w") + group = zarr.open_group(URLPipeline.from_url("memory:pipe-share|wrap:"), mode="w") group.create_array("x", shape=(2,), dtype="i4") - reopened = zarr.open_group("memory://pipe-share|wrap:", mode="r") + reopened = zarr.open_group(URLPipeline.from_url("memory://pipe-share|wrap:"), mode="r") assert "x" in reopened async def test_memory_root_rejects_query(self) -> None: @@ -627,10 +623,10 @@ async def test_file_root_localhost_authority(self, tmp_path: Path) -> None: # spell the path URL-style: on Windows C:/... gains a leading slash posix = tmp_path.as_posix() url_path = posix if posix.startswith("/") else f"/{posix}" - zarr.open_group(f"file://localhost{url_path}|wrap:", mode="w") - opened = zarr.open_group(f"file:{tmp_path}|wrap:", mode="r") + zarr.open_group(URLPipeline.from_url(f"file://localhost{url_path}|wrap:"), mode="w") + opened = zarr.open_group(URLPipeline.from_url(f"file:{tmp_path}|wrap:"), mode="r") assert isinstance(opened, zarr.Group) - zarr.open_group(f"file://{url_path}|wrap:", mode="r") + zarr.open_group(URLPipeline.from_url(f"file://{url_path}|wrap:"), mode="r") async def test_file_root_windows_drive_spellings( self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch @@ -676,7 +672,7 @@ async def test_memory_root_is_distinct_from_fsspec_memory_url(self) -> None: # is zarr's ManagedMemoryStore; the two do not share data register_url_adapter("wrap", WrapperAdapter) plain = await make_store("memory://distinct-store") - piped = await make_store("memory://distinct-store|wrap:") + piped = await make_store(URLPipeline.from_url("memory://distinct-store|wrap:")) assert not isinstance(plain, ManagedMemoryStore) assert isinstance(piped, TracingStore) assert isinstance(piped._store, ManagedMemoryStore) @@ -691,15 +687,26 @@ async def test_file_root_does_not_percent_decode(self, tmp_path: Path) -> None: class TestMakeStoreIntegration: + """`make_store` / `make_store_path` accept a `URLPipeline` like any other StoreLike.""" + async def test_make_store_path_combines_paths(self, tmp_path: Path) -> None: + """Adapter residual path, format-segment node path and the caller's path join in order.""" register_url_adapter("wrap", WrapperAdapter) - store_path = await make_store_path(f"{tmp_path}|wrap:residual", path="user/sub") - assert store_path.path == "residual/user/sub" + pipeline = URLPipeline.from_url(f"{tmp_path}|wrap:residual|zarr3:node") + store_path = await make_store_path(pipeline, path="user/sub") + assert store_path.path == "residual/node/user/sub" + assert isinstance(store_path.store, TracingStore) async def test_make_store_rejects_residual_path(self, tmp_path: Path) -> None: register_url_adapter("wrap", WrapperAdapter) with pytest.raises(URLPipelineError, match="resolves to a path inside a store"): - await make_store(f"{tmp_path}|wrap:residual") + await make_store(URLPipeline.from_url(f"{tmp_path}|wrap:residual")) + + async def test_make_store_rejects_format_segment_node_path(self, tmp_path: Path) -> None: + """A node path given on the format segment also addresses a path inside the store.""" + register_url_adapter("wrap", WrapperAdapter) + with pytest.raises(URLPipelineError, match="resolves to a path inside a store"): + await make_store(URLPipeline.from_url(f"{tmp_path}|wrap:|zarr3:node")) async def test_make_store_rejects_slash_root_path(self, tmp_path: Path) -> None: # an adapter returning path="/" addresses the store root; the raw @@ -713,7 +720,7 @@ async def open_pipeline_segment( return dataclasses.replace(preceding, path="/") register_url_adapter("slashy", SlashRootAdapter) - store = await make_store(f"{tmp_path}|slashy:") + store = await make_store(URLPipeline.from_url(f"{tmp_path}|slashy:")) assert store is not None async def test_make_store_closes_store_on_residual_path_error(self) -> None: @@ -733,12 +740,12 @@ async def open_pipeline_segment( register_url_adapter("leaky", LeakProbe) with pytest.raises(URLPipelineError, match="resolves to a path inside a store"): - await make_store("memory://base|leaky:") + await make_store(URLPipeline.from_url("memory://base|leaky:")) assert closed == [True] async def test_make_store_no_residual_path(self, tmp_path: Path) -> None: register_url_adapter("wrap", WrapperAdapter) - store = await make_store(f"{tmp_path}|wrap:") + store = await make_store(URLPipeline.from_url(f"{tmp_path}|wrap:")) assert isinstance(store, TracingStore) async def test_make_store_rejects_invalid_mode_before_resolving(self, tmp_path: Path) -> None: @@ -746,18 +753,18 @@ async def test_make_store_rejects_invalid_mode_before_resolving(self, tmp_path: # rather than being handed to adapters register_url_adapter("wrap", WrapperAdapter) with pytest.raises(ValueError, match="Invalid mode"): - await make_store(f"{tmp_path}|wrap:", mode="invalid") # type: ignore[arg-type] + await make_store(URLPipeline.from_url(f"{tmp_path}|wrap:"), mode="invalid") # type: ignore[arg-type] async def test_zarr_open_end_to_end(self, tmp_path: Path) -> None: register_url_adapter("wrap", WrapperAdapter) - group = zarr.open_group(f"{tmp_path}|wrap:", mode="w") + group = zarr.open_group(URLPipeline.from_url(f"{tmp_path}|wrap:"), mode="w") array = group.create_array("x", shape=(4,), dtype="i4") array[:] = [1, 2, 3, 4] - assert zarr.open_array(f"{tmp_path}|wrap:x")[2] == 3 + assert zarr.open_array(URLPipeline.from_url(f"{tmp_path}|wrap:x"))[2] == 3 async def test_pipeline_store_read_only_mode(self) -> None: register_url_adapter("native", NativeAdapter) - store_path = await make_store_path("memory://base|native:", mode="r") + store_path = await make_store_path(URLPipeline.from_url("memory://base|native:"), mode="r") assert store_path.read_only async def test_mode_a_downgrades_to_read_only_open(self) -> None: @@ -765,20 +772,19 @@ async def test_mode_a_downgrades_to_read_only_open(self) -> None: # that resolves to a read-only store serves the "open" half instead # of failing outright. register_url_adapter("rooty", RootAdapter) - store_path = await make_store_path("rooty://org/repo", mode="a") + store_path = await make_store_path(URLPipeline.from_url("rooty://org/repo"), mode="a") assert store_path.read_only - async def test_mode_a_downgrade_applies_to_plain_stores_too(self) -> None: - # the open-or-create downgrade lives in StorePath.open, so it is - # uniform across pipeline and non-pipeline stores + async def test_mode_a_on_plain_read_only_store_still_raises(self) -> None: + """The open-or-create downgrade is specific to pipelines; plain stores behave as before.""" store = await MemoryStore.open(read_only=True) - store_path = await make_store_path(store, mode="a") - assert store_path.read_only + with pytest.raises(ValueError, match="Store is read-only but mode is 'a'"): + await make_store_path(store, mode="a") async def test_explicit_write_mode_reaches_adapter(self) -> None: register_url_adapter("rooty", RootAdapter) with pytest.raises(ValueError, match="read-only scheme"): - await make_store_path("rooty://org/repo", mode="w") + await make_store_path(URLPipeline.from_url("rooty://org/repo"), mode="w") async def test_storage_options_forwarded_to_root(self) -> None: # storage_options reach the root sub-URL via resolve_preceding -> @@ -786,96 +792,133 @@ async def test_storage_options_forwarded_to_root(self) -> None: # forwarding them must raise the same TypeError as a non-pipeline open. register_url_adapter("wrap", WrapperAdapter) with pytest.raises(TypeError, match="'storage_options' was provided but unused"): - await make_store("/tmp/some/path|wrap:", storage_options={"anon": True}) + await make_store( + URLPipeline.from_url("/tmp/some/path|wrap:"), storage_options={"anon": True} + ) - async def test_store_path_truediv_keeps_format(self, tmp_path: Path) -> None: - # StorePath.__truediv__ constructs a new instance; the pipeline- - # selected format must survive the division - register_url_adapter("fmt2", FormatAdapter) - store_path = await make_store_path(f"{tmp_path}|fmt2:") - assert store_path.zarr_format == 2 - assert (store_path / "sub/node").zarr_format == 2 + async def test_store_path_has_no_format(self, tmp_path: Path) -> None: + """The pipeline's format is consumed by the open call; it does not linger on the StorePath.""" + store_path = await make_store_path(URLPipeline.from_url(f"{tmp_path}|zarr2:")) + assert not hasattr(store_path, "zarr_format") async def test_zarr_format_merges_into_open(self, tmp_path: Path) -> None: - register_url_adapter("fmt2", FormatAdapter) - group = zarr.open_group(f"{tmp_path}|fmt2:", mode="w") + group = zarr.open_group(URLPipeline.from_url(f"{tmp_path}|zarr2:"), mode="w") assert group.metadata.zarr_format == 2 async def test_conflicting_explicit_format_raises(self, tmp_path: Path) -> None: - register_url_adapter("fmt2", FormatAdapter) with pytest.raises(ValueError, match="conflicts with"): - zarr.open_group(f"{tmp_path}|fmt2:", mode="w", zarr_format=3) + zarr.open_group(URLPipeline.from_url(f"{tmp_path}|zarr2:"), mode="w", zarr_format=3) async def test_matching_explicit_format_ok(self, tmp_path: Path) -> None: - register_url_adapter("fmt2", FormatAdapter) - group = zarr.open_group(f"{tmp_path}|fmt2:", mode="w", zarr_format=2) + group = zarr.open_group(URLPipeline.from_url(f"{tmp_path}|zarr2:"), mode="w", zarr_format=2) + assert group.metadata.zarr_format == 2 + + +class TestOpenURL: + """`zarr.open_url` interprets its string argument as a URL pipeline, and nothing else does.""" + + async def test_open_url_resolves_pipeline_and_format(self, tmp_path: Path) -> None: + register_url_adapter("wrap", WrapperAdapter) + group = zarr.open_url(f"{tmp_path}|wrap:|zarr2:", mode="w") + assert isinstance(group, zarr.Group) + assert group.metadata.zarr_format == 2 + assert isinstance(group.store, TracingStore) + + async def test_open_url_path_and_kwargs_pass_through(self, tmp_path: Path) -> None: + register_url_adapter("wrap", WrapperAdapter) + array = zarr.open_url( + f"{tmp_path}|wrap:|zarr3:sub", path="x", mode="w", shape=(3,), dtype="i4" + ) + assert isinstance(array, zarr.Array) + assert array.metadata.zarr_format == 3 + assert array.store_path.path == "sub/x" + + async def test_open_url_conflicting_format_raises(self, tmp_path: Path) -> None: + with pytest.raises(ValueError, match="conflicts with"): + zarr.open_url(f"{tmp_path}|zarr2:", mode="w", zarr_format=3) + + async def test_async_open_url(self, tmp_path: Path) -> None: + group = await zarr.api.asynchronous.open_url(f"{tmp_path}|zarr2:", mode="w") assert group.metadata.zarr_format == 2 class TestZarrFormatMergeInCore: """ - Every make_store_path caller honors the pipeline-selected zarr format, not - only the top-level zarr.api functions. + Every function taking a StoreLike honors the format selected by a + `URLPipeline`, not only the top-level zarr.api functions. Functions whose + `zarr_format` defaults to 3 need an explicit `zarr_format=None` to defer to + the pipeline, and raise when the default conflicts with it. """ - @pytest.fixture(autouse=True) - def _fmt2(self) -> None: - register_url_adapter("fmt2", FormatAdapter) + @pytest.fixture + def v2_pipeline(self, tmp_path: Path) -> URLPipeline: + return URLPipeline.from_url(f"{tmp_path}|zarr2:") - async def test_create_array(self, tmp_path: Path) -> None: - arr = zarr.create_array(f"{tmp_path}|fmt2:", name="x", shape=(2,), dtype="i4") + async def test_create_array(self, v2_pipeline: URLPipeline) -> None: + arr = zarr.create_array(v2_pipeline, name="x", shape=(2,), dtype="i4", zarr_format=None) assert arr.metadata.zarr_format == 2 with pytest.raises(ValueError, match="conflicts with"): - zarr.create_array(f"{tmp_path}|fmt2:", name="y", shape=(2,), dtype="i4", zarr_format=3) + zarr.create_array(v2_pipeline, name="y", shape=(2,), dtype="i4") - async def test_create_array_from_data(self, tmp_path: Path) -> None: + async def test_create_array_from_data(self, v2_pipeline: URLPipeline) -> None: # the data path of create_array goes through from_array - arr = zarr.create_array(f"{tmp_path}|fmt2:", name="x", data=np.arange(3)) + arr = zarr.create_array(v2_pipeline, name="x", data=np.arange(3), zarr_format=None) assert arr.metadata.zarr_format == 2 with pytest.raises(ValueError, match="conflicts with"): - zarr.create_array(f"{tmp_path}|fmt2:", name="y", data=np.arange(3), zarr_format=3) + zarr.create_array(v2_pipeline, name="y", data=np.arange(3)) - async def test_from_array(self, tmp_path: Path) -> None: - arr = zarr.from_array(f"{tmp_path}|fmt2:", name="x", data=np.arange(3)) + async def test_create_array_default_without_pipeline(self, tmp_path: Path) -> None: + """The default format for plain stores is still 3, independent of the config default.""" + with zarr.config.set({"default_zarr_format": 2}): + arr = zarr.create_array(tmp_path, shape=(2,), dtype="i4") + assert arr.metadata.zarr_format == 3 + + async def test_from_array(self, v2_pipeline: URLPipeline) -> None: + arr = zarr.from_array(v2_pipeline, name="x", data=np.arange(3)) assert arr.metadata.zarr_format == 2 - async def test_group_from_store(self, tmp_path: Path) -> None: - group = zarr.Group.from_store(f"{tmp_path}|fmt2:") + async def test_group_from_store(self, v2_pipeline: URLPipeline, tmp_path: Path) -> None: + group = zarr.Group.from_store(v2_pipeline, zarr_format=None) assert group.metadata.zarr_format == 2 with pytest.raises(ValueError, match="conflicts with"): - zarr.Group.from_store(f"{tmp_path}/other|fmt2:", zarr_format=3) + zarr.Group.from_store(URLPipeline.from_url(f"{tmp_path}/other|zarr2:")) async def test_group_from_store_default_without_pipeline(self, tmp_path: Path) -> None: - # the default is still the configured format (3) for plain stores - group = zarr.Group.from_store(tmp_path) + """The default format for plain stores is still 3, independent of the config default.""" + with zarr.config.set({"default_zarr_format": 2}): + group = zarr.Group.from_store(tmp_path) assert group.metadata.zarr_format == 3 - async def test_group_open(self, tmp_path: Path) -> None: - zarr.Group.from_store(f"{tmp_path}|fmt2:") - # explicit None defers to the pipeline; the default (3) conflicts - group = zarr.Group.open(f"{tmp_path}|fmt2:", zarr_format=None) + async def test_group_open(self, v2_pipeline: URLPipeline) -> None: + zarr.Group.from_store(v2_pipeline, zarr_format=None) + group = zarr.Group.open(v2_pipeline, zarr_format=None) assert group.metadata.zarr_format == 2 with pytest.raises(ValueError, match="conflicts with"): - zarr.Group.open(f"{tmp_path}|fmt2:") + zarr.Group.open(v2_pipeline) - async def test_array_open(self, tmp_path: Path) -> None: - zarr.create_array(f"{tmp_path}|fmt2:", name="x", shape=(2,), dtype="i4") - arr = zarr.Array.open(f"{tmp_path}/x|fmt2:", zarr_format=None) + async def test_array_open(self, v2_pipeline: URLPipeline, tmp_path: Path) -> None: + zarr.create_array(v2_pipeline, name="x", shape=(2,), dtype="i4", zarr_format=None) + array_pipeline = URLPipeline.from_url(f"{tmp_path}|zarr2:x") + arr = zarr.Array.open(array_pipeline, zarr_format=None) assert arr.metadata.zarr_format == 2 with pytest.raises(ValueError, match="conflicts with"): - zarr.Array.open(f"{tmp_path}/x|fmt2:") + zarr.Array.open(array_pipeline) - async def test_open_like(self, tmp_path: Path) -> None: + async def test_open_like(self, v2_pipeline: URLPipeline) -> None: # open_like inherits v2 filters/compressor from the reference; the # pipeline supplies the format, which open_array then creates with ref = zarr.create_array({}, shape=(2,), dtype="i4", zarr_format=2) - arr = zarr.open_like(ref, store=f"{tmp_path}|fmt2:", path="x") + arr = zarr.open_like(ref, store=v2_pipeline, path="x") assert arr.metadata.zarr_format == 2 - async def test_deprecated_async_array_create(self, tmp_path: Path) -> None: + async def test_deprecated_async_array_create( + self, v2_pipeline: URLPipeline, tmp_path: Path + ) -> None: from zarr.core.array import AsyncArray - arr = await AsyncArray._create(f"{tmp_path}|fmt2:", shape=(2,), dtype="i4", zarr_format=2) + arr = await AsyncArray._create(v2_pipeline, shape=(2,), dtype="i4", zarr_format=2) assert arr.metadata.zarr_format == 2 with pytest.raises(ValueError, match="conflicts with"): - await AsyncArray._create(f"{tmp_path}/o|fmt2:", shape=(2,), dtype="i4", zarr_format=3) + await AsyncArray._create( + URLPipeline.from_url(f"{tmp_path}/o|zarr2:"), shape=(2,), dtype="i4", zarr_format=3 + ) From 72682644b1edd93935edfdf678090bb50f1ead2e Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Tue, 6 Oct 2026 11:06:55 +0200 Subject: [PATCH 2/4] refactor(url-pipeline): make open_url open existing nodes only `zarr.open_url(url, *, mode="r" | "r+", storage_options=None)` no longer mirrors `zarr.open`: it never creates or overwrites, takes no `path` (the node is addressed by the URL, via the format segment's body), no `zarr_format` (taken from the pipeline or inferred), and no `**kwargs`. Creating a node at a pipeline-addressed location is done by passing a `URLPipeline` to `create_array` / `create_group`. Assisted-by: ClaudeCode:claude-fable-5-1 Co-Authored-By: Claude Fable 5.1 --- changes/4192.feature.md | 7 +-- docs/user-guide/storage.md | 9 ++-- src/zarr/api/asynchronous.py | 55 ++++++++++++------------ src/zarr/api/synchronous.py | 54 +++++++++++------------ tests/test_url_pipeline/test_resolver.py | 45 ++++++++++++++----- 5 files changed, 97 insertions(+), 73 deletions(-) diff --git a/changes/4192.feature.md b/changes/4192.feature.md index 91a4748417..dc3f224045 100644 --- a/changes/4192.feature.md +++ b/changes/4192.feature.md @@ -1,8 +1,9 @@ Added core support for URL pipelines (https://github.com/jbms/url-pipeline): `|`-chained URLs that address zarr data through nested storage layers, e.g. -`s3://bucket/data.zip|zip:|zarr3:`. A pipeline is opened with the new `zarr.open_url` -function, or by passing a parsed `zarr.storage.URLPipeline` (built with -`URLPipeline.from_url`) to any function that accepts a `StoreLike`. Adapters for the +`s3://bucket/data.zip|zip:|zarr3:`. A parsed `zarr.storage.URLPipeline` (built with +`URLPipeline.from_url`) can be passed to any function that accepts a `StoreLike`, and +the new `zarr.open_url(url, mode="r" | "r+")` opens the existing node a pipeline string +addresses without ever creating or overwriting anything. Adapters for the sub-URLs after the first are provided by third-party packages (e.g. Icechunk) through the `zarr.url_adapters` entry-point group and the single-method `zarr.abc.url_pipeline.URLPipelineAdapter` interface; they are loaded lazily and diff --git a/docs/user-guide/storage.md b/docs/user-guide/storage.md index 4d4571c7e3..95a55743d0 100644 --- a/docs/user-guide/storage.md +++ b/docs/user-guide/storage.md @@ -118,9 +118,9 @@ sub-URL locates a resource with a conventional URL; each subsequent sub-URL name [`zarr.abc.url_pipeline`][zarr.abc.url_pipeline] for the adapter interface. A builtin `zip:` adapter is under development and will expand this section. -A pipeline is opened either with [`zarr.open_url`][], which is `zarr.open` with its -string argument read as a pipeline, or by passing a parsed -[`URLPipeline`][zarr.storage.URLPipeline] to any function that accepts a `StoreLike`: +A parsed [`URLPipeline`][zarr.storage.URLPipeline] can be passed to any function that +accepts a `StoreLike`, and [`zarr.open_url`][] opens the *existing* node a pipeline +string addresses, read-only unless `mode="r+"` is given: ```python exec="true" session="storage" source="above" result="ansi" from zarr.storage import URLPipeline @@ -135,6 +135,9 @@ print(zarr.open_url("memory://pipeline-demo|zarr2:").metadata.zarr_format) Only these two entry points read a string as a pipeline. Plain string store specifications keep their existing meaning everywhere else, so a local path that happens to contain `|` still names a directory, whatever adapters are installed. +Unlike `zarr.open`, `zarr.open_url` never creates or overwrites: to create a node at a +pipeline-addressed location, pass the `URLPipeline` to `zarr.create_array` or +`zarr.create_group`. A trailing `zarr2:` or `zarr3:` segment is a *format segment*. It is not an adapter: zarr-python reads it from the pipeline itself and uses it as the `zarr_format` of the diff --git a/src/zarr/api/asynchronous.py b/src/zarr/api/asynchronous.py index 9b110ed63e..4fc0abd8da 100644 --- a/src/zarr/api/asynchronous.py +++ b/src/zarr/api/asynchronous.py @@ -440,54 +440,53 @@ async def open( async def open_url( url: str, *, - mode: AccessModeLiteral | None = None, - zarr_format: ZarrFormat | None = None, - path: str | None = None, + mode: Literal["r", "r+"] = "r", storage_options: dict[str, Any] | None = None, - **kwargs: Any, ) -> AnyAsyncArray | AsyncGroup: - """Open a group or array addressed by a URL pipeline. + """Open the existing group or array addressed by a URL pipeline. + + This is the only entry point that reads a string as a + [URL pipeline][user-guide-url-pipelines]; [`open`][zarr.api.asynchronous.open] and the other + `StoreLike`-taking functions treat strings as they always have. Unlike + `open`, `open_url` never creates or overwrites anything: to create a node at + a pipeline-addressed location, pass a `URLPipeline` to + [`create_array`][zarr.api.asynchronous.create_array] or + [`create_group`][zarr.api.asynchronous.create_group]. - This is [`open`][zarr.api.asynchronous.open] with its `store` argument - interpreted as a [URL pipeline][user-guide-url-pipelines] string, i.e. - `open(URLPipeline.from_url(url), ...)`. It is the only entry point that - reads a string as a pipeline; `open` and the other `StoreLike`-taking - functions treat strings as they always have. + The zarr format is taken from a trailing `zarr2:`/`zarr3:` segment when + present, and inferred from the stored metadata otherwise. Parameters ---------- url : str A URL pipeline such as `"s3://bucket/data.zip|zip:|zarr3:"`. A URL - without `|` is a trivial pipeline consisting of its root alone. - mode : {'r', 'r+', 'a', 'w', 'w-'}, optional - Persistence mode, with the same meaning as for `zarr.open`. - zarr_format : {2, 3, None}, optional - The zarr format to use. None (the default) defers to a trailing - `zarr2:`/`zarr3:` segment of the pipeline; a value that conflicts - with such a segment raises `ValueError`. - path : str or None, optional - The path within the store to open, appended to any node path the - pipeline itself addresses. - storage_options : dict + without `|` is a trivial pipeline consisting of its root alone. The + node to open is addressed by the URL itself: the body of a trailing + `zarr2:`/`zarr3:` segment is the path of the node within the store. + mode : {'r', 'r+'}, optional + `'r'` (the default) opens read-only; `'r+'` opens for reading and + writing. The node must exist in either case. + storage_options : dict, optional Options for the pipeline's root sub-URL (e.g. fsspec options). - **kwargs - Additional parameters are passed through to `zarr.open_array` or - `zarr.open_group`. Returns ------- z : array or group - Return type depends on what exists in the given store. + Return type depends on what exists at the URL. For a statically typed + result, pass a `URLPipeline` to `open_array` or `open_group` instead. """ from zarr.storage import URLPipeline + if mode not in ("r", "r+"): + raise ValueError( + f"Invalid mode: {mode!r}. open_url opens existing nodes only and accepts " + "'r' or 'r+'; to create a node at a URL pipeline, pass " + "URLPipeline.from_url(url) to create_array or create_group." + ) return await open( store=URLPipeline.from_url(url), mode=mode, - zarr_format=zarr_format, - path=path, storage_options=storage_options, - **kwargs, ) diff --git a/src/zarr/api/synchronous.py b/src/zarr/api/synchronous.py index a599f6b8e1..5959316686 100644 --- a/src/zarr/api/synchronous.py +++ b/src/zarr/api/synchronous.py @@ -250,54 +250,52 @@ def open( def open_url( url: str, *, - mode: AccessModeLiteral | None = None, - zarr_format: ZarrFormat | None = None, - path: str | None = None, + mode: Literal["r", "r+"] = "r", storage_options: dict[str, Any] | None = None, - **kwargs: Any, ) -> AnyArray | Group: - """Open a group or array addressed by a URL pipeline. + """Open the existing group or array addressed by a URL pipeline. - This is [`open`][zarr.open] with its `store` argument interpreted as a - [URL pipeline][user-guide-url-pipelines] string, i.e. - `zarr.open(URLPipeline.from_url(url), ...)`. It is the only entry point - that reads a string as a pipeline; `zarr.open` and the other - `StoreLike`-taking functions treat strings as they always have. + This is the only entry point that reads a string as a + [URL pipeline][user-guide-url-pipelines]; [`open`][zarr.open] and the other + `StoreLike`-taking functions treat strings as they always have. Unlike + `open`, `open_url` never creates or overwrites anything: to create a node at + a pipeline-addressed location, pass a `URLPipeline` to + [`create_array`][zarr.create_array] or [`create_group`][zarr.create_group]. + + The zarr format is taken from a trailing `zarr2:`/`zarr3:` segment when + present, and inferred from the stored metadata otherwise. Parameters ---------- url : str A URL pipeline such as `"s3://bucket/data.zip|zip:|zarr3:"`. A URL - without `|` is a trivial pipeline consisting of its root alone. - mode : {'r', 'r+', 'a', 'w', 'w-'}, optional - Persistence mode, with the same meaning as for `zarr.open`. - zarr_format : {2, 3, None}, optional - The zarr format to use. None (the default) defers to a trailing - `zarr2:`/`zarr3:` segment of the pipeline; a value that conflicts - with such a segment raises `ValueError`. - path : str or None, optional - The path within the store to open, appended to any node path the - pipeline itself addresses. - storage_options : dict + without `|` is a trivial pipeline consisting of its root alone. The + node to open is addressed by the URL itself: the body of a trailing + `zarr2:`/`zarr3:` segment is the path of the node within the store. + mode : {'r', 'r+'}, optional + `'r'` (the default) opens read-only; `'r+'` opens for reading and + writing. The node must exist in either case. + storage_options : dict, optional Options for the pipeline's root sub-URL (e.g. fsspec options). - **kwargs - Additional parameters are passed through to `zarr.open_array` or - `zarr.open_group`. Returns ------- z : array or group - Return type depends on what exists in the given store. + Return type depends on what exists at the URL. For a statically typed + result, pass a `URLPipeline` to `open_array` or `open_group` instead. """ from zarr.storage import URLPipeline + if mode not in ("r", "r+"): + raise ValueError( + f"Invalid mode: {mode!r}. open_url opens existing nodes only and accepts " + "'r' or 'r+'; to create a node at a URL pipeline, pass " + "URLPipeline.from_url(url) to create_array or create_group." + ) return open( store=URLPipeline.from_url(url), mode=mode, - zarr_format=zarr_format, - path=path, storage_options=storage_options, - **kwargs, ) diff --git a/tests/test_url_pipeline/test_resolver.py b/tests/test_url_pipeline/test_resolver.py index d290a12dee..a642402ae2 100644 --- a/tests/test_url_pipeline/test_resolver.py +++ b/tests/test_url_pipeline/test_resolver.py @@ -815,30 +815,53 @@ async def test_matching_explicit_format_ok(self, tmp_path: Path) -> None: class TestOpenURL: - """`zarr.open_url` interprets its string argument as a URL pipeline, and nothing else does.""" + """ + `zarr.open_url` interprets its string argument as a URL pipeline and opens + the existing node it addresses. It never creates: creation goes through + `create_array` / `create_group` with a `URLPipeline`. + """ - async def test_open_url_resolves_pipeline_and_format(self, tmp_path: Path) -> None: + async def test_open_url_opens_existing_node_with_pipeline_format(self, tmp_path: Path) -> None: register_url_adapter("wrap", WrapperAdapter) - group = zarr.open_url(f"{tmp_path}|wrap:|zarr2:", mode="w") + zarr.create_group(URLPipeline.from_url(f"{tmp_path}|wrap:|zarr2:")) + group = zarr.open_url(f"{tmp_path}|wrap:|zarr2:") assert isinstance(group, zarr.Group) assert group.metadata.zarr_format == 2 assert isinstance(group.store, TracingStore) + assert group.read_only - async def test_open_url_path_and_kwargs_pass_through(self, tmp_path: Path) -> None: + async def test_open_url_format_segment_body_addresses_node(self, tmp_path: Path) -> None: + """The node path lives in the URL, as the body of the format segment.""" register_url_adapter("wrap", WrapperAdapter) - array = zarr.open_url( - f"{tmp_path}|wrap:|zarr3:sub", path="x", mode="w", shape=(3,), dtype="i4" + zarr.create_array( + URLPipeline.from_url(f"{tmp_path}|wrap:"), name="sub/x", shape=(3,), dtype="i4" ) + array = zarr.open_url(f"{tmp_path}|wrap:|zarr3:sub/x") assert isinstance(array, zarr.Array) - assert array.metadata.zarr_format == 3 assert array.store_path.path == "sub/x" - async def test_open_url_conflicting_format_raises(self, tmp_path: Path) -> None: - with pytest.raises(ValueError, match="conflicts with"): - zarr.open_url(f"{tmp_path}|zarr2:", mode="w", zarr_format=3) + async def test_open_url_read_write(self, tmp_path: Path) -> None: + zarr.create_group(URLPipeline.from_url(f"{tmp_path}|zarr3:")) + group = zarr.open_url(f"{tmp_path}|zarr3:", mode="r+") + assert not group.read_only + group.attrs["answer"] = 42 + assert zarr.open_url(f"{tmp_path}|zarr3:").attrs["answer"] == 42 + + async def test_open_url_missing_node_raises(self, tmp_path: Path) -> None: + """Nothing is created on open: a URL to a missing node is an error in every mode.""" + with pytest.raises(FileNotFoundError): + zarr.open_url(f"{tmp_path}/missing|zarr3:") + with pytest.raises(FileNotFoundError): + zarr.open_url(f"{tmp_path}/missing|zarr3:", mode="r+") + + async def test_open_url_rejects_create_modes(self, tmp_path: Path) -> None: + """The creating modes of `zarr.open` are not accepted.""" + with pytest.raises(ValueError, match="opens existing nodes only"): + zarr.open_url(f"{tmp_path}|zarr3:", mode="a") # type: ignore[arg-type] async def test_async_open_url(self, tmp_path: Path) -> None: - group = await zarr.api.asynchronous.open_url(f"{tmp_path}|zarr2:", mode="w") + zarr.create_group(URLPipeline.from_url(f"{tmp_path}|zarr2:")) + group = await zarr.api.asynchronous.open_url(f"{tmp_path}|zarr2:") assert group.metadata.zarr_format == 2 From 2ec816cc21050f88b19aea7b386a6591deb3783a Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Tue, 6 Oct 2026 14:15:59 +0200 Subject: [PATCH 3/4] refactor(url-pipeline): drop the string-routing leftovers from the resolver With strings never routed as pipelines, four pieces of the resolver only served the old design: - The schemeless-root extension (`data.zip|zip:`). The root of a pipeline now carries a URL scheme, as the specification requires; a bare path is rejected with a message pointing at `file:` URLs and `%7C`. This also removes the one place where `?` and `#` were treated as filename characters. - The negative lookahead that kept fsspec's `zip::...` chained URLs out of root-scheme detection; it protected `zarr.open` strings that no longer reach the resolver. - The mode "a" downgrade for read-only pipelines in `make_store_path`; `open_url` defaults to "r", and a read-only pipeline now raises on "a" exactly like a plain read-only store. - The "non-dict `storage_options` reserved for per-segment configuration" reservation in docstrings and the user guide. Assisted-by: ClaudeCode:claude-fable-5-1 Co-Authored-By: Claude Fable 5.1 --- changes/4192.feature.md | 2 - docs/user-guide/storage.md | 13 +- src/zarr/abc/url_pipeline.py | 7 +- src/zarr/storage/_common.py | 5 - src/zarr/storage/_url_pipeline.py | 56 ++++----- tests/test_url_pipeline/test_parser.py | 41 +------ .../test_url_pipeline/test_pipeline_object.py | 2 - tests/test_url_pipeline/test_resolver.py | 111 ++++++++++-------- 8 files changed, 98 insertions(+), 139 deletions(-) diff --git a/changes/4192.feature.md b/changes/4192.feature.md index dc3f224045..21e87736a3 100644 --- a/changes/4192.feature.md +++ b/changes/4192.feature.md @@ -20,8 +20,6 @@ Behavior notes: (`zarr.create_array`, `Group.from_store`, `Group.open`, `Array.open`) need an explicit `zarr_format=None` to defer to the pipeline. `Array.open` gains a `zarr_format` parameter mirroring `Group.open`. -- With the default mode `"a"`, a pipeline that resolves to a read-only resource is - opened for reading instead of raising; other stores behave as before. - Inside a pipeline, `memory:` and `file:` roots follow the URL pipeline spec's spelling rules (`memory:` ≡ `memory:/` ≡ `memory://`; `file:` must carry an absolute path with at most a `localhost` authority). Percent-escapes are not decoded, matching diff --git a/docs/user-guide/storage.md b/docs/user-guide/storage.md index 95a55743d0..bb294353aa 100644 --- a/docs/user-guide/storage.md +++ b/docs/user-guide/storage.md @@ -147,20 +147,17 @@ open. A `zarr_format` argument that conflicts with the format segment raises and `Group.open`) need an explicit `zarr_format=None` to defer to the pipeline. `storage_options` apply to the *root* sub-URL (e.g. fsspec options for `s3://...`); -adapters may consume adapter-specific, namespaced keys. Non-dict forms of -`storage_options` are reserved for future per-segment configuration. No percent-escape -is decoded anywhere in a pipeline. Registered adapters cannot intercept zarr's native -`file:` and `memory:` root schemes. With the default mode `"a"`, a pipeline that -resolves to a read-only resource is opened for reading instead of raising. +adapters may consume adapter-specific, namespaced keys. No percent-escape is decoded +anywhere in a pipeline. Registered adapters cannot intercept zarr's native `file:` and +`memory:` root schemes. Inside a pipeline, a `memory:` root is zarr's managed in-memory store (`memory:`, `memory:/` and `memory://` are equivalent, and `memory:name` selects a named store). When fsspec is installed, a plain `memory://name` string store specification is still routed to fsspec's in-memory filesystem, which is a different store. A `file:` root must carry an absolute path; percent-escapes are not decoded, matching the -[local store](#local-store). As a zarr-python extension, the root may also be a bare -local path (`data/example.zip|zip:`); such pipelines are not portable to other -implementations. +[local store](#local-store). The root always carries a scheme: a local path is spelled +as a `file:` URL, never as a bare path. ## Explicit Store Creation diff --git a/src/zarr/abc/url_pipeline.py b/src/zarr/abc/url_pipeline.py index e1cf96db41..f0f58140c8 100644 --- a/src/zarr/abc/url_pipeline.py +++ b/src/zarr/abc/url_pipeline.py @@ -50,8 +50,7 @@ class PipelineSegment: Attributes ---------- scheme : str - The lowercased URL scheme. Empty string only for a schemeless root - (a bare local path), which is treated as opaque text. + The lowercased URL scheme. body : str The text after `scheme:` and before any `?`. Interpretation is scheme-defined; it is **not** URL-normalized, so case-significant @@ -130,10 +129,6 @@ class PipelineContext: backends. An adapter that consumes keys should strip them before resolving the rest of the pipeline, by passing the reduced mapping to [`resolve_preceding`][zarr.abc.url_pipeline.PipelineContext.resolve_preceding]. - Non-dict forms of the caller-facing `storage_options` argument are - reserved for future per-segment configuration (one mapping per - pipeline segment); this attribute will remain a single mapping — - the one addressed to this adapter's segment. """ preceding: tuple[PipelineSegment, ...] diff --git a/src/zarr/storage/_common.py b/src/zarr/storage/_common.py index 3886e9d0f8..24d0dd4122 100644 --- a/src/zarr/storage/_common.py +++ b/src/zarr/storage/_common.py @@ -488,11 +488,6 @@ async def make_store_path( if isinstance(store_like, URLPipeline): result = await resolve_pipeline(store_like, mode=mode, storage_options=storage_options) combined_path = _join_paths([normalize_path(result.path), path_normalized]) - if mode == "a" and result.store.read_only: - # "a" (open-or-create, the zarr.open default) on a pipeline that - # resolves to a read-only resource serves the "open" half; any - # later write fails at the store level. - mode = "r" return await StorePath.open(result.store, path=combined_path, mode=mode) if isinstance(store_like, StorePath): diff --git a/src/zarr/storage/_url_pipeline.py b/src/zarr/storage/_url_pipeline.py index ec8c9294eb..a040c61441 100644 --- a/src/zarr/storage/_url_pipeline.py +++ b/src/zarr/storage/_url_pipeline.py @@ -11,12 +11,8 @@ store specifications, including local paths that happen to contain `|`, keep their pre-existing meaning. No percent-escape is decoded. -As a zarr-python extension to the specification (which requires the root -sub-URL of an absolute pipeline to carry a scheme), the root sub-URL may be -a schemeless local filesystem path, e.g. `data/example.zip|zip:`. Schemeless -roots are treated as opaque text — no query or fragment splitting is applied -to them. Such pipelines are not portable to other URL pipeline -implementations; portable pipelines should spell the root as a `file:` URL. +As the specification requires, the root sub-URL carries a URL scheme: a +local path is spelled as an absolute `file:` URL. """ from __future__ import annotations @@ -50,10 +46,8 @@ # nonstandard schemes (e.g. "earthmover.myscheme"). _SCHEME_RE = re.compile(r"^[a-zA-Z][a-zA-Z0-9+.\-]*$") -# Root scheme at the very start of the sub-URL. The negative lookahead -# excludes fsspec's chained-URL syntax (``zip::file://...``), which is not a -# URL pipeline and keeps flowing through the fsspec machinery. -_ROOT_SCHEME_RE = re.compile(r"([a-zA-Z][a-zA-Z0-9+.\-]*):(?!:)") +# Root scheme at the very start of the sub-URL. +_ROOT_SCHEME_RE = re.compile(r"([a-zA-Z][a-zA-Z0-9+.\-]*):") # Schemes zarr resolves natively at the pipeline root. These are never # dispatched to a registered root adapter, so an installed package cannot @@ -67,12 +61,12 @@ def _root_scheme(root_sub_url: str) -> str: """ - Detect the scheme of the root sub-URL, or `""` for an opaque root. + Detect the scheme of the root sub-URL, or `""` when it has none. - Scheme extraction happens on the raw string (so nothing `urlparse` - would strip or reject — whitespace, exotic authorities — can desync the - detected scheme from the body). Single-letter candidates are delegated - to `parse_store_url`, which knows Windows drive letters are not schemes. + Scheme extraction happens on the raw string, so nothing `urlparse` + would strip or reject can desync the detected scheme from the body. + Single-letter candidates are delegated to `parse_store_url`, which + knows Windows drive letters are not schemes. """ match = _ROOT_SCHEME_RE.match(root_sub_url) if match is None: @@ -107,10 +101,9 @@ def parse_pipeline(url: str) -> tuple[PipelineSegment, ...]: `scheme:body` where the trailing colon is optional when the body is empty (`zip` is equivalent to `zip:`). - A schemeless root (a bare local path) is accepted as a zarr-python - extension to the specification and treated as opaque text: no query or - fragment splitting applies, since `?` and `#` are ordinary filename - characters there. See the module docstring. + The root must carry a scheme; a bare local path is rejected, since the + specification has no schemeless sub-URLs and `|` could not be escaped + in one. Segment text is preserved verbatim (no case or percent-encoding normalization) except that schemes are lowercased. @@ -124,9 +117,11 @@ def parse_pipeline(url: str) -> tuple[PipelineSegment, ...]: if index == 0: scheme = _root_scheme(part) if not scheme: - # opaque root (bare local path, fsspec chained URL, ...) - segments.append(PipelineSegment(scheme="", body=part, query=None, raw=part)) - continue + raise URLPipelineError( + f"the root sub-URL {part!r} of the URL pipeline {url!r} has no URL scheme. " + "Spell a local path as an absolute 'file:' URL, and a literal '|' in " + "it as '%7C'." + ) body_and_scheme, query = _split_query(part) body = body_and_scheme[len(scheme) + 1 :] segments.append(PipelineSegment(scheme=scheme, body=body, query=query, raw=part)) @@ -279,13 +274,11 @@ def __str__(self) -> str: def _root_routes_to_adapter(scheme: str) -> bool: """ Whether a root sub-URL with this scheme is dispatched to a registered - root adapter. Schemes zarr resolves natively (`file:`, `memory:`) and - opaque roots are excluded; the registry check inspects entry-point - names only — no adapter code is imported here. + root adapter. Schemes zarr resolves natively (`file:`, `memory:`) are + excluded; the registry check inspects entry-point names only — no + adapter code is imported here. """ - return ( - bool(scheme) and scheme not in _NATIVE_ROOT_SCHEMES and scheme in list_url_adapter_schemes() - ) + return scheme not in _NATIVE_ROOT_SCHEMES and scheme in list_url_adapter_schemes() async def resolve_pipeline( @@ -313,8 +306,7 @@ async def resolve_pipeline( resolver enforces this on whatever the final adapter returns. storage_options : dict | None Options forwarded to the root sub-URL's store (and visible to - adapters via the context). Non-dict forms are reserved for future - per-segment configuration (one mapping per pipeline segment). + adapters via the context). Raises ------ @@ -388,8 +380,8 @@ async def _resolve_root( `memory:` and `file:` roots are resolved here with the URL pipeline spec's semantics (spelling equivalences, mandatory absolute `file:` - paths); everything else — bare local paths, fsspec URLs — delegates to - the existing `StoreLike` machinery unchanged. + paths); every other scheme (e.g. fsspec URLs) delegates to the existing + `StoreLike` machinery unchanged. """ if segment.scheme == "memory": return _resolve_memory_root(segment, mode=mode, storage_options=storage_options) diff --git a/tests/test_url_pipeline/test_parser.py b/tests/test_url_pipeline/test_parser.py index 13d7846e85..3dc2480ca8 100644 --- a/tests/test_url_pipeline/test_parser.py +++ b/tests/test_url_pipeline/test_parser.py @@ -19,11 +19,11 @@ def test_single_root_url() -> None: assert str(segment) == "s3://bucket/key" -def test_schemeless_root() -> None: - (segment,) = parse_pipeline("/local/path") - assert segment.scheme == "" - assert segment.body == "/local/path" - assert segment.raw == "/local/path" +@pytest.mark.parametrize("url", ["/local/path", "/local/path|zip:", "\tfile:/tmp/x|zip:"]) +def test_schemeless_root_rejected(url: str) -> None: + """The root sub-URL must carry a URL scheme; a bare path (or anything urlparse would not read as a scheme) is an error.""" + with pytest.raises(URLPipelineError, match="URL scheme"): + parse_pipeline(url) def test_adapter_chain() -> None: @@ -111,32 +111,6 @@ def test_single_letter_scheme_with_exotic_authority() -> None: assert segments[0].raw == "x://[authority]/path" -def test_fsspec_chained_root_is_opaque() -> None: - # fsspec's ``scheme::`` chaining is not pipeline syntax; such roots are - # schemeless/opaque and flow through the ordinary store machinery - (segment,) = parse_pipeline("zip::file:///tmp/data.zip") - assert segment.scheme == "" - assert segment.body == "zip::file:///tmp/data.zip" - - -def test_whitespace_prefixed_root_is_opaque() -> None: - # urlparse strips leading whitespace when sniffing a scheme; the parser - # must not, or the detected scheme desyncs from the body slice - segments = parse_pipeline("\tfile:/tmp/x|zip:") - assert segments[0].scheme == "" - assert segments[0].body == "\tfile:/tmp/x" - - -def test_schemeless_root_keeps_query_and_fragment_chars() -> None: - # ? and # are ordinary filename characters in a bare local path - segments = parse_pipeline("/tmp/d?v=1|zip:") - assert segments[0].scheme == "" - assert segments[0].body == "/tmp/d?v=1" - assert segments[0].query is None - segments = parse_pipeline("/tmp/d#frag|zip:") - assert segments[0].body == "/tmp/d#frag" - - def test_round_trip() -> None: url = "s3://bucket/a.zip?v=2|zip:b/inner.zip|zip:c|zarr3:" segments = parse_pipeline(url) @@ -160,9 +134,6 @@ def test_round_trip() -> None: # the root scheme as a local path on one platform but not another _ROOT_SCHEME = st.from_regex(r"[a-zA-Z]{2}[a-zA-Z0-9+.\-]{0,14}", fullmatch=True) _BODY = st.text(st.characters(min_codepoint=32, max_codepoint=126, exclude_characters="|#?")) -# a root body starting with ":" would spell fsspec's chained-URL syntax -# ("scheme::..."), which the parser deliberately treats as an opaque root -_ROOT_BODY = _BODY.filter(lambda body: not body.startswith(":")) _QUERY = st.text(st.characters(min_codepoint=32, max_codepoint=126, exclude_characters="|#")) @@ -176,7 +147,7 @@ def pipelines( parts = [] for index in range(depth): scheme = draw(_ROOT_SCHEME if index == 0 else _ADAPTER_SCHEME) - body = draw(_ROOT_BODY if index == 0 else _BODY) + body = draw(_BODY) query = draw(st.none() | _QUERY) raw = f"{scheme}:{body}" + (f"?{query}" if query is not None else "") parts.append(raw) diff --git a/tests/test_url_pipeline/test_pipeline_object.py b/tests/test_url_pipeline/test_pipeline_object.py index c21e166810..4c0d2779a5 100644 --- a/tests/test_url_pipeline/test_pipeline_object.py +++ b/tests/test_url_pipeline/test_pipeline_object.py @@ -31,8 +31,6 @@ ("memory://x|wrap:a", ("memory", "wrap"), ("memory", "wrap"), None, ""), # a single URL is a trivial pipeline ("s3://bucket/key", ("s3",), ("s3",), None, ""), - # schemeless local root (zarr-python extension) - ("/local/data.zip|zip:", ("", "zip"), ("", "zip"), None, ""), ], ) def test_from_url( diff --git a/tests/test_url_pipeline/test_resolver.py b/tests/test_url_pipeline/test_resolver.py index a642402ae2..e00c29cb90 100644 --- a/tests/test_url_pipeline/test_resolver.py +++ b/tests/test_url_pipeline/test_resolver.py @@ -353,10 +353,10 @@ def test_native_schemes_do_not_route_to_adapters(self) -> None: class TestResolve: async def test_wrapper_adapter_chain(self, tmp_path: Path) -> None: register_url_adapter("wrap", WrapperAdapter) - result = await resolve_pipeline(f"{tmp_path}|wrap:inner/path") + result = await resolve_pipeline(f"file:{tmp_path.as_posix()}|wrap:inner/path") assert isinstance(result.store, TracingStore) assert result.path == "inner/path" - assert result.store.context.preceding_url == str(tmp_path) + assert result.store.context.preceding_url == f"file:{tmp_path.as_posix()}" assert result.store.segment.scheme == "wrap" assert result.store.segment.body == "inner/path" @@ -364,7 +364,7 @@ async def test_nested_wrappers_preserve_residual_paths(self, tmp_path: Path) -> # each wrapper joins the preceding residual path with its own, so # no segment's path is lost in root|wrap:a|wrap:b register_url_adapter("wrap", WrapperAdapter) - result = await resolve_pipeline(f"{tmp_path}|wrap:a|wrap:b") + result = await resolve_pipeline(f"file:{tmp_path.as_posix()}|wrap:a|wrap:b") assert result.path == "a/b" async def test_native_adapter_gets_preceding_url(self) -> None: @@ -395,11 +395,11 @@ async def test_root_adapter_composes_with_chain(self) -> None: async def test_read_only_flag(self, tmp_path: Path) -> None: register_url_adapter("wrap", WrapperAdapter) - result = await resolve_pipeline(f"{tmp_path}|wrap:", mode="r") + result = await resolve_pipeline(f"file:{tmp_path.as_posix()}|wrap:", mode="r") assert isinstance(result.store, TracingStore) assert result.store.context.read_only assert result.store.context.mode == "r" - result = await resolve_pipeline(f"{tmp_path}|wrap:") + result = await resolve_pipeline(f"file:{tmp_path.as_posix()}|wrap:") assert isinstance(result.store, TracingStore) assert not result.store.context.read_only assert result.store.context.mode is None @@ -467,7 +467,7 @@ async def open_pipeline_segment( return dataclasses.replace(preceding, path=segment.body) register_url_adapter("rowrap", ReadOnlyRootWrapper) - result = await resolve_pipeline(f"{tmp_path}|rowrap:x", mode="w") + result = await resolve_pipeline(f"file:{tmp_path.as_posix()}|rowrap:x", mode="w") assert result.path == "x" async def test_wrapper_on_local_file_root_read_only(self, tmp_path: Path) -> None: @@ -477,7 +477,7 @@ async def test_wrapper_on_local_file_root_read_only(self, tmp_path: Path) -> Non target = tmp_path / "data.bin" target.write_bytes(b"payload") register_url_adapter("wrap", WrapperAdapter) - result = await resolve_pipeline(f"{target}|wrap:", mode="r") + result = await resolve_pipeline(f"file:{target.as_posix()}|wrap:", mode="r") assert isinstance(result.store, TracingStore) assert result.store.read_only @@ -496,7 +496,7 @@ async def test_wrapper_on_local_file_root_writable_modes( target = tmp_path / "data.bin" target.write_bytes(b"payload") register_url_adapter("wrap", WrapperAdapter) - await resolve_pipeline(f"{target}|wrap:", mode=mode) + await resolve_pipeline(f"file:{target.as_posix()}|wrap:", mode=mode) async def test_resolve_preceding_storage_options_override(self, tmp_path: Path) -> None: # an adapter that consumed its namespaced keys strips them before @@ -517,7 +517,8 @@ async def open_pipeline_segment( # the local-path root rejects any surviving storage_options, so this # passing proves the adapter's keys were stripped before resolution result = await resolve_pipeline( - f"{tmp_path}|consuming:x", storage_options={"consuming_secret": "s3cr3t"} + f"file:{tmp_path.as_posix()}|consuming:x", + storage_options={"consuming_secret": "s3cr3t"}, ) assert result.path == "x" assert ConsumingWrapper.seen == "s3cr3t" @@ -555,14 +556,14 @@ async def test_single_url_is_a_trivial_pipeline(self) -> None: async def test_accepts_pipeline_object(self, tmp_path: Path) -> None: """`resolve_pipeline` takes a parsed `URLPipeline` as well as a string.""" register_url_adapter("wrap", WrapperAdapter) - result = await resolve_pipeline(URLPipeline.from_url(f"{tmp_path}|wrap:a")) + result = await resolve_pipeline(URLPipeline.from_url(f"file:{tmp_path.as_posix()}|wrap:a")) assert isinstance(result.store, TracingStore) assert result.path == "a" async def test_format_segment_path_joins_residual_path(self, tmp_path: Path) -> None: """The node path of a trailing format segment extends the adapters' residual path.""" register_url_adapter("wrap", WrapperAdapter) - result = await resolve_pipeline(f"{tmp_path}|wrap:a|zarr3:b") + result = await resolve_pipeline(f"file:{tmp_path.as_posix()}|wrap:a|zarr3:b") assert result.path == "a/b" async def test_unknown_adapter_scheme_raises(self) -> None: @@ -692,7 +693,7 @@ class TestMakeStoreIntegration: async def test_make_store_path_combines_paths(self, tmp_path: Path) -> None: """Adapter residual path, format-segment node path and the caller's path join in order.""" register_url_adapter("wrap", WrapperAdapter) - pipeline = URLPipeline.from_url(f"{tmp_path}|wrap:residual|zarr3:node") + pipeline = URLPipeline.from_url(f"file:{tmp_path.as_posix()}|wrap:residual|zarr3:node") store_path = await make_store_path(pipeline, path="user/sub") assert store_path.path == "residual/node/user/sub" assert isinstance(store_path.store, TracingStore) @@ -700,13 +701,13 @@ async def test_make_store_path_combines_paths(self, tmp_path: Path) -> None: async def test_make_store_rejects_residual_path(self, tmp_path: Path) -> None: register_url_adapter("wrap", WrapperAdapter) with pytest.raises(URLPipelineError, match="resolves to a path inside a store"): - await make_store(URLPipeline.from_url(f"{tmp_path}|wrap:residual")) + await make_store(URLPipeline.from_url(f"file:{tmp_path.as_posix()}|wrap:residual")) async def test_make_store_rejects_format_segment_node_path(self, tmp_path: Path) -> None: """A node path given on the format segment also addresses a path inside the store.""" register_url_adapter("wrap", WrapperAdapter) with pytest.raises(URLPipelineError, match="resolves to a path inside a store"): - await make_store(URLPipeline.from_url(f"{tmp_path}|wrap:|zarr3:node")) + await make_store(URLPipeline.from_url(f"file:{tmp_path.as_posix()}|wrap:|zarr3:node")) async def test_make_store_rejects_slash_root_path(self, tmp_path: Path) -> None: # an adapter returning path="/" addresses the store root; the raw @@ -720,7 +721,7 @@ async def open_pipeline_segment( return dataclasses.replace(preceding, path="/") register_url_adapter("slashy", SlashRootAdapter) - store = await make_store(URLPipeline.from_url(f"{tmp_path}|slashy:")) + store = await make_store(URLPipeline.from_url(f"file:{tmp_path.as_posix()}|slashy:")) assert store is not None async def test_make_store_closes_store_on_residual_path_error(self) -> None: @@ -745,7 +746,7 @@ async def open_pipeline_segment( async def test_make_store_no_residual_path(self, tmp_path: Path) -> None: register_url_adapter("wrap", WrapperAdapter) - store = await make_store(URLPipeline.from_url(f"{tmp_path}|wrap:")) + store = await make_store(URLPipeline.from_url(f"file:{tmp_path.as_posix()}|wrap:")) assert isinstance(store, TracingStore) async def test_make_store_rejects_invalid_mode_before_resolving(self, tmp_path: Path) -> None: @@ -753,30 +754,28 @@ async def test_make_store_rejects_invalid_mode_before_resolving(self, tmp_path: # rather than being handed to adapters register_url_adapter("wrap", WrapperAdapter) with pytest.raises(ValueError, match="Invalid mode"): - await make_store(URLPipeline.from_url(f"{tmp_path}|wrap:"), mode="invalid") # type: ignore[arg-type] + await make_store( + URLPipeline.from_url(f"file:{tmp_path.as_posix()}|wrap:"), + mode="invalid", # type: ignore[arg-type] + ) async def test_zarr_open_end_to_end(self, tmp_path: Path) -> None: register_url_adapter("wrap", WrapperAdapter) - group = zarr.open_group(URLPipeline.from_url(f"{tmp_path}|wrap:"), mode="w") + group = zarr.open_group(URLPipeline.from_url(f"file:{tmp_path.as_posix()}|wrap:"), mode="w") array = group.create_array("x", shape=(4,), dtype="i4") array[:] = [1, 2, 3, 4] - assert zarr.open_array(URLPipeline.from_url(f"{tmp_path}|wrap:x"))[2] == 3 + assert zarr.open_array(URLPipeline.from_url(f"file:{tmp_path.as_posix()}|wrap:x"))[2] == 3 async def test_pipeline_store_read_only_mode(self) -> None: register_url_adapter("native", NativeAdapter) store_path = await make_store_path(URLPipeline.from_url("memory://base|native:"), mode="r") assert store_path.read_only - async def test_mode_a_downgrades_to_read_only_open(self) -> None: - # mode "a" (the zarr.open default) is open-or-create: a pipeline - # that resolves to a read-only store serves the "open" half instead - # of failing outright. + async def test_mode_a_on_read_only_store_raises(self) -> None: + """Mode "a" on a read-only store raises, whether the store comes from a pipeline or not.""" register_url_adapter("rooty", RootAdapter) - store_path = await make_store_path(URLPipeline.from_url("rooty://org/repo"), mode="a") - assert store_path.read_only - - async def test_mode_a_on_plain_read_only_store_still_raises(self) -> None: - """The open-or-create downgrade is specific to pipelines; plain stores behave as before.""" + with pytest.raises(ValueError, match="Store is read-only but mode is 'a'"): + await make_store_path(URLPipeline.from_url("rooty://org/repo"), mode="a") store = await MemoryStore.open(read_only=True) with pytest.raises(ValueError, match="Store is read-only but mode is 'a'"): await make_store_path(store, mode="a") @@ -793,24 +792,32 @@ async def test_storage_options_forwarded_to_root(self) -> None: register_url_adapter("wrap", WrapperAdapter) with pytest.raises(TypeError, match="'storage_options' was provided but unused"): await make_store( - URLPipeline.from_url("/tmp/some/path|wrap:"), storage_options={"anon": True} + URLPipeline.from_url("file:/tmp/some/path|wrap:"), storage_options={"anon": True} ) async def test_store_path_has_no_format(self, tmp_path: Path) -> None: """The pipeline's format is consumed by the open call; it does not linger on the StorePath.""" - store_path = await make_store_path(URLPipeline.from_url(f"{tmp_path}|zarr2:")) + store_path = await make_store_path( + URLPipeline.from_url(f"file:{tmp_path.as_posix()}|zarr2:") + ) assert not hasattr(store_path, "zarr_format") async def test_zarr_format_merges_into_open(self, tmp_path: Path) -> None: - group = zarr.open_group(URLPipeline.from_url(f"{tmp_path}|zarr2:"), mode="w") + group = zarr.open_group( + URLPipeline.from_url(f"file:{tmp_path.as_posix()}|zarr2:"), mode="w" + ) assert group.metadata.zarr_format == 2 async def test_conflicting_explicit_format_raises(self, tmp_path: Path) -> None: with pytest.raises(ValueError, match="conflicts with"): - zarr.open_group(URLPipeline.from_url(f"{tmp_path}|zarr2:"), mode="w", zarr_format=3) + zarr.open_group( + URLPipeline.from_url(f"file:{tmp_path.as_posix()}|zarr2:"), mode="w", zarr_format=3 + ) async def test_matching_explicit_format_ok(self, tmp_path: Path) -> None: - group = zarr.open_group(URLPipeline.from_url(f"{tmp_path}|zarr2:"), mode="w", zarr_format=2) + group = zarr.open_group( + URLPipeline.from_url(f"file:{tmp_path.as_posix()}|zarr2:"), mode="w", zarr_format=2 + ) assert group.metadata.zarr_format == 2 @@ -823,8 +830,8 @@ class TestOpenURL: async def test_open_url_opens_existing_node_with_pipeline_format(self, tmp_path: Path) -> None: register_url_adapter("wrap", WrapperAdapter) - zarr.create_group(URLPipeline.from_url(f"{tmp_path}|wrap:|zarr2:")) - group = zarr.open_url(f"{tmp_path}|wrap:|zarr2:") + zarr.create_group(URLPipeline.from_url(f"file:{tmp_path.as_posix()}|wrap:|zarr2:")) + group = zarr.open_url(f"file:{tmp_path.as_posix()}|wrap:|zarr2:") assert isinstance(group, zarr.Group) assert group.metadata.zarr_format == 2 assert isinstance(group.store, TracingStore) @@ -834,34 +841,37 @@ async def test_open_url_format_segment_body_addresses_node(self, tmp_path: Path) """The node path lives in the URL, as the body of the format segment.""" register_url_adapter("wrap", WrapperAdapter) zarr.create_array( - URLPipeline.from_url(f"{tmp_path}|wrap:"), name="sub/x", shape=(3,), dtype="i4" + URLPipeline.from_url(f"file:{tmp_path.as_posix()}|wrap:"), + name="sub/x", + shape=(3,), + dtype="i4", ) - array = zarr.open_url(f"{tmp_path}|wrap:|zarr3:sub/x") + array = zarr.open_url(f"file:{tmp_path.as_posix()}|wrap:|zarr3:sub/x") assert isinstance(array, zarr.Array) assert array.store_path.path == "sub/x" async def test_open_url_read_write(self, tmp_path: Path) -> None: - zarr.create_group(URLPipeline.from_url(f"{tmp_path}|zarr3:")) - group = zarr.open_url(f"{tmp_path}|zarr3:", mode="r+") + zarr.create_group(URLPipeline.from_url(f"file:{tmp_path.as_posix()}|zarr3:")) + group = zarr.open_url(f"file:{tmp_path.as_posix()}|zarr3:", mode="r+") assert not group.read_only group.attrs["answer"] = 42 - assert zarr.open_url(f"{tmp_path}|zarr3:").attrs["answer"] == 42 + assert zarr.open_url(f"file:{tmp_path.as_posix()}|zarr3:").attrs["answer"] == 42 async def test_open_url_missing_node_raises(self, tmp_path: Path) -> None: """Nothing is created on open: a URL to a missing node is an error in every mode.""" with pytest.raises(FileNotFoundError): - zarr.open_url(f"{tmp_path}/missing|zarr3:") + zarr.open_url(f"file:{tmp_path.as_posix()}/missing|zarr3:") with pytest.raises(FileNotFoundError): - zarr.open_url(f"{tmp_path}/missing|zarr3:", mode="r+") + zarr.open_url(f"file:{tmp_path.as_posix()}/missing|zarr3:", mode="r+") async def test_open_url_rejects_create_modes(self, tmp_path: Path) -> None: """The creating modes of `zarr.open` are not accepted.""" with pytest.raises(ValueError, match="opens existing nodes only"): - zarr.open_url(f"{tmp_path}|zarr3:", mode="a") # type: ignore[arg-type] + zarr.open_url(f"file:{tmp_path.as_posix()}|zarr3:", mode="a") # type: ignore[arg-type] async def test_async_open_url(self, tmp_path: Path) -> None: - zarr.create_group(URLPipeline.from_url(f"{tmp_path}|zarr2:")) - group = await zarr.api.asynchronous.open_url(f"{tmp_path}|zarr2:") + zarr.create_group(URLPipeline.from_url(f"file:{tmp_path.as_posix()}|zarr2:")) + group = await zarr.api.asynchronous.open_url(f"file:{tmp_path.as_posix()}|zarr2:") assert group.metadata.zarr_format == 2 @@ -875,7 +885,7 @@ class TestZarrFormatMergeInCore: @pytest.fixture def v2_pipeline(self, tmp_path: Path) -> URLPipeline: - return URLPipeline.from_url(f"{tmp_path}|zarr2:") + return URLPipeline.from_url(f"file:{tmp_path.as_posix()}|zarr2:") async def test_create_array(self, v2_pipeline: URLPipeline) -> None: arr = zarr.create_array(v2_pipeline, name="x", shape=(2,), dtype="i4", zarr_format=None) @@ -904,7 +914,7 @@ async def test_group_from_store(self, v2_pipeline: URLPipeline, tmp_path: Path) group = zarr.Group.from_store(v2_pipeline, zarr_format=None) assert group.metadata.zarr_format == 2 with pytest.raises(ValueError, match="conflicts with"): - zarr.Group.from_store(URLPipeline.from_url(f"{tmp_path}/other|zarr2:")) + zarr.Group.from_store(URLPipeline.from_url(f"file:{tmp_path.as_posix()}/other|zarr2:")) async def test_group_from_store_default_without_pipeline(self, tmp_path: Path) -> None: """The default format for plain stores is still 3, independent of the config default.""" @@ -921,7 +931,7 @@ async def test_group_open(self, v2_pipeline: URLPipeline) -> None: async def test_array_open(self, v2_pipeline: URLPipeline, tmp_path: Path) -> None: zarr.create_array(v2_pipeline, name="x", shape=(2,), dtype="i4", zarr_format=None) - array_pipeline = URLPipeline.from_url(f"{tmp_path}|zarr2:x") + array_pipeline = URLPipeline.from_url(f"file:{tmp_path.as_posix()}|zarr2:x") arr = zarr.Array.open(array_pipeline, zarr_format=None) assert arr.metadata.zarr_format == 2 with pytest.raises(ValueError, match="conflicts with"): @@ -943,5 +953,8 @@ async def test_deprecated_async_array_create( assert arr.metadata.zarr_format == 2 with pytest.raises(ValueError, match="conflicts with"): await AsyncArray._create( - URLPipeline.from_url(f"{tmp_path}/o|zarr2:"), shape=(2,), dtype="i4", zarr_format=3 + URLPipeline.from_url(f"file:{tmp_path.as_posix()}/o|zarr2:"), + shape=(2,), + dtype="i4", + zarr_format=3, ) From 7cee87ba79f0ccd68c6d4e37bf34bc66e4e75658 Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Tue, 6 Oct 2026 14:18:35 +0200 Subject: [PATCH 4/4] refactor(url-pipeline): resolve through URLPipeline.resolve; simplify root scheme detection `URLPipeline.resolve(mode=..., storage_options=...)` replaces the module-level `resolve_pipeline`, and `parse_pipeline` leaves the public `zarr.storage` surface: the object is the one entry point for parsing and resolving a pipeline. Root scheme detection no longer consults `parse_store_url` for single-letter candidates. A letter followed by `:` is a URL scheme on every platform, so parsing is deterministic across operating systems; a Windows drive path is spelled `file:/C:/...` in a pipeline, as the root must carry a scheme anyway. Assisted-by: ClaudeCode:claude-fable-5-1 Co-Authored-By: Claude Fable 5.1 --- docs/user-guide/extending.md | 5 +- src/zarr/storage/__init__.py | 4 +- src/zarr/storage/_common.py | 6 +- src/zarr/storage/_url_pipeline.py | 118 +++++++++++------------ tests/test_url_pipeline/test_parser.py | 19 ++-- tests/test_url_pipeline/test_resolver.py | 76 ++++++++------- 6 files changed, 106 insertions(+), 122 deletions(-) diff --git a/docs/user-guide/extending.md b/docs/user-guide/extending.md index 5138ba3a10..34128ce52a 100644 --- a/docs/user-guide/extending.md +++ b/docs/user-guide/extending.md @@ -126,10 +126,11 @@ inner adapter sees the outer one's residual path `a` and the pipeline resolves t import asyncio from zarr.registry import register_url_adapter -from zarr.storage import resolve_pipeline +from zarr.storage import URLPipeline register_url_adapter("mypackage.myscheme", MyAdapter) -resolution = asyncio.run(resolve_pipeline("memory://x|mypackage.myscheme:a|mypackage.myscheme:b")) +pipeline = URLPipeline.from_url("memory://x|mypackage.myscheme:a|mypackage.myscheme:b") +resolution = asyncio.run(pipeline.resolve()) assert resolution.path == "a/b" ``` diff --git a/src/zarr/storage/__init__.py b/src/zarr/storage/__init__.py index 66328d5a32..c4d8d534ac 100644 --- a/src/zarr/storage/__init__.py +++ b/src/zarr/storage/__init__.py @@ -10,7 +10,7 @@ from zarr.storage._logging import LoggingStore from zarr.storage._memory import GpuMemoryStore, ManagedMemoryStore, MemoryStore from zarr.storage._obstore import ObjectStore -from zarr.storage._url_pipeline import URLPipeline, parse_pipeline, resolve_pipeline +from zarr.storage._url_pipeline import URLPipeline from zarr.storage._wrapper import WrapperStore from zarr.storage._zip import ZipStore @@ -27,8 +27,6 @@ "URLPipeline", "WrapperStore", "ZipStore", - "parse_pipeline", - "resolve_pipeline", ] diff --git a/src/zarr/storage/_common.py b/src/zarr/storage/_common.py index 24d0dd4122..00d8f5b4f5 100644 --- a/src/zarr/storage/_common.py +++ b/src/zarr/storage/_common.py @@ -29,7 +29,7 @@ ) from zarr.storage._local import LocalStore from zarr.storage._memory import ManagedMemoryStore, MemoryStore -from zarr.storage._url_pipeline import URLPipeline, resolve_pipeline +from zarr.storage._url_pipeline import URLPipeline from zarr.storage._utils import UPath, _join_paths, normalize_path, parse_store_url _has_fsspec = importlib.util.find_spec("fsspec") @@ -360,7 +360,7 @@ async def make_store( _read_only = mode == "r" if isinstance(store_like, URLPipeline): - result = await resolve_pipeline(store_like, mode=mode, storage_options=storage_options) + result = await store_like.resolve(mode=mode, storage_options=storage_options) if normalize_path(result.path): result.store.close() raise URLPipelineError( @@ -486,7 +486,7 @@ async def make_store_path( path_normalized = normalize_path(path) if isinstance(store_like, URLPipeline): - result = await resolve_pipeline(store_like, mode=mode, storage_options=storage_options) + result = await store_like.resolve(mode=mode, storage_options=storage_options) combined_path = _join_paths([normalize_path(result.path), path_normalized]) return await StorePath.open(result.store, path=combined_path, mode=mode) diff --git a/src/zarr/storage/_url_pipeline.py b/src/zarr/storage/_url_pipeline.py index a040c61441..e53fc4f04d 100644 --- a/src/zarr/storage/_url_pipeline.py +++ b/src/zarr/storage/_url_pipeline.py @@ -29,13 +29,13 @@ from zarr.errors import URLPipelineError from zarr.registry import get_url_adapter, list_url_adapter_schemes from zarr.storage._memory import ManagedMemoryStore -from zarr.storage._utils import _join_paths, normalize_path, parse_store_url +from zarr.storage._utils import _join_paths, normalize_path if TYPE_CHECKING: from zarr.abc.store import Store from zarr.core.common import AccessModeLiteral, ZarrFormat -__all__ = ["URLPipeline", "parse_pipeline", "resolve_pipeline"] +__all__ = ["URLPipeline"] # Format segments are consumed by zarr-python itself rather than by a # registered adapter: they select the zarr format of the node the pipeline @@ -64,20 +64,14 @@ def _root_scheme(root_sub_url: str) -> str: Detect the scheme of the root sub-URL, or `""` when it has none. Scheme extraction happens on the raw string, so nothing `urlparse` - would strip or reject can desync the detected scheme from the body. - Single-letter candidates are delegated to `parse_store_url`, which - knows Windows drive letters are not schemes. + would strip or reject can desync the detected scheme from the body. A + single letter followed by `:` is a scheme on every platform: a Windows + drive path is spelled `file:/C:/...` in a pipeline, never bare. """ match = _ROOT_SCHEME_RE.match(root_sub_url) if match is None: return "" - scheme = match.group(1) - if len(scheme) == 1: - try: - return parse_store_url(root_sub_url).scheme.lower() - except ValueError: - return scheme.lower() - return scheme.lower() + return match.group(1).lower() def _split_query(sub_url: str) -> tuple[str, str | None]: @@ -95,9 +89,7 @@ def parse_pipeline(url: str) -> tuple[PipelineSegment, ...]: """ Parse a URL pipeline into its `|`-delimited segments. - The first segment is the *root* sub-URL; its scheme is detected with the - same rules as ordinary store URLs (Windows drive letters are not - schemes). Subsequent segments are *adapter* sub-URLs of the form + The first segment is the *root* sub-URL. Subsequent segments are *adapter* sub-URLs of the form `scheme:body` where the trailing colon is optional when the body is empty (`zip` is equivalent to `zip:`). @@ -142,7 +134,9 @@ class URLPipeline: """ A parsed URL pipeline, ready to be passed wherever a `StoreLike` is accepted. - Construct one with [`from_url`][zarr.storage.URLPipeline.from_url]. Because a + Construct one with [`from_url`][zarr.storage.URLPipeline.from_url] and open it + through any `StoreLike`-taking function, or resolve it directly with + [`resolve`][zarr.storage.URLPipeline.resolve]. Because a `URLPipeline` is an explicit object, no plain string store specification is ever interpreted as a pipeline: `zarr.open("data|x.zarr")` still addresses a local directory of that name, while @@ -267,6 +261,50 @@ def resolve_zarr_format(self, zarr_format: ZarrFormat | None) -> ZarrFormat | No ) return pipeline_format + async def resolve( + self, + *, + mode: AccessModeLiteral | None = None, + storage_options: dict[str, Any] | None = None, + ) -> AdapterResolution: + """ + Resolve the pipeline into a store and a residual path. + + The residual path joins the path returned by the adapters with the + node path of a trailing format segment (`...|zarr3:path/to/node`). + The format itself is not part of the resolution; read it from + `zarr_format`. + + Parameters + ---------- + mode : AccessModeLiteral | None + The caller's access mode. `"r"` requires a read-only store; the + resolver enforces this on whatever the final adapter returns. + storage_options : dict | None + Options forwarded to the root sub-URL's store (and visible to + adapters via the context). + + Raises + ------ + URLPipelineError + If a segment names a scheme with no registered adapter, names an + adapter entry point that fails to import, or the root sub-URL + cannot be resolved. + TypeError + If `storage_options` are passed to a root that does not accept + them (`file:` and `memory:`), as for non-pipeline stores. + OSError + Errors from opening the root resource (e.g. a missing local + directory in mode `"r"`) propagate unchanged, as for non-pipeline + stores. + """ + resolution = await _resolve(self.store_segments, mode=mode, storage_options=storage_options) + if self.path: + resolution = dataclasses.replace( + resolution, path=_join_paths([normalize_path(resolution.path), self.path]) + ) + return resolution + def __str__(self) -> str: return "|".join(segment.raw for segment in self.segments) @@ -281,54 +319,6 @@ def _root_routes_to_adapter(scheme: str) -> bool: return scheme not in _NATIVE_ROOT_SCHEMES and scheme in list_url_adapter_schemes() -async def resolve_pipeline( - url: str | URLPipeline, - *, - mode: AccessModeLiteral | None = None, - storage_options: dict[str, Any] | None = None, -) -> AdapterResolution: - """ - Resolve a URL pipeline into a store and a residual path. - - The residual path joins the path returned by the adapters with the node - path of a trailing format segment (`...|zarr3:path/to/node`). The format - itself is not part of the resolution; read it from - [`URLPipeline.zarr_format`][zarr.storage.URLPipeline]. - - Parameters - ---------- - url : str | URLPipeline - A URL pipeline, e.g. `"s3://bucket/data.zip|zip:|zarr3:"`, or its - parsed form. A URL without `|` is a trivial pipeline consisting of - its root alone. - mode : AccessModeLiteral | None - The caller's access mode. `"r"` requires a read-only store; the - resolver enforces this on whatever the final adapter returns. - storage_options : dict | None - Options forwarded to the root sub-URL's store (and visible to - adapters via the context). - - Raises - ------ - URLPipelineError - If the URL cannot be parsed, names a scheme with no registered adapter, names an adapter entry point that fails to - import, or has a root sub-URL that cannot be resolved. - TypeError - If `storage_options` are passed to a root that does not accept them - (local paths and `memory:`), as for non-pipeline stores. - OSError - Errors from opening the root resource (e.g. a missing local directory - in mode `"r"`) propagate unchanged, as for non-pipeline stores. - """ - pipeline = URLPipeline.from_url(url) if isinstance(url, str) else url - resolution = await _resolve(pipeline.store_segments, mode=mode, storage_options=storage_options) - if pipeline.path: - resolution = dataclasses.replace( - resolution, path=_join_paths([normalize_path(resolution.path), pipeline.path]) - ) - return resolution - - async def _resolve( segments: tuple[PipelineSegment, ...], *, diff --git a/tests/test_url_pipeline/test_parser.py b/tests/test_url_pipeline/test_parser.py index 3dc2480ca8..bcc2dd3283 100644 --- a/tests/test_url_pipeline/test_parser.py +++ b/tests/test_url_pipeline/test_parser.py @@ -1,7 +1,5 @@ from __future__ import annotations -import sys - import pytest from hypothesis import given from hypothesis import strategies as st @@ -74,16 +72,12 @@ def test_empty_query() -> None: assert segment.query == "" -def test_windows_drive_path_is_not_a_scheme() -> None: - # On Windows parse_store_url treats C:\... as a local path; elsewhere the - # single-letter scheme is preserved but must not crash the parser. +def test_single_letter_scheme_is_a_scheme_everywhere() -> None: + """A drive-letter-like prefix is a URL scheme on every platform; bare Windows paths are not pipelines.""" (segment,) = parse_pipeline(r"C:\data\store") + assert segment.scheme == "c" + assert segment.body == r"\data\store" assert segment.raw == r"C:\data\store" - if sys.platform == "win32": - assert segment.scheme == "" - assert segment.body == r"C:\data\store" - else: - assert segment.scheme == "c" @pytest.mark.parametrize("url", ["a||b:", "|zip:", "file:/tmp|", ""]) @@ -104,10 +98,9 @@ def test_invalid_adapter_scheme_rejected(segment: str) -> None: def test_single_letter_scheme_with_exotic_authority() -> None: - # single-letter candidates are delegated to parse_store_url, which may - # reject an exotic authority; the raw scheme is used as the fallback + """Scheme detection works on the raw text, so an authority urlparse would reject is no obstacle.""" segments = parse_pipeline("x://[authority]/path|zip:") - assert segments[0].scheme in ("", "x") + assert segments[0].scheme == "x" assert segments[0].raw == "x://[authority]/path" diff --git a/tests/test_url_pipeline/test_resolver.py b/tests/test_url_pipeline/test_resolver.py index e00c29cb90..ecfa37a37f 100644 --- a/tests/test_url_pipeline/test_resolver.py +++ b/tests/test_url_pipeline/test_resolver.py @@ -29,7 +29,6 @@ ) from zarr.storage import LocalStore, ManagedMemoryStore, MemoryStore, URLPipeline, WrapperStore from zarr.storage._common import _has_fsspec, make_store, make_store_path -from zarr.storage._url_pipeline import resolve_pipeline from zarr.storage._utils import _join_paths # fsspec < 2024.12.0 has no AsyncFileSystemWrapper, so a plain memory:// URL @@ -140,7 +139,9 @@ def test_entrypoint_discovery(self) -> None: @pytest.mark.usefixtures("set_path") async def test_entrypoint_end_to_end(self) -> None: - result = await resolve_pipeline("memory://src|example-pkg.entrypoint-scheme:sub/path") + result = await URLPipeline.from_url( + "memory://src|example-pkg.entrypoint-scheme:sub/path" + ).resolve() assert result.path == "sub/path" @pytest.mark.usefixtures("set_path") @@ -353,7 +354,7 @@ def test_native_schemes_do_not_route_to_adapters(self) -> None: class TestResolve: async def test_wrapper_adapter_chain(self, tmp_path: Path) -> None: register_url_adapter("wrap", WrapperAdapter) - result = await resolve_pipeline(f"file:{tmp_path.as_posix()}|wrap:inner/path") + result = await URLPipeline.from_url(f"file:{tmp_path.as_posix()}|wrap:inner/path").resolve() assert isinstance(result.store, TracingStore) assert result.path == "inner/path" assert result.store.context.preceding_url == f"file:{tmp_path.as_posix()}" @@ -364,25 +365,25 @@ async def test_nested_wrappers_preserve_residual_paths(self, tmp_path: Path) -> # each wrapper joins the preceding residual path with its own, so # no segment's path is lost in root|wrap:a|wrap:b register_url_adapter("wrap", WrapperAdapter) - result = await resolve_pipeline(f"file:{tmp_path.as_posix()}|wrap:a|wrap:b") + result = await URLPipeline.from_url(f"file:{tmp_path.as_posix()}|wrap:a|wrap:b").resolve() assert result.path == "a/b" async def test_native_adapter_gets_preceding_url(self) -> None: register_url_adapter("native", NativeAdapter) NativeAdapter.seen_urls.clear() - await resolve_pipeline("s3://bucket/repo|native:") + await URLPipeline.from_url("s3://bucket/repo|native:").resolve() assert NativeAdapter.seen_urls == ["s3://bucket/repo"] async def test_multi_segment_preceding_url(self) -> None: register_url_adapter("wrap", WrapperAdapter) register_url_adapter("native", NativeAdapter) NativeAdapter.seen_urls.clear() - await resolve_pipeline("memory://base|wrap:a|native:x") + await URLPipeline.from_url("memory://base|wrap:a|native:x").resolve() assert NativeAdapter.seen_urls == ["memory://base|wrap:a"] async def test_root_adapter(self) -> None: register_url_adapter("rooty", RootAdapter) - result = await resolve_pipeline("rooty://org/repo") + result = await URLPipeline.from_url("rooty://org/repo").resolve() assert result.path == "org/repo" assert result.store.read_only @@ -390,16 +391,16 @@ async def test_root_adapter_composes_with_chain(self) -> None: register_url_adapter("rooty", RootAdapter) register_url_adapter("native", NativeAdapter) NativeAdapter.seen_urls.clear() - await resolve_pipeline("rooty://org/repo|native:x") + await URLPipeline.from_url("rooty://org/repo|native:x").resolve() assert NativeAdapter.seen_urls == ["rooty://org/repo"] async def test_read_only_flag(self, tmp_path: Path) -> None: register_url_adapter("wrap", WrapperAdapter) - result = await resolve_pipeline(f"file:{tmp_path.as_posix()}|wrap:", mode="r") + result = await URLPipeline.from_url(f"file:{tmp_path.as_posix()}|wrap:").resolve(mode="r") assert isinstance(result.store, TracingStore) assert result.store.context.read_only assert result.store.context.mode == "r" - result = await resolve_pipeline(f"file:{tmp_path.as_posix()}|wrap:") + result = await URLPipeline.from_url(f"file:{tmp_path.as_posix()}|wrap:").resolve() assert isinstance(result.store, TracingStore) assert not result.store.context.read_only assert result.store.context.mode is None @@ -408,7 +409,7 @@ async def test_read_only_is_enforced_on_disobedient_adapters(self) -> None: # the resolver downgrades a writable store returned under mode "r" # instead of trusting the adapter to have honored context.read_only register_url_adapter("bad", DisobedientAdapter) - result = await resolve_pipeline("memory://base|bad:", mode="r") + result = await URLPipeline.from_url("memory://base|bad:").resolve(mode="r") assert result.store.read_only store = await make_store(URLPipeline.from_url("memory://base|bad:"), mode="r") assert store.read_only @@ -428,7 +429,7 @@ async def open_pipeline_segment( register_url_adapter("stubborn", StubbornAdapter) with pytest.raises(URLPipelineError, match="does not support read-only conversion"): - await resolve_pipeline("memory://base|stubborn:", mode="r") + await URLPipeline.from_url("memory://base|stubborn:").resolve(mode="r") async def test_read_only_enforcement_failure_closes_store(self) -> None: # the adapter's store must not leak when it cannot be made read-only @@ -451,7 +452,7 @@ async def open_pipeline_segment( register_url_adapter("stubborn", StubbornAdapter) with pytest.raises(URLPipelineError): - await resolve_pipeline("memory://base|stubborn:", mode="r") + await URLPipeline.from_url("memory://base|stubborn:").resolve(mode="r") assert closed == [True] async def test_resolve_preceding_mode_override(self, tmp_path: Path) -> None: @@ -467,7 +468,9 @@ async def open_pipeline_segment( return dataclasses.replace(preceding, path=segment.body) register_url_adapter("rowrap", ReadOnlyRootWrapper) - result = await resolve_pipeline(f"file:{tmp_path.as_posix()}|rowrap:x", mode="w") + result = await URLPipeline.from_url(f"file:{tmp_path.as_posix()}|rowrap:x").resolve( + mode="w" + ) assert result.path == "x" async def test_wrapper_on_local_file_root_read_only(self, tmp_path: Path) -> None: @@ -477,7 +480,7 @@ async def test_wrapper_on_local_file_root_read_only(self, tmp_path: Path) -> Non target = tmp_path / "data.bin" target.write_bytes(b"payload") register_url_adapter("wrap", WrapperAdapter) - result = await resolve_pipeline(f"file:{target.as_posix()}|wrap:", mode="r") + result = await URLPipeline.from_url(f"file:{target.as_posix()}|wrap:").resolve(mode="r") assert isinstance(result.store, TracingStore) assert result.store.read_only @@ -496,7 +499,7 @@ async def test_wrapper_on_local_file_root_writable_modes( target = tmp_path / "data.bin" target.write_bytes(b"payload") register_url_adapter("wrap", WrapperAdapter) - await resolve_pipeline(f"file:{target.as_posix()}|wrap:", mode=mode) + await URLPipeline.from_url(f"file:{target.as_posix()}|wrap:").resolve(mode=mode) async def test_resolve_preceding_storage_options_override(self, tmp_path: Path) -> None: # an adapter that consumed its namespaced keys strips them before @@ -516,8 +519,7 @@ async def open_pipeline_segment( register_url_adapter("consuming", ConsumingWrapper) # the local-path root rejects any surviving storage_options, so this # passing proves the adapter's keys were stripped before resolution - result = await resolve_pipeline( - f"file:{tmp_path.as_posix()}|consuming:x", + result = await URLPipeline.from_url(f"file:{tmp_path.as_posix()}|consuming:x").resolve( storage_options={"consuming_secret": "s3cr3t"}, ) assert result.path == "x" @@ -538,44 +540,44 @@ async def open_pipeline_segment( register_url_adapter("probe", OptionsProbe) opts = {"anon": True} - await resolve_pipeline("s3://bucket/x|probe:", storage_options=opts) + await URLPipeline.from_url("s3://bucket/x|probe:").resolve(storage_options=opts) assert OptionsProbe.seen_options == opts async def test_wrapper_at_root_position_raises(self) -> None: # a wrapper adapter used as the pipeline root has nothing to wrap register_url_adapter("wrap", WrapperAdapter) with pytest.raises(URLPipelineError, match="no preceding sub-URL"): - await resolve_pipeline("wrap:whatever") + await URLPipeline.from_url("wrap:whatever").resolve() async def test_single_url_is_a_trivial_pipeline(self) -> None: """A URL without `|` resolves as a pipeline consisting of its root alone.""" - result = await resolve_pipeline("memory://plain") + result = await URLPipeline.from_url("memory://plain").resolve() assert isinstance(result.store, ManagedMemoryStore) assert result.path == "" - async def test_accepts_pipeline_object(self, tmp_path: Path) -> None: - """`resolve_pipeline` takes a parsed `URLPipeline` as well as a string.""" + async def test_resolve_method(self, tmp_path: Path) -> None: + """`URLPipeline.resolve` yields the adapter's store and residual path.""" register_url_adapter("wrap", WrapperAdapter) - result = await resolve_pipeline(URLPipeline.from_url(f"file:{tmp_path.as_posix()}|wrap:a")) + result = await URLPipeline.from_url(f"file:{tmp_path.as_posix()}|wrap:a").resolve() assert isinstance(result.store, TracingStore) assert result.path == "a" async def test_format_segment_path_joins_residual_path(self, tmp_path: Path) -> None: """The node path of a trailing format segment extends the adapters' residual path.""" register_url_adapter("wrap", WrapperAdapter) - result = await resolve_pipeline(f"file:{tmp_path.as_posix()}|wrap:a|zarr3:b") + result = await URLPipeline.from_url(f"file:{tmp_path.as_posix()}|wrap:a|zarr3:b").resolve() assert result.path == "a/b" async def test_unknown_adapter_scheme_raises(self) -> None: with pytest.raises(URLPipelineError, match="no URL pipeline adapter is registered"): - await resolve_pipeline("memory://base|no-such-adapter:") + await URLPipeline.from_url("memory://base|no-such-adapter:").resolve() async def test_base_case_value_error_is_wrapped(self) -> None: # a root that only the fallback scheme detection accepts must not # leak urlparse's ValueError out of the base case register_url_adapter("wrap", WrapperAdapter) with pytest.raises(URLPipelineError, match="could not resolve the pipeline root"): - await resolve_pipeline("vendor.x://[authority]/path|wrap:") + await URLPipeline.from_url("vendor.x://[authority]/path|wrap:").resolve() class TestSpecRoots: @@ -584,7 +586,7 @@ class TestSpecRoots: @pytest.mark.parametrize("root", ["memory:", "memory:/", "memory://"]) async def test_memory_root_spellings_equivalent(self, root: str) -> None: register_url_adapter("wrap", WrapperAdapter) - result = await resolve_pipeline(f"{root}|wrap:") + result = await URLPipeline.from_url(f"{root}|wrap:").resolve() assert isinstance(result.store, TracingStore) inner = result.store._store assert isinstance(inner, ManagedMemoryStore) @@ -594,7 +596,7 @@ async def test_memory_root_spellings_equivalent(self, root: str) -> None: @pytest.mark.parametrize("root", ["memory:a/b", "memory:/a/b", "memory://a/b"]) async def test_memory_root_named_spellings_equivalent(self, root: str) -> None: register_url_adapter("wrap", WrapperAdapter) - result = await resolve_pipeline(f"{root}|wrap:") + result = await URLPipeline.from_url(f"{root}|wrap:").resolve() assert isinstance(result.store, TracingStore) inner = result.store._store assert isinstance(inner, ManagedMemoryStore) @@ -611,12 +613,12 @@ async def test_memory_root_shares_data_across_spellings(self) -> None: async def test_memory_root_rejects_query(self) -> None: register_url_adapter("wrap", WrapperAdapter) with pytest.raises(URLPipelineError, match="do not accept a query"): - await resolve_pipeline("memory:a?opt=1|wrap:") + await URLPipeline.from_url("memory:a?opt=1|wrap:").resolve() async def test_memory_root_rejects_storage_options(self) -> None: register_url_adapter("wrap", WrapperAdapter) with pytest.raises(TypeError, match="'storage_options' was provided but unused"): - await resolve_pipeline("memory:a|wrap:", storage_options={"anon": True}) + await URLPipeline.from_url("memory:a|wrap:").resolve(storage_options={"anon": True}) async def test_file_root_localhost_authority(self, tmp_path: Path) -> None: # file://localhost/p and file:///p are equivalent to file:/p @@ -642,26 +644,26 @@ async def test_file_root_windows_drive_spellings( # chdir into tmp so it is created there monkeypatch.chdir(tmp_path) drive_path = "C:/drive/data" - r1 = await resolve_pipeline(f"file:{drive_path}|wrap:") - r2 = await resolve_pipeline(f"file:///{drive_path}|wrap:") + r1 = await URLPipeline.from_url(f"file:{drive_path}|wrap:").resolve() + r2 = await URLPipeline.from_url(f"file:///{drive_path}|wrap:").resolve() assert isinstance(r1.store, TracingStore) assert isinstance(r2.store, TracingStore) async def test_file_root_rejects_other_authority(self) -> None: register_url_adapter("wrap", WrapperAdapter) with pytest.raises(URLPipelineError, match="unsupported authority"): - await resolve_pipeline("file://example.com/tmp/x|wrap:") + await URLPipeline.from_url("file://example.com/tmp/x|wrap:").resolve() @pytest.mark.parametrize("root", ["file:relative/path", "file://localhost"]) async def test_file_root_rejects_relative_paths(self, root: str) -> None: register_url_adapter("wrap", WrapperAdapter) with pytest.raises(URLPipelineError, match="absolute path"): - await resolve_pipeline(f"{root}|wrap:") + await URLPipeline.from_url(f"{root}|wrap:").resolve() async def test_file_root_rejects_query(self, tmp_path: Path) -> None: register_url_adapter("wrap", WrapperAdapter) with pytest.raises(URLPipelineError, match="do not accept a query"): - await resolve_pipeline(f"file:{tmp_path}?v=1|wrap:") + await URLPipeline.from_url(f"file:{tmp_path}?v=1|wrap:").resolve() @pytest.mark.skipif( not _has_async_fsspec_wrapper, @@ -682,7 +684,7 @@ async def test_file_root_does_not_percent_decode(self, tmp_path: Path) -> None: # documented divergence from RFC 8089: consistent with LocalStore, no # percent-escape is decoded, so %20 names a literal directory register_url_adapter("wrap", WrapperAdapter) - await resolve_pipeline(f"file:{tmp_path.as_posix()}/a%20b|wrap:") + await URLPipeline.from_url(f"file:{tmp_path.as_posix()}/a%20b|wrap:").resolve() assert (tmp_path / "a%20b").is_dir() assert not (tmp_path / "a b").exists()