diff --git a/changes/4192.feature.md b/changes/4192.feature.md
index 3ea14b30d6..21e87736a3 100644
--- a/changes/4192.feature.md
+++ b/changes/4192.feature.md
@@ -1,46 +1,34 @@
Added core support for URL pipelines (https://github.com/jbms/url-pipeline):
`|`-chained URLs that address zarr data through nested storage layers, e.g.
-`s3://bucket/data.zip|zip:|zarr3:`. This includes the parser
-(`zarr.storage.parse_pipeline` / `zarr.storage.resolve_pipeline`), the
-single-method `zarr.abc.url_pipeline.URLPipelineAdapter` interface, and the
-`zarr.url_adapters` entry-point group through which third-party packages
-(e.g. Icechunk) register adapters for their own schemes. Adapters for a scheme
-are loaded lazily and individually. Builtin adapters (`zip:`,
-`zarr2:`/`zarr3:`) will follow separately.
+`s3://bucket/data.zip|zip:|zarr3:`. A parsed `zarr.storage.URLPipeline` (built with
+`URLPipeline.from_url`) can be passed to any function that accepts a `StoreLike`, and
+the new `zarr.open_url(url, mode="r" | "r+")` opens the existing node a pipeline string
+addresses without ever creating or overwriting anything. Adapters for the
+sub-URLs after the first are provided by third-party packages (e.g. Icechunk) through
+the `zarr.url_adapters` entry-point group and the single-method
+`zarr.abc.url_pipeline.URLPipelineAdapter` interface; they are loaded lazily and
+individually. A builtin `zip:` adapter will follow separately.
Behavior notes:
-- The `|` character is now reserved as the pipeline delimiter in every string
- store specification, and no percent-escape is decoded; pass a `pathlib.Path`
- to address a local file whose name contains `|`. URLs without a `|` (and
- without a registered root adapter scheme) are handled exactly as before —
- registered adapters cannot intercept zarr's native `file:`/`memory:`
- routing, and fsspec chained URLs (`zip::s3://...`) keep flowing to fsspec.
-- Inside a pipeline, `memory:` and `file:` roots follow the URL pipeline
- spec's spelling rules (`memory:` ≡ `memory:/` ≡ `memory://`; `file:` must
- carry an absolute path with at most a `localhost` authority). Percent-escapes
- are *not* decoded in `file:` roots, matching `LocalStore`; this is a
- documented divergence from RFC 8089. When fsspec is installed, a plain
- `memory://name` URL is still routed to fsspec's in-memory filesystem, whereas
- a pipeline's `memory:` root is zarr's `ManagedMemoryStore`; the two do not
- share data.
+- Plain strings are never interpreted as pipelines: `zarr.open("data|x.zarr")` still
+ addresses a local directory of that name, whatever adapters are installed. Only
+ `zarr.open_url` and `URLPipeline.from_url` read a string as a pipeline.
+- A trailing `zarr2:`/`zarr3:` segment selects the zarr format of the open or create
+ call, and its body is the path of the node to open. A `zarr_format` argument that
+ conflicts with it raises `ValueError`; functions whose `zarr_format` defaults to 3
+ (`zarr.create_array`, `Group.from_store`, `Group.open`, `Array.open`) need an explicit
+ `zarr_format=None` to defer to the pipeline. `Array.open` gains a `zarr_format`
+ parameter mirroring `Group.open`.
+- Inside a pipeline, `memory:` and `file:` roots follow the URL pipeline spec's spelling
+ rules (`memory:` ≡ `memory:/` ≡ `memory://`; `file:` must carry an absolute path with
+ at most a `localhost` authority). Percent-escapes are not decoded, matching
+ `LocalStore`. A pipeline's `memory:` root is zarr's `ManagedMemoryStore`, which does
+ not share data with the fsspec in-memory filesystem a plain `memory://name` string
+ resolves to.
- Wrapper adapters can open a local *file* root only in mode `"r"` (via
- `PipelineContext.resolve_preceding(mode="r")`); other modes create a
- directory at the root and fail. Writing through a container file is not yet
- supported.
-- A `zarr.url_adapters` entry point whose scheme is already registered (e.g. by
- a builtin adapter) is ignored with a `ZarrUserWarning`, as are duplicate
- entry-point names; an entry point that fails to import raises
- `URLPipelineError` naming it, and stays discoverable for a retry.
-- `zarr.create_array` and `Group.from_store` now default `zarr_format` to
- `None`, resolving to the configured default (3) as before unless a URL
- pipeline selects a format; `Array.open` gains a `zarr_format` parameter
- mirroring `Group.open`. `Group.open` and `Array.open` keep their default of
- 3, so pass `zarr_format=None` to defer to a pipeline's format. A format
- selected by a pipeline that conflicts with an explicit `zarr_format` raises
- `ValueError` in every `open*`/`create*`/`save*` entry point.
-- Mode `"a"` (open-or-create, the `zarr.open` default) on a *read-only* store
- now serves the "open" half instead of raising upfront, for all stores;
- unambiguous write modes (`"w"`, `"w-"`, `"r+"`) still raise.
-- For root-adapter URLs (e.g. `gh://org/repo`), `storage_options` are handed
- to the adapter and are not validated as used by `make_store`.
+ `PipelineContext.resolve_preceding(mode="r")`); writing through a container file is
+ not yet supported.
+- A `zarr.url_adapters` entry point whose scheme is already registered is ignored with a
+ `ZarrUserWarning`, as are duplicate entry-point names; an entry point that fails to
+ import raises `URLPipelineError` naming it, and stays discoverable for a retry.
diff --git a/docs/api/zarr/functions/open_url.md b/docs/api/zarr/functions/open_url.md
new file mode 100644
index 0000000000..84774a1710
--- /dev/null
+++ b/docs/api/zarr/functions/open_url.md
@@ -0,0 +1,5 @@
+---
+title: zarr.open_url
+---
+
+::: zarr.open_url
diff --git a/docs/user-guide/extending.md b/docs/user-guide/extending.md
index d5fe5726ca..34128ce52a 100644
--- a/docs/user-guide/extending.md
+++ b/docs/user-guide/extending.md
@@ -112,7 +112,7 @@ class MyAdapter(URLPipelineAdapter):
) -> AdapterResolution:
# A *wrapper* adapter opens the resource to its left and wraps it. Join the
# preceding residual path with this segment's own path, and keep every other
- # field (e.g. zarr_format) via dataclasses.replace.
+ # field via dataclasses.replace.
preceding = await context.resolve_preceding()
store = WrapperStore(preceding.store)
path = "/".join(part.strip("/") for part in (preceding.path, segment.body) if part)
@@ -126,10 +126,11 @@ inner adapter sees the outer one's residual path `a` and the pipeline resolves t
import asyncio
from zarr.registry import register_url_adapter
-from zarr.storage import resolve_pipeline
+from zarr.storage import URLPipeline
register_url_adapter("mypackage.myscheme", MyAdapter)
-resolution = asyncio.run(resolve_pipeline("memory://x|mypackage.myscheme:a|mypackage.myscheme:b"))
+pipeline = URLPipeline.from_url("memory://x|mypackage.myscheme:a|mypackage.myscheme:b")
+resolution = asyncio.run(pipeline.resolve())
assert resolution.path == "a/b"
```
@@ -144,7 +145,8 @@ scheme as the entry-point name. Nonstandard schemes should be vendor-prefixed
Adapters are loaded lazily, only when a pipeline naming their scheme is resolved. An
adapter may also be used as a *root* scheme (e.g. `mypackage.myscheme://org/repo`), in
-which case `context.preceding` is empty. The returned store must honor
+which case `context.preceding` is empty; such a URL reaches the adapter only through
+`zarr.open_url` or a `URLPipeline`, never as a plain string store specification. The returned store must honor
`context.read_only`, and `open_pipeline_segment` runs on zarr's internal event loop, so
it must not block or call zarr's synchronous API. See
[`zarr.abc.url_pipeline`][] for the full contract.
diff --git a/docs/user-guide/storage.md b/docs/user-guide/storage.md
index 010abc08a4..bb294353aa 100644
--- a/docs/user-guide/storage.md
+++ b/docs/user-guide/storage.md
@@ -100,8 +100,9 @@ print(group)
group = zarr.open_group(UPath('s3://noaa-nwm-retro-v2-zarr-pds', anon=True), mode='r')
```
-- a [URL pipeline](#user-guide-url-pipelines) string containing `|`, such as
- `s3://bucket/data.zip|zip:`, which is resolved through registered adapters.
+- a [`URLPipeline`][zarr.storage.URLPipeline], the parsed form of a
+ [URL pipeline](#user-guide-url-pipelines) such as `s3://bucket/data.zip|zip:`, which
+ is resolved through registered adapters. Strings are never interpreted as pipelines.
- a [`Store`][zarr.abc.store.Store] or [`StorePath`][zarr.storage.StorePath] -
see explicit store creation below.
@@ -114,28 +115,49 @@ sub-URL locates a resource with a conventional URL; each subsequent sub-URL name
*adapter* that reinterprets everything to its left (e.g.
`s3://bucket/data.zip|zip:|zarr3:`). Adapters are provided by packages through the
`zarr.url_adapters` entry-point group — see
-[`zarr.abc.url_pipeline`][zarr.abc.url_pipeline] for the adapter interface. Builtin
-adapters (`zip:`, `zarr2:`/`zarr3:`) are under development and will expand this
-section. URLs without a `|` (and without a registered root scheme) are handled
-exactly as before.
-
-`storage_options` passed to `zarr.open` apply to the *root* sub-URL (e.g. fsspec
-options for `s3://...`); adapters may consume adapter-specific, namespaced keys.
-Non-dict forms of `storage_options` are reserved for future per-segment
-configuration.
-
-The `|` character is reserved as the pipeline delimiter in every string store
-specification, and no percent-escape is decoded: to address a local file whose
-*name* contains `|` (or `#`), pass a `pathlib.Path` instead of a string.
-Registered adapters cannot intercept zarr's native `file:` and `memory:` root
-schemes, and fsspec's chained-URL syntax (`zip::s3://...`) keeps flowing to
-fsspec.
+[`zarr.abc.url_pipeline`][zarr.abc.url_pipeline] for the adapter interface. A builtin
+`zip:` adapter is under development and will expand this section.
+
+A parsed [`URLPipeline`][zarr.storage.URLPipeline] can be passed to any function that
+accepts a `StoreLike`, and [`zarr.open_url`][] opens the *existing* node a pipeline
+string addresses, read-only unless `mode="r+"` is given:
+
+```python exec="true" session="storage" source="above" result="ansi"
+from zarr.storage import URLPipeline
+
+pipeline = URLPipeline.from_url("memory://pipeline-demo|zarr2:")
+group = zarr.open_group(pipeline, mode="w")
+print(group.metadata.zarr_format)
+# the same thing, as a one-liner
+print(zarr.open_url("memory://pipeline-demo|zarr2:").metadata.zarr_format)
+```
+
+Only these two entry points read a string as a pipeline. Plain string store
+specifications keep their existing meaning everywhere else, so a local path that
+happens to contain `|` still names a directory, whatever adapters are installed.
+Unlike `zarr.open`, `zarr.open_url` never creates or overwrites: to create a node at a
+pipeline-addressed location, pass the `URLPipeline` to `zarr.create_array` or
+`zarr.create_group`.
+
+A trailing `zarr2:` or `zarr3:` segment is a *format segment*. It is not an adapter:
+zarr-python reads it from the pipeline itself and uses it as the `zarr_format` of the
+open or create call, and its body (`zarr3:path/to/node`) is the path of the node to
+open. A `zarr_format` argument that conflicts with the format segment raises
+`ValueError`; functions whose `zarr_format` defaults to 3 (such as `zarr.create_array`
+and `Group.open`) need an explicit `zarr_format=None` to defer to the pipeline.
+
+`storage_options` apply to the *root* sub-URL (e.g. fsspec options for `s3://...`);
+adapters may consume adapter-specific, namespaced keys. No percent-escape is decoded
+anywhere in a pipeline. Registered adapters cannot intercept zarr's native `file:` and
+`memory:` root schemes.
Inside a pipeline, a `memory:` root is zarr's managed in-memory store (`memory:`,
`memory:/` and `memory://` are equivalent, and `memory:name` selects a named store).
-When fsspec is installed, a plain `memory://name` URL *without* a `|` is still routed to
-fsspec's in-memory filesystem, which is a different store. A `file:` root must carry an
-absolute path; percent-escapes are not decoded, matching the [local store](#local-store).
+When fsspec is installed, a plain `memory://name` string store specification is still
+routed to fsspec's in-memory filesystem, which is a different store. A `file:` root
+must carry an absolute path; percent-escapes are not decoded, matching the
+[local store](#local-store). The root always carries a scheme: a local path is spelled
+as a `file:` URL, never as a bare path.
## Explicit Store Creation
diff --git a/mkdocs.yml b/mkdocs.yml
index 58b5d3344a..c0b7a96fa0 100644
--- a/mkdocs.yml
+++ b/mkdocs.yml
@@ -81,6 +81,7 @@ nav:
- 'zarr.open_consolidated': api/zarr/functions/open_consolidated.md
- 'zarr.open_group': api/zarr/functions/open_group.md
- 'zarr.open_like': api/zarr/functions/open_like.md
+ - 'zarr.open_url': api/zarr/functions/open_url.md
- 'zarr.print_debug_info': api/zarr/functions/print_debug_info.md
- 'zarr.registry': api/zarr/registry.md
- 'zarr.save': api/zarr/functions/save.md
diff --git a/src/zarr/__init__.py b/src/zarr/__init__.py
index f305245f4e..b8ac1fb11e 100644
--- a/src/zarr/__init__.py
+++ b/src/zarr/__init__.py
@@ -27,6 +27,7 @@
open_consolidated,
open_group,
open_like,
+ open_url,
save,
save_array,
save_group,
@@ -175,6 +176,7 @@ def set_format(log_format: str) -> None:
"open_consolidated",
"open_group",
"open_like",
+ "open_url",
"print_debug_info",
"save",
"save_array",
diff --git a/src/zarr/abc/url_pipeline.py b/src/zarr/abc/url_pipeline.py
index 337563685d..f0f58140c8 100644
--- a/src/zarr/abc/url_pipeline.py
+++ b/src/zarr/abc/url_pipeline.py
@@ -25,7 +25,7 @@
if TYPE_CHECKING:
from zarr.abc.store import Store
- from zarr.core.common import AccessModeLiteral, ZarrFormat
+ from zarr.core.common import AccessModeLiteral
__all__ = [
"AdapterResolution",
@@ -50,8 +50,7 @@ class PipelineSegment:
Attributes
----------
scheme : str
- The lowercased URL scheme. Empty string only for a schemeless root
- (a bare local path), which is treated as opaque text.
+ The lowercased URL scheme.
body : str
The text after `scheme:` and before any `?`. Interpretation is
scheme-defined; it is **not** URL-normalized, so case-significant
@@ -89,17 +88,17 @@ class AdapterResolution:
Residual path *within* the store that the pipeline addresses
(e.g. `"path/to/node"` for `...|icechunk://tag.v1/path/to/node`).
Empty string when the pipeline addresses the store root.
- zarr_format : ZarrFormat | None
- Zarr format selected by a format segment (`zarr2:`/`zarr3:`),
- or None if unspecified. A wrapper adapter that re-wraps a preceding
- resolution must carry every field it does not change forward —
- prefer `dataclasses.replace(preceding, store=..., path=...)` over
- reconstructing, so fields added later are never silently dropped.
+
+ A wrapper adapter that re-wraps a preceding resolution must carry every
+ field it does not change forward: prefer
+ `dataclasses.replace(preceding, store=..., path=...)` over
+ reconstructing, so fields added later are never silently dropped.
+ Format segments (`zarr2:`/`zarr3:`) are not adapters; they are consumed
+ by [`URLPipeline`][zarr.storage.URLPipeline] before resolution.
"""
store: Store
path: str = ""
- zarr_format: ZarrFormat | None = None
@dataclass(frozen=True, eq=False)
@@ -130,10 +129,6 @@ class PipelineContext:
backends. An adapter that consumes keys should strip them before
resolving the rest of the pipeline, by passing the reduced mapping
to [`resolve_preceding`][zarr.abc.url_pipeline.PipelineContext.resolve_preceding].
- Non-dict forms of the caller-facing `storage_options` argument are
- reserved for future per-segment configuration (one mapping per
- pipeline segment); this attribute will remain a single mapping —
- the one addressed to this adapter's segment.
"""
preceding: tuple[PipelineSegment, ...]
diff --git a/src/zarr/api/asynchronous.py b/src/zarr/api/asynchronous.py
index 4bb3ce621d..4fc0abd8da 100644
--- a/src/zarr/api/asynchronous.py
+++ b/src/zarr/api/asynchronous.py
@@ -48,7 +48,7 @@
ZarrUserWarning,
)
from zarr.storage import StorePath
-from zarr.storage._common import make_store_path
+from zarr.storage._common import make_store_path, resolve_zarr_format
if TYPE_CHECKING:
from collections.abc import Iterable
@@ -88,6 +88,7 @@
"open_consolidated",
"open_group",
"open_like",
+ "open_url",
"save",
"save_array",
"save_group",
@@ -403,8 +404,8 @@ async def open(
mode = "r"
else:
mode = "a"
+ zarr_format = resolve_zarr_format(store, zarr_format)
store_path = await make_store_path(store, mode=mode, path=path, storage_options=storage_options)
- zarr_format = store_path.resolve_zarr_format(zarr_format)
# TODO: the mode check below seems wrong!
if "shape" not in kwargs and mode in (*_READ_MODES, "w"):
@@ -436,6 +437,59 @@ async def open(
return await open_group(store=store_path, zarr_format=zarr_format, mode=mode, **kwargs)
+async def open_url(
+ url: str,
+ *,
+ mode: Literal["r", "r+"] = "r",
+ storage_options: dict[str, Any] | None = None,
+) -> AnyAsyncArray | AsyncGroup:
+ """Open the existing group or array addressed by a URL pipeline.
+
+ This is the only entry point that reads a string as a
+ [URL pipeline][user-guide-url-pipelines]; [`open`][zarr.api.asynchronous.open] and the other
+ `StoreLike`-taking functions treat strings as they always have. Unlike
+ `open`, `open_url` never creates or overwrites anything: to create a node at
+ a pipeline-addressed location, pass a `URLPipeline` to
+ [`create_array`][zarr.api.asynchronous.create_array] or
+ [`create_group`][zarr.api.asynchronous.create_group].
+
+ The zarr format is taken from a trailing `zarr2:`/`zarr3:` segment when
+ present, and inferred from the stored metadata otherwise.
+
+ Parameters
+ ----------
+ url : str
+ A URL pipeline such as `"s3://bucket/data.zip|zip:|zarr3:"`. A URL
+ without `|` is a trivial pipeline consisting of its root alone. The
+ node to open is addressed by the URL itself: the body of a trailing
+ `zarr2:`/`zarr3:` segment is the path of the node within the store.
+ mode : {'r', 'r+'}, optional
+ `'r'` (the default) opens read-only; `'r+'` opens for reading and
+ writing. The node must exist in either case.
+ storage_options : dict, optional
+ Options for the pipeline's root sub-URL (e.g. fsspec options).
+
+ Returns
+ -------
+ z : array or group
+ Return type depends on what exists at the URL. For a statically typed
+ result, pass a `URLPipeline` to `open_array` or `open_group` instead.
+ """
+ from zarr.storage import URLPipeline
+
+ if mode not in ("r", "r+"):
+ raise ValueError(
+ f"Invalid mode: {mode!r}. open_url opens existing nodes only and accepts "
+ "'r' or 'r+'; to create a node at a URL pipeline, pass "
+ "URLPipeline.from_url(url) to create_array or create_group."
+ )
+ return await open(
+ store=URLPipeline.from_url(url),
+ mode=mode,
+ storage_options=storage_options,
+ )
+
+
async def open_consolidated(
*args: Any, use_consolidated: Literal[True] = True, **kwargs: Any
) -> AsyncGroup:
@@ -531,8 +585,8 @@ async def save_array(
raise TypeError("arr argument must be numpy or other NDArrayLike array")
mode = kwargs.pop("mode", "a")
+ zarr_format = resolve_zarr_format(store, zarr_format)
store_path = await make_store_path(store, path=path, mode=mode, storage_options=storage_options)
- zarr_format = store_path.resolve_zarr_format(zarr_format)
if zarr_format is None:
zarr_format = _default_zarr_format()
if np.isscalar(arr):
@@ -600,8 +654,8 @@ async def save_group(
for arr in (*args, *kwargs.values()):
get_data_type_from_native_dtype(arr.dtype)
+ zarr_format = resolve_zarr_format(store, zarr_format)
store_path = await make_store_path(store, path=path, mode="w", storage_options=storage_options)
- zarr_format = store_path.resolve_zarr_format(zarr_format)
if zarr_format is None:
zarr_format = _default_zarr_format()
# The group replaces whatever is stored under the path, now that the arguments are known
@@ -800,8 +854,8 @@ async def create_group(
mode: Literal["a"] = "a"
+ zarr_format = resolve_zarr_format(store, zarr_format)
store_path = await make_store_path(store, path=path, mode=mode, storage_options=storage_options)
- zarr_format = store_path.resolve_zarr_format(zarr_format)
if zarr_format is None:
zarr_format = _default_zarr_format()
@@ -893,8 +947,8 @@ async def open_group(
}
)
+ zarr_format = resolve_zarr_format(store, zarr_format)
store_path = await make_store_path(store, mode=mode, storage_options=storage_options, path=path)
- zarr_format = store_path.resolve_zarr_format(zarr_format)
if attributes is None:
attributes = {}
@@ -1119,8 +1173,8 @@ async def create(
if mode is None:
mode = "a"
overwrite = overwrite or _infer_overwrite(mode)
+ zarr_format = resolve_zarr_format(store, zarr_format)
store_path = await make_store_path(store, path=path, mode=mode, storage_options=storage_options)
- zarr_format = store_path.resolve_zarr_format(zarr_format)
if zarr_format is None:
zarr_format = _default_zarr_format()
@@ -1336,8 +1390,8 @@ async def open_array(
"""
mode = kwargs.pop("mode", None)
+ zarr_format = resolve_zarr_format(store, zarr_format)
store_path = await make_store_path(store, path=path, mode=mode, storage_options=storage_options)
- zarr_format = store_path.resolve_zarr_format(zarr_format)
if "write_empty_chunks" in kwargs:
_warn_write_empty_chunks_kwarg()
diff --git a/src/zarr/api/synchronous.py b/src/zarr/api/synchronous.py
index 6778f8d907..5959316686 100644
--- a/src/zarr/api/synchronous.py
+++ b/src/zarr/api/synchronous.py
@@ -66,6 +66,7 @@
"open_consolidated",
"open_group",
"open_like",
+ "open_url",
"save",
"save_array",
"save_group",
@@ -246,6 +247,58 @@ def open(
return Group(obj)
+def open_url(
+ url: str,
+ *,
+ mode: Literal["r", "r+"] = "r",
+ storage_options: dict[str, Any] | None = None,
+) -> AnyArray | Group:
+ """Open the existing group or array addressed by a URL pipeline.
+
+ This is the only entry point that reads a string as a
+ [URL pipeline][user-guide-url-pipelines]; [`open`][zarr.open] and the other
+ `StoreLike`-taking functions treat strings as they always have. Unlike
+ `open`, `open_url` never creates or overwrites anything: to create a node at
+ a pipeline-addressed location, pass a `URLPipeline` to
+ [`create_array`][zarr.create_array] or [`create_group`][zarr.create_group].
+
+ The zarr format is taken from a trailing `zarr2:`/`zarr3:` segment when
+ present, and inferred from the stored metadata otherwise.
+
+ Parameters
+ ----------
+ url : str
+ A URL pipeline such as `"s3://bucket/data.zip|zip:|zarr3:"`. A URL
+ without `|` is a trivial pipeline consisting of its root alone. The
+ node to open is addressed by the URL itself: the body of a trailing
+ `zarr2:`/`zarr3:` segment is the path of the node within the store.
+ mode : {'r', 'r+'}, optional
+ `'r'` (the default) opens read-only; `'r+'` opens for reading and
+ writing. The node must exist in either case.
+ storage_options : dict, optional
+ Options for the pipeline's root sub-URL (e.g. fsspec options).
+
+ Returns
+ -------
+ z : array or group
+ Return type depends on what exists at the URL. For a statically typed
+ result, pass a `URLPipeline` to `open_array` or `open_group` instead.
+ """
+ from zarr.storage import URLPipeline
+
+ if mode not in ("r", "r+"):
+ raise ValueError(
+ f"Invalid mode: {mode!r}. open_url opens existing nodes only and accepts "
+ "'r' or 'r+'; to create a node at a URL pipeline, pass "
+ "URLPipeline.from_url(url) to create_array or create_group."
+ )
+ return open(
+ store=URLPipeline.from_url(url),
+ mode=mode,
+ storage_options=storage_options,
+ )
+
+
def open_consolidated(*args: Any, use_consolidated: Literal[True] = True, **kwargs: Any) -> Group:
"""
Alias for [`open_group`][zarr.api.synchronous.open_group] with `use_consolidated=True`.
@@ -866,7 +919,7 @@ def create_array(
serializer: SerializerLike = "auto",
fill_value: Any | None = DEFAULT_FILL_VALUE,
order: MemoryOrder | None = None,
- zarr_format: ZarrFormat | None = None,
+ zarr_format: ZarrFormat | None = 3,
attributes: dict[str, JSON] | None = None,
chunk_key_encoding: ChunkKeyEncodingLike | None = None,
dimension_names: DimensionNamesLike = None,
diff --git a/src/zarr/core/array.py b/src/zarr/core/array.py
index f32ee7a94b..feedea7528 100644
--- a/src/zarr/core/array.py
+++ b/src/zarr/core/array.py
@@ -154,7 +154,7 @@
_parse_bytes_bytes_codec,
get_pipeline_class,
)
-from zarr.storage._common import StorePath, make_store_path
+from zarr.storage._common import StorePath, make_store_path, resolve_zarr_format
from zarr.storage._utils import _relativize_path
if TYPE_CHECKING:
@@ -466,8 +466,8 @@ async def _create(
Deprecated in favor of [`zarr.api.asynchronous.create_array`][].
"""
+ zarr_format = resolve_zarr_format(store, zarr_format)
store_path = await make_store_path(store)
- zarr_format = store_path.resolve_zarr_format(zarr_format) or zarr_format
dtype_parsed = parse_dtype(dtype, zarr_format=zarr_format)
shape = parse_shapelike(shape)
@@ -819,8 +819,8 @@ async def example():
#
```
"""
+ zarr_format = resolve_zarr_format(store, zarr_format)
store_path = await make_store_path(store)
- zarr_format = store_path.resolve_zarr_format(zarr_format)
metadata_dict = await get_array_metadata(store_path, zarr_format=zarr_format)
# TODO: remove this cast when we have better type hints
_metadata_dict = cast("ArrayMetadataJSON_V3", metadata_dict)
@@ -4387,8 +4387,8 @@ async def from_array(
"""
mode: Literal["a"] = "a"
config_parsed = parse_array_config(config)
+ zarr_format = resolve_zarr_format(store, zarr_format)
store_path = await make_store_path(store, path=name, mode=mode, storage_options=storage_options)
- zarr_format = store_path.resolve_zarr_format(zarr_format)
(
chunks,
@@ -4714,7 +4714,7 @@ async def create_array(
serializer: SerializerLike = "auto",
fill_value: Any | None = DEFAULT_FILL_VALUE,
order: MemoryOrder | None = None,
- zarr_format: ZarrFormat | None = None,
+ zarr_format: ZarrFormat | None = 3,
attributes: dict[str, JSON] | None = None,
chunk_key_encoding: ChunkKeyEncodingLike | None = None,
dimension_names: DimensionNamesLike = None,
@@ -4874,10 +4874,10 @@ async def create_array(
else:
mode: Literal["a"] = "a"
+ zarr_format = resolve_zarr_format(store, zarr_format)
store_path = await make_store_path(
store, path=name, mode=mode, storage_options=storage_options
)
- zarr_format = store_path.resolve_zarr_format(zarr_format)
return await init_array(
store_path=store_path,
shape=shape_parsed,
diff --git a/src/zarr/core/group.py b/src/zarr/core/group.py
index b541aff437..01d6e05df9 100644
--- a/src/zarr/core/group.py
+++ b/src/zarr/core/group.py
@@ -68,7 +68,7 @@
ZarrUserWarning,
)
from zarr.storage import StoreLike, StorePath
-from zarr.storage._common import make_store_path
+from zarr.storage._common import make_store_path, resolve_zarr_format
from zarr.storage._utils import _join_paths, _normalize_path_keys, normalize_path
if TYPE_CHECKING:
@@ -531,10 +531,10 @@ async def from_store(
*,
attributes: dict[str, Any] | None = None,
overwrite: bool = False,
- zarr_format: ZarrFormat | None = None,
+ zarr_format: ZarrFormat | None = 3,
) -> AsyncGroup:
+ zarr_format = resolve_zarr_format(store, zarr_format)
store_path = await make_store_path(store)
- zarr_format = store_path.resolve_zarr_format(zarr_format)
if zarr_format is None:
zarr_format = _default_zarr_format()
attributes = attributes or {}
@@ -578,8 +578,8 @@ async def open(
(``.zmetadata`` by default). Specify the custom key as ``use_consolidated``
to load consolidated metadata from a non-default key.
"""
+ zarr_format = resolve_zarr_format(store, zarr_format)
store_path = await make_store_path(store)
- zarr_format = store_path.resolve_zarr_format(zarr_format)
if not store_path.store.supports_consolidated_metadata:
# Fail if consolidated metadata was requested but the Store doesn't support it
if use_consolidated:
@@ -1959,7 +1959,7 @@ def from_store(
store: StoreLike,
*,
attributes: dict[str, Any] | None = None,
- zarr_format: ZarrFormat | None = None,
+ zarr_format: ZarrFormat | None = 3,
overwrite: bool = False,
) -> Group:
"""Instantiate a group from an initialized store.
@@ -1973,8 +1973,8 @@ def from_store(
attributes : dict, optional
A dictionary of JSON-serializable values with user-defined attributes.
zarr_format : {2, 3, None}, optional
- Zarr storage format version. If None, the format selected by a URL
- pipeline segment is used, falling back to the configured default.
+ Zarr storage format version. None defers to the format selected by a
+ `URLPipeline` store, falling back to the configured default.
overwrite : bool, optional
If True, do not raise an error if the group already exists.
diff --git a/src/zarr/storage/__init__.py b/src/zarr/storage/__init__.py
index edc01f1a5b..c4d8d534ac 100644
--- a/src/zarr/storage/__init__.py
+++ b/src/zarr/storage/__init__.py
@@ -10,7 +10,7 @@
from zarr.storage._logging import LoggingStore
from zarr.storage._memory import GpuMemoryStore, ManagedMemoryStore, MemoryStore
from zarr.storage._obstore import ObjectStore
-from zarr.storage._url_pipeline import parse_pipeline, resolve_pipeline
+from zarr.storage._url_pipeline import URLPipeline
from zarr.storage._wrapper import WrapperStore
from zarr.storage._zip import ZipStore
@@ -24,10 +24,9 @@
"ObjectStore",
"StoreLike",
"StorePath",
+ "URLPipeline",
"WrapperStore",
"ZipStore",
- "parse_pipeline",
- "resolve_pipeline",
]
diff --git a/src/zarr/storage/_common.py b/src/zarr/storage/_common.py
index f649a7eea6..00d8f5b4f5 100644
--- a/src/zarr/storage/_common.py
+++ b/src/zarr/storage/_common.py
@@ -2,7 +2,7 @@
import importlib.util
from pathlib import Path
-from typing import TYPE_CHECKING, Any, Literal, Self
+from typing import TYPE_CHECKING, Any, Literal, Self, overload
from zarr.abc.store import (
ByteRequest,
@@ -29,7 +29,7 @@
)
from zarr.storage._local import LocalStore
from zarr.storage._memory import ManagedMemoryStore, MemoryStore
-from zarr.storage._url_pipeline import is_url_pipeline, resolve_pipeline
+from zarr.storage._url_pipeline import URLPipeline
from zarr.storage._utils import UPath, _join_paths, normalize_path, parse_store_url
_has_fsspec = importlib.util.find_spec("fsspec")
@@ -53,57 +53,19 @@ class StorePath:
The store to use.
path : str
The path within the store.
-
- Attributes
- ----------
- zarr_format : ZarrFormat | None
- Zarr format selected by a `zarr2:`/`zarr3:` URL pipeline segment,
- or None. Not part of the StorePath's identity (ignored by `__eq__`).
"""
store: Store
path: str
- zarr_format: ZarrFormat | None
def __init__(self, store: Store, path: str = "") -> None:
self.store = store
self.path = normalize_path(path)
- self.zarr_format = None
@property
def read_only(self) -> bool:
return self.store.read_only
- def resolve_zarr_format(self, zarr_format: ZarrFormat | None) -> ZarrFormat | None:
- """
- Combine the zarr format selected by a URL pipeline segment (`zarr2:` /
- `zarr3:`) with a caller-supplied `zarr_format`.
-
- Parameters
- ----------
- zarr_format : ZarrFormat | None
- The format requested by the caller, or None when unspecified.
-
- Returns
- -------
- ZarrFormat | None
- The pipeline's format when the caller did not specify one, otherwise the
- caller's format. None when neither is set.
-
- Raises
- ------
- ValueError
- If the caller's format differs from the one selected by the pipeline.
- """
- if self.zarr_format is None:
- return zarr_format
- if zarr_format is not None and zarr_format != self.zarr_format:
- raise ValueError(
- f"zarr_format={zarr_format} conflicts with the 'zarr{self.zarr_format}:' "
- "segment of the URL pipeline; pass zarr_format=None to use the pipeline's format"
- )
- return self.zarr_format
-
@classmethod
async def _create_open_instance(cls, store: Store, path: str) -> Self:
"""Helper to create and return a StorePath instance."""
@@ -116,10 +78,7 @@ async def open(cls, store: Store, path: str, mode: AccessModeLiteral | None = No
Open StorePath based on the provided mode.
* If the mode is None, return an opened version of the store with no changes.
- * If the mode is 'r+', 'w-', or 'w' and the store is read-only, raise a ValueError.
- * If the mode is 'a' (open-or-create) and the store is read-only, serve the
- "open" half: the StorePath is opened read-only, and any subsequent write
- fails at the store level.
+ * If the mode is 'r+', 'w-', 'w', or 'a' and the store is read-only, raise a ValueError.
* If the mode is 'r' and the store is not read-only, return a copy of the store with read_only set to True.
* If the mode is 'w-' and the store is not read-only and the StorePath contains keys, raise a FileExistsError.
@@ -145,8 +104,7 @@ async def open(cls, store: Store, path: str, mode: AccessModeLiteral | None = No
FileExistsError
If the mode is 'w-' and the store path already exists.
ValueError
- If the mode is "r+", "w-", or "w" and the store is read-only, or if the
- mode is not a valid access mode.
+ If the mode is not "r" and the store is read-only, or
"""
# fastpath if mode is None
@@ -157,9 +115,8 @@ async def open(cls, store: Store, path: str, mode: AccessModeLiteral | None = No
raise ValueError(f"Invalid mode: {mode}, expected one of {ANY_ACCESS_MODE}")
if store.read_only:
- # mode "a" (open-or-create) on a read-only store means open-only;
- # unambiguous write modes are rejected outright.
- if mode not in ("r", "a"):
+ # Don't allow write operations on a read-only store
+ if mode != "r":
raise ValueError(
f"Store is read-only but mode is {mode!r}. Create a writable store or use 'r' mode."
)
@@ -320,9 +277,7 @@ def delete_sync(self) -> None:
def __truediv__(self, other: str) -> StorePath:
"""Combine this store path with another path"""
- result = self.__class__(self.store, _join_paths([self.path, other]))
- result.zarr_format = self.zarr_format
- return result
+ return self.__class__(self.store, _join_paths([self.path, other]))
def __str__(self) -> str:
return _join_paths([str(self.store), self.path])
@@ -350,7 +305,7 @@ def __eq__(self, other: object) -> bool:
return False
-type StoreLike = Store | StorePath | FSMap | Path | UPath | str | dict[str, Buffer]
+type StoreLike = Store | StorePath | URLPipeline | FSMap | Path | UPath | str | dict[str, Buffer]
async def make_store(
@@ -371,9 +326,9 @@ async def make_store(
- `None` = `MemoryStore` object.
- `FSMap` = `FsspecStore` object.
- `UPath` = `FsspecStore` object, or `LocalStore` for a local `UPath`.
- - `str` containing `|` (or whose scheme has a registered root adapter) = the store
- resolved by the [URL pipeline][user-guide-url-pipelines]; the pipeline must address
- the store root (use `make_store_path` for pipelines that address a path within a store).
+ - `URLPipeline` = the store resolved by the [URL pipeline][user-guide-url-pipelines];
+ the pipeline must address the store root (use `make_store_path` for pipelines that
+ address a path within a store). Plain strings are never interpreted as pipelines.
Parameters
----------
@@ -404,12 +359,12 @@ async def make_store(
raise ValueError(f"Invalid mode: {mode}, expected one of {ANY_ACCESS_MODE}")
_read_only = mode == "r"
- if isinstance(store_like, str) and is_url_pipeline(store_like):
- result = await resolve_pipeline(store_like, mode=mode, storage_options=storage_options)
+ if isinstance(store_like, URLPipeline):
+ result = await store_like.resolve(mode=mode, storage_options=storage_options)
if normalize_path(result.path):
result.store.close()
raise URLPipelineError(
- f"the URL pipeline {store_like!r} resolves to a path inside a store; "
+ f"the URL pipeline {str(store_like)!r} resolves to a path inside a store; "
"use zarr.open() or make_store_path() instead of make_store()"
)
return result.store
@@ -521,8 +476,8 @@ async def make_store_path(
ValueError
If path is provided for a store that does not support it.
URLPipelineError
- If `store_like` is a [URL pipeline][user-guide-url-pipelines] string that
- cannot be resolved.
+ If `store_like` is a [URL pipeline][user-guide-url-pipelines] that cannot be
+ resolved.
See Also
--------
@@ -530,12 +485,10 @@ async def make_store_path(
"""
path_normalized = normalize_path(path)
- if isinstance(store_like, str) and is_url_pipeline(store_like):
- result = await resolve_pipeline(store_like, mode=mode, storage_options=storage_options)
+ if isinstance(store_like, URLPipeline):
+ result = await store_like.resolve(mode=mode, storage_options=storage_options)
combined_path = _join_paths([normalize_path(result.path), path_normalized])
- store_path = await StorePath.open(result.store, path=combined_path, mode=mode)
- store_path.zarr_format = result.zarr_format
- return store_path
+ return await StorePath.open(result.store, path=combined_path, mode=mode)
if isinstance(store_like, StorePath):
# Already a StorePath
@@ -556,6 +509,36 @@ async def make_store_path(
return await StorePath.open(store, path=path_normalized, mode=mode)
+@overload
+def resolve_zarr_format(store_like: StoreLike | None, zarr_format: ZarrFormat) -> ZarrFormat: ...
+
+
+@overload
+def resolve_zarr_format(
+ store_like: StoreLike | None, zarr_format: ZarrFormat | None
+) -> ZarrFormat | None: ...
+
+
+def resolve_zarr_format(
+ store_like: StoreLike | None, zarr_format: ZarrFormat | None
+) -> ZarrFormat | None:
+ """
+ Merge the zarr format selected by a `URLPipeline` store specification with
+ the format a caller asked for.
+
+ For any other `StoreLike`, `zarr_format` is returned unchanged. For a
+ pipeline, see [`URLPipeline.resolve_zarr_format`][zarr.storage.URLPipeline].
+
+ Raises
+ ------
+ ValueError
+ If `zarr_format` is not None and differs from the pipeline's format.
+ """
+ if isinstance(store_like, URLPipeline):
+ return store_like.resolve_zarr_format(zarr_format)
+ return zarr_format
+
+
async def ensure_no_existing_node(
store_path: StorePath,
zarr_format: ZarrFormat,
diff --git a/src/zarr/storage/_url_pipeline.py b/src/zarr/storage/_url_pipeline.py
index 41a2114daf..e53fc4f04d 100644
--- a/src/zarr/storage/_url_pipeline.py
+++ b/src/zarr/storage/_url_pipeline.py
@@ -5,18 +5,14 @@
(registered through the `zarr.url_adapters` entry-point group) are loaded
only when a pipeline URL naming their scheme is actually resolved.
-The `|` character is reserved as the pipeline delimiter in every string
-store specification: a string containing `|` is always routed through the
-pipeline machinery, and no percent-escape is decoded. To address a local
-file whose *name* contains `|` (or `#`), pass a `pathlib.Path` instead of a
-string.
-
-As a zarr-python extension to the specification (which requires the root
-sub-URL of an absolute pipeline to carry a scheme), the root sub-URL may be
-a schemeless local filesystem path, e.g. `data/example.zip|zip:`. Schemeless
-roots are treated as opaque text — no query or fragment splitting is applied
-to them. Such pipelines are not portable to other URL pipeline
-implementations; portable pipelines should spell the root as a `file:` URL.
+A string is never interpreted as a pipeline on its own: the user wraps it
+in a [`URLPipeline`][zarr.storage.URLPipeline] (or calls `zarr.open_url`),
+and that object is what the `StoreLike` machinery resolves. Plain string
+store specifications, including local paths that happen to contain `|`,
+keep their pre-existing meaning. No percent-escape is decoded.
+
+As the specification requires, the root sub-URL carries a URL scheme: a
+local path is spelled as an absolute `file:` URL.
"""
from __future__ import annotations
@@ -33,22 +29,25 @@
from zarr.errors import URLPipelineError
from zarr.registry import get_url_adapter, list_url_adapter_schemes
from zarr.storage._memory import ManagedMemoryStore
-from zarr.storage._utils import parse_store_url
+from zarr.storage._utils import _join_paths, normalize_path
if TYPE_CHECKING:
from zarr.abc.store import Store
- from zarr.core.common import AccessModeLiteral
+ from zarr.core.common import AccessModeLiteral, ZarrFormat
-__all__ = ["is_url_pipeline", "parse_pipeline", "resolve_pipeline"]
+__all__ = ["URLPipeline"]
+
+# Format segments are consumed by zarr-python itself rather than by a
+# registered adapter: they select the zarr format of the node the pipeline
+# addresses, and their body is a path to that node.
+_FORMAT_SCHEMES: dict[str, ZarrFormat] = {"zarr2": 2, "zarr3": 3}
# Adapter scheme per RFC 3986 plus "." to permit vendor-prefixed
# nonstandard schemes (e.g. "earthmover.myscheme").
_SCHEME_RE = re.compile(r"^[a-zA-Z][a-zA-Z0-9+.\-]*$")
-# Root scheme at the very start of the sub-URL. The negative lookahead
-# excludes fsspec's chained-URL syntax (``zip::file://...``), which is not a
-# URL pipeline and keeps flowing through the fsspec machinery.
-_ROOT_SCHEME_RE = re.compile(r"([a-zA-Z][a-zA-Z0-9+.\-]*):(?!:)")
+# Root scheme at the very start of the sub-URL.
+_ROOT_SCHEME_RE = re.compile(r"([a-zA-Z][a-zA-Z0-9+.\-]*):")
# Schemes zarr resolves natively at the pipeline root. These are never
# dispatched to a registered root adapter, so an installed package cannot
@@ -62,23 +61,17 @@
def _root_scheme(root_sub_url: str) -> str:
"""
- Detect the scheme of the root sub-URL, or `""` for an opaque root.
+ Detect the scheme of the root sub-URL, or `""` when it has none.
- Scheme extraction happens on the raw string (so nothing `urlparse`
- would strip or reject — whitespace, exotic authorities — can desync the
- detected scheme from the body). Single-letter candidates are delegated
- to `parse_store_url`, which knows Windows drive letters are not schemes.
+ Scheme extraction happens on the raw string, so nothing `urlparse`
+ would strip or reject can desync the detected scheme from the body. A
+ single letter followed by `:` is a scheme on every platform: a Windows
+ drive path is spelled `file:/C:/...` in a pipeline, never bare.
"""
match = _ROOT_SCHEME_RE.match(root_sub_url)
if match is None:
return ""
- scheme = match.group(1)
- if len(scheme) == 1:
- try:
- return parse_store_url(root_sub_url).scheme.lower()
- except ValueError:
- return scheme.lower()
- return scheme.lower()
+ return match.group(1).lower()
def _split_query(sub_url: str) -> tuple[str, str | None]:
@@ -96,16 +89,13 @@ def parse_pipeline(url: str) -> tuple[PipelineSegment, ...]:
"""
Parse a URL pipeline into its `|`-delimited segments.
- The first segment is the *root* sub-URL; its scheme is detected with the
- same rules as ordinary store URLs (Windows drive letters are not
- schemes). Subsequent segments are *adapter* sub-URLs of the form
+ The first segment is the *root* sub-URL. Subsequent segments are *adapter* sub-URLs of the form
`scheme:body` where the trailing colon is optional when the body is
empty (`zip` is equivalent to `zip:`).
- A schemeless root (a bare local path) is accepted as a zarr-python
- extension to the specification and treated as opaque text: no query or
- fragment splitting applies, since `?` and `#` are ordinary filename
- characters there. See the module docstring.
+ The root must carry a scheme; a bare local path is rejected, since the
+ specification has no schemeless sub-URLs and `|` could not be escaped
+ in one.
Segment text is preserved verbatim (no case or percent-encoding
normalization) except that schemes are lowercased.
@@ -119,9 +109,11 @@ def parse_pipeline(url: str) -> tuple[PipelineSegment, ...]:
if index == 0:
scheme = _root_scheme(part)
if not scheme:
- # opaque root (bare local path, fsspec chained URL, ...)
- segments.append(PipelineSegment(scheme="", body=part, query=None, raw=part))
- continue
+ raise URLPipelineError(
+ f"the root sub-URL {part!r} of the URL pipeline {url!r} has no URL scheme. "
+ "Spell a local path as an absolute 'file:' URL, and a literal '|' in "
+ "it as '%7C'."
+ )
body_and_scheme, query = _split_query(part)
body = body_and_scheme[len(scheme) + 1 :]
segments.append(PipelineSegment(scheme=scheme, body=body, query=query, raw=part))
@@ -137,73 +129,194 @@ def parse_pipeline(url: str) -> tuple[PipelineSegment, ...]:
return tuple(segments)
-def _root_routes_to_adapter(scheme: str) -> bool:
+@dataclasses.dataclass(frozen=True)
+class URLPipeline:
"""
- Whether a root sub-URL with this scheme is dispatched to a registered
- root adapter. Schemes zarr resolves natively (`file:`, `memory:`) and
- opaque roots are excluded; the registry check inspects entry-point
- names only — no adapter code is imported here.
- """
- return (
- bool(scheme) and scheme not in _NATIVE_ROOT_SCHEMES and scheme in list_url_adapter_schemes()
- )
-
+ A parsed URL pipeline, ready to be passed wherever a `StoreLike` is accepted.
+
+ Construct one with [`from_url`][zarr.storage.URLPipeline.from_url] and open it
+ through any `StoreLike`-taking function, or resolve it directly with
+ [`resolve`][zarr.storage.URLPipeline.resolve]. Because a
+ `URLPipeline` is an explicit object, no plain string store specification is
+ ever interpreted as a pipeline: `zarr.open("data|x.zarr")` still addresses a
+ local directory of that name, while
+ `zarr.open(URLPipeline.from_url("s3://bucket/data.zip|zip:|zarr3:"))` (or the
+ equivalent `zarr.open_url(...)`) resolves the pipeline through registered
+ adapters.
+
+ A pipeline is a value: instances are immutable, hashable, and compare equal
+ when they hold the same segments. Nothing is resolved until the pipeline is
+ opened; resolution needs the access mode and storage options of the open
+ call.
+
+ Attributes
+ ----------
+ segments : tuple[PipelineSegment, ...]
+ Every `|`-delimited sub-URL of the pipeline, in order: the root, any
+ adapter segments, and an optional trailing format segment.
-def is_url_pipeline(url: str) -> bool:
+ Raises
+ ------
+ URLPipelineError
+ If there are no segments, if a format segment (`zarr2:`/`zarr3:`)
+ appears anywhere but last, or if a format segment carries a query.
"""
- Whether `url` should be routed through the URL pipeline machinery.
- True when the URL contains a `|` separator, or when its scheme has a
- registered URL pipeline adapter (a *root adapter* such as `gh:`).
- fsspec chained URLs (`zip::file://...`) and zarr's native `file:` /
- `memory:` schemes are never routed to a root adapter.
- """
- if "|" in url:
- return True
- return _root_routes_to_adapter(_root_scheme(url))
+ segments: tuple[PipelineSegment, ...]
+
+ def __post_init__(self) -> None:
+ if not self.segments:
+ raise URLPipelineError("a URL pipeline needs at least one segment")
+ for segment in self.segments[1:-1]:
+ if segment.scheme in _FORMAT_SCHEMES:
+ raise URLPipelineError(
+ f"the format segment {segment.raw!r} must be the last segment "
+ f"of the URL pipeline {str(self)!r}"
+ )
+ format_segment = self._format_segment
+ if format_segment is not None and format_segment.query is not None:
+ raise URLPipelineError(
+ f"'{format_segment.scheme}:' segments do not accept a query: {format_segment.raw!r}"
+ )
+ @classmethod
+ def from_url(cls, url: str) -> URLPipeline:
+ """
+ Parse a URL pipeline string.
+
+ A URL without a `|` is a trivial pipeline consisting of its root alone.
+
+ Raises
+ ------
+ URLPipelineError
+ If the string cannot be parsed or the resulting pipeline is invalid
+ (see the class docstring).
+ """
+ return cls(parse_pipeline(url))
+
+ @property
+ def _format_segment(self) -> PipelineSegment | None:
+ last = self.segments[-1]
+ if len(self.segments) > 1 and last.scheme in _FORMAT_SCHEMES:
+ return last
+ return None
+
+ @property
+ def store_segments(self) -> tuple[PipelineSegment, ...]:
+ """
+ The segments that address a store: the root and any adapter segments,
+ i.e. `segments` without a trailing format segment.
+ """
+ if self._format_segment is None:
+ return self.segments
+ return self.segments[:-1]
+
+ @property
+ def zarr_format(self) -> ZarrFormat | None:
+ """The format selected by a trailing `zarr2:`/`zarr3:` segment, or None."""
+ format_segment = self._format_segment
+ if format_segment is None:
+ return None
+ return _FORMAT_SCHEMES[format_segment.scheme]
+
+ @property
+ def path(self) -> str:
+ """
+ The node path given as the body of a trailing format segment,
+ normalized. Empty when there is no format segment or its body is empty.
+ """
+ format_segment = self._format_segment
+ if format_segment is None:
+ return ""
+ return normalize_path(format_segment.body)
+
+ def resolve_zarr_format(self, zarr_format: ZarrFormat | None) -> ZarrFormat | None:
+ """
+ Combine the format selected by this pipeline with a caller-supplied one.
+
+ Parameters
+ ----------
+ zarr_format : ZarrFormat | None
+ The format requested by the caller, or None when unspecified.
+
+ Returns
+ -------
+ ZarrFormat | None
+ The pipeline's format when the caller did not specify one, otherwise
+ the caller's format. None when neither is set.
+
+ Raises
+ ------
+ ValueError
+ If the caller's format differs from the one selected by the pipeline.
+ """
+ pipeline_format = self.zarr_format
+ if pipeline_format is None:
+ return zarr_format
+ if zarr_format is not None and zarr_format != pipeline_format:
+ raise ValueError(
+ f"zarr_format={zarr_format} conflicts with the 'zarr{pipeline_format}:' "
+ f"segment of the URL pipeline {str(self)!r}; pass zarr_format=None to use "
+ "the pipeline's format"
+ )
+ return pipeline_format
+
+ async def resolve(
+ self,
+ *,
+ mode: AccessModeLiteral | None = None,
+ storage_options: dict[str, Any] | None = None,
+ ) -> AdapterResolution:
+ """
+ Resolve the pipeline into a store and a residual path.
+
+ The residual path joins the path returned by the adapters with the
+ node path of a trailing format segment (`...|zarr3:path/to/node`).
+ The format itself is not part of the resolution; read it from
+ `zarr_format`.
+
+ Parameters
+ ----------
+ mode : AccessModeLiteral | None
+ The caller's access mode. `"r"` requires a read-only store; the
+ resolver enforces this on whatever the final adapter returns.
+ storage_options : dict | None
+ Options forwarded to the root sub-URL's store (and visible to
+ adapters via the context).
+
+ Raises
+ ------
+ URLPipelineError
+ If a segment names a scheme with no registered adapter, names an
+ adapter entry point that fails to import, or the root sub-URL
+ cannot be resolved.
+ TypeError
+ If `storage_options` are passed to a root that does not accept
+ them (`file:` and `memory:`), as for non-pipeline stores.
+ OSError
+ Errors from opening the root resource (e.g. a missing local
+ directory in mode `"r"`) propagate unchanged, as for non-pipeline
+ stores.
+ """
+ resolution = await _resolve(self.store_segments, mode=mode, storage_options=storage_options)
+ if self.path:
+ resolution = dataclasses.replace(
+ resolution, path=_join_paths([normalize_path(resolution.path), self.path])
+ )
+ return resolution
-async def resolve_pipeline(
- url: str,
- *,
- mode: AccessModeLiteral | None = None,
- storage_options: dict[str, Any] | None = None,
-) -> AdapterResolution:
- """
- Resolve a URL pipeline into a store and a residual path.
+ def __str__(self) -> str:
+ return "|".join(segment.raw for segment in self.segments)
- Parameters
- ----------
- url : str
- A URL pipeline, e.g. `"s3://bucket/data.zip|zip:|zarr3:"`.
- mode : AccessModeLiteral | None
- The caller's access mode. `"r"` requires a read-only store; the
- resolver enforces this on whatever the final adapter returns.
- storage_options : dict | None
- Options forwarded to the root sub-URL's store (and visible to
- adapters via the context). Non-dict forms are reserved for future
- per-segment configuration (one mapping per pipeline segment).
- Raises
- ------
- URLPipelineError
- If the URL cannot be parsed, is not a pipeline, names a scheme with
- no registered adapter, names an adapter entry point that fails to
- import, or has a root sub-URL that cannot be resolved.
- TypeError
- If `storage_options` are passed to a root that does not accept them
- (local paths and `memory:`), as for non-pipeline stores.
- OSError
- Errors from opening the root resource (e.g. a missing local directory
- in mode `"r"`) propagate unchanged, as for non-pipeline stores.
+def _root_routes_to_adapter(scheme: str) -> bool:
"""
- segments = parse_pipeline(url)
- if len(segments) == 1 and not _root_routes_to_adapter(segments[0].scheme):
- raise URLPipelineError(
- f"{url!r} is not a URL pipeline: it has no '|' separator and no "
- f"URL pipeline adapter is registered for scheme {segments[0].scheme!r}"
- )
- return await _resolve(segments, mode=mode, storage_options=storage_options)
+ Whether a root sub-URL with this scheme is dispatched to a registered
+ root adapter. Schemes zarr resolves natively (`file:`, `memory:`) are
+ excluded; the registry check inspects entry-point names only — no
+ adapter code is imported here.
+ """
+ return scheme not in _NATIVE_ROOT_SCHEMES and scheme in list_url_adapter_schemes()
async def _resolve(
@@ -218,14 +331,7 @@ async def _resolve(
)
*preceding, last = segments
- try:
- adapter_cls = get_url_adapter(last.scheme)
- except URLPipelineError as exc:
- raise URLPipelineError(
- f"{exc} Note: '|' is reserved as the URL pipeline delimiter; to "
- "address a local file whose name contains '|', pass a "
- "pathlib.Path instead of a string."
- ) from None
+ adapter_cls = get_url_adapter(last.scheme)
context = PipelineContext(
preceding=tuple(preceding),
@@ -264,8 +370,8 @@ async def _resolve_root(
`memory:` and `file:` roots are resolved here with the URL pipeline
spec's semantics (spelling equivalences, mandatory absolute `file:`
- paths); everything else — bare local paths, fsspec URLs — delegates to
- the existing `StoreLike` machinery unchanged.
+ paths); every other scheme (e.g. fsspec URLs) delegates to the existing
+ `StoreLike` machinery unchanged.
"""
if segment.scheme == "memory":
return _resolve_memory_root(segment, mode=mode, storage_options=storage_options)
diff --git a/tests/test_api.py b/tests/test_api.py
index 6a2c4f85f8..f874bd45b7 100644
--- a/tests/test_api.py
+++ b/tests/test_api.py
@@ -311,12 +311,9 @@ async def test_open_like_creates_array_by_default(
assert np.all(Array(new_arr)[:] == ref_arr.fill_value)
-async def test_open_like_default_mode_on_read_only_store_raises_not_found(
+async def test_open_like_default_mode_rejects_read_only_store(
zarr_format: ZarrFormat,
) -> None:
- # mode "a" (open-or-create) on a read-only store serves the "open" half:
- # since the node does not exist and creating is impossible, the open
- # fails with a not-found error rather than rejecting the mode upfront.
ref_arr = zarr.create_array(
store={},
shape=(11, 12),
@@ -325,7 +322,7 @@ async def test_open_like_default_mode_on_read_only_store_raises_not_found(
zarr_format=zarr_format,
)
- with pytest.raises(ValueError, match="No array found"):
+ with pytest.raises(ValueError, match="Store is read-only but mode is 'a'"):
await zarr.api.asynchronous.open_like(
ref_arr,
path="foo",
diff --git a/tests/test_url_pipeline/test_parser.py b/tests/test_url_pipeline/test_parser.py
index 13d7846e85..bcc2dd3283 100644
--- a/tests/test_url_pipeline/test_parser.py
+++ b/tests/test_url_pipeline/test_parser.py
@@ -1,7 +1,5 @@
from __future__ import annotations
-import sys
-
import pytest
from hypothesis import given
from hypothesis import strategies as st
@@ -19,11 +17,11 @@ def test_single_root_url() -> None:
assert str(segment) == "s3://bucket/key"
-def test_schemeless_root() -> None:
- (segment,) = parse_pipeline("/local/path")
- assert segment.scheme == ""
- assert segment.body == "/local/path"
- assert segment.raw == "/local/path"
+@pytest.mark.parametrize("url", ["/local/path", "/local/path|zip:", "\tfile:/tmp/x|zip:"])
+def test_schemeless_root_rejected(url: str) -> None:
+ """The root sub-URL must carry a URL scheme; a bare path (or anything urlparse would not read as a scheme) is an error."""
+ with pytest.raises(URLPipelineError, match="URL scheme"):
+ parse_pipeline(url)
def test_adapter_chain() -> None:
@@ -74,16 +72,12 @@ def test_empty_query() -> None:
assert segment.query == ""
-def test_windows_drive_path_is_not_a_scheme() -> None:
- # On Windows parse_store_url treats C:\... as a local path; elsewhere the
- # single-letter scheme is preserved but must not crash the parser.
+def test_single_letter_scheme_is_a_scheme_everywhere() -> None:
+ """A drive-letter-like prefix is a URL scheme on every platform; bare Windows paths are not pipelines."""
(segment,) = parse_pipeline(r"C:\data\store")
+ assert segment.scheme == "c"
+ assert segment.body == r"\data\store"
assert segment.raw == r"C:\data\store"
- if sys.platform == "win32":
- assert segment.scheme == ""
- assert segment.body == r"C:\data\store"
- else:
- assert segment.scheme == "c"
@pytest.mark.parametrize("url", ["a||b:", "|zip:", "file:/tmp|", ""])
@@ -104,39 +98,12 @@ def test_invalid_adapter_scheme_rejected(segment: str) -> None:
def test_single_letter_scheme_with_exotic_authority() -> None:
- # single-letter candidates are delegated to parse_store_url, which may
- # reject an exotic authority; the raw scheme is used as the fallback
+ """Scheme detection works on the raw text, so an authority urlparse would reject is no obstacle."""
segments = parse_pipeline("x://[authority]/path|zip:")
- assert segments[0].scheme in ("", "x")
+ assert segments[0].scheme == "x"
assert segments[0].raw == "x://[authority]/path"
-def test_fsspec_chained_root_is_opaque() -> None:
- # fsspec's ``scheme::`` chaining is not pipeline syntax; such roots are
- # schemeless/opaque and flow through the ordinary store machinery
- (segment,) = parse_pipeline("zip::file:///tmp/data.zip")
- assert segment.scheme == ""
- assert segment.body == "zip::file:///tmp/data.zip"
-
-
-def test_whitespace_prefixed_root_is_opaque() -> None:
- # urlparse strips leading whitespace when sniffing a scheme; the parser
- # must not, or the detected scheme desyncs from the body slice
- segments = parse_pipeline("\tfile:/tmp/x|zip:")
- assert segments[0].scheme == ""
- assert segments[0].body == "\tfile:/tmp/x"
-
-
-def test_schemeless_root_keeps_query_and_fragment_chars() -> None:
- # ? and # are ordinary filename characters in a bare local path
- segments = parse_pipeline("/tmp/d?v=1|zip:")
- assert segments[0].scheme == ""
- assert segments[0].body == "/tmp/d?v=1"
- assert segments[0].query is None
- segments = parse_pipeline("/tmp/d#frag|zip:")
- assert segments[0].body == "/tmp/d#frag"
-
-
def test_round_trip() -> None:
url = "s3://bucket/a.zip?v=2|zip:b/inner.zip|zip:c|zarr3:"
segments = parse_pipeline(url)
@@ -160,9 +127,6 @@ def test_round_trip() -> None:
# the root scheme as a local path on one platform but not another
_ROOT_SCHEME = st.from_regex(r"[a-zA-Z]{2}[a-zA-Z0-9+.\-]{0,14}", fullmatch=True)
_BODY = st.text(st.characters(min_codepoint=32, max_codepoint=126, exclude_characters="|#?"))
-# a root body starting with ":" would spell fsspec's chained-URL syntax
-# ("scheme::..."), which the parser deliberately treats as an opaque root
-_ROOT_BODY = _BODY.filter(lambda body: not body.startswith(":"))
_QUERY = st.text(st.characters(min_codepoint=32, max_codepoint=126, exclude_characters="|#"))
@@ -176,7 +140,7 @@ def pipelines(
parts = []
for index in range(depth):
scheme = draw(_ROOT_SCHEME if index == 0 else _ADAPTER_SCHEME)
- body = draw(_ROOT_BODY if index == 0 else _BODY)
+ body = draw(_BODY)
query = draw(st.none() | _QUERY)
raw = f"{scheme}:{body}" + (f"?{query}" if query is not None else "")
parts.append(raw)
diff --git a/tests/test_url_pipeline/test_pipeline_object.py b/tests/test_url_pipeline/test_pipeline_object.py
new file mode 100644
index 0000000000..4c0d2779a5
--- /dev/null
+++ b/tests/test_url_pipeline/test_pipeline_object.py
@@ -0,0 +1,113 @@
+from __future__ import annotations
+
+import pytest
+
+from zarr.errors import URLPipelineError
+from zarr.storage import URLPipeline
+from zarr.storage._url_pipeline import parse_pipeline
+
+
+@pytest.mark.parametrize(
+ ("url", "schemes", "store_schemes", "zarr_format", "path"),
+ [
+ (
+ "s3://bucket/data.zip|zip:inner|zarr3:",
+ ("s3", "zip", "zarr3"),
+ ("s3", "zip"),
+ 3,
+ "",
+ ),
+ # the trailing colon of a format segment is optional
+ ("file:///data/node.zarr|zarr2", ("file", "zarr2"), ("file",), 2, ""),
+ # the body of a format segment is a node path, normalized
+ (
+ "file:///repo|icechunk:|zarr3:/path/to/array/",
+ ("file", "icechunk", "zarr3"),
+ ("file", "icechunk"),
+ 3,
+ "path/to/array",
+ ),
+ # no format segment
+ ("memory://x|wrap:a", ("memory", "wrap"), ("memory", "wrap"), None, ""),
+ # a single URL is a trivial pipeline
+ ("s3://bucket/key", ("s3",), ("s3",), None, ""),
+ ],
+)
+def test_from_url(
+ url: str,
+ schemes: tuple[str, ...],
+ store_schemes: tuple[str, ...],
+ zarr_format: int | None,
+ path: str,
+) -> None:
+ """
+ `URLPipeline.from_url` holds every parsed segment of the URL, `str()`
+ reproduces the URL exactly, and the zarr format, node path and
+ store-addressing segments are derived from a trailing `zarr2:`/`zarr3:`
+ segment.
+ """
+ pipeline = URLPipeline.from_url(url)
+ assert pipeline.segments == parse_pipeline(url)
+ assert tuple(segment.scheme for segment in pipeline.segments) == schemes
+ assert tuple(segment.scheme for segment in pipeline.store_segments) == store_schemes
+ assert pipeline.zarr_format == zarr_format
+ assert pipeline.path == path
+ assert str(pipeline) == url
+ assert URLPipeline(pipeline.segments) == pipeline
+
+
+def test_format_segment_must_be_last() -> None:
+ """A `zarr2:`/`zarr3:` segment anywhere but the end of the pipeline is rejected."""
+ with pytest.raises(URLPipelineError, match="must be the last segment"):
+ URLPipeline.from_url("file:/data|zarr3:|zip:")
+
+
+def test_format_segment_rejects_query() -> None:
+ """A `zarr2:`/`zarr3:` segment takes a node path only; a query is an error."""
+ with pytest.raises(URLPipelineError, match="do not accept a query"):
+ URLPipeline.from_url("file:/data|zarr3:x?opt=1")
+
+
+def test_direct_construction_is_validated() -> None:
+ """Validation lives in the constructor, so a hand-built pipeline is checked too."""
+ segments = parse_pipeline("file:/data|zarr3:|zip:")
+ with pytest.raises(URLPipelineError, match="must be the last segment"):
+ URLPipeline(segments)
+
+
+def test_empty_pipeline_rejected() -> None:
+ """A pipeline needs at least a root segment."""
+ with pytest.raises(URLPipelineError, match="at least one segment"):
+ URLPipeline(())
+
+
+def test_hashable() -> None:
+ """Pipelines are values: equal pipelines hash equal and can key a dict."""
+ a = URLPipeline.from_url("s3://b/d.zip|zip:|zarr3:")
+ b = URLPipeline.from_url("s3://b/d.zip|zip:|zarr3:")
+ assert a == b
+ assert hash(a) == hash(b)
+ assert {a: 1}[b] == 1
+
+
+@pytest.mark.parametrize(
+ ("pipeline_format", "requested", "expected"),
+ [(None, None, None), (None, 3, 3), (2, None, 2), (2, 2, 2)],
+)
+def test_resolve_zarr_format(
+ pipeline_format: int | None, requested: int | None, expected: int | None
+) -> None:
+ """
+ `resolve_zarr_format` returns the pipeline's format when the caller did not
+ specify one, and the caller's format otherwise.
+ """
+ suffix = "" if pipeline_format is None else f"|zarr{pipeline_format}:"
+ pipeline = URLPipeline.from_url(f"memory://x{suffix}")
+ assert pipeline.resolve_zarr_format(requested) == expected # type: ignore[arg-type]
+
+
+def test_resolve_zarr_format_conflict_raises() -> None:
+ """A caller-supplied format that differs from the pipeline's segment is an error."""
+ pipeline = URLPipeline.from_url("memory://x|zarr2:")
+ with pytest.raises(ValueError, match="conflicts with the 'zarr2:' segment"):
+ pipeline.resolve_zarr_format(3)
diff --git a/tests/test_url_pipeline/test_resolver.py b/tests/test_url_pipeline/test_resolver.py
index f9941b5101..ecfa37a37f 100644
--- a/tests/test_url_pipeline/test_resolver.py
+++ b/tests/test_url_pipeline/test_resolver.py
@@ -1,5 +1,6 @@
from __future__ import annotations
+import contextlib
import dataclasses
import sys
import threading
@@ -26,9 +27,8 @@
list_url_adapter_schemes,
register_url_adapter,
)
-from zarr.storage import ManagedMemoryStore, MemoryStore, WrapperStore
+from zarr.storage import LocalStore, ManagedMemoryStore, MemoryStore, URLPipeline, WrapperStore
from zarr.storage._common import _has_fsspec, make_store, make_store_path
-from zarr.storage._url_pipeline import is_url_pipeline, resolve_pipeline
from zarr.storage._utils import _join_paths
# fsspec < 2024.12.0 has no AsyncFileSystemWrapper, so a plain memory:// URL
@@ -112,17 +112,6 @@ async def open_pipeline_segment(
return AdapterResolution(store=await MemoryStore.open(read_only=False))
-class FormatAdapter(URLPipelineAdapter):
- """A dummy format-selecting adapter, like the builtin `zarr2:`."""
-
- @classmethod
- async def open_pipeline_segment(
- cls, segment: PipelineSegment, context: PipelineContext
- ) -> AdapterResolution:
- preceding = await context.resolve_preceding()
- return dataclasses.replace(preceding, zarr_format=2)
-
-
class TestRegistry:
def test_register_and_get(self) -> None:
register_url_adapter("demo", WrapperAdapter)
@@ -150,7 +139,9 @@ def test_entrypoint_discovery(self) -> None:
@pytest.mark.usefixtures("set_path")
async def test_entrypoint_end_to_end(self) -> None:
- result = await resolve_pipeline("memory://src|example-pkg.entrypoint-scheme:sub/path")
+ result = await URLPipeline.from_url(
+ "memory://src|example-pkg.entrypoint-scheme:sub/path"
+ ).resolve()
assert result.path == "sub/path"
@pytest.mark.usefixtures("set_path")
@@ -318,50 +309,55 @@ def test_duplicate_entry_points_warn_and_first_wins(
assert not any(e.name == "dup" for e in pending)
-class TestIsURLPipeline:
- def test_pipe_routes(self) -> None:
- assert is_url_pipeline("memory://x|demo:")
+class TestStringsAreNeverPipelines:
+ """
+ Only a `URLPipeline` object is resolved as a pipeline. A plain string
+ keeps the meaning it had before URL pipelines existed, whatever adapters
+ are registered.
+ """
- def test_registered_root_scheme_routes(self) -> None:
- register_url_adapter("rooty", RootAdapter)
- assert is_url_pipeline("rooty://org/repo")
+ async def test_local_path_containing_pipe_is_a_local_store(self, tmp_path: Path) -> None:
+ """A string path with `|` and a registered adapter's name is still a local directory."""
+ register_url_adapter("zip", WrapperAdapter)
+ store_path = await make_store_path(f"{tmp_path}/a|zip:x", mode="w")
+ assert isinstance(store_path.store, LocalStore)
+ assert store_path.store.root == tmp_path / "a|zip:x"
- @pytest.mark.parametrize("url", ["s3://bucket/key", "/local/path", "memory://x", "C:.zarr"])
- def test_plain_urls_do_not_route(self, url: str) -> None:
- assert not is_url_pipeline(url)
+ async def test_registered_root_scheme_string_is_not_routed(self) -> None:
+ """A URL whose scheme names a registered root adapter is not handed to that adapter."""
+ calls: list[PipelineSegment] = []
+
+ class CountingRootAdapter(RootAdapter):
+ @classmethod
+ async def open_pipeline_segment(
+ cls, segment: PipelineSegment, context: PipelineContext
+ ) -> AdapterResolution:
+ calls.append(segment)
+ return await super().open_pipeline_segment(segment, context)
- def test_fsspec_chained_urls_do_not_route(self) -> None:
- # fsspec's ``scheme::`` chaining is not a URL pipeline: even with a
- # same-named root adapter registered, these keep flowing to fsspec.
- register_url_adapter("zip", RootAdapter)
- assert not is_url_pipeline("zip::file:///tmp/data.zip")
- assert not is_url_pipeline("simplecache::s3://bucket/key")
+ register_url_adapter("rooty", CountingRootAdapter)
+ # the string falls through to the fsspec route, which may raise for
+ # an unknown protocol; whatever happens, the adapter is not consulted
+ with contextlib.suppress(Exception):
+ await make_store_path("rooty://org/repo")
+ assert calls == []
def test_native_schemes_do_not_route_to_adapters(self) -> None:
- # an installed package must not be able to intercept zarr's own
- # local-path and in-memory routing by registering file:/memory:
+ """An installed package cannot intercept `file:`/`memory:` roots by registering them."""
register_url_adapter("file", RootAdapter)
register_url_adapter("memory", RootAdapter)
- assert not is_url_pipeline("file:/tmp/x")
- assert not is_url_pipeline("memory://x")
-
- def test_exotic_authority_root_scheme(self) -> None:
- # The spec's own root-URL example: urlparse rejects the bracketed
- # non-IP authority, so scheme detection must work on the raw string
- # rather than raising before routing.
- url = "vendor1-2.custom-1+proto.ext://[authority]/path?query/part?x"
- assert not is_url_pipeline(url)
- register_url_adapter("vendor1-2.custom-1+proto.ext", RootAdapter)
- assert is_url_pipeline(url)
+ pipeline = URLPipeline.from_url("memory://x")
+ store = zarr.core.sync.sync(make_store(pipeline))
+ assert isinstance(store, ManagedMemoryStore)
class TestResolve:
async def test_wrapper_adapter_chain(self, tmp_path: Path) -> None:
register_url_adapter("wrap", WrapperAdapter)
- result = await resolve_pipeline(f"{tmp_path}|wrap:inner/path")
+ result = await URLPipeline.from_url(f"file:{tmp_path.as_posix()}|wrap:inner/path").resolve()
assert isinstance(result.store, TracingStore)
assert result.path == "inner/path"
- assert result.store.context.preceding_url == str(tmp_path)
+ assert result.store.context.preceding_url == f"file:{tmp_path.as_posix()}"
assert result.store.segment.scheme == "wrap"
assert result.store.segment.body == "inner/path"
@@ -369,25 +365,25 @@ async def test_nested_wrappers_preserve_residual_paths(self, tmp_path: Path) ->
# each wrapper joins the preceding residual path with its own, so
# no segment's path is lost in root|wrap:a|wrap:b
register_url_adapter("wrap", WrapperAdapter)
- result = await resolve_pipeline(f"{tmp_path}|wrap:a|wrap:b")
+ result = await URLPipeline.from_url(f"file:{tmp_path.as_posix()}|wrap:a|wrap:b").resolve()
assert result.path == "a/b"
async def test_native_adapter_gets_preceding_url(self) -> None:
register_url_adapter("native", NativeAdapter)
NativeAdapter.seen_urls.clear()
- await resolve_pipeline("s3://bucket/repo|native:")
+ await URLPipeline.from_url("s3://bucket/repo|native:").resolve()
assert NativeAdapter.seen_urls == ["s3://bucket/repo"]
async def test_multi_segment_preceding_url(self) -> None:
register_url_adapter("wrap", WrapperAdapter)
register_url_adapter("native", NativeAdapter)
NativeAdapter.seen_urls.clear()
- await resolve_pipeline("memory://base|wrap:a|native:x")
+ await URLPipeline.from_url("memory://base|wrap:a|native:x").resolve()
assert NativeAdapter.seen_urls == ["memory://base|wrap:a"]
async def test_root_adapter(self) -> None:
register_url_adapter("rooty", RootAdapter)
- result = await resolve_pipeline("rooty://org/repo")
+ result = await URLPipeline.from_url("rooty://org/repo").resolve()
assert result.path == "org/repo"
assert result.store.read_only
@@ -395,16 +391,16 @@ async def test_root_adapter_composes_with_chain(self) -> None:
register_url_adapter("rooty", RootAdapter)
register_url_adapter("native", NativeAdapter)
NativeAdapter.seen_urls.clear()
- await resolve_pipeline("rooty://org/repo|native:x")
+ await URLPipeline.from_url("rooty://org/repo|native:x").resolve()
assert NativeAdapter.seen_urls == ["rooty://org/repo"]
async def test_read_only_flag(self, tmp_path: Path) -> None:
register_url_adapter("wrap", WrapperAdapter)
- result = await resolve_pipeline(f"{tmp_path}|wrap:", mode="r")
+ result = await URLPipeline.from_url(f"file:{tmp_path.as_posix()}|wrap:").resolve(mode="r")
assert isinstance(result.store, TracingStore)
assert result.store.context.read_only
assert result.store.context.mode == "r"
- result = await resolve_pipeline(f"{tmp_path}|wrap:")
+ result = await URLPipeline.from_url(f"file:{tmp_path.as_posix()}|wrap:").resolve()
assert isinstance(result.store, TracingStore)
assert not result.store.context.read_only
assert result.store.context.mode is None
@@ -413,9 +409,9 @@ async def test_read_only_is_enforced_on_disobedient_adapters(self) -> None:
# the resolver downgrades a writable store returned under mode "r"
# instead of trusting the adapter to have honored context.read_only
register_url_adapter("bad", DisobedientAdapter)
- result = await resolve_pipeline("memory://base|bad:", mode="r")
+ result = await URLPipeline.from_url("memory://base|bad:").resolve(mode="r")
assert result.store.read_only
- store = await make_store("memory://base|bad:", mode="r")
+ store = await make_store(URLPipeline.from_url("memory://base|bad:"), mode="r")
assert store.read_only
async def test_read_only_enforcement_without_conversion_raises(self) -> None:
@@ -433,7 +429,7 @@ async def open_pipeline_segment(
register_url_adapter("stubborn", StubbornAdapter)
with pytest.raises(URLPipelineError, match="does not support read-only conversion"):
- await resolve_pipeline("memory://base|stubborn:", mode="r")
+ await URLPipeline.from_url("memory://base|stubborn:").resolve(mode="r")
async def test_read_only_enforcement_failure_closes_store(self) -> None:
# the adapter's store must not leak when it cannot be made read-only
@@ -456,7 +452,7 @@ async def open_pipeline_segment(
register_url_adapter("stubborn", StubbornAdapter)
with pytest.raises(URLPipelineError):
- await resolve_pipeline("memory://base|stubborn:", mode="r")
+ await URLPipeline.from_url("memory://base|stubborn:").resolve(mode="r")
assert closed == [True]
async def test_resolve_preceding_mode_override(self, tmp_path: Path) -> None:
@@ -472,7 +468,9 @@ async def open_pipeline_segment(
return dataclasses.replace(preceding, path=segment.body)
register_url_adapter("rowrap", ReadOnlyRootWrapper)
- result = await resolve_pipeline(f"{tmp_path}|rowrap:x", mode="w")
+ result = await URLPipeline.from_url(f"file:{tmp_path.as_posix()}|rowrap:x").resolve(
+ mode="w"
+ )
assert result.path == "x"
async def test_wrapper_on_local_file_root_read_only(self, tmp_path: Path) -> None:
@@ -482,7 +480,7 @@ async def test_wrapper_on_local_file_root_read_only(self, tmp_path: Path) -> Non
target = tmp_path / "data.bin"
target.write_bytes(b"payload")
register_url_adapter("wrap", WrapperAdapter)
- result = await resolve_pipeline(f"{target}|wrap:", mode="r")
+ result = await URLPipeline.from_url(f"file:{target.as_posix()}|wrap:").resolve(mode="r")
assert isinstance(result.store, TracingStore)
assert result.store.read_only
@@ -501,7 +499,7 @@ async def test_wrapper_on_local_file_root_writable_modes(
target = tmp_path / "data.bin"
target.write_bytes(b"payload")
register_url_adapter("wrap", WrapperAdapter)
- await resolve_pipeline(f"{target}|wrap:", mode=mode)
+ await URLPipeline.from_url(f"file:{target.as_posix()}|wrap:").resolve(mode=mode)
async def test_resolve_preceding_storage_options_override(self, tmp_path: Path) -> None:
# an adapter that consumed its namespaced keys strips them before
@@ -521,8 +519,8 @@ async def open_pipeline_segment(
register_url_adapter("consuming", ConsumingWrapper)
# the local-path root rejects any surviving storage_options, so this
# passing proves the adapter's keys were stripped before resolution
- result = await resolve_pipeline(
- f"{tmp_path}|consuming:x", storage_options={"consuming_secret": "s3cr3t"}
+ result = await URLPipeline.from_url(f"file:{tmp_path.as_posix()}|consuming:x").resolve(
+ storage_options={"consuming_secret": "s3cr3t"},
)
assert result.path == "x"
assert ConsumingWrapper.seen == "s3cr3t"
@@ -542,43 +540,44 @@ async def open_pipeline_segment(
register_url_adapter("probe", OptionsProbe)
opts = {"anon": True}
- await resolve_pipeline("s3://bucket/x|probe:", storage_options=opts)
+ await URLPipeline.from_url("s3://bucket/x|probe:").resolve(storage_options=opts)
assert OptionsProbe.seen_options == opts
async def test_wrapper_at_root_position_raises(self) -> None:
# a wrapper adapter used as the pipeline root has nothing to wrap
register_url_adapter("wrap", WrapperAdapter)
with pytest.raises(URLPipelineError, match="no preceding sub-URL"):
- await resolve_pipeline("wrap:whatever")
+ await URLPipeline.from_url("wrap:whatever").resolve()
+
+ async def test_single_url_is_a_trivial_pipeline(self) -> None:
+ """A URL without `|` resolves as a pipeline consisting of its root alone."""
+ result = await URLPipeline.from_url("memory://plain").resolve()
+ assert isinstance(result.store, ManagedMemoryStore)
+ assert result.path == ""
+
+ async def test_resolve_method(self, tmp_path: Path) -> None:
+ """`URLPipeline.resolve` yields the adapter's store and residual path."""
+ register_url_adapter("wrap", WrapperAdapter)
+ result = await URLPipeline.from_url(f"file:{tmp_path.as_posix()}|wrap:a").resolve()
+ assert isinstance(result.store, TracingStore)
+ assert result.path == "a"
- async def test_not_a_pipeline_raises(self) -> None:
- with pytest.raises(URLPipelineError, match="is not a URL pipeline"):
- await resolve_pipeline("s3://bucket/plain")
+ async def test_format_segment_path_joins_residual_path(self, tmp_path: Path) -> None:
+ """The node path of a trailing format segment extends the adapters' residual path."""
+ register_url_adapter("wrap", WrapperAdapter)
+ result = await URLPipeline.from_url(f"file:{tmp_path.as_posix()}|wrap:a|zarr3:b").resolve()
+ assert result.path == "a/b"
async def test_unknown_adapter_scheme_raises(self) -> None:
with pytest.raises(URLPipelineError, match="no URL pipeline adapter is registered"):
- await resolve_pipeline("memory://base|no-such-adapter:")
-
- async def test_unknown_adapter_error_hints_at_reserved_pipe(self, tmp_path: Path) -> None:
- # a filename containing '|' routes here; the error points at the fix
- with pytest.raises(URLPipelineError, match="pass a\\s+pathlib.Path"):
- await resolve_pipeline(f"{tmp_path}/a|b")
+ await URLPipeline.from_url("memory://base|no-such-adapter:").resolve()
async def test_base_case_value_error_is_wrapped(self) -> None:
# a root that only the fallback scheme detection accepts must not
# leak urlparse's ValueError out of the base case
register_url_adapter("wrap", WrapperAdapter)
with pytest.raises(URLPipelineError, match="could not resolve the pipeline root"):
- await resolve_pipeline("vendor.x://[authority]/path|wrap:")
-
- async def test_zarr_format_flows_from_adapter(self, tmp_path: Path) -> None:
- register_url_adapter("fmt2", FormatAdapter)
- result = await resolve_pipeline(f"{tmp_path}|fmt2:")
- assert result.zarr_format == 2
- # wrapper adapters carry the format through (dataclasses.replace)
- register_url_adapter("wrap", WrapperAdapter)
- result = await resolve_pipeline(f"{tmp_path}|fmt2:|wrap:")
- assert result.zarr_format == 2
+ await URLPipeline.from_url("vendor.x://[authority]/path|wrap:").resolve()
class TestSpecRoots:
@@ -587,7 +586,7 @@ class TestSpecRoots:
@pytest.mark.parametrize("root", ["memory:", "memory:/", "memory://"])
async def test_memory_root_spellings_equivalent(self, root: str) -> None:
register_url_adapter("wrap", WrapperAdapter)
- result = await resolve_pipeline(f"{root}|wrap:")
+ result = await URLPipeline.from_url(f"{root}|wrap:").resolve()
assert isinstance(result.store, TracingStore)
inner = result.store._store
assert isinstance(inner, ManagedMemoryStore)
@@ -597,7 +596,7 @@ async def test_memory_root_spellings_equivalent(self, root: str) -> None:
@pytest.mark.parametrize("root", ["memory:a/b", "memory:/a/b", "memory://a/b"])
async def test_memory_root_named_spellings_equivalent(self, root: str) -> None:
register_url_adapter("wrap", WrapperAdapter)
- result = await resolve_pipeline(f"{root}|wrap:")
+ result = await URLPipeline.from_url(f"{root}|wrap:").resolve()
assert isinstance(result.store, TracingStore)
inner = result.store._store
assert isinstance(inner, ManagedMemoryStore)
@@ -606,20 +605,20 @@ async def test_memory_root_named_spellings_equivalent(self, root: str) -> None:
async def test_memory_root_shares_data_across_spellings(self) -> None:
register_url_adapter("wrap", WrapperAdapter)
- group = zarr.open_group("memory:pipe-share|wrap:", mode="w")
+ group = zarr.open_group(URLPipeline.from_url("memory:pipe-share|wrap:"), mode="w")
group.create_array("x", shape=(2,), dtype="i4")
- reopened = zarr.open_group("memory://pipe-share|wrap:", mode="r")
+ reopened = zarr.open_group(URLPipeline.from_url("memory://pipe-share|wrap:"), mode="r")
assert "x" in reopened
async def test_memory_root_rejects_query(self) -> None:
register_url_adapter("wrap", WrapperAdapter)
with pytest.raises(URLPipelineError, match="do not accept a query"):
- await resolve_pipeline("memory:a?opt=1|wrap:")
+ await URLPipeline.from_url("memory:a?opt=1|wrap:").resolve()
async def test_memory_root_rejects_storage_options(self) -> None:
register_url_adapter("wrap", WrapperAdapter)
with pytest.raises(TypeError, match="'storage_options' was provided but unused"):
- await resolve_pipeline("memory:a|wrap:", storage_options={"anon": True})
+ await URLPipeline.from_url("memory:a|wrap:").resolve(storage_options={"anon": True})
async def test_file_root_localhost_authority(self, tmp_path: Path) -> None:
# file://localhost/p and file:///p are equivalent to file:/p
@@ -627,10 +626,10 @@ async def test_file_root_localhost_authority(self, tmp_path: Path) -> None:
# spell the path URL-style: on Windows C:/... gains a leading slash
posix = tmp_path.as_posix()
url_path = posix if posix.startswith("/") else f"/{posix}"
- zarr.open_group(f"file://localhost{url_path}|wrap:", mode="w")
- opened = zarr.open_group(f"file:{tmp_path}|wrap:", mode="r")
+ zarr.open_group(URLPipeline.from_url(f"file://localhost{url_path}|wrap:"), mode="w")
+ opened = zarr.open_group(URLPipeline.from_url(f"file:{tmp_path}|wrap:"), mode="r")
assert isinstance(opened, zarr.Group)
- zarr.open_group(f"file://{url_path}|wrap:", mode="r")
+ zarr.open_group(URLPipeline.from_url(f"file://{url_path}|wrap:"), mode="r")
async def test_file_root_windows_drive_spellings(
self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch
@@ -645,26 +644,26 @@ async def test_file_root_windows_drive_spellings(
# chdir into tmp so it is created there
monkeypatch.chdir(tmp_path)
drive_path = "C:/drive/data"
- r1 = await resolve_pipeline(f"file:{drive_path}|wrap:")
- r2 = await resolve_pipeline(f"file:///{drive_path}|wrap:")
+ r1 = await URLPipeline.from_url(f"file:{drive_path}|wrap:").resolve()
+ r2 = await URLPipeline.from_url(f"file:///{drive_path}|wrap:").resolve()
assert isinstance(r1.store, TracingStore)
assert isinstance(r2.store, TracingStore)
async def test_file_root_rejects_other_authority(self) -> None:
register_url_adapter("wrap", WrapperAdapter)
with pytest.raises(URLPipelineError, match="unsupported authority"):
- await resolve_pipeline("file://example.com/tmp/x|wrap:")
+ await URLPipeline.from_url("file://example.com/tmp/x|wrap:").resolve()
@pytest.mark.parametrize("root", ["file:relative/path", "file://localhost"])
async def test_file_root_rejects_relative_paths(self, root: str) -> None:
register_url_adapter("wrap", WrapperAdapter)
with pytest.raises(URLPipelineError, match="absolute path"):
- await resolve_pipeline(f"{root}|wrap:")
+ await URLPipeline.from_url(f"{root}|wrap:").resolve()
async def test_file_root_rejects_query(self, tmp_path: Path) -> None:
register_url_adapter("wrap", WrapperAdapter)
with pytest.raises(URLPipelineError, match="do not accept a query"):
- await resolve_pipeline(f"file:{tmp_path}?v=1|wrap:")
+ await URLPipeline.from_url(f"file:{tmp_path}?v=1|wrap:").resolve()
@pytest.mark.skipif(
not _has_async_fsspec_wrapper,
@@ -676,7 +675,7 @@ async def test_memory_root_is_distinct_from_fsspec_memory_url(self) -> None:
# is zarr's ManagedMemoryStore; the two do not share data
register_url_adapter("wrap", WrapperAdapter)
plain = await make_store("memory://distinct-store")
- piped = await make_store("memory://distinct-store|wrap:")
+ piped = await make_store(URLPipeline.from_url("memory://distinct-store|wrap:"))
assert not isinstance(plain, ManagedMemoryStore)
assert isinstance(piped, TracingStore)
assert isinstance(piped._store, ManagedMemoryStore)
@@ -685,21 +684,32 @@ async def test_file_root_does_not_percent_decode(self, tmp_path: Path) -> None:
# documented divergence from RFC 8089: consistent with LocalStore, no
# percent-escape is decoded, so %20 names a literal directory
register_url_adapter("wrap", WrapperAdapter)
- await resolve_pipeline(f"file:{tmp_path.as_posix()}/a%20b|wrap:")
+ await URLPipeline.from_url(f"file:{tmp_path.as_posix()}/a%20b|wrap:").resolve()
assert (tmp_path / "a%20b").is_dir()
assert not (tmp_path / "a b").exists()
class TestMakeStoreIntegration:
+ """`make_store` / `make_store_path` accept a `URLPipeline` like any other StoreLike."""
+
async def test_make_store_path_combines_paths(self, tmp_path: Path) -> None:
+ """Adapter residual path, format-segment node path and the caller's path join in order."""
register_url_adapter("wrap", WrapperAdapter)
- store_path = await make_store_path(f"{tmp_path}|wrap:residual", path="user/sub")
- assert store_path.path == "residual/user/sub"
+ pipeline = URLPipeline.from_url(f"file:{tmp_path.as_posix()}|wrap:residual|zarr3:node")
+ store_path = await make_store_path(pipeline, path="user/sub")
+ assert store_path.path == "residual/node/user/sub"
+ assert isinstance(store_path.store, TracingStore)
async def test_make_store_rejects_residual_path(self, tmp_path: Path) -> None:
register_url_adapter("wrap", WrapperAdapter)
with pytest.raises(URLPipelineError, match="resolves to a path inside a store"):
- await make_store(f"{tmp_path}|wrap:residual")
+ await make_store(URLPipeline.from_url(f"file:{tmp_path.as_posix()}|wrap:residual"))
+
+ async def test_make_store_rejects_format_segment_node_path(self, tmp_path: Path) -> None:
+ """A node path given on the format segment also addresses a path inside the store."""
+ register_url_adapter("wrap", WrapperAdapter)
+ with pytest.raises(URLPipelineError, match="resolves to a path inside a store"):
+ await make_store(URLPipeline.from_url(f"file:{tmp_path.as_posix()}|wrap:|zarr3:node"))
async def test_make_store_rejects_slash_root_path(self, tmp_path: Path) -> None:
# an adapter returning path="/" addresses the store root; the raw
@@ -713,7 +723,7 @@ async def open_pipeline_segment(
return dataclasses.replace(preceding, path="/")
register_url_adapter("slashy", SlashRootAdapter)
- store = await make_store(f"{tmp_path}|slashy:")
+ store = await make_store(URLPipeline.from_url(f"file:{tmp_path.as_posix()}|slashy:"))
assert store is not None
async def test_make_store_closes_store_on_residual_path_error(self) -> None:
@@ -733,12 +743,12 @@ async def open_pipeline_segment(
register_url_adapter("leaky", LeakProbe)
with pytest.raises(URLPipelineError, match="resolves to a path inside a store"):
- await make_store("memory://base|leaky:")
+ await make_store(URLPipeline.from_url("memory://base|leaky:"))
assert closed == [True]
async def test_make_store_no_residual_path(self, tmp_path: Path) -> None:
register_url_adapter("wrap", WrapperAdapter)
- store = await make_store(f"{tmp_path}|wrap:")
+ store = await make_store(URLPipeline.from_url(f"file:{tmp_path.as_posix()}|wrap:"))
assert isinstance(store, TracingStore)
async def test_make_store_rejects_invalid_mode_before_resolving(self, tmp_path: Path) -> None:
@@ -746,39 +756,36 @@ async def test_make_store_rejects_invalid_mode_before_resolving(self, tmp_path:
# rather than being handed to adapters
register_url_adapter("wrap", WrapperAdapter)
with pytest.raises(ValueError, match="Invalid mode"):
- await make_store(f"{tmp_path}|wrap:", mode="invalid") # type: ignore[arg-type]
+ await make_store(
+ URLPipeline.from_url(f"file:{tmp_path.as_posix()}|wrap:"),
+ mode="invalid", # type: ignore[arg-type]
+ )
async def test_zarr_open_end_to_end(self, tmp_path: Path) -> None:
register_url_adapter("wrap", WrapperAdapter)
- group = zarr.open_group(f"{tmp_path}|wrap:", mode="w")
+ group = zarr.open_group(URLPipeline.from_url(f"file:{tmp_path.as_posix()}|wrap:"), mode="w")
array = group.create_array("x", shape=(4,), dtype="i4")
array[:] = [1, 2, 3, 4]
- assert zarr.open_array(f"{tmp_path}|wrap:x")[2] == 3
+ assert zarr.open_array(URLPipeline.from_url(f"file:{tmp_path.as_posix()}|wrap:x"))[2] == 3
async def test_pipeline_store_read_only_mode(self) -> None:
register_url_adapter("native", NativeAdapter)
- store_path = await make_store_path("memory://base|native:", mode="r")
+ store_path = await make_store_path(URLPipeline.from_url("memory://base|native:"), mode="r")
assert store_path.read_only
- async def test_mode_a_downgrades_to_read_only_open(self) -> None:
- # mode "a" (the zarr.open default) is open-or-create: a pipeline
- # that resolves to a read-only store serves the "open" half instead
- # of failing outright.
+ async def test_mode_a_on_read_only_store_raises(self) -> None:
+ """Mode "a" on a read-only store raises, whether the store comes from a pipeline or not."""
register_url_adapter("rooty", RootAdapter)
- store_path = await make_store_path("rooty://org/repo", mode="a")
- assert store_path.read_only
-
- async def test_mode_a_downgrade_applies_to_plain_stores_too(self) -> None:
- # the open-or-create downgrade lives in StorePath.open, so it is
- # uniform across pipeline and non-pipeline stores
+ with pytest.raises(ValueError, match="Store is read-only but mode is 'a'"):
+ await make_store_path(URLPipeline.from_url("rooty://org/repo"), mode="a")
store = await MemoryStore.open(read_only=True)
- store_path = await make_store_path(store, mode="a")
- assert store_path.read_only
+ with pytest.raises(ValueError, match="Store is read-only but mode is 'a'"):
+ await make_store_path(store, mode="a")
async def test_explicit_write_mode_reaches_adapter(self) -> None:
register_url_adapter("rooty", RootAdapter)
with pytest.raises(ValueError, match="read-only scheme"):
- await make_store_path("rooty://org/repo", mode="w")
+ await make_store_path(URLPipeline.from_url("rooty://org/repo"), mode="w")
async def test_storage_options_forwarded_to_root(self) -> None:
# storage_options reach the root sub-URL via resolve_preceding ->
@@ -786,96 +793,170 @@ async def test_storage_options_forwarded_to_root(self) -> None:
# forwarding them must raise the same TypeError as a non-pipeline open.
register_url_adapter("wrap", WrapperAdapter)
with pytest.raises(TypeError, match="'storage_options' was provided but unused"):
- await make_store("/tmp/some/path|wrap:", storage_options={"anon": True})
+ await make_store(
+ URLPipeline.from_url("file:/tmp/some/path|wrap:"), storage_options={"anon": True}
+ )
- async def test_store_path_truediv_keeps_format(self, tmp_path: Path) -> None:
- # StorePath.__truediv__ constructs a new instance; the pipeline-
- # selected format must survive the division
- register_url_adapter("fmt2", FormatAdapter)
- store_path = await make_store_path(f"{tmp_path}|fmt2:")
- assert store_path.zarr_format == 2
- assert (store_path / "sub/node").zarr_format == 2
+ async def test_store_path_has_no_format(self, tmp_path: Path) -> None:
+ """The pipeline's format is consumed by the open call; it does not linger on the StorePath."""
+ store_path = await make_store_path(
+ URLPipeline.from_url(f"file:{tmp_path.as_posix()}|zarr2:")
+ )
+ assert not hasattr(store_path, "zarr_format")
async def test_zarr_format_merges_into_open(self, tmp_path: Path) -> None:
- register_url_adapter("fmt2", FormatAdapter)
- group = zarr.open_group(f"{tmp_path}|fmt2:", mode="w")
+ group = zarr.open_group(
+ URLPipeline.from_url(f"file:{tmp_path.as_posix()}|zarr2:"), mode="w"
+ )
assert group.metadata.zarr_format == 2
async def test_conflicting_explicit_format_raises(self, tmp_path: Path) -> None:
- register_url_adapter("fmt2", FormatAdapter)
with pytest.raises(ValueError, match="conflicts with"):
- zarr.open_group(f"{tmp_path}|fmt2:", mode="w", zarr_format=3)
+ zarr.open_group(
+ URLPipeline.from_url(f"file:{tmp_path.as_posix()}|zarr2:"), mode="w", zarr_format=3
+ )
async def test_matching_explicit_format_ok(self, tmp_path: Path) -> None:
- register_url_adapter("fmt2", FormatAdapter)
- group = zarr.open_group(f"{tmp_path}|fmt2:", mode="w", zarr_format=2)
+ group = zarr.open_group(
+ URLPipeline.from_url(f"file:{tmp_path.as_posix()}|zarr2:"), mode="w", zarr_format=2
+ )
+ assert group.metadata.zarr_format == 2
+
+
+class TestOpenURL:
+ """
+ `zarr.open_url` interprets its string argument as a URL pipeline and opens
+ the existing node it addresses. It never creates: creation goes through
+ `create_array` / `create_group` with a `URLPipeline`.
+ """
+
+ async def test_open_url_opens_existing_node_with_pipeline_format(self, tmp_path: Path) -> None:
+ register_url_adapter("wrap", WrapperAdapter)
+ zarr.create_group(URLPipeline.from_url(f"file:{tmp_path.as_posix()}|wrap:|zarr2:"))
+ group = zarr.open_url(f"file:{tmp_path.as_posix()}|wrap:|zarr2:")
+ assert isinstance(group, zarr.Group)
+ assert group.metadata.zarr_format == 2
+ assert isinstance(group.store, TracingStore)
+ assert group.read_only
+
+ async def test_open_url_format_segment_body_addresses_node(self, tmp_path: Path) -> None:
+ """The node path lives in the URL, as the body of the format segment."""
+ register_url_adapter("wrap", WrapperAdapter)
+ zarr.create_array(
+ URLPipeline.from_url(f"file:{tmp_path.as_posix()}|wrap:"),
+ name="sub/x",
+ shape=(3,),
+ dtype="i4",
+ )
+ array = zarr.open_url(f"file:{tmp_path.as_posix()}|wrap:|zarr3:sub/x")
+ assert isinstance(array, zarr.Array)
+ assert array.store_path.path == "sub/x"
+
+ async def test_open_url_read_write(self, tmp_path: Path) -> None:
+ zarr.create_group(URLPipeline.from_url(f"file:{tmp_path.as_posix()}|zarr3:"))
+ group = zarr.open_url(f"file:{tmp_path.as_posix()}|zarr3:", mode="r+")
+ assert not group.read_only
+ group.attrs["answer"] = 42
+ assert zarr.open_url(f"file:{tmp_path.as_posix()}|zarr3:").attrs["answer"] == 42
+
+ async def test_open_url_missing_node_raises(self, tmp_path: Path) -> None:
+ """Nothing is created on open: a URL to a missing node is an error in every mode."""
+ with pytest.raises(FileNotFoundError):
+ zarr.open_url(f"file:{tmp_path.as_posix()}/missing|zarr3:")
+ with pytest.raises(FileNotFoundError):
+ zarr.open_url(f"file:{tmp_path.as_posix()}/missing|zarr3:", mode="r+")
+
+ async def test_open_url_rejects_create_modes(self, tmp_path: Path) -> None:
+ """The creating modes of `zarr.open` are not accepted."""
+ with pytest.raises(ValueError, match="opens existing nodes only"):
+ zarr.open_url(f"file:{tmp_path.as_posix()}|zarr3:", mode="a") # type: ignore[arg-type]
+
+ async def test_async_open_url(self, tmp_path: Path) -> None:
+ zarr.create_group(URLPipeline.from_url(f"file:{tmp_path.as_posix()}|zarr2:"))
+ group = await zarr.api.asynchronous.open_url(f"file:{tmp_path.as_posix()}|zarr2:")
assert group.metadata.zarr_format == 2
class TestZarrFormatMergeInCore:
"""
- Every make_store_path caller honors the pipeline-selected zarr format, not
- only the top-level zarr.api functions.
+ Every function taking a StoreLike honors the format selected by a
+ `URLPipeline`, not only the top-level zarr.api functions. Functions whose
+ `zarr_format` defaults to 3 need an explicit `zarr_format=None` to defer to
+ the pipeline, and raise when the default conflicts with it.
"""
- @pytest.fixture(autouse=True)
- def _fmt2(self) -> None:
- register_url_adapter("fmt2", FormatAdapter)
+ @pytest.fixture
+ def v2_pipeline(self, tmp_path: Path) -> URLPipeline:
+ return URLPipeline.from_url(f"file:{tmp_path.as_posix()}|zarr2:")
- async def test_create_array(self, tmp_path: Path) -> None:
- arr = zarr.create_array(f"{tmp_path}|fmt2:", name="x", shape=(2,), dtype="i4")
+ async def test_create_array(self, v2_pipeline: URLPipeline) -> None:
+ arr = zarr.create_array(v2_pipeline, name="x", shape=(2,), dtype="i4", zarr_format=None)
assert arr.metadata.zarr_format == 2
with pytest.raises(ValueError, match="conflicts with"):
- zarr.create_array(f"{tmp_path}|fmt2:", name="y", shape=(2,), dtype="i4", zarr_format=3)
+ zarr.create_array(v2_pipeline, name="y", shape=(2,), dtype="i4")
- async def test_create_array_from_data(self, tmp_path: Path) -> None:
+ async def test_create_array_from_data(self, v2_pipeline: URLPipeline) -> None:
# the data path of create_array goes through from_array
- arr = zarr.create_array(f"{tmp_path}|fmt2:", name="x", data=np.arange(3))
+ arr = zarr.create_array(v2_pipeline, name="x", data=np.arange(3), zarr_format=None)
assert arr.metadata.zarr_format == 2
with pytest.raises(ValueError, match="conflicts with"):
- zarr.create_array(f"{tmp_path}|fmt2:", name="y", data=np.arange(3), zarr_format=3)
+ zarr.create_array(v2_pipeline, name="y", data=np.arange(3))
- async def test_from_array(self, tmp_path: Path) -> None:
- arr = zarr.from_array(f"{tmp_path}|fmt2:", name="x", data=np.arange(3))
+ async def test_create_array_default_without_pipeline(self, tmp_path: Path) -> None:
+ """The default format for plain stores is still 3, independent of the config default."""
+ with zarr.config.set({"default_zarr_format": 2}):
+ arr = zarr.create_array(tmp_path, shape=(2,), dtype="i4")
+ assert arr.metadata.zarr_format == 3
+
+ async def test_from_array(self, v2_pipeline: URLPipeline) -> None:
+ arr = zarr.from_array(v2_pipeline, name="x", data=np.arange(3))
assert arr.metadata.zarr_format == 2
- async def test_group_from_store(self, tmp_path: Path) -> None:
- group = zarr.Group.from_store(f"{tmp_path}|fmt2:")
+ async def test_group_from_store(self, v2_pipeline: URLPipeline, tmp_path: Path) -> None:
+ group = zarr.Group.from_store(v2_pipeline, zarr_format=None)
assert group.metadata.zarr_format == 2
with pytest.raises(ValueError, match="conflicts with"):
- zarr.Group.from_store(f"{tmp_path}/other|fmt2:", zarr_format=3)
+ zarr.Group.from_store(URLPipeline.from_url(f"file:{tmp_path.as_posix()}/other|zarr2:"))
async def test_group_from_store_default_without_pipeline(self, tmp_path: Path) -> None:
- # the default is still the configured format (3) for plain stores
- group = zarr.Group.from_store(tmp_path)
+ """The default format for plain stores is still 3, independent of the config default."""
+ with zarr.config.set({"default_zarr_format": 2}):
+ group = zarr.Group.from_store(tmp_path)
assert group.metadata.zarr_format == 3
- async def test_group_open(self, tmp_path: Path) -> None:
- zarr.Group.from_store(f"{tmp_path}|fmt2:")
- # explicit None defers to the pipeline; the default (3) conflicts
- group = zarr.Group.open(f"{tmp_path}|fmt2:", zarr_format=None)
+ async def test_group_open(self, v2_pipeline: URLPipeline) -> None:
+ zarr.Group.from_store(v2_pipeline, zarr_format=None)
+ group = zarr.Group.open(v2_pipeline, zarr_format=None)
assert group.metadata.zarr_format == 2
with pytest.raises(ValueError, match="conflicts with"):
- zarr.Group.open(f"{tmp_path}|fmt2:")
+ zarr.Group.open(v2_pipeline)
- async def test_array_open(self, tmp_path: Path) -> None:
- zarr.create_array(f"{tmp_path}|fmt2:", name="x", shape=(2,), dtype="i4")
- arr = zarr.Array.open(f"{tmp_path}/x|fmt2:", zarr_format=None)
+ async def test_array_open(self, v2_pipeline: URLPipeline, tmp_path: Path) -> None:
+ zarr.create_array(v2_pipeline, name="x", shape=(2,), dtype="i4", zarr_format=None)
+ array_pipeline = URLPipeline.from_url(f"file:{tmp_path.as_posix()}|zarr2:x")
+ arr = zarr.Array.open(array_pipeline, zarr_format=None)
assert arr.metadata.zarr_format == 2
with pytest.raises(ValueError, match="conflicts with"):
- zarr.Array.open(f"{tmp_path}/x|fmt2:")
+ zarr.Array.open(array_pipeline)
- async def test_open_like(self, tmp_path: Path) -> None:
+ async def test_open_like(self, v2_pipeline: URLPipeline) -> None:
# open_like inherits v2 filters/compressor from the reference; the
# pipeline supplies the format, which open_array then creates with
ref = zarr.create_array({}, shape=(2,), dtype="i4", zarr_format=2)
- arr = zarr.open_like(ref, store=f"{tmp_path}|fmt2:", path="x")
+ arr = zarr.open_like(ref, store=v2_pipeline, path="x")
assert arr.metadata.zarr_format == 2
- async def test_deprecated_async_array_create(self, tmp_path: Path) -> None:
+ async def test_deprecated_async_array_create(
+ self, v2_pipeline: URLPipeline, tmp_path: Path
+ ) -> None:
from zarr.core.array import AsyncArray
- arr = await AsyncArray._create(f"{tmp_path}|fmt2:", shape=(2,), dtype="i4", zarr_format=2)
+ arr = await AsyncArray._create(v2_pipeline, shape=(2,), dtype="i4", zarr_format=2)
assert arr.metadata.zarr_format == 2
with pytest.raises(ValueError, match="conflicts with"):
- await AsyncArray._create(f"{tmp_path}/o|fmt2:", shape=(2,), dtype="i4", zarr_format=3)
+ await AsyncArray._create(
+ URLPipeline.from_url(f"file:{tmp_path.as_posix()}/o|zarr2:"),
+ shape=(2,),
+ dtype="i4",
+ zarr_format=3,
+ )