feat: url-pipeline core — parser, adapter ABC, registry, store hooks

Implements URL pipeline support (https://github.com/jbms/url-pipeline):
'|'-chained URLs resolve through pluggable adapters registered under the
'zarr.url_adapters' entry-point group (entry-point name = URL scheme).

- zarr.abc.url_pipeline: PipelineSegment, AdapterResolution,
  PipelineContext, URLPipelineAdapter (single-classmethod contract)
- zarr.storage._url_pipeline: parse_pipeline / resolve_pipeline; the root
  sub-URL delegates to make_store so existing file/memory/fsspec routing
  is unchanged
- registry: register_url_adapter / get_url_adapter /
  list_url_adapter_schemes (name check only; no adapter imports)
- make_store/make_store_path route strings containing '|' (or a
  registered root scheme) through the resolver; residual store paths
  combine with the user-supplied path
- StorePath gains a zarr_format attribute (populated by format segments
  in a follow-up)

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

Author: jhamman

changes/2943.feature.md

@@ -0,0 +1,9 @@
+Added core support for URL pipelines (https://github.com/jbms/url-pipeline):
+`|`-chained URLs that address zarr data through nested storage layers, e.g.
+`s3://bucket/data.zip|zip:|zarr3:`. This PR adds the parser, the single-method
+`zarr.abc.url_pipeline.URLPipelineAdapter` interface, and the
+`zarr.url_adapters` entry-point group through which third-party packages
+(e.g. Icechunk) register adapters for their own schemes. Adapters for a scheme
+are loaded lazily and individually; URLs without a `|` separator (and without
+a registered root scheme) are handled exactly as before. Builtin adapters
+(`zip:`, `zarr2:`/`zarr3:`) follow in separate pull requests.

src/zarr/abc/url_pipeline.py

@@ -0,0 +1,174 @@
+"""
+Abstract base class and data model for URL pipeline adapters.
+
+A URL pipeline is a `|`-separated chain of sub-URLs, read outer-to-inner,
+as specified by https://github.com/jbms/url-pipeline. The first sub-URL (the
+*root*) locates a resource using a conventional URL, and each subsequent
+sub-URL names an *adapter* that reinterprets everything to its left:
+
+    s3://bucket/data.zip|zip:path/inside|zarr3:
+
+Third-party packages provide adapters by subclassing
+[`URLPipelineAdapter`][zarr.abc.url_pipeline.URLPipelineAdapter] and
+registering the class under the `zarr.url_adapters` entry-point group,
+using the URL scheme as the entry-point name.
+"""
+
+from __future__ import annotations
+
+from abc import ABC, abstractmethod
+from dataclasses import dataclass, field
+from typing import TYPE_CHECKING, Any
+
+if TYPE_CHECKING:
+    from collections.abc import Awaitable, Callable
+
+    from zarr.abc.store import Store
+    from zarr.core.common import AccessModeLiteral, ZarrFormat
+
+__all__ = [
+    "AdapterResolution",
+    "PipelineContext",
+    "PipelineSegment",
+    "URLPipelineAdapter",
+]
+
+
+@dataclass(frozen=True)
+class PipelineSegment:
+    """
+    One `|`-delimited sub-URL of a URL pipeline.
+
+    Attributes
+    ----------
+    scheme : str
+        The lowercased URL scheme. Empty string only for a schemeless root
+        (a bare local path).
+    body : str
+        The text after `scheme:` and before any `?`. Interpretation is
+        scheme-defined; it is **not** URL-normalized, so case-significant
+        content (e.g. icechunk snapshot IDs) is preserved.
+    query : str | None
+        The raw query string after `?`, or None. Interpretation is
+        scheme-defined.
+    raw : str
+        The exact original sub-URL text, preserved for lossless
+        reconstruction of the pipeline.
+    """
+
+    scheme: str
+    body: str
+    query: str | None
+    raw: str
+
+    def __str__(self) -> str:
+        return self.raw
+
+
+@dataclass(frozen=True)
+class AdapterResolution:
+    """
+    The result of resolving a URL pipeline (or a prefix of one).
+
+    Attributes
+    ----------
+    store : Store
+        The resolved store.
+    path : str
+        Residual path *within* the store that the pipeline addresses
+        (e.g. `"path/to/node"` for `...|icechunk://tag.v1/path/to/node`).
+        Empty string when the pipeline addresses the store root.
+    zarr_format : ZarrFormat | None
+        Zarr format selected by a format segment (`zarr2:`/`zarr3:`),
+        or None if unspecified.
+    """
+
+    store: Store
+    path: str = ""
+    zarr_format: ZarrFormat | None = None
+
+
+@dataclass(frozen=True)
+class PipelineContext:
+    """
+    Context handed to a [`URLPipelineAdapter`][zarr.abc.url_pipeline.URLPipelineAdapter]
+    describing the pipeline to the left of its segment.
+
+    Attributes
+    ----------
+    preceding : tuple[PipelineSegment, ...]
+        The parsed sub-URLs to the left of the adapter's segment, outer to
+        inner. Empty when the adapter's segment is the pipeline root.
+    mode : AccessModeLiteral | None
+        The access mode requested by the caller (e.g. `zarr.open(mode=...)`),
+        or None when unspecified. Adapters for read-only resources should
+        raise for explicit write modes (`"w"`, `"w-"`, `"a"`, `"r+"`)
+        and open read-only for `None` and `"r"`.
+    read_only : bool
+        True when `mode == "r"`. Adapters must construct their store
+        read-only when this is set.
+    storage_options : dict[str, Any] | None
+        Options passed by the caller. By convention these configure the
+        *root* sub-URL (e.g. fsspec options), but adapters may consume
+        adapter-specific keys.
+    """
+
+    preceding: tuple[PipelineSegment, ...]
+    mode: AccessModeLiteral | None
+    read_only: bool
+    storage_options: dict[str, Any] | None
+    _resolver: Callable[[tuple[PipelineSegment, ...]], Awaitable[AdapterResolution]] = field(
+        repr=False
+    )
+
+    @property
+    def preceding_url(self) -> str:
+        """The pipeline to the left of this segment, reconstructed exactly."""
+        return "|".join(segment.raw for segment in self.preceding)
+
+    async def resolve_preceding(self) -> AdapterResolution:
+        """
+        Resolve the preceding pipeline into a store.
+
+        This is the entry point for *wrapper* adapters (e.g. `zip:`) that
+        operate on the resource produced by the segments to their left.
+        Adapters backed by their own I/O machinery (e.g. `icechunk:`)
+        should use [`preceding_url`][zarr.abc.url_pipeline.PipelineContext.preceding_url]
+        instead and never materialize the intermediate store.
+        """
+        return await self._resolver(self.preceding)
+
+
+class URLPipelineAdapter(ABC):
+    """
+    Handler for one URL pipeline scheme.
+
+    Subclasses implement a single classmethod,
+    [`open_pipeline_segment`][zarr.abc.url_pipeline.URLPipelineAdapter.open_pipeline_segment],
+    and are registered under the `zarr.url_adapters` entry-point group with
+    the URL scheme as the entry-point name:
+
+        [project.entry-points."zarr.url_adapters"]
+        myscheme = "mypackage.zarr_adapter:MyAdapter"
+
+    An adapter is used in two positions:
+
+    - as an *adapter segment*: `s3://bucket/repo|icechunk://tag.v1` — the
+      context carries the preceding sub-URLs;
+    - as a *root scheme*: `gh://org/repo` — `context.preceding` is empty.
+    """
+
+    @classmethod
+    @abstractmethod
+    async def open_pipeline_segment(
+        cls, segment: PipelineSegment, context: PipelineContext
+    ) -> AdapterResolution:
+        """
+        Resolve `segment` (in the context of the pipeline to its left)
+        into a store, an optional residual path within that store, and an
+        optional zarr format.
+
+        The returned store must already be open and must honor
+        `context.read_only`.
+        """
+        ...

src/zarr/api/asynchronous.py

@@ -103,7 +103,7 @@
 
 def _infer_overwrite(mode: AccessModeLiteral) -> bool:
     """
-    Check that an ``AccessModeLiteral`` is compatible with overwriting an existing Zarr node.
+    Check that an `AccessModeLiteral` is compatible with overwriting an existing Zarr node.
     """
     return mode in _OVERWRITE_MODES
 
@@ -112,9 +112,9 @@ def _warn_unimplemented_kwargs(kwargs: dict[str, Any]) -> None:
     """
     Emit a "not yet implemented" warning for each provided keyword argument that is not None.
 
-    ``kwargs`` maps a keyword argument name to its supplied value. The ``stacklevel`` is chosen
+    `kwargs` maps a keyword argument name to its supplied value. The `stacklevel` is chosen
     so the warning points at the caller of the public API function (the same location as an
-    inline ``warnings.warn(..., stacklevel=2)`` would).
+    inline `warnings.warn(..., stacklevel=2)` would).
     """
     for name, value in kwargs.items():
         if value is not None:
@@ -194,7 +194,7 @@ async def consolidate_metadata(
 
     Upon completion, the metadata of the root node in the Zarr hierarchy will be
     updated to include all the metadata of child nodes. For Stores that do
-    not support consolidated metadata, this operation raises a ``TypeError``.
+    not support consolidated metadata, this operation raises a `TypeError`.
 
     Parameters
     ----------
@@ -215,10 +215,10 @@ async def consolidate_metadata(
     Returns
     -------
     group: AsyncGroup
-        The group, with the ``consolidated_metadata`` field set to include
+        The group, with the `consolidated_metadata` field set to include
         the metadata of each child node. If the Store doesn't support
         consolidated metadata, this function raises a `TypeError`.
-        See ``Store.supports_consolidated_metadata``.
+        See `Store.supports_consolidated_metadata`.
     """
     store_path = await make_store_path(store, path=path)
 
@@ -420,7 +420,7 @@ async def open_consolidated(
     *args: Any, use_consolidated: Literal[True] = True, **kwargs: Any
 ) -> AsyncGroup:
     """
-    Alias for [`open_group`][zarr.api.asynchronous.open_group] with ``use_consolidated=True``.
+    Alias for [`open_group`][zarr.api.asynchronous.open_group] with `use_consolidated=True`.
     """
     if use_consolidated is not True:
         raise TypeError(
@@ -484,7 +484,7 @@ async def save_array(
     arr : ndarray
         NumPy array with data to save.
     zarr_format : {2, 3, None}, optional
-        The zarr format to use when saving. The default is ``None``, which will
+        The zarr format to use when saving. The default is `None`, which will
         use the default Zarr format defined in the global configuration object.
     path : str or None, optional
         The path within the store where the array will be saved.
@@ -745,12 +745,12 @@ async def create_group(
     path : str, optional
         Group path within store.
     overwrite : bool, optional
-        If True, pre-existing data at ``path`` will be deleted before
+        If True, pre-existing data at `path` will be deleted before
         creating the group.
     zarr_format : {2, 3, None}, optional
         The zarr format to use when saving.
-        If no ``zarr_format`` is provided, the default format will be used.
-        This default can be changed by modifying the value of ``default_zarr_format``
+        If no `zarr_format` is provided, the default format will be used.
+        This default can be changed by modifying the value of `default_zarr_format`
         in [`zarr.config`][zarr.config].
     storage_options : dict
         If using an fsspec URL to create the store, these will be passed to
@@ -828,17 +828,17 @@ async def open_group(
         Whether to use consolidated metadata.
 
         By default, consolidated metadata is used if it's present in the
-        store (in the ``zarr.json`` for Zarr format 3 and in the ``.zmetadata`` file
+        store (in the `zarr.json` for Zarr format 3 and in the `.zmetadata` file
         for Zarr format 2).
 
-        To explicitly require consolidated metadata, set ``use_consolidated=True``,
+        To explicitly require consolidated metadata, set `use_consolidated=True`,
         which will raise an exception if consolidated metadata is not found.
 
-        To explicitly *not* use consolidated metadata, set ``use_consolidated=False``,
+        To explicitly *not* use consolidated metadata, set `use_consolidated=False`,
         which will fall back to using the regular, non consolidated metadata.
 
         Zarr format 2 allowed configuring the key storing the consolidated metadata
-        (``.zmetadata`` by default). Specify the custom key as ``use_consolidated``
+        (`.zmetadata` by default). Specify the custom key as `use_consolidated`
         to load consolidated metadata from a non-default key.
 
     Returns
@@ -924,40 +924,40 @@ async def create(
     shape : int or tuple of ints
         Array shape.
     chunks : int or tuple of ints, optional
-        Chunk shape. If True, will be guessed from ``shape`` and ``dtype``. If
-        False, will be set to ``shape``, i.e., single chunk for the whole array.
+        Chunk shape. If True, will be guessed from `shape` and `dtype`. If
+        False, will be set to `shape`, i.e., single chunk for the whole array.
         If an int, the chunk size in each dimension will be given by the value
-        of ``chunks``. Default is True.
+        of `chunks`. Default is True.
     dtype : str or dtype, optional
         NumPy dtype.
     compressor : Codec, optional
         Primary compressor to compress chunk data.
-        Zarr format 2 only. Zarr format 3 arrays should use ``codecs`` instead.
+        Zarr format 2 only. Zarr format 3 arrays should use `codecs` instead.
 
-        If neither ``compressor`` nor ``filters`` are provided, the default compressor
+        If neither `compressor` nor `filters` are provided, the default compressor
         [`zarr.codecs.ZstdCodec`][] is used.
 
-        If ``compressor`` is set to ``None``, no compression is used.
+        If `compressor` is set to `None`, no compression is used.
     fill_value : Any, optional
         Fill value for the array.
     order : {'C', 'F'}, optional
-        Deprecated in favor of the ``config`` keyword argument.
-        Pass ``{'order': <value>}`` to ``create`` instead of using this parameter.
+        Deprecated in favor of the `config` keyword argument.
+        Pass `{'order': <value>}` to `create` instead of using this parameter.
         Memory layout to be used within each chunk.
-        If not specified, the ``array.order`` parameter in the global config will be used.
+        If not specified, the `array.order` parameter in the global config will be used.
     store : StoreLike or None, default=None
         StoreLike object to open. See the
         [storage documentation in the user guide][user-guide-store-like]
         for a description of all valid StoreLike values.
     synchronizer : object, optional
         Array synchronizer.
     overwrite : bool, optional
-        If True, delete all pre-existing data in ``store`` at ``path`` before
+        If True, delete all pre-existing data in `store` at `path` before
         creating the array.
     path : str, optional
         Path under which array is stored.
     chunk_store : StoreLike or None, default=None
-        Separate storage for chunks. If not provided, ``store`` will be used
+        Separate storage for chunks. If not provided, `store` will be used
         for storage of both chunks and metadata.
     filters : Iterable[Codec] | Literal["auto"], optional
         Iterable of filters to apply to each chunk of the array, in order, before serializing that
@@ -970,14 +970,14 @@ async def create(
         For Zarr format 2, a "filter" can be any numcodecs codec; you should ensure that the
         order of your filters is consistent with the behavior of each filter.
 
-        The default value of ``"auto"`` instructs Zarr to use a default based on the data
+        The default value of `"auto"` instructs Zarr to use a default based on the data
         type of the array and the Zarr format specified. For all data types in Zarr V3, and most
         data types in Zarr V2, the default filters are empty. The only cases where default filters
         are not empty is when the Zarr format is 2, and the data type is a variable-length data type like
         [`zarr.dtype.VariableLengthUTF8`][] or [`zarr.dtype.VariableLengthUTF8`][]. In these cases,
         the default filters contains a single element which is a codec specific to that particular data type.
 
-        To create an array with no filters, provide an empty iterable or the value ``None``.
+        To create an array with no filters, provide an empty iterable or the value `None`.
     cache_metadata : bool, optional
         If True, array configuration metadata will be cached for the
         lifetime of the object. If False, array metadata will be reloaded
@@ -993,17 +993,17 @@ async def create(
         A codec to encode object arrays, only needed if dtype=object.
     dimension_separator : {'.', '/'}, optional
         Separator placed between the dimensions of a chunk.
-        Zarr format 2 only. Zarr format 3 arrays should use ``chunk_key_encoding`` instead.
+        Zarr format 2 only. Zarr format 3 arrays should use `chunk_key_encoding` instead.
     write_empty_chunks : bool, optional
-        Deprecated in favor of the ``config`` keyword argument.
-        Pass ``{'write_empty_chunks': <value>}`` to ``create`` instead of using this parameter.
+        Deprecated in favor of the `config` keyword argument.
+        Pass `{'write_empty_chunks': <value>}` to `create` instead of using this parameter.
         If True, all chunks will be stored regardless of their
         contents. If False, each chunk is compared to the array's fill value
         prior to storing. If a chunk is uniformly equal to the fill value, then
         that chunk is not be stored, and the store entry for that chunk's key
         is deleted.
     zarr_format : {2, 3, None}, optional
-        The Zarr format to use when creating an array. The default is ``None``,
+        The Zarr format to use when creating an array. The default is `None`,
         which instructs Zarr to choose the default Zarr format value defined in the
         runtime configuration.
     meta_array : array-like, optional
@@ -1016,15 +1016,15 @@ async def create(
     chunk_key_encoding : ChunkKeyEncoding, optional
         A specification of how the chunk keys are represented in storage.
         Zarr format 3 only. Zarr format 2 arrays should use `dimension_separator` instead.
-        Default is ``("default", "/")``.
+        Default is `("default", "/")`.
     codecs : Sequence of Codecs or dicts, optional
         An iterable of Codec or dict serializations of Codecs. Zarr V3 only.
 
-        The elements of ``codecs`` specify the transformation from array values to stored bytes.
-        Zarr format 3 only. Zarr format 2 arrays should use ``filters`` and ``compressor`` instead.
+        The elements of `codecs` specify the transformation from array values to stored bytes.
+        Zarr format 3 only. Zarr format 2 arrays should use `filters` and `compressor` instead.
 
         If no codecs are provided, default codecs will be used based on the data type of the array.
-        For most data types, the default codecs are the tuple ``(BytesCodec(), ZstdCodec())``;
+        For most data types, the default codecs are the tuple `(BytesCodec(), ZstdCodec())`;
         data types that require a special [`zarr.abc.codec.ArrayBytesCodec`][], like variable-length strings or bytes,
         will use the [`zarr.abc.codec.ArrayBytesCodec`][] required for the data type instead of [`zarr.codecs.BytesCodec`][].
     dimension_names : Iterable[str | None] | None = None