From a74a6cd11f3dd3444821b48bce2f45a199eb4224 Mon Sep 17 00:00:00 2001 From: Sun Dapeng Date: Mon, 5 Oct 2026 13:12:22 +0800 Subject: [PATCH 01/27] [python] Route reads to io-cache targets per file type When a REST table token vends io-cache endpoints, RESTTokenFileIO now wraps the OSS FileIO so that immutable files are read through a cache target chosen by file type, and every other request stays on the OSS endpoint. The rules are the same as in the Java Jindo FileIO. - io-cache.endpoint names one target. io-cache.targets with io-cache.target..endpoint (and optional .region and .path-style-access) names several, and io-cache.routes maps file types to them. io-cache.whitelist and io-cache.policy keep their meaning, and io-cache.origin.endpoint names the OSS endpoint when fs.oss.endpoint points at a cache for older clients. io-cache.enabled set in the catalog options overrides the token. - Only read and meta requests use a target; exists counts as meta. Writes, listings, deletes, renames, copies and non-oss paths stay on OSS, and a client-side dlf.oss-endpoint turns routing off. - Files rewritten in place (FileType.is_mutable) and metadata named by a sequence id never use a target. - Data files are recognized by name: a data or changelog file prefix, including a custom one, followed by {uuid}-{count}.{extension}. Manifests, indexes and statistics share the {uuid}-{count} shape but have no extension, so a custom prefix such as manifest- cannot mix them up; the manifest sidecar is excluded explicitly. Other names keep their FileType, and names Paimon does not write stay on OSS. - Errors from a target are raised as they are, without retrying on OSS. - PyArrow, Lance and Vortex readers use the target's endpoint, scheme and addressing style (fs.oss.second.level.domain.enable for path-style). The routing cases in tests/resources/io_cache are shared with the Java Jindo FileIO and paimon-rust. --- .../catalog/rest/rest_token_file_io.py | 30 +- .../pypaimon/common/options/config.py | 7 + .../pypaimon/filesystem/io_cache_file_io.py | 275 ++++++++ .../pypaimon/filesystem/io_cache_routing.py | 226 +++++++ .../pypaimon/filesystem/pyarrow_file_io.py | 18 +- .../read/reader/format_lance_reader.py | 2 +- .../read/reader/format_vortex_reader.py | 2 +- .../pypaimon/read/reader/lance_utils.py | 17 +- .../pypaimon/read/reader/vortex_utils.py | 17 +- paimon-python/pypaimon/tests/file_io_test.py | 40 ++ .../pypaimon/tests/io_cache_file_io_test.py | 600 ++++++++++++++++++ .../pypaimon/tests/io_cache_routing_test.py | 245 +++++++ .../pypaimon/tests/lance_utils_test.py | 28 + .../resources/io_cache/routable-types.json | 69 ++ .../tests/resources/io_cache/routing.json | 90 +++ .../tests/rest/rest_token_file_io_test.py | 108 ++++ 16 files changed, 1752 insertions(+), 22 deletions(-) create mode 100644 paimon-python/pypaimon/filesystem/io_cache_file_io.py create mode 100644 paimon-python/pypaimon/filesystem/io_cache_routing.py create mode 100644 paimon-python/pypaimon/tests/io_cache_file_io_test.py create mode 100644 paimon-python/pypaimon/tests/io_cache_routing_test.py create mode 100644 paimon-python/pypaimon/tests/resources/io_cache/routable-types.json create mode 100644 paimon-python/pypaimon/tests/resources/io_cache/routing.json diff --git a/paimon-python/pypaimon/catalog/rest/rest_token_file_io.py b/paimon-python/pypaimon/catalog/rest/rest_token_file_io.py index 293171285181..1057f88b6c13 100644 --- a/paimon-python/pypaimon/catalog/rest/rest_token_file_io.py +++ b/paimon-python/pypaimon/catalog/rest/rest_token_file_io.py @@ -20,6 +20,7 @@ import time from datetime import timedelta from typing import Optional, Union +from urllib.parse import urlparse from cachetools import TTLCache @@ -33,6 +34,8 @@ from pypaimon.common.options import Options from pypaimon.common.options.config import CatalogOptions, OssOptions from pypaimon.common.uri_reader import UriReaderFactory +from pypaimon.filesystem.io_cache_file_io import IoCacheRoutingFileIO +from pypaimon.filesystem.io_cache_routing import IoCacheRouting class RESTTokenFileIO(FileIO): @@ -111,20 +114,27 @@ def _file_io_with_token(self, minimum_validity_millis: int = 0): if file_io is not None: return file_io, cache_key - merged_properties = RESTUtil.merge( + merged_properties = self._merge_token_with_catalog_options(RESTUtil.merge( self.catalog_options.to_map() if self.catalog_options else {}, cache_key.token - ) - if self.catalog_options: - dlf_oss_endpoint = self.catalog_options.get(CatalogOptions.DLF_OSS_ENDPOINT) - if dlf_oss_endpoint and dlf_oss_endpoint.strip(): - merged_properties[OssOptions.OSS_ENDPOINT.key()] = dlf_oss_endpoint - merged_options = Options(merged_properties) + )) - file_io = FileIO.get(self.path, merged_options) + file_io = self._create_file_io(merged_properties) cache[cache_key] = file_io return file_io, cache_key + def _create_file_io(self, properties: dict) -> FileIO: + routing = IoCacheRouting.create(properties) + if routing is None or urlparse(self.path).scheme != "oss": + return FileIO.get(self.path, Options(properties)) + self.log.info( + "io-cache routing enabled for [%s], targets %s, origin endpoint %s", + self.identifier, routing.targets(), routing.origin_endpoint()) + origin = FileIO.get(self.path, Options(routing.origin_options(properties))) + return IoCacheRoutingFileIO( + routing, origin, + lambda name: FileIO.get(self.path, Options(routing.target_options(properties, name)))) + def _merge_token_with_catalog_options(self, token: dict) -> dict: """Merge token with catalog options, DLF OSS endpoint should override the standard OSS endpoint.""" merged_token = dict(token) @@ -132,6 +142,10 @@ def _merge_token_with_catalog_options(self, token: dict) -> dict: dlf_oss_endpoint = self.catalog_options.get(CatalogOptions.DLF_OSS_ENDPOINT) if dlf_oss_endpoint and dlf_oss_endpoint.strip(): merged_token[OssOptions.OSS_ENDPOINT.key()] = dlf_oss_endpoint + # An explicit io-cache.enabled in the catalog options wins over the token, as in Java. + if self.catalog_options.contains(CatalogOptions.IO_CACHE_ENABLED): + merged_token[CatalogOptions.IO_CACHE_ENABLED.key()] = str( + self.catalog_options.get(CatalogOptions.IO_CACHE_ENABLED)).lower() return merged_token def new_input_stream(self, path: str): diff --git a/paimon-python/pypaimon/common/options/config.py b/paimon-python/pypaimon/common/options/config.py index e67030957954..ce9b3c570785 100644 --- a/paimon-python/pypaimon/common/options/config.py +++ b/paimon-python/pypaimon/common/options/config.py @@ -31,6 +31,10 @@ class OssOptions: OSS_ENDPOINT = ConfigOptions.key("fs.oss.endpoint").string_type().no_default_value().with_description( "OSS endpoint") OSS_REGION = ConfigOptions.key("fs.oss.region").string_type().no_default_value().with_description("OSS region") + OSS_SECOND_LEVEL_DOMAIN_ENABLE = ConfigOptions.key( + "fs.oss.second.level.domain.enable").boolean_type().default_value(False).with_description( + "Path-style addressing (endpoint/bucket/key) instead of bucket.endpoint, " + "for IP or non-wildcard endpoints") OSS_SSE_METHOD = ConfigOptions.key( "fs.oss.server-side-encryption").string_type().no_default_value().with_description( "OSS atomic metadata encryption method: AES256, KMS or SM4") @@ -124,6 +128,9 @@ class CatalogOptions: "DLF token file path") DLF_OSS_ENDPOINT = ConfigOptions.key("dlf.oss-endpoint").string_type().no_default_value().with_description( "DLF OSS endpoint") + IO_CACHE_ENABLED = ConfigOptions.key("io-cache.enabled").boolean_type().default_value(False).with_description( + "Route reads to the io-cache targets vended with the table token; " + "a value set in the catalog options overrides the token") DLF_TOKEN_LOADER = ConfigOptions.key("dlf.token-loader").string_type().no_default_value().with_description( "DLF token loader") DLF_TOKEN_ECS_ROLE_NAME = ConfigOptions.key( diff --git a/paimon-python/pypaimon/filesystem/io_cache_file_io.py b/paimon-python/pypaimon/filesystem/io_cache_file_io.py new file mode 100644 index 000000000000..e9c6dbe91718 --- /dev/null +++ b/paimon-python/pypaimon/filesystem/io_cache_file_io.py @@ -0,0 +1,275 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""FileIO that sends each request to an io-cache target or the origin endpoint.""" + +import threading +from typing import Callable, Dict, List, Optional + +import pyarrow.fs as pafs +from pyarrow._fs import FileSystemHandler + +from pypaimon.common.file_io import FileIO +from pypaimon.filesystem.io_cache_routing import IoCacheRouting, Op + + +class IoCacheRoutingFileIO(FileIO): + """Sends each call to origin or to a target FileIO, all built for the table path. + Calls and attributes that are not routed go to origin.""" + + def __init__(self, routing: IoCacheRouting, origin: FileIO, + create_target: Callable[[str], FileIO]): + self._routing = routing + self._origin = origin + self._create_target = create_target + self._targets: Dict[str, FileIO] = {} + self._targets_lock = threading.Lock() + self._filesystem = pafs.PyFileSystem(_IoCacheRoutingHandler(self)) + + @property + def properties(self): + return self._origin.properties + + @property + def filesystem(self): + """Filesystem for readers; it takes the URIs returned by to_filesystem_path.""" + return self._filesystem + + def _route(self, op: Op, path: str) -> Optional[str]: + return self._routing.route(op, path) + + def _file_io(self, name: Optional[str]) -> FileIO: + """Origin, or the FileIO of a target, created on first use.""" + if name is None: + return self._origin + file_io = self._targets.get(name) + if file_io is not None: + return file_io + with self._targets_lock: + file_io = self._targets.get(name) + if file_io is None: + file_io = self._create_target(name) + self._targets[name] = file_io + return file_io + + def _call(self, op: Op, path: str, action): + return action(self._file_io(self._route(op, path))) + + def read_file_io(self, path: str) -> FileIO: + """FileIO for native readers that open files using their own storage client.""" + return self._file_io(self._route(Op.READ, path)) + + def _call_batch(self, op: Op, paths: List[str], action) -> list: + """Runs action(file_io, paths) once per routed group; results follow the order of paths.""" + results = [None] * len(paths) + groups: Dict[Optional[str], List[int]] = {} + for index, path in enumerate(paths): + groups.setdefault(self._route(op, path), []).append(index) + for name, indices in groups.items(): + group = [paths[index] for index in indices] + values = action(self._file_io(name), group) + for index, value in zip(indices, values): + results[index] = value + return results + + def new_input_stream(self, path: str): + return self._call(Op.READ, path, lambda io: io.new_input_stream(path)) + + def get_file_status(self, path: str): + return self._call(Op.META, path, lambda io: io.get_file_status(path)) + + def get_file_size(self, path: str) -> int: + return self._call(Op.META, path, lambda io: io.get_file_size(path)) + + def new_output_stream(self, path: str): + return self._origin.new_output_stream(path) + + def exists(self, path: str) -> bool: + return self._call(Op.EXISTS, path, lambda io: io.exists(path)) + + def exists_batch(self, paths: List[str]) -> Dict[str, bool]: + paths = list(paths) + found = self._call_batch( + Op.EXISTS, paths, lambda io, group: [io.exists_batch(group)[p] for p in group]) + return dict(zip(paths, found)) + + def is_dir(self, path: str) -> bool: + return self._origin.is_dir(path) + + def list_status(self, path: str): + return self._origin.list_status(path) + + def delete(self, path: str, recursive: bool = False) -> bool: + return self._origin.delete(path, recursive) + + def mkdirs(self, path: str) -> bool: + return self._origin.mkdirs(path) + + def rename(self, src: str, dst: str) -> bool: + return self._origin.rename(src, dst) + + def copy_file(self, source_path: str, target_path: str, overwrite: bool = False): + return self._origin.copy_file(source_path, target_path, overwrite) + + def try_to_write_atomic(self, path: str, content: str) -> bool: + return self._origin.try_to_write_atomic(path, content) + + def create_blob_presigned_url(self, table_root, descriptor, validity) -> str: + return self._origin.create_blob_presigned_url(table_root, descriptor, validity) + + def to_filesystem_path(self, path: str) -> str: + # The routing filesystem needs the full path, it converts it for the chosen filesystem. + return path + + def write_parquet(self, path: str, data, compression: str = 'zstd', + zstd_level: int = 1, **kwargs): + return self._origin.write_parquet(path, data, compression, zstd_level, **kwargs) + + def write_orc(self, path: str, data, compression: str = 'zstd', + zstd_level: int = 1, **kwargs): + return self._origin.write_orc(path, data, compression, zstd_level, **kwargs) + + def write_avro(self, path: str, data, avro_schema=None, + compression: str = 'zstd', zstd_level: int = 1, **kwargs): + return self._origin.write_avro(path, data, avro_schema, compression, zstd_level, **kwargs) + + def write_lance(self, path: str, data, **kwargs): + return self._origin.write_lance(path, data, **kwargs) + + def write_blob(self, path: str, data, **kwargs): + return self._origin.write_blob(path, data, **kwargs) + + def write_vortex(self, path: str, data, **kwargs): + return self._origin.write_vortex(path, data, **kwargs) + + def write_row(self, path: str, data, fields=None, zstd_level: int = 1, **kwargs): + return self._origin.write_row(path, data, fields, zstd_level, **kwargs) + + def close(self): + with self._targets_lock: + targets = list(self._targets.values()) + self._targets.clear() + try: + self._origin.close() + finally: + for target in targets: + target.close() + + def __getattr__(self, name): + origin = self.__dict__.get('_origin') + if origin is None: + raise AttributeError(name) + return getattr(origin, name) + + +class _IoCacheRoutingHandler(FileSystemHandler): + """Sends each pyarrow filesystem call to the filesystem of origin or of a target FileIO.""" + + def __init__(self, file_io: IoCacheRoutingFileIO): + self._routing_io = file_io + + def __eq__(self, other): + if isinstance(other, _IoCacheRoutingHandler): + return self._routing_io is other._routing_io + return NotImplemented + + def __ne__(self, other): + if isinstance(other, _IoCacheRoutingHandler): + return self._routing_io is not other._routing_io + return NotImplemented + + def get_type_name(self) -> str: + return "io-cache-routing" + + def normalize_path(self, path: str) -> str: + return path + + def register_file_size(self, path: str, file_size: int): + file_io = self._routing_io._file_io(self._routing_io._route(Op.READ, path)) + handler = getattr(file_io.filesystem, "handler", None) + register = getattr(handler, "register_file_size", None) + if register is not None: + register(file_io.to_filesystem_path(path), file_size) + + def get_file_info(self, paths) -> list: + infos = self._routing_io._call_batch( + Op.META, list(paths), + lambda io, group: io.filesystem.get_file_info([io.to_filesystem_path(path) for path in group])) + return [self._file_info(path, info) for path, info in zip(paths, infos)] + + @staticmethod + def _file_info(path, info): + return pafs.FileInfo(path, info.type, mtime_ns=info.mtime_ns, size=info.size) + + def open_input_file(self, path: str): + return self._routing_io._call( + Op.READ, path, + lambda io: io.filesystem.open_input_file(io.to_filesystem_path(path))) + + def open_input_stream(self, path: str): + # The outer PyFileSystem already applied the requested decompression. + return self._routing_io._call( + Op.READ, path, + lambda io: io.filesystem.open_input_stream(io.to_filesystem_path(path), compression=None)) + + def open_output_stream(self, path: str, metadata): + origin = self._routing_io._origin + return origin.filesystem.open_output_stream( + origin.to_filesystem_path(path), compression=None, metadata=metadata) + + def open_append_stream(self, path: str, metadata): + origin = self._routing_io._origin + return origin.filesystem.open_append_stream( + origin.to_filesystem_path(path), compression=None, metadata=metadata) + + def get_file_info_selector(self, selector) -> list: + origin = self._routing_io._origin + base = origin.to_filesystem_path(selector.base_dir).rstrip("/") + infos = origin.filesystem.get_file_info(pafs.FileSelector( + base, + allow_not_found=selector.allow_not_found, recursive=selector.recursive)) + # Backend paths must stay in the full-URI namespace accepted by this filesystem. + return [self._file_info(selector.base_dir.rstrip("/") + info.path[len(base):], info) + for info in infos] + + def create_dir(self, path: str, recursive: bool): + origin = self._routing_io._origin + origin.filesystem.create_dir(origin.to_filesystem_path(path), recursive=recursive) + + def delete_dir(self, path: str): + origin = self._routing_io._origin + origin.filesystem.delete_dir(origin.to_filesystem_path(path)) + + def delete_dir_contents(self, path: str, missing_dir_ok: bool = False): + origin = self._routing_io._origin + origin.filesystem.delete_dir_contents( + origin.to_filesystem_path(path), missing_dir_ok=missing_dir_ok) + + def delete_root_dir_contents(self): + raise NotImplementedError("io-cache routing filesystem has no root directory") + + def delete_file(self, path: str): + origin = self._routing_io._origin + origin.filesystem.delete_file(origin.to_filesystem_path(path)) + + def move(self, src: str, dest: str): + origin = self._routing_io._origin + origin.filesystem.move(origin.to_filesystem_path(src), origin.to_filesystem_path(dest)) + + def copy_file(self, src: str, dest: str): + origin = self._routing_io._origin + origin.filesystem.copy_file(origin.to_filesystem_path(src), origin.to_filesystem_path(dest)) diff --git a/paimon-python/pypaimon/filesystem/io_cache_routing.py b/paimon-python/pypaimon/filesystem/io_cache_routing.py new file mode 100644 index 000000000000..9c183a7969cc --- /dev/null +++ b/paimon-python/pypaimon/filesystem/io_cache_routing.py @@ -0,0 +1,226 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Chooses the endpoint of each FileIO request from the io-cache.* keys of a table token.""" + +import os +import re +from enum import Enum +from typing import Dict, FrozenSet, List, Mapping, NamedTuple, Optional, Tuple + +from pypaimon.common.options.config import CatalogOptions, OssOptions +from pypaimon.utils.file_type import FileType + +FS_OSS_ENDPOINT = OssOptions.OSS_ENDPOINT.key() +FS_OSS_REGION = OssOptions.OSS_REGION.key() +FS_OSS_PATH_STYLE = OssOptions.OSS_SECOND_LEVEL_DOMAIN_ENABLE.key() +FS_OSS_HTTPS_ENABLE = "fs.oss.https.enable" +DLF_OSS_ENDPOINT = CatalogOptions.DLF_OSS_ENDPOINT.key() +IO_CACHE_ENABLED = CatalogOptions.IO_CACHE_ENABLED.key() +IO_CACHE_TARGET_PREFIX = "io-cache.target." + +_TARGET_NAME = re.compile(r"[a-z][a-z0-9-]*") +# Data files are named {prefix}{uuid}-{count}.{extension}; this matches what follows the prefix. +_DATA_FILE_SUFFIX = re.compile( + r"[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}-[0-9]+\..+") +_MANIFEST_SIDECAR_SUFFIX = ".avro.sidecar" +# JindoSDK cache client keys; the origin option set must not talk to a cache. +_ORIGIN_DROPPED_PREFIXES = ("fs.oss.dlf-cache.", "fs.jindocache.") + + +class Op(Enum): + """FileIO operations; only READ, META and EXISTS can go to a target.""" + READ = "read" + META = "meta" + EXISTS = "exists" + WRITE = "write" + LIST = "list" + DELETE = "delete" + RENAME = "rename" + MKDIRS = "mkdirs" + COPY = "copy" + ATOMIC_WRITE = "atomic-write" + TWO_PHASE_WRITE = "two-phase-write" + PRESIGN = "presign" + + +def _flag(value) -> bool: + return value is not None and str(value).strip().lower() == "true" + + +def _parse_policy(value) -> FrozenSet[Op]: + # read and meta can use a target; write only applies to JindoCache, none turns it off + tokens = {token.strip().lower() for token in str(value or "").split(",")} + if "none" in tokens: + return frozenset() + return frozenset(op for op in (Op.READ, Op.META) if op.value in tokens) + + +def _parse_routes(value: str) -> List[Tuple[set, str]]: + # "types=target;...": the first rule containing a type picks its target + rules = [] + for rule in value.lower().split(";"): + types, eq, target = rule.partition("=") + types = FileType.parse_whitelist(types) if eq and target.strip() else set() + if types: + rules.append((types, target.strip())) + return rules + + +def _declared_targets(options: Mapping[str, str]) -> Dict[str, Optional[str]]: + # io-cache.targets with io-cache.target..endpoint, else io-cache.endpoint as "default" + names = options.get("io-cache.targets") + if names is None: + endpoint = options.get("io-cache.endpoint") + return {} if endpoint is None else {"default": endpoint} + endpoints = {} + for name in str(names).lower().split(","): + name = name.strip() + if _TARGET_NAME.fullmatch(name) and name not in endpoints: + endpoints[name] = options.get(IO_CACHE_TARGET_PREFIX + name + ".endpoint") + return endpoints + + +def data_prefixes(options: Mapping[str, str]) -> Tuple[str, ...]: + """Data file name prefixes: data- and changelog-, plus the table's own prefixes.""" + prefixes = ["data-", "changelog-"] + for key in ("data-file.prefix", "changelog-file.prefix"): + value = options.get(key) + if value and value.strip() and value not in prefixes: + prefixes.append(value) + return tuple(prefixes) + + +def routable_type(path: str, prefixes: Tuple[str, ...]) -> Optional[FileType]: + """The type a cache may serve, or None when the file must be read from the origin.""" + path = path.rstrip("/") + name = os.path.basename(path) + if _is_temporary(name) or FileType.is_mutable(path): + return None + if _is_data_file_name(name, prefixes): + return FileType.FILE_INDEX if name.endswith(".index") else FileType.DATA + # Sequential metadata may be read before it exists; cached NotFound would hide new commits. + parent = os.path.basename(os.path.dirname(path)) + if name.startswith(("snapshot-", "schema-")) or ( + name.startswith("changelog-") and parent == "changelog"): + return None + file_type = FileType.classify(path) + return None if file_type == FileType.DATA else file_type + + +def _is_data_file_name(name: str, prefixes: Tuple[str, ...]) -> bool: + """Manifests, indexes and statistics share the uuid-count shape but have no extension.""" + if name.endswith(_MANIFEST_SIDECAR_SUFFIX): + return False + return any(name.startswith(p) and _DATA_FILE_SUFFIX.fullmatch(name[len(p):]) for p in prefixes) + + +def _is_temporary(name: str) -> bool: + return name.endswith(".tmp") or ".tmp-" in name or ".tmp." in name + + +def _with_endpoint(options: Dict[str, str], endpoint: Optional[str]) -> Dict[str, str]: + if not endpoint: + return options + # JindoSDK drops the scheme of a URL endpoint, so carry it in fs.oss.https.enable. + scheme, separator, host = endpoint.partition("://") + if separator: + scheme = scheme.lower() + options[FS_OSS_HTTPS_ENABLE] = "true" if scheme == "https" else "false" + endpoint = scheme + separator + host + options[FS_OSS_ENDPOINT] = endpoint.rstrip("/") + return options + + +class CacheTarget(NamedTuple): + """A named cache target with its endpoint and addressing settings.""" + name: str + endpoint: str + path_style_access: bool + region: Optional[str] + + +class IoCacheRouting: + """Per-request choice between the io-cache targets and the origin endpoint.""" + + def __init__(self, options: Mapping[str, str], policy: FrozenSet[Op], + endpoints: Dict[str, Optional[str]]): + origin = options.get("io-cache.origin.endpoint") + self._origin = origin if origin and origin.strip() else options.get(FS_OSS_ENDPOINT) + self._policy = policy + self._targets = { + name: CacheTarget(name, endpoint.strip(), + _flag(options.get(IO_CACHE_TARGET_PREFIX + name + ".path-style-access")), + options.get(IO_CACHE_TARGET_PREFIX + name + ".region")) + for name, endpoint in endpoints.items() if endpoint and endpoint.strip()} + whitelist = options.get("io-cache.whitelist") + self._whitelist = set(FileType) if whitelist is None else FileType.parse_whitelist( + str(whitelist).lower()) + routes = options.get("io-cache.routes") + self._routes = None if routes is None else _parse_routes(str(routes)) + self._prefixes = data_prefixes(options) + + @staticmethod + def create(options: Mapping[str, str]) -> Optional['IoCacheRouting']: + """The routing of the options, or None when requests keep using fs.oss.endpoint.""" + if str(options.get(DLF_OSS_ENDPOINT) or "").strip() or not _flag(options.get(IO_CACHE_ENABLED)): + return None + policy = _parse_policy(options.get("io-cache.policy")) + endpoints = _declared_targets(options) + return IoCacheRouting(options, policy, endpoints) if policy and endpoints else None + + def targets(self) -> Dict[str, str]: + """Endpoints of the targets that have one, by name, in declaration order.""" + return {name: target.endpoint for name, target in self._targets.items()} + + def origin_endpoint(self) -> Optional[str]: + return self._origin + + def route(self, op: Op, path: str) -> Optional[str]: + """Name of the target of one request, or None when it goes to origin.""" + # An existence check is a file status lookup. + if op is Op.EXISTS: + op = Op.META + # Only OSS paths can use cache targets. + if op not in self._policy or not path.startswith("oss://"): + return None + file_type = routable_type(path, self._prefixes) + if file_type is None or file_type not in self._whitelist: + return None + if self._routes is None: + return next(iter(self._targets), None) + name = next((target for types, target in self._routes if file_type in types), None) + return name if name in self._targets else None + + def origin_options(self, options: Mapping[str, str]) -> Dict[str, str]: + """Options of the origin FileIO: the origin endpoint, without cache client keys.""" + result = {key: value for key, value in options.items() + if not str(key).startswith(_ORIGIN_DROPPED_PREFIXES)} + return _with_endpoint(result, self._origin) + + def target_options(self, options: Mapping[str, str], name: str) -> Dict[str, str]: + """Options of the FileIO of one target: its endpoint, addressing and region.""" + target = self._targets[name] + endpoint = target.endpoint + # A target endpoint without a scheme means https. + if "://" not in endpoint: + endpoint = "https://" + endpoint + result = _with_endpoint(dict(options), endpoint) + result[FS_OSS_PATH_STYLE] = "true" if target.path_style_access else "false" + if target.region and target.region.strip(): + result[FS_OSS_REGION] = target.region + return result diff --git a/paimon-python/pypaimon/filesystem/pyarrow_file_io.py b/paimon-python/pypaimon/filesystem/pyarrow_file_io.py index ad5d3c8a4ea9..e91e0de7defc 100644 --- a/paimon-python/pypaimon/filesystem/pyarrow_file_io.py +++ b/paimon-python/pypaimon/filesystem/pyarrow_file_io.py @@ -59,8 +59,10 @@ def __init__(self, path: str, catalog_options: Options): # goes into endpoint_override, so keys must omit it (init + path share # this flag so they can't drift). self._pyarrow_gte_16 = parse(pyarrow.__version__) >= parse("16.0.0") - self._oss_bucket_in_endpoint = not self._pyarrow_gte_16 scheme, netloc, _ = self.parse_location(path) + # Path-style addressing sends bucket/key to the endpoint itself, so the bucket stays out of it. + self._oss_path_style = scheme == "oss" and self.properties.get(OssOptions.OSS_SECOND_LEVEL_DOMAIN_ENABLE) + self._oss_bucket_in_endpoint = not self._pyarrow_gte_16 and not self._oss_path_style self.uri_reader_factory = UriReaderFactory(catalog_options) self._is_oss = scheme in {"oss"} self._oss_bucket = None @@ -209,12 +211,18 @@ def _initialize_oss_fs(self, path) -> FileSystem: "region": self.properties.get(OssOptions.OSS_REGION), } + endpoint = self.properties.get(OssOptions.OSS_ENDPOINT) + if endpoint and '://' in endpoint: + # endpoint_override takes host[:port]; a URL endpoint carries its scheme separately + scheme, _, endpoint = endpoint.partition('://') + client_kwargs['scheme'] = scheme.lower() + endpoint = endpoint.rstrip('/') if not self._oss_bucket_in_endpoint: - client_kwargs['force_virtual_addressing'] = True - client_kwargs['endpoint_override'] = self.properties.get(OssOptions.OSS_ENDPOINT) + if self._pyarrow_gte_16: + client_kwargs['force_virtual_addressing'] = not self._oss_path_style + client_kwargs['endpoint_override'] = endpoint else: - client_kwargs['endpoint_override'] = (self._oss_bucket + "." + - self.properties.get(OssOptions.OSS_ENDPOINT)) + client_kwargs['endpoint_override'] = self._oss_bucket + "." + endpoint retry_config = self._create_s3_retry_config() client_kwargs.update(retry_config) diff --git a/paimon-python/pypaimon/read/reader/format_lance_reader.py b/paimon-python/pypaimon/read/reader/format_lance_reader.py index 67edfbd02530..a827245805e0 100644 --- a/paimon-python/pypaimon/read/reader/format_lance_reader.py +++ b/paimon-python/pypaimon/read/reader/format_lance_reader.py @@ -45,7 +45,7 @@ def __init__(self, file_io: FileIO, file_path: str, read_fields: List[DataField] self._read_field_names = [f.name for f in read_fields] - file_path_for_lance, storage_options = to_lance_specified(file_io, file_path) + file_path_for_lance, storage_options = to_lance_specified(file_io, file_path, is_read=True) # Read file metadata (footer only) to get actual schema file_schema_names = set( diff --git a/paimon-python/pypaimon/read/reader/format_vortex_reader.py b/paimon-python/pypaimon/read/reader/format_vortex_reader.py index fcd50a24f005..3787717209da 100644 --- a/paimon-python/pypaimon/read/reader/format_vortex_reader.py +++ b/paimon-python/pypaimon/read/reader/format_vortex_reader.py @@ -42,7 +42,7 @@ def __init__(self, file_io: FileIO, file_path: str, read_fields: List[DataField] import vortex from pypaimon.read.reader.vortex_utils import to_vortex_specified - file_path_for_vortex, store_kwargs = to_vortex_specified(file_io, file_path) + file_path_for_vortex, store_kwargs = to_vortex_specified(file_io, file_path, is_read=True) if store_kwargs: from vortex import store diff --git a/paimon-python/pypaimon/read/reader/lance_utils.py b/paimon-python/pypaimon/read/reader/lance_utils.py index 1c4df400e612..b593b15ea670 100644 --- a/paimon-python/pypaimon/read/reader/lance_utils.py +++ b/paimon-python/pypaimon/read/reader/lance_utils.py @@ -21,9 +21,11 @@ from pypaimon.common.file_io import FileIO from pypaimon.common.options.config import OssOptions +from pypaimon.filesystem.io_cache_file_io import IoCacheRoutingFileIO -def to_lance_specified(file_io: FileIO, file_path: str) -> Tuple[str, Optional[Dict[str, str]]]: +def to_lance_specified(file_io: FileIO, file_path: str, + is_read: bool = False) -> Tuple[str, Optional[Dict[str, str]]]: """Convert path and extract storage options for Lance format.""" # For RESTTokenFileIO, get underlying FileIO which already has latest token merged # This follows Java implementation: ((RESTTokenFileIO) fileIO).fileIO() @@ -32,6 +34,9 @@ def to_lance_specified(file_io: FileIO, file_path: str) -> Tuple[str, Optional[D # Call file_io() to get underlying FileIO with latest token # This ensures token is refreshed and merged with catalog options file_io = file_io.file_io() + + if is_read and isinstance(file_io, IoCacheRoutingFileIO): + file_io = file_io.read_file_io(file_path) # Now get properties from the underlying FileIO (which has latest token) if hasattr(file_io, 'get_merged_properties'): @@ -62,9 +67,13 @@ def to_lance_specified(file_io: FileIO, file_path: str) -> Tuple[str, Optional[D storage_options[key] = value endpoint = properties.get(OssOptions.OSS_ENDPOINT) + path_style = properties.get(OssOptions.OSS_SECOND_LEVEL_DOMAIN_ENABLE) if endpoint: - endpoint_clean = endpoint.replace('http://', '').replace('https://', '') - storage_options['endpoint'] = f"https://{bucket}.{endpoint_clean}" + parsed_endpoint = urlparse(endpoint if '://' in endpoint else 'https://' + endpoint) + host = parsed_endpoint.netloc if path_style else f"{bucket}.{parsed_endpoint.netloc}" + storage_options['endpoint'] = f"{parsed_endpoint.scheme}://{host}" + if parsed_endpoint.scheme == 'http': + storage_options['allow_http'] = 'true' if properties.contains(OssOptions.OSS_ACCESS_KEY_ID): storage_options['access_key_id'] = properties.get(OssOptions.OSS_ACCESS_KEY_ID) @@ -78,7 +87,7 @@ def to_lance_specified(file_io: FileIO, file_path: str) -> Tuple[str, Optional[D if properties.contains(OssOptions.OSS_ENDPOINT): storage_options['oss_endpoint'] = properties.get(OssOptions.OSS_ENDPOINT) - storage_options['virtual_hosted_style_request'] = 'true' + storage_options['virtual_hosted_style_request'] = 'false' if path_style else 'true' if bucket and path: file_path_for_lance = f"oss://{bucket}/{path}" diff --git a/paimon-python/pypaimon/read/reader/vortex_utils.py b/paimon-python/pypaimon/read/reader/vortex_utils.py index d66682898f9a..5f4fb0d554a8 100644 --- a/paimon-python/pypaimon/read/reader/vortex_utils.py +++ b/paimon-python/pypaimon/read/reader/vortex_utils.py @@ -21,9 +21,11 @@ from pypaimon.common.file_io import FileIO from pypaimon.common.options.config import OssOptions +from pypaimon.filesystem.io_cache_file_io import IoCacheRoutingFileIO -def to_vortex_specified(file_io: FileIO, file_path: str) -> Tuple[str, Optional[Dict[str, str]]]: +def to_vortex_specified(file_io: FileIO, file_path: str, + is_read: bool = False) -> Tuple[str, Optional[Dict[str, str]]]: """Convert path and extract storage options for Vortex store.from_url(). Returns (url, store_kwargs) where store_kwargs can be passed as @@ -33,6 +35,9 @@ def to_vortex_specified(file_io: FileIO, file_path: str) -> Tuple[str, Optional[ if hasattr(file_io, 'file_io'): file_io = file_io.file_io() + if is_read and isinstance(file_io, IoCacheRoutingFileIO): + file_io = file_io.read_file_io(file_path) + if hasattr(file_io, 'get_merged_properties'): properties = file_io.get_merged_properties() else: @@ -54,13 +59,19 @@ def to_vortex_specified(file_io: FileIO, file_path: str) -> Tuple[str, Optional[ if scheme == 'oss' and properties: parsed = urlparse(file_path) bucket = parsed.netloc + endpoint = properties.get(OssOptions.OSS_ENDPOINT) + parsed_endpoint = urlparse(endpoint if '://' in endpoint else 'https://' + endpoint) + path_style = properties.get(OssOptions.OSS_SECOND_LEVEL_DOMAIN_ENABLE) + host = parsed_endpoint.netloc if path_style else f"{bucket}.{parsed_endpoint.netloc}" store_kwargs = { - 'endpoint': f"https://{bucket}.{properties.get(OssOptions.OSS_ENDPOINT)}", + 'endpoint': f"{parsed_endpoint.scheme}://{host}", 'access_key_id': properties.get(OssOptions.OSS_ACCESS_KEY_ID), 'secret_access_key': properties.get(OssOptions.OSS_ACCESS_KEY_SECRET), - 'virtual_hosted_style_request': 'true', + 'virtual_hosted_style_request': 'false' if path_style else 'true', } + if parsed_endpoint.scheme == 'http': + store_kwargs['allow_http'] = 'true' if properties.contains(OssOptions.OSS_SECURITY_TOKEN): store_kwargs['session_token'] = properties.get(OssOptions.OSS_SECURITY_TOKEN) diff --git a/paimon-python/pypaimon/tests/file_io_test.py b/paimon-python/pypaimon/tests/file_io_test.py index 53c8e8d29698..dbd6cacf4ee9 100644 --- a/paimon-python/pypaimon/tests/file_io_test.py +++ b/paimon-python/pypaimon/tests/file_io_test.py @@ -149,6 +149,46 @@ def record_get_file_info(paths): "OSS+PyArrow<16 must pass key only to get_file_info, not bucket/key. Got: %r" % (p,) ) + def test_legacy_oss_url_endpoint_passes_scheme(self): + options = { + OssOptions.OSS_ACCESS_KEY_ID.key(): 'test-key', + OssOptions.OSS_ACCESS_KEY_SECRET.key(): 'test-secret', + OssOptions.OSS_IMPL.key(): 'legacy', + } + cases = (('http://127.0.0.1:9000/', 'http', '127.0.0.1:9000'), + ('HTTPS://oss-cn-hangzhou.aliyuncs.com', 'https', 'oss-cn-hangzhou.aliyuncs.com'), + ('oss-cn-hangzhou.aliyuncs.com', None, 'oss-cn-hangzhou.aliyuncs.com')) + for endpoint, scheme, host in cases: + with patch('pypaimon.filesystem.pyarrow_file_io.pafs.S3FileSystem') as s3_file_system: + oss_io = OssFileIO("oss://test-bucket/warehouse", Options( + {**options, OssOptions.OSS_ENDPOINT.key(): endpoint})) + kwargs = s3_file_system.call_args[1] + self.assertEqual(scheme, kwargs.get('scheme')) + expected = 'test-bucket.' + host if oss_io._oss_bucket_in_endpoint else host + self.assertEqual(expected, kwargs['endpoint_override']) + + def test_legacy_oss_path_style_keeps_bucket_out_of_endpoint(self): + options = Options({ + OssOptions.OSS_ENDPOINT.key(): 'http://10.0.0.1:8080', + OssOptions.OSS_ACCESS_KEY_ID.key(): 'test-key', + OssOptions.OSS_ACCESS_KEY_SECRET.key(): 'test-secret', + OssOptions.OSS_IMPL.key(): 'legacy', + OssOptions.OSS_SECOND_LEVEL_DOMAIN_ENABLE.key(): 'true', + }) + for version, virtual in (('15.0.0', None), ('16.0.0', False)): + # the retry strategy is missing from the old pyarrow some CI jobs install + with patch('pyarrow.__version__', version), \ + patch('pypaimon.filesystem.pyarrow_file_io.pafs.AwsStandardS3RetryStrategy', create=True), \ + patch('pypaimon.filesystem.pyarrow_file_io.pafs.S3FileSystem') as s3_file_system: + oss_io = OssFileIO("oss://test-bucket/warehouse", options) + kwargs = s3_file_system.call_args[1] + self.assertEqual('10.0.0.1:8080', kwargs['endpoint_override']) + self.assertEqual(virtual, kwargs.get('force_virtual_addressing')) + self.assertFalse(oss_io._oss_bucket_in_endpoint) + oss_io = OssFileIO("oss://test-bucket/warehouse", options) + self.assertEqual("test-bucket/warehouse/data-1.parquet", + oss_io.to_filesystem_path("oss://test-bucket/warehouse/data-1.parquet")) + def test_exists(self): lt7 = _pyarrow_lt_7() with tempfile.TemporaryDirectory(prefix="file_io_nonexistent_") as tmpdir: diff --git a/paimon-python/pypaimon/tests/io_cache_file_io_test.py b/paimon-python/pypaimon/tests/io_cache_file_io_test.py new file mode 100644 index 000000000000..eb1787a8e014 --- /dev/null +++ b/paimon-python/pypaimon/tests/io_cache_file_io_test.py @@ -0,0 +1,600 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Tests for IoCacheRoutingFileIO and its pyarrow filesystem.""" + +import gzip +import io +import os +import shutil +import tempfile +import unittest +from datetime import timedelta +from pathlib import Path +from urllib.parse import urlparse + +import pyarrow as pa +import pyarrow.dataset as ds +import pyarrow.fs as pafs +import pyarrow.parquet as pq +import pytest +from pyarrow._fs import FileSystemHandler + +from pypaimon.common.file_io import FileIO +from pypaimon.filesystem.caching_file_io import (CachingFileIO, + LocalMemoryCacheManager) +from pypaimon.filesystem.io_cache_file_io import IoCacheRoutingFileIO +from pypaimon.filesystem.io_cache_routing import IoCacheRouting +from pypaimon.filesystem.local_file_io import LocalFileIO +from pypaimon.read.reader.format_avro_reader import FormatAvroReader +from pypaimon.read.reader.format_pyarrow_reader import FormatPyArrowReader +from pypaimon.schema.data_types import AtomicType, DataField +from pypaimon.table.row.blob import BlobDescriptor +from pypaimon.tests.io_cache_routing_test import ROUTING +from pypaimon.utils.file_type import FileType + +TABLE = "oss://bkt/db1.db/t1" +UUID = "8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b" +PARTITION = TABLE + "/dt=1/bucket-0" +DATA = PARTITION + "/data-" + UUID + "-0.parquet" +OTHER_DATA = PARTITION + "/data-" + UUID + "-1.parquet" +UNKNOWN = PARTITION + "/part-" + UUID + "-0.parquet" +MANIFEST = TABLE + "/manifest/manifest-" + UUID + "-0" +SNAPSHOT = TABLE + "/snapshot/snapshot-1" +LATEST = TABLE + "/snapshot/LATEST" + + +def _routing(policy="meta,read", whitelist="*", **overrides): + """Manifests go to the accel target, data files to the cluster target.""" + options = { + "fs.oss.endpoint": "http://accel.example.com", + "io-cache.enabled": "true", + "io-cache.origin.endpoint": "https://origin.example.com", + "io-cache.policy": policy, + "io-cache.whitelist": whitelist, + "io-cache.targets": "accel,cluster", + "io-cache.target.accel.endpoint": "http://accel.example.com", + "io-cache.target.cluster.endpoint": "http://10.0.0.1:8080", + "io-cache.routes": "meta=accel;data=cluster", + } + options.update(overrides) + return IoCacheRouting.create(options) + + +class _DirFileIO(LocalFileIO): + """LocalFileIO that keeps oss://bucket/key under root/bucket/key, standing in for one endpoint.""" + + def __init__(self, root): + super().__init__() + self.root = root + self.error = None + self.attempts = 0 + self.closed = False + + def _to_file(self, path): + parsed = urlparse(path) + if parsed.scheme != "oss": + return super()._to_file(path) + return Path(self.root, parsed.netloc, parsed.path.lstrip("/")) + + def _check(self): + self.attempts += 1 + if self.error is not None: + raise self.error + + def new_input_stream(self, path): + self._check() + return super().new_input_stream(path) + + def new_output_stream(self, path): + self._check() + return super().new_output_stream(path) + + def get_file_status(self, path): + self._check() + return super().get_file_status(path) + + def exists(self, path): + self._check() + return super().exists(path) + + def to_filesystem_path(self, path): + # Only the routing filesystem calls this, right before it calls the filesystem. + self._check() + return super().to_filesystem_path(path) + + def close(self): + self.closed = True + + +class IoCacheRoutingFileIOTest(unittest.TestCase): + + def setUp(self): + self.temp_dir = tempfile.mkdtemp(prefix="io_cache_file_io_test_") + self.origin = _DirFileIO(os.path.join(self.temp_dir, "origin")) + self.accel = _DirFileIO(os.path.join(self.temp_dir, "accel")) + self.cluster = _DirFileIO(os.path.join(self.temp_dir, "cluster")) + self.created = [] + self.creation_error = None + self.file_io = self._file_io(_routing()) + + def tearDown(self): + shutil.rmtree(self.temp_dir, ignore_errors=True) + + def _create_target(self, name): + self.created.append(name) + if self.creation_error is not None: + raise self.creation_error + return {"accel": self.accel, "cluster": self.cluster}[name] + + def _file_io(self, routing): + return IoCacheRoutingFileIO(routing, self.origin, self._create_target) + + @staticmethod + def _put(file_io, path, content): + file_io._to_file(path).parent.mkdir(parents=True, exist_ok=True) + file_io._to_file(path).write_bytes(content) + + def _put_all(self, *paths): + for path in paths: + for name, file_io in (("origin", self.origin), ("accel", self.accel), ("cluster", self.cluster)): + self._put(file_io, path, name.encode()) + + @staticmethod + def _content(file_io, path): + file = file_io._to_file(path) + return file.read_bytes() if file.exists() else None + + def _attempts(self): + return self.origin.attempts, self.accel.attempts, self.cluster.attempts + + def test_reads_follow_the_route(self): + self._put_all(DATA, MANIFEST, LATEST, UNKNOWN, SNAPSHOT) + self.assertEqual(b"cluster", self.file_io.new_input_stream(DATA).read()) + self.assertEqual("cluster", self.file_io.read_file_utf8(DATA)) + self.assertEqual(b"lus", self.file_io.read_file_range(DATA, 1, 3)) + self.assertEqual(b"accel", self.file_io.new_input_stream(MANIFEST).read()) + for path in (LATEST, UNKNOWN, SNAPSHOT): + with self.subTest(path=path): + self._put(self.origin, path, b"origin") + self.assertEqual(b"origin", self.file_io.new_input_stream(path).read()) + + def test_single_target_takes_every_routed_type(self): + self._put_all(DATA, MANIFEST) + routing = IoCacheRouting.create({ + "io-cache.enabled": "true", + "io-cache.endpoint": "http://cluster.example.com", + "io-cache.policy": "read", + }) + file_io = IoCacheRoutingFileIO(routing, self.origin, lambda name: {"default": self.cluster}[name]) + self.assertEqual(b"cluster", file_io.new_input_stream(DATA).read()) + self.assertEqual(b"cluster", file_io.filesystem.open_input_file(MANIFEST).read()) + self.assertEqual(6, file_io.get_file_size(DATA)) + + def test_targets_are_created_on_first_use(self): + self._put_all(DATA, MANIFEST, LATEST) + self.file_io.new_input_stream(LATEST) + self.assertEqual([], self.created) + self.file_io.new_input_stream(DATA) + self.file_io.get_file_status(DATA) + self.assertEqual(["cluster"], self.created) + self.file_io.new_input_stream(MANIFEST) + self.assertEqual(["cluster", "accel"], self.created) + + def test_reads_without_read_policy_use_origin(self): + self._put_all(DATA) + file_io = self._file_io(_routing(policy="meta")) + self.assertEqual(b"origin", file_io.new_input_stream(DATA).read()) + self.assertEqual(7, file_io.get_file_size(DATA)) + self.assertEqual(b"origin", self._file_io(_routing(whitelist="meta")).new_input_stream(DATA).read()) + + def test_meta_follows_the_route(self): + self._put(self.cluster, DATA, b"cached!") + self._put(self.accel, MANIFEST, b"m") + self.assertEqual(7, self.file_io.get_file_size(DATA)) + self.assertEqual(1, self.file_io.get_file_status(MANIFEST).size) + with self.assertRaises(FileNotFoundError): + self._file_io(_routing(policy="read")).get_file_size(DATA) + + def test_exists_follows_the_route(self): + self._put(self.cluster, DATA, b"cached!") + self._put(self.accel, MANIFEST, b"m") + self._put(self.origin, OTHER_DATA, b"origin") + self._put(self.origin, SNAPSHOT, b"origin") + self._put(self.origin, TABLE + "/manifest/manifest-list-" + UUID + "-0", b"origin") + self.assertTrue(self.file_io.exists(DATA)) + # OTHER_DATA is routed to the cluster, which does not have it + self.assertFalse(self.file_io.exists(OTHER_DATA)) + self.assertTrue(self.file_io.exists(SNAPSHOT)) + self.assertEqual({MANIFEST: True, DATA: True, SNAPSHOT: True, OTHER_DATA: False}, + self.file_io.exists_batch([MANIFEST, DATA, SNAPSHOT, OTHER_DATA])) + self.assertTrue(self._file_io(_routing(policy="read")).exists(OTHER_DATA)) + self.assertTrue(self.file_io.is_dir(PARTITION)) + self.assertTrue(self.file_io.is_dir(TABLE + "/manifest")) + + def test_listing_and_mutations_use_origin(self): + self._put_all(DATA, SNAPSHOT) + listed = self.file_io.list_status(PARTITION) + self.assertEqual([str(self.origin._to_file(DATA))], [status.path for status in listed]) + + self.assertTrue(self.file_io.rename(DATA, OTHER_DATA)) + self.assertIsNone(self._content(self.origin, DATA)) + self.assertEqual(b"origin", self._content(self.origin, OTHER_DATA)) + self.assertEqual(b"cluster", self._content(self.cluster, DATA)) + + self.file_io.copy_file(OTHER_DATA, DATA) + self.assertEqual(b"origin", self._content(self.origin, DATA)) + self.assertIsNone(self._content(self.cluster, OTHER_DATA)) + + self.assertTrue(self.file_io.delete(DATA)) + self.assertIsNone(self._content(self.origin, DATA)) + self.assertEqual(b"cluster", self._content(self.cluster, DATA)) + + self.assertTrue(self.file_io.mkdirs(TABLE + "/new-dir")) + self.assertTrue(self.origin._to_file(TABLE + "/new-dir").is_dir()) + self.assertFalse(self.cluster._to_file(TABLE + "/new-dir").exists()) + + self.assertTrue(self.file_io.try_to_write_atomic(TABLE + "/snapshot/snapshot-2", "{}")) + self.assertEqual(b"{}", self._content(self.origin, TABLE + "/snapshot/snapshot-2")) + self.assertEqual([], self.created) + + def test_writes_use_origin(self): + table = pa.table({"a": [1, 2]}) + file_io = self._file_io(_routing(policy="read,meta,write")) + with file_io.new_output_stream(DATA) as out: + out.write(b"written") + file_io.write_parquet(OTHER_DATA, table) + file_io.overwrite_file_utf8(MANIFEST, "3") + with file_io.filesystem.open_output_stream(UNKNOWN) as out: + out.write(b"streamed") + self.assertEqual(b"written", self._content(self.origin, DATA)) + self.assertEqual(table, pq.ParquetFile(str(self.origin._to_file(OTHER_DATA))).read()) + self.assertEqual(b"3", self._content(self.origin, MANIFEST)) + self.assertEqual(b"streamed", self._content(self.origin, UNKNOWN)) + self.assertEqual([], self.created) + + def test_filesystem_reads_follow_the_route(self): + for file_io, value in ((self.origin, 1), (self.cluster, 2)): + sink = pa.BufferOutputStream() + pq.write_table(pa.table({"a": [value]}), sink) + self._put(file_io, DATA, sink.getvalue().to_pybytes()) + filesystem = self.file_io.filesystem + self.assertIs(filesystem, self.file_io.filesystem) + path = self.file_io.to_filesystem_path(DATA) + self.assertEqual([2], pq.read_table(path, filesystem=filesystem).column("a").to_pylist()) + with filesystem.open_input_file(path) as f: + self.assertEqual([2], pq.read_table(f).column("a").to_pylist()) + origin_only = self._file_io(_routing(policy="meta")) + self.assertEqual([1], pq.read_table(DATA, filesystem=origin_only.filesystem).column("a").to_pylist()) + + def test_readers_use_the_routing_filesystem(self): + for file_io, value in ((self.origin, 1), (self.accel, 2), (self.cluster, 3)): + file_io.write_parquet(DATA, pa.table({"a": pa.array([value], pa.int64())})) + file_io.write_avro(MANIFEST, pa.table({"a": pa.array([value], pa.int64())})) + fields = [DataField(0, "a", AtomicType("BIGINT"))] + size = self.cluster._to_file(DATA).stat().st_size + + reader = FormatPyArrowReader(self.file_io, "parquet", DATA, fields, None, file_size=size) + self.assertEqual([3], reader.read_arrow_batch().column(0).to_pylist()) + reader = FormatAvroReader(self.file_io, MANIFEST, ["a"], fields, None) + self.assertEqual([2], reader.read_arrow_batch().column(0).to_pylist()) + reader = FormatPyArrowReader(self._file_io(_routing(policy="meta")), "parquet", DATA, fields, None) + self.assertEqual([1], reader.read_arrow_batch().column(0).to_pylist()) + + def test_filesystem_file_info_follows_the_route(self): + self._put(self.cluster, DATA, b"cached!") + self._put(self.accel, MANIFEST, b"m") + self._put(self.origin, LATEST, b"1") + infos = self.file_io.filesystem.get_file_info([LATEST, DATA, MANIFEST, OTHER_DATA]) + self.assertEqual([pafs.FileType.File, pafs.FileType.File, pafs.FileType.File, pafs.FileType.NotFound], + [info.type for info in infos]) + self.assertEqual([1, 7, 1], [info.size for info in infos[:3]]) + self.assertEqual([LATEST, DATA, MANIFEST, OTHER_DATA], [info.path for info in infos]) + + def test_directory_dataset_keeps_oss_paths_and_routes_reads(self): + for file_io, value in ((self.origin, 1), (self.cluster, 2)): + file_io.write_parquet(DATA, pa.table({"a": [value]})) + dataset = ds.dataset(PARTITION, format="parquet", filesystem=self.file_io.filesystem) + self.assertEqual([2], dataset.to_table().column("a").to_pylist()) + + def test_filesystem_mutations_use_origin(self): + self._put_all(DATA) + filesystem = self.file_io.filesystem + selected = filesystem.get_file_info(pafs.FileSelector(PARTITION)) + self.assertEqual([DATA], [info.path for info in selected]) + filesystem.copy_file(DATA, OTHER_DATA) + self.assertEqual(b"origin", self._content(self.origin, OTHER_DATA)) + filesystem.delete_file(OTHER_DATA) + filesystem.move(DATA, OTHER_DATA) + self.assertEqual(b"origin", self._content(self.origin, OTHER_DATA)) + self.assertEqual(b"cluster", self._content(self.cluster, DATA)) + filesystem.create_dir(TABLE + "/new-dir") + self.assertTrue(self.origin._to_file(TABLE + "/new-dir").is_dir()) + filesystem.delete_dir(TABLE + "/new-dir") + filesystem.delete_dir_contents(PARTITION) + self.assertEqual([], list(self.origin._to_file(PARTITION).iterdir())) + self.assertEqual(b"cluster", self._content(self.cluster, DATA)) + self.assertEqual([], self.created) + + def test_filesystem_streams_are_compressed_once(self): + path = PARTITION + "/data-" + UUID + "-0.json.gz" + self.origin._to_file(path).parent.mkdir(parents=True) + with self.file_io.filesystem.open_output_stream(path) as out: + out.write(b"payload") + self.assertEqual(b"payload", gzip.decompress(self._content(self.origin, path))) + self._put(self.cluster, path, self._content(self.origin, path)) + self.origin._to_file(path).unlink() + with self.file_io.filesystem.open_input_stream(path) as stream: + self.assertEqual(b"payload", stream.read()) + + def test_target_errors_are_raised(self): + self._put(self.origin, DATA, b"origin") + with self.assertRaises(FileNotFoundError): + self.file_io.new_input_stream(DATA) + with self.assertRaises(FileNotFoundError): + self.file_io.get_file_status(DATA) + with self.assertRaises(FileNotFoundError): + self.file_io.filesystem.open_input_file(DATA) + with self.assertRaises(FileNotFoundError): + self.file_io.filesystem.open_input_stream(DATA) + # Target errors propagate without retrying the request on origin. + for error in (OSError("[E1010]HTTP Status: 503 Error Code: SlowDown"), ConnectionRefusedError("refused"), + TimeoutError("timed out"), OSError("[E1010]HTTP Status: 403 Error Code: AccessDenied")): + self.cluster.error = error + with self.assertRaises(type(error)): + self.file_io.new_input_stream(DATA) + with self.assertRaises(type(error)): + self.file_io.filesystem.open_input_file(DATA) + self.assertEqual(0, self.origin.attempts) + + def test_target_that_cannot_be_created_raises(self): + self._put_all(DATA, MANIFEST) + self.creation_error = RuntimeError("cannot connect") + with self.assertRaisesRegex(RuntimeError, "cannot connect"): + self.file_io.new_input_stream(DATA) + self.creation_error = None + self.assertEqual(b"cluster", self.file_io.new_input_stream(DATA).read()) + self.assertEqual(["cluster", "cluster"], self.created) + + def test_register_file_size_reaches_the_routed_filesystem(self): + origin, targets = _RecordingFileIO(), {} + + def create(name): + targets[name] = _RecordingFileIO() + return targets[name] + + file_io = IoCacheRoutingFileIO(_routing(), origin, create) + handler = file_io.filesystem.handler + handler.register_file_size(DATA, 7) + handler.register_file_size(LATEST, 1) + self.assertEqual(["register_file_size"], origin.calls) + self.assertEqual({"cluster": ["register_file_size"]}, {name: io.calls for name, io in targets.items()}) + + def test_close_closes_created_targets(self): + self._put_all(DATA) + self.file_io.new_input_stream(DATA) + self.file_io.close() + self.assertEqual((True, False, True), (self.origin.closed, self.accel.closed, self.cluster.closed)) + + def test_local_block_cache_on_top(self): + self._put_all(DATA) + caching = CachingFileIO(self.file_io, LocalMemoryCacheManager(1 << 20), {FileType.DATA}) + self.assertEqual(b"cluster", caching.new_input_stream(DATA).read()) + self.assertIs(self.file_io.filesystem, caching.filesystem) + self.assertEqual(DATA, caching.to_filesystem_path(DATA)) + + def test_unrouted_attributes_come_from_origin(self): + self.assertIs(self.origin.properties, self.file_io.properties) + self.assertIs(self.origin.uri_reader_factory, self.file_io.uri_reader_factory) + self.assertFalse(hasattr(self.file_io, "file_io")) + + +class _RecordingHandler(FileSystemHandler): + + def __init__(self, calls): + self.calls = calls + + def _record(self, name, result=None): + self.calls.append(name) + return result + + def __eq__(self, other): + return self is other + + def __ne__(self, other): + return self is not other + + def get_type_name(self): + return "recording" + + def normalize_path(self, path): + return path + + def register_file_size(self, path, file_size): + self._record("register_file_size") + + def get_file_info(self, paths): + return self._record("get_file_info", [pafs.FileInfo(path, pafs.FileType.File, size=0) for path in paths]) + + def get_file_info_selector(self, selector): + return self._record("get_file_info_selector", []) + + def create_dir(self, path, recursive): + self._record("create_dir") + + def delete_dir(self, path): + self._record("delete_dir") + + def delete_dir_contents(self, path, missing_dir_ok=False): + self._record("delete_dir_contents") + + def delete_root_dir_contents(self): + self._record("delete_root_dir_contents") + + def delete_file(self, path): + self._record("delete_file") + + def move(self, src, dest): + self._record("move") + + def copy_file(self, src, dest): + self._record("copy_file") + + def open_input_stream(self, path): + return self._record("open_input_stream", pa.BufferReader(b"")) + + def open_input_file(self, path): + return self._record("open_input_file", pa.BufferReader(b"")) + + def open_output_stream(self, path, metadata): + return self._record("open_output_stream", pa.BufferOutputStream()) + + def open_append_stream(self, path, metadata): + return self._record("open_append_stream", pa.BufferOutputStream()) + + +class _RecordingFileIO(FileIO): + """Records every call; the vector test only checks which FileIO was called.""" + + def __init__(self): + self.calls = [] + self.filesystem = pafs.PyFileSystem(_RecordingHandler(self.calls)) + + def _record(self, name, result=None): + self.calls.append(name) + return result + + def new_input_stream(self, path): + return self._record("new_input_stream", io.BytesIO(b"")) + + def new_output_stream(self, path): + return self._record("new_output_stream", io.BytesIO()) + + def get_file_status(self, path): + return self._record("get_file_status", pafs.FileInfo(path, pafs.FileType.File, size=0)) + + def list_status(self, path): + return self._record("list_status", []) + + def exists(self, path): + return self._record("exists", True) + + def exists_batch(self, paths): + return self._record("exists_batch", {path: True for path in paths}) + + def delete(self, path, recursive=False): + return self._record("delete", True) + + def mkdirs(self, path): + return self._record("mkdirs", True) + + def rename(self, src, dst): + return self._record("rename", True) + + def copy_file(self, source_path, target_path, overwrite=False): + self._record("copy_file") + + def try_to_write_atomic(self, path, content): + return self._record("try_to_write_atomic", True) + + def create_blob_presigned_url(self, table_root, descriptor, validity): + return self._record("create_blob_presigned_url", "https://signed") + + def write_parquet(self, path, data, compression='zstd', zstd_level=1, **kwargs): + self._record("write_parquet") + + def to_filesystem_path(self, path): + return path + + +# The FileIO and filesystem calls of each routing operation; pypaimon has no two-phase writer. +OP_CALLS = { + "read": { + "new_input_stream": lambda f, p: f.new_input_stream(p), + "read_file_range": lambda f, p: f.read_file_range(p, 0, 1), + "fs.open_input_file": lambda f, p: f.filesystem.open_input_file(p), + "fs.open_input_stream": lambda f, p: f.filesystem.open_input_stream(p), + }, + "meta": { + "get_file_status": lambda f, p: f.get_file_status(p), + "get_file_size": lambda f, p: f.get_file_size(p), + "fs.get_file_info": lambda f, p: f.filesystem.get_file_info([p]), + }, + "exists": { + "exists": lambda f, p: f.exists(p), + "exists_batch": lambda f, p: f.exists_batch([p]), + }, + "write": { + "new_output_stream": lambda f, p: f.new_output_stream(p), + "write_parquet": lambda f, p: f.write_parquet(p, None), + "fs.open_output_stream": lambda f, p: f.filesystem.open_output_stream(p), + "fs.open_append_stream": lambda f, p: f.filesystem.open_append_stream(p), + }, + "list": { + "list_status": lambda f, p: f.list_status(p), + "fs.get_file_info_selector": lambda f, p: f.filesystem.get_file_info(pafs.FileSelector(p)), + }, + "delete": { + "delete": lambda f, p: f.delete(p), + "fs.delete_file": lambda f, p: f.filesystem.delete_file(p), + "fs.delete_dir": lambda f, p: f.filesystem.delete_dir(p), + }, + "rename": { + "rename": lambda f, p: f.rename(p, p + "-renamed"), + "fs.move": lambda f, p: f.filesystem.move(p, p + "-renamed"), + }, + "mkdirs": { + "mkdirs": lambda f, p: f.mkdirs(p), + "fs.create_dir": lambda f, p: f.filesystem.create_dir(p), + }, + "copy": { + "copy_file": lambda f, p: f.copy_file(p, p + "-copy"), + "fs.copy_file": lambda f, p: f.filesystem.copy_file(p, p + "-copy"), + }, + "atomic-write": { + "try_to_write_atomic": lambda f, p: f.try_to_write_atomic(p, "{}"), + }, + "two-phase-write": {}, + "presign": { + "create_blob_presigned_url": lambda f, p: f.create_blob_presigned_url( + ROUTING["table_root"], BlobDescriptor(p, 0, 1), timedelta(minutes=1)), + }, +} + +ROUTED_CALLS = [ + (case, name, call) + for case in ROUTING["cases"] + if IoCacheRouting.create(case["options"]) is not None + for name, call in OP_CALLS[case["op"]].items() +] + + +@pytest.mark.parametrize("case,name,call", ROUTED_CALLS, + ids=["{} / {}".format(case["name"], name) for case, name, _ in ROUTED_CALLS]) +def test_routing_vector_through_file_io(case, name, call): + routing = IoCacheRouting.create(case["options"]) + endpoints = dict(routing.targets(), origin=routing.origin_endpoint()) + assert len(set(endpoints.values())) == len(endpoints) + file_ios = {key: _RecordingFileIO() for key in endpoints} + file_io = IoCacheRoutingFileIO(routing, file_ios["origin"], file_ios.__getitem__) + call(file_io, case["path"]) + called = {key for key, recorder in file_ios.items() if recorder.calls} + assert called == {key for key, endpoint in endpoints.items() if endpoint == case["expect"]} + + +if __name__ == "__main__": + unittest.main() diff --git a/paimon-python/pypaimon/tests/io_cache_routing_test.py b/paimon-python/pypaimon/tests/io_cache_routing_test.py new file mode 100644 index 000000000000..14811b07c25d --- /dev/null +++ b/paimon-python/pypaimon/tests/io_cache_routing_test.py @@ -0,0 +1,245 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Tests for the io-cache routing decision, including the shared routing cases.""" + +import json +import os +import unittest + +import pytest + +from pypaimon.filesystem.io_cache_routing import (IoCacheRouting, Op, + data_prefixes, routable_type) +from pypaimon.utils.file_type import FileType + +_RESOURCES = os.path.join(os.path.dirname(__file__), "resources", "io_cache") + +TYPE_NAMES = { + FileType.META: "meta", + FileType.DATA: "data", + FileType.BUCKET_INDEX: "bucket-index", + FileType.GLOBAL_INDEX: "global-index", + FileType.FILE_INDEX: "file-index", + None: None, +} + + +def load_vectors(name): + with open(os.path.join(_RESOURCES, name), encoding="utf-8") as f: + return json.load(f) + + +ROUTING = load_vectors("routing.json") +ROUTABLE_TYPES = load_vectors("routable-types.json") + + +def endpoint(options, op, path): + """The endpoint of one request as a client sees it: override, old endpoint, origin or a target.""" + if options.get("dlf.oss-endpoint"): + return options["dlf.oss-endpoint"] + routing = IoCacheRouting.create(options) + if routing is None: + return options.get("fs.oss.endpoint") + name = routing.route(op, path) + return routing.origin_endpoint() if name is None else routing.targets()[name] + + +@pytest.mark.parametrize("case", ROUTING["cases"], ids=[c["name"] for c in ROUTING["cases"]]) +def test_routing_vector(case): + assert endpoint(case["options"], Op(case["op"]), case["path"]) == case["expect"] + + +@pytest.mark.parametrize("case", ROUTABLE_TYPES["cases"], ids=[c["path"] for c in ROUTABLE_TYPES["cases"]]) +def test_routable_type_vector(case): + assert TYPE_NAMES[routable_type(case["path"], data_prefixes(case["options"]))] == case["type"] + + +class IoCacheRoutingTest(unittest.TestCase): + + TABLE = "oss://bkt/db1.db/t1" + DATA = TABLE + "/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet" + MANIFEST = TABLE + "/manifest/manifest-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0" + + def _options(self, **overrides): + options = { + "fs.oss.endpoint": "http://legacy.example.com", + "io-cache.enabled": "true", + "io-cache.endpoint": "http://cache.example.com", + "io-cache.origin.endpoint": "https://origin.example.com", + "io-cache.policy": "meta,read", + } + options.update(overrides) + return options + + def _multi(self, **overrides): + options = self._options(**{ + "io-cache.targets": "accel,cluster", + "io-cache.target.accel.endpoint": "http://accel.example.com", + "io-cache.target.cluster.endpoint": "http://10.0.0.1:8080", + "io-cache.routes": "meta=accel;data=cluster", + }) + options.update(overrides) + return options + + def test_create(self): + self.assertIsNotNone(IoCacheRouting.create(self._options())) + self.assertIsNotNone(IoCacheRouting.create(self._options(**{"io-cache.enabled": " TRUE "}))) + for overrides in ({"io-cache.enabled": "false"}, {"io-cache.enabled": "yes"}, + {"io-cache.policy": "none"}, {"io-cache.policy": "read,none"}, + {"io-cache.policy": "write,exists"}, {"io-cache.targets": ""}, + {"io-cache.targets": "Bad_Name,1x"}, + {"dlf.oss-endpoint": "oss-cn-hangzhou.aliyuncs.com"}): + with self.subTest(overrides=overrides): + self.assertIsNone(IoCacheRouting.create(self._options(**overrides))) + options = self._options() + del options["io-cache.enabled"] + self.assertIsNone(IoCacheRouting.create(options)) + + def test_policy_routes_only_read_and_meta(self): + routing = IoCacheRouting.create(self._options(**{"io-cache.policy": " Meta , READ ,prefetch,write"})) + for op in Op: + with self.subTest(op=op): + expected = "default" if op in (Op.READ, Op.META, Op.EXISTS) else None + self.assertEqual(expected, routing.route(op, self.DATA)) + read_only = IoCacheRouting.create(self._options(**{"io-cache.policy": "read"})) + self.assertIsNone(read_only.route(Op.META, self.DATA)) + self.assertIsNone(read_only.route(Op.EXISTS, self.DATA)) + + def test_whitelist(self): + index = self.TABLE + "/index/btree-global-index-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b.index" + routing = IoCacheRouting.create(self._options(**{"io-cache.whitelist": "Global-Index,DATA,video"})) + self.assertEqual("default", routing.route(Op.READ, index)) + self.assertEqual("default", routing.route(Op.READ, self.DATA)) + self.assertIsNone(routing.route(Op.READ, self.MANIFEST)) + self.assertIsNone(IoCacheRouting.create(self._options(**{"io-cache.whitelist": ""})).route( + Op.READ, self.DATA)) + + def test_targets_and_routes(self): + routing = IoCacheRouting.create(self._multi(**{ + "io-cache.targets": " Accel ,cluster,1x,accel,c_d,missing,", + "io-cache.routes": " Meta = ACCEL ;; data= ; =x ; video=y ; data ; *=cluster", + })) + self.assertEqual({"accel": "http://accel.example.com", "cluster": "http://10.0.0.1:8080"}, + routing.targets()) + self.assertEqual("accel", routing.route(Op.META, self.MANIFEST)) + self.assertEqual("cluster", routing.route(Op.READ, self.DATA)) + to_missing = IoCacheRouting.create(self._multi(**{"io-cache.routes": "data=missing"})) + self.assertIsNone(to_missing.route(Op.READ, self.DATA)) + + def test_first_target_with_an_endpoint_without_routes(self): + options = self._multi() + del options["io-cache.routes"] + del options["io-cache.target.accel.endpoint"] + routing = IoCacheRouting.create(options) + self.assertEqual({"cluster": "http://10.0.0.1:8080"}, routing.targets()) + self.assertEqual("cluster", routing.route(Op.READ, self.MANIFEST)) + + def test_target_without_endpoint_keeps_routing_on_origin(self): + routing = IoCacheRouting.create(self._options(**{"io-cache.endpoint": ""})) + self.assertEqual({}, routing.targets()) + self.assertIsNone(routing.route(Op.READ, self.DATA)) + self.assertEqual("https://origin.example.com", endpoint( + self._options(**{"io-cache.endpoint": ""}), Op.READ, self.DATA)) + + def test_origin_defaults_to_fs_oss_endpoint(self): + options = self._options() + del options["io-cache.origin.endpoint"] + self.assertEqual("http://legacy.example.com", IoCacheRouting.create(options).origin_endpoint()) + + def test_directory_with_trailing_slash_uses_its_name(self): + routing = IoCacheRouting.create(self._options()) + self.assertIsNone(routing.route(Op.META, self.TABLE + "/tag/tag-1/")) + self.assertEqual("default", routing.route(Op.META, self.MANIFEST + "/")) + self.assertIsNone(routing.route(Op.META, self.TABLE + "/snapshot/snapshot-1/")) + + def test_table_prefixes_add_data_prefixes(self): + path = self.TABLE + "/bucket-0/part-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet" + self.assertIsNone(IoCacheRouting.create(self._options()).route(Op.READ, path)) + for key in ("data-file.prefix", "changelog-file.prefix"): + routing = IoCacheRouting.create(self._options(**{key: "part-"})) + self.assertEqual("default", routing.route(Op.READ, path)) + + def test_origin_options(self): + options = self._options(**{ + "fs.oss.dlf-cache.consistent-hash.enabled": "true", + "fs.jindocache.namespace.rpc.address": "10.0.0.1:8101", + "fs.oss.accessKeyId": "ak", + "fs.oss.retry.count": "5", + }) + origin = IoCacheRouting.create(options).origin_options(options) + self.assertEqual("https://origin.example.com", origin["fs.oss.endpoint"]) + self.assertEqual("true", origin["fs.oss.https.enable"]) + self.assertEqual("ak", origin["fs.oss.accessKeyId"]) + self.assertEqual("5", origin["fs.oss.retry.count"]) + self.assertNotIn("fs.oss.dlf-cache.consistent-hash.enabled", origin) + self.assertNotIn("fs.jindocache.namespace.rpc.address", origin) + self.assertNotIn("fs.oss.second.level.domain.enable", origin) + self.assertEqual("http://legacy.example.com", options["fs.oss.endpoint"]) + + def test_target_options(self): + options = self._multi(**{ + "io-cache.target.accel.endpoint": "HTTP://accel.example.com:8080/", + "io-cache.target.cluster.endpoint": "10.0.0.1:8443", + "io-cache.target.cluster.path-style-access": "true", + "io-cache.target.cluster.region": "cn-shanghai", + "fs.oss.region": "cn-hangzhou", + "fs.oss.https.enable": "true", + "fs.oss.second.level.domain.enable": "true", + "fs.oss.dlf-cache.consistent-hash.enabled": "true", + }) + routing = IoCacheRouting.create(options) + accel = routing.target_options(options, "accel") + self.assertEqual("http://accel.example.com:8080", accel["fs.oss.endpoint"]) + self.assertEqual("false", accel["fs.oss.https.enable"]) + self.assertEqual("false", accel["fs.oss.second.level.domain.enable"]) + self.assertEqual("cn-hangzhou", accel["fs.oss.region"]) + self.assertEqual("true", accel["fs.oss.dlf-cache.consistent-hash.enabled"]) + cluster = routing.target_options(options, "cluster") + self.assertEqual("https://10.0.0.1:8443", cluster["fs.oss.endpoint"]) + self.assertEqual("true", cluster["fs.oss.https.enable"]) + self.assertEqual("true", cluster["fs.oss.second.level.domain.enable"]) + self.assertEqual("cn-shanghai", cluster["fs.oss.region"]) + self.assertEqual("cn-hangzhou", options["fs.oss.region"]) + + def test_blank_region_and_file_prefix_are_ignored(self): + options = self._options(**{ + "io-cache.target.default.region": " ", + "fs.oss.region": "cn-hangzhou", + "data-file.prefix": " ", + }) + routing = IoCacheRouting.create(options) + self.assertEqual("cn-hangzhou", routing.target_options(options, "default")["fs.oss.region"]) + self.assertEqual(("data-", "changelog-"), data_prefixes(options)) + + def test_default_target_options(self): + options = self._options(**{ + "io-cache.endpoint": "cache.example.com", + "io-cache.target.default.path-style-access": "true", + "io-cache.target.default.region": "cn-beijing", + "fs.oss.https.enable": "false", + }) + default = IoCacheRouting.create(options).target_options(options, "default") + self.assertEqual("https://cache.example.com", default["fs.oss.endpoint"]) + self.assertEqual("true", default["fs.oss.https.enable"]) + self.assertEqual("true", default["fs.oss.second.level.domain.enable"]) + self.assertEqual("cn-beijing", default["fs.oss.region"]) + + def test_target_keeps_sdk_settings(self): + options = self._options(**{"fs.oss.retry.count": "5"}) + target = IoCacheRouting.create(options).target_options(options, "default") + self.assertEqual("5", target["fs.oss.retry.count"]) diff --git a/paimon-python/pypaimon/tests/lance_utils_test.py b/paimon-python/pypaimon/tests/lance_utils_test.py index 2ffb220efa46..c66b259a36a6 100644 --- a/paimon-python/pypaimon/tests/lance_utils_test.py +++ b/paimon-python/pypaimon/tests/lance_utils_test.py @@ -20,11 +20,39 @@ from pypaimon.common.file_io import FileIO from pypaimon.common.options import Options from pypaimon.common.options.config import OssOptions +from pypaimon.filesystem.io_cache_file_io import IoCacheRoutingFileIO +from pypaimon.filesystem.io_cache_routing import IoCacheRouting from pypaimon.read.reader.lance_utils import to_lance_specified +from pypaimon.read.reader.vortex_utils import to_vortex_specified class LanceUtilsTest(unittest.TestCase): + def test_native_reads_use_target_addressing_and_writes_use_origin(self): + path = "oss://bkt/t/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-1.lance" + options = { + "fs.oss.endpoint": "https://origin.example.com", + "fs.oss.accessKeyId": "ak", + "fs.oss.accessKeySecret": "sk", + "io-cache.enabled": "true", + "io-cache.policy": "read,meta", + "io-cache.endpoint": "http://cache.example.com:8080", + "io-cache.target.default.path-style-access": "true", + } + routing = IoCacheRouting.create(options) + origin = FileIO.get(path, Options(routing.origin_options(options))) + file_io = IoCacheRoutingFileIO(routing, origin, lambda name: FileIO.get( + path, Options(routing.target_options(options, name)))) + for convert in (to_lance_specified, to_vortex_specified): + with self.subTest(convert=convert.__name__): + _, read = convert(file_io, path, is_read=True) + self.assertEqual("http://cache.example.com:8080", read["endpoint"]) + self.assertEqual("false", read["virtual_hosted_style_request"]) + self.assertEqual("true", read["allow_http"]) + _, write = convert(file_io, path) + self.assertEqual("https://bkt.origin.example.com", write["endpoint"]) + self.assertEqual("true", write["virtual_hosted_style_request"]) + def test_oss_url_bucket_extraction_correctness(self): file_path = "oss://test-bucket/db-name.db/table-name/bucket-0/data.lance" diff --git a/paimon-python/pypaimon/tests/resources/io_cache/routable-types.json b/paimon-python/pypaimon/tests/resources/io_cache/routable-types.json new file mode 100644 index 000000000000..bb780c885329 --- /dev/null +++ b/paimon-python/pypaimon/tests/resources/io_cache/routable-types.json @@ -0,0 +1,69 @@ +{ + "table_root": "oss://bkt/db1.db/t1", + "cases": [ + {"path": "oss://bkt/db1.db/t1/snapshot/snapshot-12", "options": {}, "type": null, "reason": "snapshot: sequential metadata may be read before it exists"}, + {"path": "oss://bkt/db1.db/t1/snapshot/LATEST", "options": {}, "type": null, "reason": "hint file, rewritten in place"}, + {"path": "oss://bkt/db1.db/t1/snapshot/EARLIEST", "options": {}, "type": null, "reason": "hint file, rewritten in place"}, + {"path": "oss://bkt/db1.db/t1/branch/branch-dev/snapshot/LATEST", "options": {}, "type": null, "reason": "hint file on a branch"}, + {"path": "oss://bkt/db1.db/t1/schema/schema-3", "options": {}, "type": null, "reason": "schema: named by sequence id"}, + {"path": "oss://bkt/db1.db/t1/manifest/manifest-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0", "options": {}, "type": "meta", "reason": "manifest"}, + {"path": "oss://bkt/db1.db/t1/manifest/manifest-list-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-1", "options": {}, "type": "meta", "reason": "manifest list"}, + {"path": "oss://bkt/db1.db/t1/manifest/index-manifest-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0", "options": {}, "type": "meta", "reason": "index manifest; 'manifest' wins over 'index-'"}, + {"path": "oss://bkt/db1.db/t1/manifest/manifest-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.avro.sidecar", "options": {}, "type": "meta", "reason": "manifest sidecar"}, + {"path": "oss://bkt/db1.db/t1/statistics/stat-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0", "options": {}, "type": "meta", "reason": "statistics"}, + {"path": "oss://bkt/db1.db/t1/tag/tag-2026-09-30", "options": {}, "type": null, "reason": "tags can be replaced"}, + {"path": "oss://bkt/db1.db/t1/tag/tag-success-file/t1_SUCCESS", "options": {}, "type": null, "reason": "tag success file"}, + {"path": "oss://bkt/db1.db/t1/consumer/consumer-job1", "options": {}, "type": null, "reason": "consumer progress, rewritten"}, + {"path": "oss://bkt/db1.db/t1/service/service-primary-key-lookup", "options": {}, "type": null, "reason": "service file, rewritten"}, + {"path": "oss://bkt/db1.db/t1/dt=2026-09-30/_SUCCESS", "options": {}, "type": null, "reason": "partition mark-done file"}, + {"path": "oss://bkt/db1.db/t1/changelog/changelog-5", "options": {}, "type": null, "reason": "decoupled changelog metadata, named by sequence id"}, + {"path": "oss://bkt/db1.db/t1/changelog/LATEST", "options": {}, "type": null, "reason": "changelog hint file"}, + {"path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "options": {}, "type": "data", "reason": "data file"}, + {"path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/changelog-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "options": {}, "type": "data", "reason": "changelog data file in a bucket"}, + {"path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.blob", "options": {}, "type": "data", "reason": "blob file"}, + {"path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet.index", "options": {}, "type": "file-index", "reason": "data-file index sidecar"}, + {"path": "oss://bkt/db1.db/t1/index/index-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0", "options": {}, "type": "bucket-index", "reason": "hash index or deletion vectors"}, + {"path": "oss://bkt/db1.db/t1/index/btree-global-index-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b.index", "options": {}, "type": "global-index", "reason": "global btree index"}, + {"path": "oss://bkt/db1.db/t1/index/lumina-global-index-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b.index", "options": {}, "type": "global-index", "reason": "global vector index"}, + {"path": "oss://bkt/db1.db/t1/snapshot/.snapshot-13.8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b.tmp", "options": {}, "type": null, "reason": "temp file of an atomic write"}, + {"path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/.data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet.8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b.tmp", "options": {}, "type": null, "reason": "temp data file"}, + {"path": "oss://bkt/db1.db/t1/metadata/version-hint.text", "options": {}, "type": null, "reason": "Iceberg-compatible metadata hint"}, + {"path": "oss://bkt/db1.db/t1/metadata/v3.metadata.json", "options": {}, "type": null, "reason": "Iceberg-compatible metadata, not a Paimon name"}, + {"path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/000000_0", "options": {}, "type": null, "reason": "migrated Hive file: unknown name"}, + {"path": "oss://bkt/db1.db/t1/dt=2026-09-30/part-00000-1a2b.snappy.parquet", "options": {}, "type": null, "reason": "format-table style name: unknown"}, + {"path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/custom-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.orc", "options": {}, "type": null, "reason": "custom data-file.prefix without the option"}, + {"path": "oss://bkt/db1.db/t1/README", "options": {}, "type": null, "reason": "unknown name"}, + {"path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/manifest-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.orc", "options": {"data-file.prefix": "manifest-"}, "type": "data", "reason": "a data file named like a manifest by data-file.prefix is data"}, + {"path": "oss://bkt/db1.db/t1/manifest/manifest-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0", "options": {"data-file.prefix": "manifest-"}, "type": "meta", "reason": "manifests stay meta with data-file.prefix manifest-"}, + {"path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/index-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.orc", "options": {"data-file.prefix": "index-"}, "type": "data", "reason": "a data file named like a bucket index by data-file.prefix is data"}, + {"path": "oss://bkt/db1.db/t1/index/index-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0", "options": {"data-file.prefix": "index-"}, "type": "bucket-index", "reason": "bucket indexes stay bucket-index with data-file.prefix index-"}, + {"path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/stat-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.orc", "options": {"data-file.prefix": "stat-"}, "type": "data", "reason": "a data file named like statistics by data-file.prefix is data"}, + {"path": "oss://bkt/db1.db/t1/statistics/stat-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0", "options": {"data-file.prefix": "stat-"}, "type": "meta", "reason": "statistics stay meta with data-file.prefix stat-"}, + {"path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/manifest-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.orc.index", "options": {"data-file.prefix": "manifest-"}, "type": "file-index", "reason": "the file index of such a data file stays file-index"}, + {"path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/ unknown.parquet", "options": {"data-file.prefix": " "}, "type": null, "reason": "a whitespace-only prefix does not make unknown files cacheable"}, + {"path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/custom-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.orc", "options": {"data-file.prefix": "custom-"}, "type": "data", "reason": "custom data-file.prefix from the merged options"}, + {"path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/cl-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.orc", "options": {"changelog-file.prefix": "cl-"}, "type": "data", "reason": "custom changelog-file.prefix from the merged options"}, + {"path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.orc", "options": {"data-file.prefix": "custom-"}, "type": "data", "reason": "default prefixes stay routable next to a custom one"}, + {"path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/7f3a/manifest-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.orc", "options": {"data-file.prefix": "manifest-"}, "type": "data", "reason": "an external data path may put an entropy directory between the bucket and the file"}, + {"path": "oss://bkt/db1.db/t1/bucket-postpone/index-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.orc", "options": {"data-file.prefix": "index-"}, "type": "data", "reason": "data file of a postpone bucket"}, + {"path": "oss://bkt/db1.db/t1/postpone/stat-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.orc", "options": {"data-file.prefix": "stat-"}, "type": "data", "reason": "data file of a legacy postpone bucket"}, + {"path": "oss://bkt/db1.db/t1/dt=2026-09-30/8b1f7c2e/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "options": {}, "type": "data", "reason": "the name decides, not the directory"}, + {"path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/snapshot-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.orc", "options": {"data-file.prefix": "snapshot-"}, "type": "data", "reason": "snapshots are named by sequence id, so a uuid name with this prefix is data"}, + {"path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/schema-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.orc", "options": {"data-file.prefix": "schema-"}, "type": "data", "reason": "schemas are named by sequence id, so a uuid name with this prefix is data"}, + {"path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/snapshot-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.orc.index", "options": {"data-file.prefix": "snapshot-"}, "type": "file-index", "reason": "file index of a data file named like a snapshot"}, + {"path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/global-index-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.orc.index", "options": {"data-file.prefix": "global-index-"}, "type": "file-index", "reason": "file index of a data file named like a global index"}, + {"path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/tag-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.orc", "options": {"data-file.prefix": "tag-"}, "type": null, "reason": "names rewritten in place never use a cache, whatever the prefix"}, + {"path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/consumer-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.orc", "options": {"data-file.prefix": "consumer-"}, "type": null, "reason": "names rewritten in place never use a cache, whatever the prefix"}, + {"path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/index-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0", "options": {"data-file.prefix": "index-"}, "type": "bucket-index", "reason": "no extension: a bucket index, not a data file"}, + {"path": "oss://bkt/db1.db/t1/manifest/manifest-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.avro.sidecar", "options": {"data-file.prefix": "manifest-"}, "type": "meta", "reason": "a manifest sidecar is metadata whatever the data prefix"}, + {"path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet.tmp-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b", "options": {}, "type": null, "reason": "temp name with a .tmp- suffix"}, + {"path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet.tmp.8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b", "options": {}, "type": null, "reason": "temp name with a .tmp. infix"}, + {"path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-1.parquet", "options": {}, "type": null, "reason": "not a name Paimon writers produce"}, + {"path": "oss://bkt/bucket-0/db/t/changelog/changelog-5", "options": {}, "type": null, "reason": "decoupled changelog metadata below a bucket-* warehouse directory"}, + {"path": "oss://bkt/warehouse/bucket-0/db/t1/tag/tag-file", "options": {"data-file.prefix": "tag-"}, "type": null, "reason": "tag metadata with a matching data prefix"}, + {"path": "oss://bkt/warehouse/bucket-0/db/t1/consumer/consumer-file", "options": {"data-file.prefix": "consumer-"}, "type": null, "reason": "consumer metadata with a matching data prefix"}, + {"path": "oss://bkt/warehouse/bucket-0/db/t1/service/service-file", "options": {"data-file.prefix": "service-"}, "type": null, "reason": "service metadata with a matching data prefix"}, + {"path": "oss://bkt/warehouse/bucket-0/db/t1/metadata/version-hint.text", "options": {"data-file.prefix": "version-"}, "type": null, "reason": "Iceberg metadata hint with a matching data prefix"}, + {"path": "oss://bkt/warehouse/bucket-0/db/t1/branch/branch-feature", "options": {"data-file.prefix": "branch-"}, "type": null, "reason": "branch directory with a matching data prefix"} + ] +} diff --git a/paimon-python/pypaimon/tests/resources/io_cache/routing.json b/paimon-python/pypaimon/tests/resources/io_cache/routing.json new file mode 100644 index 000000000000..0614fff87ee9 --- /dev/null +++ b/paimon-python/pypaimon/tests/resources/io_cache/routing.json @@ -0,0 +1,90 @@ +{ + "table_root": "oss://bkt/db1.db/t1", + "cases": [ + {"name": "multi: a data file named like a manifest goes to the data target", "options": {"fs.oss.endpoint": "https://accelerator.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.targets": "accel,cluster", "io-cache.target.accel.endpoint": "https://accelerator.example.com", "io-cache.target.accel.region": "cn-hangzhou", "io-cache.target.cluster.endpoint": "http://cluster.example.com", "io-cache.target.cluster.path-style-access": "true", "io-cache.policy": "meta,read", "io-cache.whitelist": "*", "io-cache.routes": "meta=accel;data,bucket-index,global-index,file-index=cluster", "data-file.prefix": "manifest-"}, "op": "read", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/manifest-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.orc", "expect": "http://cluster.example.com"}, + {"name": "multi: manifests keep the meta target with data-file.prefix manifest-", "options": {"fs.oss.endpoint": "https://accelerator.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.targets": "accel,cluster", "io-cache.target.accel.endpoint": "https://accelerator.example.com", "io-cache.target.accel.region": "cn-hangzhou", "io-cache.target.cluster.endpoint": "http://cluster.example.com", "io-cache.target.cluster.path-style-access": "true", "io-cache.policy": "meta,read", "io-cache.whitelist": "*", "io-cache.routes": "meta=accel;data,bucket-index,global-index,file-index=cluster", "data-file.prefix": "manifest-"}, "op": "read", "path": "oss://bkt/db1.db/t1/manifest/manifest-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0", "expect": "https://accelerator.example.com"}, + {"name": "multi: a data file named like a bucket index goes to the data target", "options": {"fs.oss.endpoint": "https://accelerator.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.targets": "accel,cluster", "io-cache.target.accel.endpoint": "https://accelerator.example.com", "io-cache.target.accel.region": "cn-hangzhou", "io-cache.target.cluster.endpoint": "http://cluster.example.com", "io-cache.target.cluster.path-style-access": "true", "io-cache.policy": "meta,read", "io-cache.whitelist": "*", "io-cache.routes": "data=cluster;bucket-index=accel", "data-file.prefix": "index-"}, "op": "read", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/index-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.orc", "expect": "http://cluster.example.com"}, + {"name": "multi: bucket indexes keep their target with data-file.prefix index-", "options": {"fs.oss.endpoint": "https://accelerator.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.targets": "accel,cluster", "io-cache.target.accel.endpoint": "https://accelerator.example.com", "io-cache.target.accel.region": "cn-hangzhou", "io-cache.target.cluster.endpoint": "http://cluster.example.com", "io-cache.target.cluster.path-style-access": "true", "io-cache.policy": "meta,read", "io-cache.whitelist": "*", "io-cache.routes": "data=cluster;bucket-index=accel", "data-file.prefix": "index-"}, "op": "read", "path": "oss://bkt/db1.db/t1/index/index-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0", "expect": "https://accelerator.example.com"}, + {"name": "multi: a data file named like a snapshot goes to the data target", "options": {"fs.oss.endpoint": "https://accelerator.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.targets": "accel,cluster", "io-cache.target.accel.endpoint": "https://accelerator.example.com", "io-cache.target.accel.region": "cn-hangzhou", "io-cache.target.cluster.endpoint": "http://cluster.example.com", "io-cache.target.cluster.path-style-access": "true", "io-cache.policy": "meta,read", "io-cache.whitelist": "*", "io-cache.routes": "meta=accel;data,bucket-index,global-index,file-index=cluster", "data-file.prefix": "snapshot-"}, "op": "read", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/snapshot-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.orc", "expect": "http://cluster.example.com"}, + {"name": "multi: a data file below an entropy directory goes to the data target", "options": {"fs.oss.endpoint": "https://accelerator.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.targets": "accel,cluster", "io-cache.target.accel.endpoint": "https://accelerator.example.com", "io-cache.target.accel.region": "cn-hangzhou", "io-cache.target.cluster.endpoint": "http://cluster.example.com", "io-cache.target.cluster.path-style-access": "true", "io-cache.policy": "meta,read", "io-cache.whitelist": "*", "io-cache.routes": "meta=accel;data,bucket-index,global-index,file-index=cluster", "data-file.prefix": "manifest-"}, "op": "read", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/7f3a/manifest-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.orc", "expect": "http://cluster.example.com"}, + {"name": "multi: a manifest sidecar keeps the meta target with data-file.prefix manifest-", "options": {"fs.oss.endpoint": "https://accelerator.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.targets": "accel,cluster", "io-cache.target.accel.endpoint": "https://accelerator.example.com", "io-cache.target.accel.region": "cn-hangzhou", "io-cache.target.cluster.endpoint": "http://cluster.example.com", "io-cache.target.cluster.path-style-access": "true", "io-cache.policy": "meta,read", "io-cache.whitelist": "*", "io-cache.routes": "meta=accel;data,bucket-index,global-index,file-index=cluster", "data-file.prefix": "manifest-"}, "op": "read", "path": "oss://bkt/db1.db/t1/manifest/manifest-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.avro.sidecar", "expect": "https://accelerator.example.com"}, + {"name": "multi: a data file named like statistics goes to the data target", "options": {"fs.oss.endpoint": "https://accelerator.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.targets": "accel,cluster", "io-cache.target.accel.endpoint": "https://accelerator.example.com", "io-cache.target.accel.region": "cn-hangzhou", "io-cache.target.cluster.endpoint": "http://cluster.example.com", "io-cache.target.cluster.path-style-access": "true", "io-cache.policy": "meta,read", "io-cache.whitelist": "*", "io-cache.routes": "meta=accel;data,bucket-index,global-index,file-index=cluster", "data-file.prefix": "stat-"}, "op": "read", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/stat-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.orc", "expect": "http://cluster.example.com"}, + {"name": "single: a dls path is not routed", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": "http://cache.example.com", "io-cache.policy": "meta,read", "io-cache.whitelist": "meta,data"}, "op": "read", "path": "dls://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "expect": "https://oss-cn-hangzhou-internal.aliyuncs.com"}, + {"name": "policy: trimmed case-insensitive tokens", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": "http://cache.example.com", "io-cache.policy": " READ , Meta ", "io-cache.whitelist": "meta,data"}, "op": "meta", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "expect": "http://cache.example.com"}, + {"name": "policy: none overrides read", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": "http://cache.example.com", "io-cache.policy": "read,NONE", "io-cache.whitelist": "meta,data"}, "op": "exists", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "expect": "http://cache.example.com"}, + {"name": "policy: substrings are not operations", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": "http://cache.example.com", "io-cache.policy": "thread,metadata", "io-cache.whitelist": "meta,data"}, "op": "exists", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "expect": "http://cache.example.com"}, + {"name": "policy: unknown tokens do not disable read", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": "http://cache.example.com", "io-cache.policy": "read,nonetheless", "io-cache.whitelist": "meta,data"}, "op": "exists", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "expect": "https://oss-cn-hangzhou-internal.aliyuncs.com"}, + {"name": "single: whitespace endpoint is missing", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": " ", "io-cache.policy": "meta,read", "io-cache.whitelist": "meta,data"}, "op": "read", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "expect": "https://oss-cn-hangzhou-internal.aliyuncs.com"}, + {"name": "multi: whitespace endpoint is skipped", "options": {"fs.oss.endpoint": "https://accelerator.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.targets": "accel,cluster", "io-cache.target.accel.endpoint": " ", "io-cache.target.accel.region": "cn-hangzhou", "io-cache.target.cluster.endpoint": "http://cluster.example.com", "io-cache.target.cluster.path-style-access": "true", "io-cache.policy": "meta,read", "io-cache.whitelist": "*"}, "op": "read", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "expect": "http://cluster.example.com"}, + {"name": "multi: target endpoint is trimmed", "options": {"fs.oss.endpoint": "https://accelerator.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.targets": "accel,cluster", "io-cache.target.accel.endpoint": "https://accelerator.example.com", "io-cache.target.accel.region": "cn-hangzhou", "io-cache.target.cluster.endpoint": " http://cluster.example.com ", "io-cache.target.cluster.path-style-access": "true", "io-cache.policy": "meta,read", "io-cache.whitelist": "*", "io-cache.routes": "meta=accel;data,bucket-index,global-index,file-index=cluster"}, "op": "read", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "expect": "http://cluster.example.com"}, + {"name": "single: whitespace origin uses legacy endpoint", "options": {"fs.oss.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": " ", "io-cache.endpoint": "http://cache.example.com", "io-cache.policy": "meta,read", "io-cache.whitelist": "meta,data"}, "op": "list", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "expect": "https://oss-cn-hangzhou-internal.aliyuncs.com"}, + {"name": "single: data read goes to the cache", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": "http://cache.example.com", "io-cache.policy": "meta,read", "io-cache.whitelist": "meta,data"}, "op": "read", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "expect": "http://cache.example.com"}, + {"name": "single: data status goes to the cache", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": "http://cache.example.com", "io-cache.policy": "meta,read", "io-cache.whitelist": "meta,data"}, "op": "meta", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "expect": "http://cache.example.com"}, + {"name": "single: manifest read goes to the cache", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": "http://cache.example.com", "io-cache.policy": "meta,read", "io-cache.whitelist": "meta,data"}, "op": "read", "path": "oss://bkt/db1.db/t1/manifest/manifest-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0", "expect": "http://cache.example.com"}, + {"name": "single: manifest list read goes to the cache", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": "http://cache.example.com", "io-cache.policy": "meta,read", "io-cache.whitelist": "meta,data"}, "op": "read", "path": "oss://bkt/db1.db/t1/manifest/manifest-list-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-1", "expect": "http://cache.example.com"}, + {"name": "single: snapshot read stays at origin", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": "http://cache.example.com", "io-cache.policy": "meta,read", "io-cache.whitelist": "meta,data"}, "op": "read", "path": "oss://bkt/db1.db/t1/snapshot/snapshot-12", "expect": "https://oss-cn-hangzhou-internal.aliyuncs.com"}, + {"name": "single: snapshot status stays at origin", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": "http://cache.example.com", "io-cache.policy": "meta,read", "io-cache.whitelist": "meta,data"}, "op": "meta", "path": "oss://bkt/db1.db/t1/snapshot/snapshot-12", "expect": "https://oss-cn-hangzhou-internal.aliyuncs.com"}, + {"name": "single: hint file stays at origin", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": "http://cache.example.com", "io-cache.policy": "meta,read", "io-cache.whitelist": "meta,data"}, "op": "read", "path": "oss://bkt/db1.db/t1/snapshot/LATEST", "expect": "https://oss-cn-hangzhou-internal.aliyuncs.com"}, + {"name": "single: index type outside the whitelist stays at origin", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": "http://cache.example.com", "io-cache.policy": "meta,read", "io-cache.whitelist": "meta,data"}, "op": "read", "path": "oss://bkt/db1.db/t1/index/index-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0", "expect": "https://oss-cn-hangzhou-internal.aliyuncs.com"}, + {"name": "single: unknown name stays at origin", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": "http://cache.example.com", "io-cache.policy": "meta,read", "io-cache.whitelist": "meta,data"}, "op": "read", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/000000_0", "expect": "https://oss-cn-hangzhou-internal.aliyuncs.com"}, + {"name": "single: temp file stays at origin", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": "http://cache.example.com", "io-cache.policy": "meta,read", "io-cache.whitelist": "meta,data"}, "op": "read", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/.data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet.8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b.tmp", "expect": "https://oss-cn-hangzhou-internal.aliyuncs.com"}, + {"name": "single: a path in another bucket is routed like the table's", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": "http://cache.example.com", "io-cache.policy": "meta,read", "io-cache.whitelist": "meta,data"}, "op": "read", "path": "oss://other-bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "expect": "http://cache.example.com"}, + {"name": "single: disabled by default", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": "http://cache.example.com", "io-cache.policy": "meta,read", "io-cache.whitelist": "meta,data"}, "op": "read", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "expect": "http://cache.example.com"}, + {"name": "single: disabled keeps fs.oss.endpoint for everything", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.enabled": "false", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": "http://cache.example.com", "io-cache.policy": "meta,read", "io-cache.whitelist": "meta,data"}, "op": "exists", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "expect": "http://cache.example.com"}, + {"name": "single: no policy means no routing", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": "http://cache.example.com", "io-cache.whitelist": "meta,data"}, "op": "exists", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "expect": "http://cache.example.com"}, + {"name": "single: policy none means no routing", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": "http://cache.example.com", "io-cache.policy": "none", "io-cache.whitelist": "meta,data"}, "op": "exists", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "expect": "http://cache.example.com"}, + {"name": "single: write-only policy means no routing", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": "http://cache.example.com", "io-cache.policy": "write", "io-cache.whitelist": "meta,data"}, "op": "exists", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "expect": "http://cache.example.com"}, + {"name": "single: read-only policy keeps status at origin", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": "http://cache.example.com", "io-cache.policy": "read", "io-cache.whitelist": "meta,data"}, "op": "meta", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "expect": "https://oss-cn-hangzhou-internal.aliyuncs.com"}, + {"name": "single: read-only policy still routes reads", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": "http://cache.example.com", "io-cache.policy": "read", "io-cache.whitelist": "meta,data"}, "op": "read", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "expect": "http://cache.example.com"}, + {"name": "single: origin defaults to fs.oss.endpoint", "options": {"fs.oss.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.enabled": "true", "io-cache.endpoint": "http://cache.example.com", "io-cache.policy": "meta,read", "io-cache.whitelist": "meta,data"}, "op": "list", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "expect": "https://oss-cn-hangzhou-internal.aliyuncs.com"}, + {"name": "single: client endpoint override wins", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": "http://cache.example.com", "io-cache.policy": "meta,read", "io-cache.whitelist": "meta,data", "dlf.oss-endpoint": "https://oss-cn-hangzhou.aliyuncs.com"}, "op": "read", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "expect": "https://oss-cn-hangzhou.aliyuncs.com"}, + {"name": "single: absent whitelist allows every routable type", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": "http://cache.example.com", "io-cache.policy": "meta,read"}, "op": "read", "path": "oss://bkt/db1.db/t1/index/index-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0", "expect": "http://cache.example.com"}, + {"name": "single: star whitelist allows every routable type", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": "http://cache.example.com", "io-cache.policy": "meta,read", "io-cache.whitelist": "*"}, "op": "read", "path": "oss://bkt/db1.db/t1/index/btree-global-index-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b.index", "expect": "http://cache.example.com"}, + {"name": "single: file index sidecar with star whitelist", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": "http://cache.example.com", "io-cache.policy": "meta,read", "io-cache.whitelist": "*"}, "op": "read", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet.index", "expect": "http://cache.example.com"}, + {"name": "single: custom data prefix without the option stays at origin", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": "http://cache.example.com", "io-cache.policy": "meta,read", "io-cache.whitelist": "meta,data"}, "op": "read", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/custom-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.orc", "expect": "https://oss-cn-hangzhou-internal.aliyuncs.com"}, + {"name": "single: custom data prefix from options is routed", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": "http://cache.example.com", "io-cache.policy": "meta,read", "io-cache.whitelist": "meta,data", "data-file.prefix": "custom-"}, "op": "read", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/custom-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.orc", "expect": "http://cache.example.com"}, + {"name": "single: empty cache endpoint sends routed reads to origin", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": "", "io-cache.policy": "meta,read", "io-cache.whitelist": "meta,data", "io-cache.routes": "*=default"}, "op": "read", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "expect": "https://oss-cn-hangzhou-internal.aliyuncs.com"}, + {"name": "multi: manifest read goes to the accelerator", "options": {"fs.oss.endpoint": "https://accelerator.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.targets": "accel,cluster", "io-cache.target.accel.endpoint": "https://accelerator.example.com", "io-cache.target.accel.region": "cn-hangzhou", "io-cache.target.cluster.endpoint": "http://cluster.example.com", "io-cache.target.cluster.path-style-access": "true", "io-cache.policy": "meta,read", "io-cache.whitelist": "*", "io-cache.routes": "meta=accel;data,bucket-index,global-index,file-index=cluster"}, "op": "read", "path": "oss://bkt/db1.db/t1/manifest/manifest-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0", "expect": "https://accelerator.example.com"}, + {"name": "multi: manifest status goes to the accelerator", "options": {"fs.oss.endpoint": "https://accelerator.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.targets": "accel,cluster", "io-cache.target.accel.endpoint": "https://accelerator.example.com", "io-cache.target.accel.region": "cn-hangzhou", "io-cache.target.cluster.endpoint": "http://cluster.example.com", "io-cache.target.cluster.path-style-access": "true", "io-cache.policy": "meta,read", "io-cache.whitelist": "*", "io-cache.routes": "meta=accel;data,bucket-index,global-index,file-index=cluster"}, "op": "meta", "path": "oss://bkt/db1.db/t1/manifest/manifest-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0", "expect": "https://accelerator.example.com"}, + {"name": "multi: data read goes to the cluster", "options": {"fs.oss.endpoint": "https://accelerator.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.targets": "accel,cluster", "io-cache.target.accel.endpoint": "https://accelerator.example.com", "io-cache.target.accel.region": "cn-hangzhou", "io-cache.target.cluster.endpoint": "http://cluster.example.com", "io-cache.target.cluster.path-style-access": "true", "io-cache.policy": "meta,read", "io-cache.whitelist": "*", "io-cache.routes": "meta=accel;data,bucket-index,global-index,file-index=cluster"}, "op": "read", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "expect": "http://cluster.example.com"}, + {"name": "multi: deletion vectors go to the cluster", "options": {"fs.oss.endpoint": "https://accelerator.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.targets": "accel,cluster", "io-cache.target.accel.endpoint": "https://accelerator.example.com", "io-cache.target.accel.region": "cn-hangzhou", "io-cache.target.cluster.endpoint": "http://cluster.example.com", "io-cache.target.cluster.path-style-access": "true", "io-cache.policy": "meta,read", "io-cache.whitelist": "*", "io-cache.routes": "meta=accel;data,bucket-index,global-index,file-index=cluster"}, "op": "read", "path": "oss://bkt/db1.db/t1/index/index-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0", "expect": "http://cluster.example.com"}, + {"name": "multi: global index goes to the cluster", "options": {"fs.oss.endpoint": "https://accelerator.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.targets": "accel,cluster", "io-cache.target.accel.endpoint": "https://accelerator.example.com", "io-cache.target.accel.region": "cn-hangzhou", "io-cache.target.cluster.endpoint": "http://cluster.example.com", "io-cache.target.cluster.path-style-access": "true", "io-cache.policy": "meta,read", "io-cache.whitelist": "*", "io-cache.routes": "meta=accel;data,bucket-index,global-index,file-index=cluster"}, "op": "read", "path": "oss://bkt/db1.db/t1/index/btree-global-index-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b.index", "expect": "http://cluster.example.com"}, + {"name": "multi: manifest existence goes to the accelerator", "options": {"fs.oss.endpoint": "https://accelerator.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.targets": "accel,cluster", "io-cache.target.accel.endpoint": "https://accelerator.example.com", "io-cache.target.accel.region": "cn-hangzhou", "io-cache.target.cluster.endpoint": "http://cluster.example.com", "io-cache.target.cluster.path-style-access": "true", "io-cache.policy": "meta,read", "io-cache.whitelist": "*", "io-cache.routes": "meta=accel;data,bucket-index,global-index,file-index=cluster"}, "op": "exists", "path": "oss://bkt/db1.db/t1/manifest/manifest-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0", "expect": "https://accelerator.example.com"}, + {"name": "multi: data existence goes to the cluster", "options": {"fs.oss.endpoint": "https://accelerator.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.targets": "accel,cluster", "io-cache.target.accel.endpoint": "https://accelerator.example.com", "io-cache.target.accel.region": "cn-hangzhou", "io-cache.target.cluster.endpoint": "http://cluster.example.com", "io-cache.target.cluster.path-style-access": "true", "io-cache.policy": "meta,read", "io-cache.whitelist": "*", "io-cache.routes": "meta=accel;data,bucket-index,global-index,file-index=cluster"}, "op": "exists", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "expect": "http://cluster.example.com"}, + {"name": "single: data existence goes to the cache", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": "http://cache.example.com", "io-cache.policy": "meta,read", "io-cache.whitelist": "meta,data"}, "op": "exists", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "expect": "http://cache.example.com"}, + {"name": "single: snapshot existence stays at origin", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": "http://cache.example.com", "io-cache.policy": "meta,read", "io-cache.whitelist": "meta,data"}, "op": "exists", "path": "oss://bkt/db1.db/t1/snapshot/snapshot-12", "expect": "https://oss-cn-hangzhou-internal.aliyuncs.com"}, + {"name": "single: hint file existence stays at origin", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": "http://cache.example.com", "io-cache.policy": "meta,read", "io-cache.whitelist": "meta,data"}, "op": "exists", "path": "oss://bkt/db1.db/t1/snapshot/LATEST", "expect": "https://oss-cn-hangzhou-internal.aliyuncs.com"}, + {"name": "single: read-only policy keeps existence at origin", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": "http://cache.example.com", "io-cache.policy": "read", "io-cache.whitelist": "meta,data"}, "op": "exists", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "expect": "https://oss-cn-hangzhou-internal.aliyuncs.com"}, + {"name": "multi: snapshot stays at origin", "options": {"fs.oss.endpoint": "https://accelerator.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.targets": "accel,cluster", "io-cache.target.accel.endpoint": "https://accelerator.example.com", "io-cache.target.accel.region": "cn-hangzhou", "io-cache.target.cluster.endpoint": "http://cluster.example.com", "io-cache.target.cluster.path-style-access": "true", "io-cache.policy": "meta,read", "io-cache.whitelist": "*", "io-cache.routes": "meta=accel;data,bucket-index,global-index,file-index=cluster"}, "op": "read", "path": "oss://bkt/db1.db/t1/snapshot/snapshot-12", "expect": "https://oss-cn-hangzhou-internal.aliyuncs.com"}, + {"name": "multi: without routes the first usable target takes all", "options": {"fs.oss.endpoint": "https://accelerator.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.targets": "accel,cluster", "io-cache.target.accel.endpoint": "https://accelerator.example.com", "io-cache.target.accel.region": "cn-hangzhou", "io-cache.target.cluster.endpoint": "http://cluster.example.com", "io-cache.target.cluster.path-style-access": "true", "io-cache.policy": "meta,read", "io-cache.whitelist": "*"}, "op": "read", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "expect": "https://accelerator.example.com"}, + {"name": "multi: route to an undeclared target stays at origin", "options": {"fs.oss.endpoint": "https://accelerator.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.targets": "accel,cluster", "io-cache.target.accel.endpoint": "https://accelerator.example.com", "io-cache.target.accel.region": "cn-hangzhou", "io-cache.target.cluster.endpoint": "http://cluster.example.com", "io-cache.target.cluster.path-style-access": "true", "io-cache.policy": "meta,read", "io-cache.whitelist": "*", "io-cache.routes": "data=x"}, "op": "read", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "expect": "https://oss-cn-hangzhou-internal.aliyuncs.com"}, + {"name": "multi: type without a matching route stays at origin", "options": {"fs.oss.endpoint": "https://accelerator.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.targets": "accel,cluster", "io-cache.target.accel.endpoint": "https://accelerator.example.com", "io-cache.target.accel.region": "cn-hangzhou", "io-cache.target.cluster.endpoint": "http://cluster.example.com", "io-cache.target.cluster.path-style-access": "true", "io-cache.policy": "meta,read", "io-cache.whitelist": "*", "io-cache.routes": "meta=accel;data=cluster"}, "op": "read", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet.index", "expect": "https://oss-cn-hangzhou-internal.aliyuncs.com"}, + {"name": "multi: targets win over the single-target shorthand", "options": {"fs.oss.endpoint": "https://accelerator.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.targets": "accel,cluster", "io-cache.target.accel.endpoint": "https://accelerator.example.com", "io-cache.target.accel.region": "cn-hangzhou", "io-cache.target.cluster.endpoint": "http://cluster.example.com", "io-cache.target.cluster.path-style-access": "true", "io-cache.policy": "meta,read", "io-cache.whitelist": "*", "io-cache.endpoint": "http://cache.example.com"}, "op": "read", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "expect": "https://accelerator.example.com"}, + {"name": "multi: target without endpoint is skipped when routes are absent", "options": {"fs.oss.endpoint": "https://accelerator.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.targets": "accel,cluster", "io-cache.target.accel.region": "cn-hangzhou", "io-cache.target.cluster.endpoint": "http://cluster.example.com", "io-cache.target.cluster.path-style-access": "true", "io-cache.policy": "meta,read", "io-cache.whitelist": "*"}, "op": "read", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "expect": "http://cluster.example.com"}, + {"name": "multi: route to a target without endpoint stays at origin", "options": {"fs.oss.endpoint": "https://accelerator.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.targets": "accel,cluster", "io-cache.target.accel.endpoint": "https://accelerator.example.com", "io-cache.target.accel.region": "cn-hangzhou", "io-cache.target.cluster.path-style-access": "true", "io-cache.policy": "meta,read", "io-cache.whitelist": "*", "io-cache.routes": "meta=accel;data,bucket-index,global-index,file-index=cluster"}, "op": "read", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "expect": "https://oss-cn-hangzhou-internal.aliyuncs.com"}, + {"name": "multi: invalid target names are ignored", "options": {"fs.oss.endpoint": "https://accelerator.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.targets": "Bad_Name,cluster", "io-cache.target.accel.endpoint": "https://accelerator.example.com", "io-cache.target.accel.region": "cn-hangzhou", "io-cache.target.cluster.endpoint": "http://cluster.example.com", "io-cache.target.cluster.path-style-access": "true", "io-cache.policy": "meta,read", "io-cache.whitelist": "*"}, "op": "read", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "expect": "http://cluster.example.com"}, + {"name": "multi: star route sends every type to one target", "options": {"fs.oss.endpoint": "https://accelerator.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.targets": "accel,cluster", "io-cache.target.accel.endpoint": "https://accelerator.example.com", "io-cache.target.accel.region": "cn-hangzhou", "io-cache.target.cluster.endpoint": "http://cluster.example.com", "io-cache.target.cluster.path-style-access": "true", "io-cache.policy": "meta,read", "io-cache.whitelist": "*", "io-cache.routes": "*=cluster"}, "op": "read", "path": "oss://bkt/db1.db/t1/manifest/manifest-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0", "expect": "http://cluster.example.com"}, + {"name": "multi: first matching route wins", "options": {"fs.oss.endpoint": "https://accelerator.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.targets": "accel,cluster", "io-cache.target.accel.endpoint": "https://accelerator.example.com", "io-cache.target.accel.region": "cn-hangzhou", "io-cache.target.cluster.endpoint": "http://cluster.example.com", "io-cache.target.cluster.path-style-access": "true", "io-cache.policy": "meta,read", "io-cache.whitelist": "*", "io-cache.routes": "data=accel;data,meta=cluster"}, "op": "read", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "expect": "https://accelerator.example.com"}, + {"name": "multi: malformed routes are skipped", "options": {"fs.oss.endpoint": "https://accelerator.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.targets": "accel,cluster", "io-cache.target.accel.endpoint": "https://accelerator.example.com", "io-cache.target.accel.region": "cn-hangzhou", "io-cache.target.cluster.endpoint": "http://cluster.example.com", "io-cache.target.cluster.path-style-access": "true", "io-cache.policy": "meta,read", "io-cache.whitelist": "*", "io-cache.routes": "=cluster;meta=accel;junk"}, "op": "read", "path": "oss://bkt/db1.db/t1/manifest/manifest-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0", "expect": "https://accelerator.example.com"}, + {"name": "multi: malformed routes do not match other types", "options": {"fs.oss.endpoint": "https://accelerator.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.targets": "accel,cluster", "io-cache.target.accel.endpoint": "https://accelerator.example.com", "io-cache.target.accel.region": "cn-hangzhou", "io-cache.target.cluster.endpoint": "http://cluster.example.com", "io-cache.target.cluster.path-style-access": "true", "io-cache.policy": "meta,read", "io-cache.whitelist": "*", "io-cache.routes": "=cluster;meta=accel;junk"}, "op": "read", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "expect": "https://oss-cn-hangzhou-internal.aliyuncs.com"}, + {"name": "multi: whitelist filters before routes", "options": {"fs.oss.endpoint": "https://accelerator.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.targets": "accel,cluster", "io-cache.target.accel.endpoint": "https://accelerator.example.com", "io-cache.target.accel.region": "cn-hangzhou", "io-cache.target.cluster.endpoint": "http://cluster.example.com", "io-cache.target.cluster.path-style-access": "true", "io-cache.policy": "meta,read", "io-cache.whitelist": "meta", "io-cache.routes": "meta=accel;data,bucket-index,global-index,file-index=cluster"}, "op": "read", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "expect": "https://oss-cn-hangzhou-internal.aliyuncs.com"}, + {"name": "single: write stays at origin", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": "http://cache.example.com", "io-cache.policy": "meta,read", "io-cache.whitelist": "meta,data"}, "op": "write", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "expect": "https://oss-cn-hangzhou-internal.aliyuncs.com"}, + {"name": "multi: write stays at origin", "options": {"fs.oss.endpoint": "https://accelerator.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.targets": "accel,cluster", "io-cache.target.accel.endpoint": "https://accelerator.example.com", "io-cache.target.accel.region": "cn-hangzhou", "io-cache.target.cluster.endpoint": "http://cluster.example.com", "io-cache.target.cluster.path-style-access": "true", "io-cache.policy": "meta,read", "io-cache.whitelist": "*", "io-cache.routes": "meta=accel;data,bucket-index,global-index,file-index=cluster"}, "op": "write", "path": "oss://bkt/db1.db/t1/manifest/manifest-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0", "expect": "https://oss-cn-hangzhou-internal.aliyuncs.com"}, + {"name": "single: list stays at origin", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": "http://cache.example.com", "io-cache.policy": "meta,read", "io-cache.whitelist": "meta,data"}, "op": "list", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "expect": "https://oss-cn-hangzhou-internal.aliyuncs.com"}, + {"name": "multi: list stays at origin", "options": {"fs.oss.endpoint": "https://accelerator.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.targets": "accel,cluster", "io-cache.target.accel.endpoint": "https://accelerator.example.com", "io-cache.target.accel.region": "cn-hangzhou", "io-cache.target.cluster.endpoint": "http://cluster.example.com", "io-cache.target.cluster.path-style-access": "true", "io-cache.policy": "meta,read", "io-cache.whitelist": "*", "io-cache.routes": "meta=accel;data,bucket-index,global-index,file-index=cluster"}, "op": "list", "path": "oss://bkt/db1.db/t1/manifest/manifest-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0", "expect": "https://oss-cn-hangzhou-internal.aliyuncs.com"}, + {"name": "single: delete stays at origin", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": "http://cache.example.com", "io-cache.policy": "meta,read", "io-cache.whitelist": "meta,data"}, "op": "delete", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "expect": "https://oss-cn-hangzhou-internal.aliyuncs.com"}, + {"name": "multi: delete stays at origin", "options": {"fs.oss.endpoint": "https://accelerator.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.targets": "accel,cluster", "io-cache.target.accel.endpoint": "https://accelerator.example.com", "io-cache.target.accel.region": "cn-hangzhou", "io-cache.target.cluster.endpoint": "http://cluster.example.com", "io-cache.target.cluster.path-style-access": "true", "io-cache.policy": "meta,read", "io-cache.whitelist": "*", "io-cache.routes": "meta=accel;data,bucket-index,global-index,file-index=cluster"}, "op": "delete", "path": "oss://bkt/db1.db/t1/manifest/manifest-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0", "expect": "https://oss-cn-hangzhou-internal.aliyuncs.com"}, + {"name": "single: rename stays at origin", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": "http://cache.example.com", "io-cache.policy": "meta,read", "io-cache.whitelist": "meta,data"}, "op": "rename", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "expect": "https://oss-cn-hangzhou-internal.aliyuncs.com"}, + {"name": "multi: rename stays at origin", "options": {"fs.oss.endpoint": "https://accelerator.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.targets": "accel,cluster", "io-cache.target.accel.endpoint": "https://accelerator.example.com", "io-cache.target.accel.region": "cn-hangzhou", "io-cache.target.cluster.endpoint": "http://cluster.example.com", "io-cache.target.cluster.path-style-access": "true", "io-cache.policy": "meta,read", "io-cache.whitelist": "*", "io-cache.routes": "meta=accel;data,bucket-index,global-index,file-index=cluster"}, "op": "rename", "path": "oss://bkt/db1.db/t1/manifest/manifest-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0", "expect": "https://oss-cn-hangzhou-internal.aliyuncs.com"}, + {"name": "single: mkdirs stays at origin", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": "http://cache.example.com", "io-cache.policy": "meta,read", "io-cache.whitelist": "meta,data"}, "op": "mkdirs", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "expect": "https://oss-cn-hangzhou-internal.aliyuncs.com"}, + {"name": "multi: mkdirs stays at origin", "options": {"fs.oss.endpoint": "https://accelerator.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.targets": "accel,cluster", "io-cache.target.accel.endpoint": "https://accelerator.example.com", "io-cache.target.accel.region": "cn-hangzhou", "io-cache.target.cluster.endpoint": "http://cluster.example.com", "io-cache.target.cluster.path-style-access": "true", "io-cache.policy": "meta,read", "io-cache.whitelist": "*", "io-cache.routes": "meta=accel;data,bucket-index,global-index,file-index=cluster"}, "op": "mkdirs", "path": "oss://bkt/db1.db/t1/manifest/manifest-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0", "expect": "https://oss-cn-hangzhou-internal.aliyuncs.com"}, + {"name": "single: copy stays at origin", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": "http://cache.example.com", "io-cache.policy": "meta,read", "io-cache.whitelist": "meta,data"}, "op": "copy", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "expect": "https://oss-cn-hangzhou-internal.aliyuncs.com"}, + {"name": "multi: copy stays at origin", "options": {"fs.oss.endpoint": "https://accelerator.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.targets": "accel,cluster", "io-cache.target.accel.endpoint": "https://accelerator.example.com", "io-cache.target.accel.region": "cn-hangzhou", "io-cache.target.cluster.endpoint": "http://cluster.example.com", "io-cache.target.cluster.path-style-access": "true", "io-cache.policy": "meta,read", "io-cache.whitelist": "*", "io-cache.routes": "meta=accel;data,bucket-index,global-index,file-index=cluster"}, "op": "copy", "path": "oss://bkt/db1.db/t1/manifest/manifest-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0", "expect": "https://oss-cn-hangzhou-internal.aliyuncs.com"}, + {"name": "single: atomic-write stays at origin", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": "http://cache.example.com", "io-cache.policy": "meta,read", "io-cache.whitelist": "meta,data"}, "op": "atomic-write", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "expect": "https://oss-cn-hangzhou-internal.aliyuncs.com"}, + {"name": "multi: atomic-write stays at origin", "options": {"fs.oss.endpoint": "https://accelerator.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.targets": "accel,cluster", "io-cache.target.accel.endpoint": "https://accelerator.example.com", "io-cache.target.accel.region": "cn-hangzhou", "io-cache.target.cluster.endpoint": "http://cluster.example.com", "io-cache.target.cluster.path-style-access": "true", "io-cache.policy": "meta,read", "io-cache.whitelist": "*", "io-cache.routes": "meta=accel;data,bucket-index,global-index,file-index=cluster"}, "op": "atomic-write", "path": "oss://bkt/db1.db/t1/manifest/manifest-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0", "expect": "https://oss-cn-hangzhou-internal.aliyuncs.com"}, + {"name": "single: two-phase-write stays at origin", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": "http://cache.example.com", "io-cache.policy": "meta,read", "io-cache.whitelist": "meta,data"}, "op": "two-phase-write", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "expect": "https://oss-cn-hangzhou-internal.aliyuncs.com"}, + {"name": "multi: two-phase-write stays at origin", "options": {"fs.oss.endpoint": "https://accelerator.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.targets": "accel,cluster", "io-cache.target.accel.endpoint": "https://accelerator.example.com", "io-cache.target.accel.region": "cn-hangzhou", "io-cache.target.cluster.endpoint": "http://cluster.example.com", "io-cache.target.cluster.path-style-access": "true", "io-cache.policy": "meta,read", "io-cache.whitelist": "*", "io-cache.routes": "meta=accel;data,bucket-index,global-index,file-index=cluster"}, "op": "two-phase-write", "path": "oss://bkt/db1.db/t1/manifest/manifest-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0", "expect": "https://oss-cn-hangzhou-internal.aliyuncs.com"}, + {"name": "single: presign stays at origin", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": "http://cache.example.com", "io-cache.policy": "meta,read", "io-cache.whitelist": "meta,data"}, "op": "presign", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "expect": "https://oss-cn-hangzhou-internal.aliyuncs.com"}, + {"name": "multi: presign stays at origin", "options": {"fs.oss.endpoint": "https://accelerator.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.targets": "accel,cluster", "io-cache.target.accel.endpoint": "https://accelerator.example.com", "io-cache.target.accel.region": "cn-hangzhou", "io-cache.target.cluster.endpoint": "http://cluster.example.com", "io-cache.target.cluster.path-style-access": "true", "io-cache.policy": "meta,read", "io-cache.whitelist": "*", "io-cache.routes": "meta=accel;data,bucket-index,global-index,file-index=cluster"}, "op": "presign", "path": "oss://bkt/db1.db/t1/manifest/manifest-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0", "expect": "https://oss-cn-hangzhou-internal.aliyuncs.com"} + ] +} diff --git a/paimon-python/pypaimon/tests/rest/rest_token_file_io_test.py b/paimon-python/pypaimon/tests/rest/rest_token_file_io_test.py index 4e94b21d728e..ed82c59e756f 100644 --- a/paimon-python/pypaimon/tests/rest/rest_token_file_io_test.py +++ b/paimon-python/pypaimon/tests/rest/rest_token_file_io_test.py @@ -29,6 +29,8 @@ from pypaimon.common.identifier import Identifier from pypaimon.common.options import Options from pypaimon.common.options.config import CatalogOptions, OssOptions +from pypaimon.filesystem.io_cache_file_io import IoCacheRoutingFileIO +from pypaimon.filesystem.io_cache_routing import Op from pypaimon.filesystem.local_file_io import LocalFileIO from pypaimon.table.row.blob import BlobDescriptor @@ -640,6 +642,112 @@ def test_refresh_token_keeps_normal_identifier(self): called_identifier = mock_api.load_table_token.call_args[0][0] self.assertEqual(called_identifier.get_object_name(), "my_table") + _TABLE = "oss://bkt/db1.db/t1" + _DATA = _TABLE + "/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet" + _IO_CACHE_TOKEN = { + OssOptions.OSS_ENDPOINT.key(): "http://cache.example.com", + "io-cache.enabled": "true", + "io-cache.endpoint": "http://cache.example.com", + "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", + "io-cache.policy": "meta,read", + "fs.oss.dlf-cache.consistent-hash.enabled": "true", + } + + def _resolve_file_io(self, token, catalog_options=None, path=_TABLE, targets=()): + """Returns the FileIO built for the token and the options of every FileIO.get call, + after creating the given targets.""" + file_io = RESTTokenFileIO(self.identifier, path, Options(catalog_options or {})) + file_io.token = RESTToken(token, int(time.time() * 1000) + 7_200_000) + with patch.object(RESTTokenFileIO, 'try_to_refresh_token'), \ + patch('pypaimon.catalog.rest.rest_token_file_io.FileIO.get') as get_io: + get_io.side_effect = lambda location, options: MagicMock(name=location) + resolved = file_io.file_io() + for target in targets: + resolved._file_io(target) + for call in get_io.call_args_list: + self.assertEqual(path, call[0][0]) + return resolved, [call[0][1].to_map() for call in get_io.call_args_list] + + def test_io_cache_routing_builds_origin_and_target_file_io(self): + resolved, created = self._resolve_file_io(self._IO_CACHE_TOKEN) + self.assertIsInstance(resolved, IoCacheRoutingFileIO) + self.assertEqual(1, len(created)) + origin = created[0] + self.assertEqual("https://oss-cn-hangzhou-internal.aliyuncs.com", origin[OssOptions.OSS_ENDPOINT.key()]) + self.assertEqual("true", origin["fs.oss.https.enable"]) + self.assertNotIn("fs.oss.dlf-cache.consistent-hash.enabled", origin) + self.assertEqual("default", resolved._routing.route(Op.READ, self._DATA)) + + _, (_, target) = self._resolve_file_io(self._IO_CACHE_TOKEN, targets=["default"]) + self.assertEqual("http://cache.example.com", target[OssOptions.OSS_ENDPOINT.key()]) + self.assertEqual("false", target["fs.oss.https.enable"]) + self.assertEqual("true", target["fs.oss.dlf-cache.consistent-hash.enabled"]) + + def test_io_cache_routing_builds_one_file_io_per_target(self): + token = dict(self._IO_CACHE_TOKEN, **{ + "io-cache.targets": "accel,cluster", + "io-cache.target.accel.endpoint": "https://accel.example.com", + "io-cache.target.cluster.endpoint": "http://10.0.0.1:8080", + "io-cache.target.cluster.path-style-access": "true", + "io-cache.target.cluster.region": "cn-shanghai", + "io-cache.routes": "meta=accel;data=cluster", + OssOptions.OSS_REGION.key(): "cn-hangzhou", + }) + resolved, (_, accel, cluster) = self._resolve_file_io(token, targets=["accel", "cluster"]) + self.assertEqual("cluster", resolved._routing.route(Op.READ, self._DATA)) + self.assertEqual(("https://accel.example.com", "false", "cn-hangzhou"), ( + accel[OssOptions.OSS_ENDPOINT.key()], accel["fs.oss.second.level.domain.enable"], + accel[OssOptions.OSS_REGION.key()])) + self.assertEqual(("http://10.0.0.1:8080", "true", "cn-shanghai"), ( + cluster[OssOptions.OSS_ENDPOINT.key()], cluster["fs.oss.second.level.domain.enable"], + cluster[OssOptions.OSS_REGION.key()])) + + def test_io_cache_routing_off_without_target(self): + token = dict(self._IO_CACHE_TOKEN) + del token["io-cache.endpoint"] + resolved, created = self._resolve_file_io(token) + self.assertNotIsInstance(resolved, IoCacheRoutingFileIO) + self.assertEqual([token], [{key: options[key] for key in token} for options in created]) + + def test_io_cache_routing_off_without_enabled(self): + token = dict(self._IO_CACHE_TOKEN) + del token["io-cache.enabled"] + resolved, created = self._resolve_file_io(token) + self.assertNotIsInstance(resolved, IoCacheRoutingFileIO) + self.assertEqual(["http://cache.example.com"], [options[OssOptions.OSS_ENDPOINT.key()] for options in created]) + + def test_catalog_io_cache_enabled_wins_over_token(self): + enabled = CatalogOptions.IO_CACHE_ENABLED.key() + resolved, created = self._resolve_file_io(self._IO_CACHE_TOKEN, {enabled: "False"}) + self.assertNotIsInstance(resolved, IoCacheRoutingFileIO) + self.assertEqual("false", created[0][enabled]) + + token = dict(self._IO_CACHE_TOKEN) + del token[enabled] + resolved, _ = self._resolve_file_io(token, {enabled: "TRUE"}) + self.assertIsInstance(resolved, IoCacheRoutingFileIO) + + def test_merge_token_applies_catalog_io_cache_enabled(self): + enabled = CatalogOptions.IO_CACHE_ENABLED.key() + token = {enabled: "true", "io-cache.endpoint": "http://cache.example.com"} + for catalog_options, expected in (({}, "true"), ({enabled: False}, "false"), ({enabled: "TRUE"}, "true")): + with self.subTest(catalog_options=catalog_options): + file_io = RESTTokenFileIO(self.identifier, self._TABLE, Options(catalog_options)) + self.assertEqual(expected, file_io._merge_token_with_catalog_options(token)[enabled]) + self.assertEqual("true", token[enabled]) + + def test_io_cache_routing_needs_an_oss_table(self): + resolved, created = self._resolve_file_io(self._IO_CACHE_TOKEN, path=self.warehouse_path) + self.assertNotIsInstance(resolved, IoCacheRoutingFileIO) + self.assertEqual(1, len(created)) + + def test_dlf_oss_endpoint_turns_io_cache_routing_off(self): + override = "oss-cn-hangzhou.aliyuncs.com" + resolved, created = self._resolve_file_io( + self._IO_CACHE_TOKEN, {CatalogOptions.DLF_OSS_ENDPOINT.key(): override}) + self.assertNotIsInstance(resolved, IoCacheRoutingFileIO) + self.assertEqual([override], [options[OssOptions.OSS_ENDPOINT.key()] for options in created]) + if __name__ == '__main__': unittest.main() From 7cbcd0d23c451b51d4c91d94ce5386bbb2002f37 Mon Sep 17 00:00:00 2001 From: Sun Dapeng Date: Tue, 6 Oct 2026 18:28:39 +0800 Subject: [PATCH 02/27] [python] Run write and delete helpers of the routing FileIO on origin IoCacheRoutingFileIO inherited write_file, delete_quietly and the other FileIO helpers that check existence before a write or after a delete. Since exists() of a cacheable file now goes to a cache, write_file's check could reach a cache before the file existed. These helpers now delegate to the origin FileIO, and a test fails when a new FileIO method is neither overridden nor listed as a plain read. --- .../pypaimon/filesystem/io_cache_file_io.py | 22 +++++++++++ .../pypaimon/tests/io_cache_file_io_test.py | 37 +++++++++++++++++++ 2 files changed, 59 insertions(+) diff --git a/paimon-python/pypaimon/filesystem/io_cache_file_io.py b/paimon-python/pypaimon/filesystem/io_cache_file_io.py index e9c6dbe91718..c24f86404b53 100644 --- a/paimon-python/pypaimon/filesystem/io_cache_file_io.py +++ b/paimon-python/pypaimon/filesystem/io_cache_file_io.py @@ -128,6 +128,28 @@ def copy_file(self, source_path: str, target_path: str, overwrite: bool = False) def try_to_write_atomic(self, path: str, content: str) -> bool: return self._origin.try_to_write_atomic(path, content) + # Helpers that write, delete or check right after a delete run on origin, probes included. + def write_file(self, path: str, content: str, overwrite: bool = False): + return self._origin.write_file(path, content, overwrite) + + def overwrite_file_utf8(self, path: str, content: str): + return self._origin.overwrite_file_utf8(path, content) + + def delete_quietly(self, path: str): + return self._origin.delete_quietly(path) + + def delete_files_quietly(self, files: List[str]): + return self._origin.delete_files_quietly(files) + + def delete_directory_quietly(self, directory: str): + return self._origin.delete_directory_quietly(directory) + + def check_or_mkdirs(self, path: str): + return self._origin.check_or_mkdirs(path) + + def copy_files(self, source_directory: str, target_directory: str, overwrite: bool = False): + return self._origin.copy_files(source_directory, target_directory, overwrite) + def create_blob_presigned_url(self, table_root, descriptor, validity) -> str: return self._origin.create_blob_presigned_url(table_root, descriptor, validity) diff --git a/paimon-python/pypaimon/tests/io_cache_file_io_test.py b/paimon-python/pypaimon/tests/io_cache_file_io_test.py index eb1787a8e014..1efb1983da6c 100644 --- a/paimon-python/pypaimon/tests/io_cache_file_io_test.py +++ b/paimon-python/pypaimon/tests/io_cache_file_io_test.py @@ -17,7 +17,9 @@ """Tests for IoCacheRoutingFileIO and its pyarrow filesystem.""" +import contextlib import gzip +import inspect import io import os import shutil @@ -575,6 +577,41 @@ def to_filesystem_path(self, path): }, } +# Inherited FileIO methods that only read through new_input_stream, or do no I/O. +INHERITED_READS = { + "read_blobs_concurrent", "read_file_range", "read_file_utf8", "read_overwritten_file_utf8", + "read_ranges_coalesced", "read_ranges_coalesced_views", +} +NO_IO = {"get", "parse_location"} + + +def test_every_file_io_method_is_classified(): + """A FileIO method the routing FileIO neither overrides nor lists here could reach a cache.""" + public = {name for name, value in inspect.getmembers(FileIO) + if not name.startswith("_") and callable(value)} + overridden = {name for name in public if name in IoCacheRoutingFileIO.__dict__} + assert public - overridden - INHERITED_READS - NO_IO == set() + assert not overridden & INHERITED_READS + + +@pytest.mark.parametrize("call", [ + lambda f: f.write_file(DATA, "x"), + lambda f: f.overwrite_file_utf8(DATA, "x"), + lambda f: f.delete_quietly(DATA), + lambda f: f.delete_files_quietly([DATA]), + lambda f: f.check_or_mkdirs(PARTITION), + lambda f: f.copy_files(PARTITION, PARTITION + "-copy"), +]) +def test_write_and_delete_helpers_use_origin(call): + routing = _routing() + file_ios = {key: _RecordingFileIO() for key in ("origin", "accel", "cluster")} + # the recorder says every path exists, so the existence checks may raise + with contextlib.suppress(FileExistsError, ValueError): + call(IoCacheRoutingFileIO(routing, file_ios["origin"], file_ios.__getitem__)) + assert file_ios["origin"].calls + assert not file_ios["accel"].calls and not file_ios["cluster"].calls + + ROUTED_CALLS = [ (case, name, call) for case in ROUTING["cases"] From 5c3e33b9dc56151b787c156ced6c2c9bd7dfedb3 Mon Sep 17 00:00:00 2001 From: Sun Dapeng Date: Tue, 6 Oct 2026 19:52:24 +0800 Subject: [PATCH 03/27] [python] Route writes of cacheable files when io-cache.policy has write Some cache endpoints, such as an OSS accelerator, also take writes and send them to OSS over their own write path, whose bandwidth adds to the direct one. With write in io-cache.policy, new_output_stream, the format writers (parquet, orc, avro, lance, blob, vortex, row), write_file and the pyarrow output streams of cacheable files use the file type's target. write_file still checks existence on origin first, and atomic writes, deletes, renames, copies, listings and writes of snapshots, hints and temp files stay on origin. A policy with only write routes writes only. --- .../pypaimon/filesystem/io_cache_file_io.py | 41 +++++++++++-------- .../pypaimon/filesystem/io_cache_routing.py | 10 +++-- .../pypaimon/tests/io_cache_file_io_test.py | 27 +++++++++--- .../pypaimon/tests/io_cache_routing_test.py | 10 +++-- .../tests/resources/io_cache/routing.json | 17 +++++++- 5 files changed, 74 insertions(+), 31 deletions(-) diff --git a/paimon-python/pypaimon/filesystem/io_cache_file_io.py b/paimon-python/pypaimon/filesystem/io_cache_file_io.py index c24f86404b53..094c56cac265 100644 --- a/paimon-python/pypaimon/filesystem/io_cache_file_io.py +++ b/paimon-python/pypaimon/filesystem/io_cache_file_io.py @@ -96,7 +96,7 @@ def get_file_size(self, path: str) -> int: return self._call(Op.META, path, lambda io: io.get_file_size(path)) def new_output_stream(self, path: str): - return self._origin.new_output_stream(path) + return self._call(Op.WRITE, path, lambda io: io.new_output_stream(path)) def exists(self, path: str) -> bool: return self._call(Op.EXISTS, path, lambda io: io.exists(path)) @@ -128,12 +128,16 @@ def copy_file(self, source_path: str, target_path: str, overwrite: bool = False) def try_to_write_atomic(self, path: str, content: str) -> bool: return self._origin.try_to_write_atomic(path, content) - # Helpers that write, delete or check right after a delete run on origin, probes included. def write_file(self, path: str, content: str, overwrite: bool = False): - return self._origin.write_file(path, content, overwrite) + # The existence check before a write runs on origin, so no cache sees the file before it exists. + if not overwrite and self._origin.exists(path): + raise FileExistsError(f"File {path} already exists and overwrite=False") + return self._call(Op.WRITE, path, lambda io: io.write_file(path, content, True)) def overwrite_file_utf8(self, path: str, content: str): - return self._origin.overwrite_file_utf8(path, content) + return self._call(Op.WRITE, path, lambda io: io.overwrite_file_utf8(path, content)) + + # Helpers that delete, or check right after a delete, run on origin, probes included. def delete_quietly(self, path: str): return self._origin.delete_quietly(path) @@ -159,27 +163,28 @@ def to_filesystem_path(self, path: str) -> str: def write_parquet(self, path: str, data, compression: str = 'zstd', zstd_level: int = 1, **kwargs): - return self._origin.write_parquet(path, data, compression, zstd_level, **kwargs) + return self._call(Op.WRITE, path, lambda io: io.write_parquet(path, data, compression, zstd_level, **kwargs)) def write_orc(self, path: str, data, compression: str = 'zstd', zstd_level: int = 1, **kwargs): - return self._origin.write_orc(path, data, compression, zstd_level, **kwargs) + return self._call(Op.WRITE, path, lambda io: io.write_orc(path, data, compression, zstd_level, **kwargs)) def write_avro(self, path: str, data, avro_schema=None, compression: str = 'zstd', zstd_level: int = 1, **kwargs): - return self._origin.write_avro(path, data, avro_schema, compression, zstd_level, **kwargs) + return self._call(Op.WRITE, path, lambda io: io.write_avro( + path, data, avro_schema, compression, zstd_level, **kwargs)) def write_lance(self, path: str, data, **kwargs): - return self._origin.write_lance(path, data, **kwargs) + return self._call(Op.WRITE, path, lambda io: io.write_lance(path, data, **kwargs)) def write_blob(self, path: str, data, **kwargs): - return self._origin.write_blob(path, data, **kwargs) + return self._call(Op.WRITE, path, lambda io: io.write_blob(path, data, **kwargs)) def write_vortex(self, path: str, data, **kwargs): - return self._origin.write_vortex(path, data, **kwargs) + return self._call(Op.WRITE, path, lambda io: io.write_vortex(path, data, **kwargs)) def write_row(self, path: str, data, fields=None, zstd_level: int = 1, **kwargs): - return self._origin.write_row(path, data, fields, zstd_level, **kwargs) + return self._call(Op.WRITE, path, lambda io: io.write_row(path, data, fields, zstd_level, **kwargs)) def close(self): with self._targets_lock: @@ -249,14 +254,16 @@ def open_input_stream(self, path: str): lambda io: io.filesystem.open_input_stream(io.to_filesystem_path(path), compression=None)) def open_output_stream(self, path: str, metadata): - origin = self._routing_io._origin - return origin.filesystem.open_output_stream( - origin.to_filesystem_path(path), compression=None, metadata=metadata) + return self._routing_io._call( + Op.WRITE, path, + lambda io: io.filesystem.open_output_stream( + io.to_filesystem_path(path), compression=None, metadata=metadata)) def open_append_stream(self, path: str, metadata): - origin = self._routing_io._origin - return origin.filesystem.open_append_stream( - origin.to_filesystem_path(path), compression=None, metadata=metadata) + return self._routing_io._call( + Op.WRITE, path, + lambda io: io.filesystem.open_append_stream( + io.to_filesystem_path(path), compression=None, metadata=metadata)) def get_file_info_selector(self, selector) -> list: origin = self._routing_io._origin diff --git a/paimon-python/pypaimon/filesystem/io_cache_routing.py b/paimon-python/pypaimon/filesystem/io_cache_routing.py index 9c183a7969cc..a8b38c807182 100644 --- a/paimon-python/pypaimon/filesystem/io_cache_routing.py +++ b/paimon-python/pypaimon/filesystem/io_cache_routing.py @@ -43,7 +43,7 @@ class Op(Enum): - """FileIO operations; only READ, META and EXISTS can go to a target.""" + """FileIO operations; only READ, META, EXISTS, WRITE and TWO_PHASE_WRITE can go to a target.""" READ = "read" META = "meta" EXISTS = "exists" @@ -63,11 +63,11 @@ def _flag(value) -> bool: def _parse_policy(value) -> FrozenSet[Op]: - # read and meta can use a target; write only applies to JindoCache, none turns it off + # read, meta and write can use a target; none turns it off tokens = {token.strip().lower() for token in str(value or "").split(",")} if "none" in tokens: return frozenset() - return frozenset(op for op in (Op.READ, Op.META) if op.value in tokens) + return frozenset(op for op in (Op.READ, Op.META, Op.WRITE) if op.value in tokens) def _parse_routes(value: str) -> List[Tuple[set, str]]: @@ -192,9 +192,11 @@ def origin_endpoint(self) -> Optional[str]: def route(self, op: Op, path: str) -> Optional[str]: """Name of the target of one request, or None when it goes to origin.""" - # An existence check is a file status lookup. + # An existence check is a file status lookup, a two-phase write is a write. if op is Op.EXISTS: op = Op.META + elif op is Op.TWO_PHASE_WRITE: + op = Op.WRITE # Only OSS paths can use cache targets. if op not in self._policy or not path.startswith("oss://"): return None diff --git a/paimon-python/pypaimon/tests/io_cache_file_io_test.py b/paimon-python/pypaimon/tests/io_cache_file_io_test.py index 1efb1983da6c..4d4719b9e158 100644 --- a/paimon-python/pypaimon/tests/io_cache_file_io_test.py +++ b/paimon-python/pypaimon/tests/io_cache_file_io_test.py @@ -254,20 +254,35 @@ def test_listing_and_mutations_use_origin(self): self.assertEqual(b"{}", self._content(self.origin, TABLE + "/snapshot/snapshot-2")) self.assertEqual([], self.created) - def test_writes_use_origin(self): + def test_writes_use_origin_without_write_policy(self): + file_io = self._file_io(_routing(policy="read,meta")) + with file_io.new_output_stream(DATA) as out: + out.write(b"written") + file_io.write_file(MANIFEST, "3") + self.assertEqual(b"written", self._content(self.origin, DATA)) + self.assertEqual(b"3", self._content(self.origin, MANIFEST)) + self.assertEqual([], self.created) + + def test_writes_follow_the_route_with_write_policy(self): table = pa.table({"a": [1, 2]}) file_io = self._file_io(_routing(policy="read,meta,write")) with file_io.new_output_stream(DATA) as out: out.write(b"written") file_io.write_parquet(OTHER_DATA, table) - file_io.overwrite_file_utf8(MANIFEST, "3") + file_io.write_file(MANIFEST, "3") + file_io.write_file(SNAPSHOT, "{}") + self.assertEqual(b"written", self._content(self.cluster, DATA)) + self.assertEqual(table, pq.ParquetFile(str(self.cluster._to_file(OTHER_DATA))).read()) + self.assertEqual(b"3", self._content(self.accel, MANIFEST)) + self.assertEqual(b"{}", self._content(self.origin, SNAPSHOT)) + # write_file checks existence on origin, not on the target + self._put(self.origin, OTHER_DATA, b"origin") + # unknown names are written on origin with file_io.filesystem.open_output_stream(UNKNOWN) as out: out.write(b"streamed") - self.assertEqual(b"written", self._content(self.origin, DATA)) - self.assertEqual(table, pq.ParquetFile(str(self.origin._to_file(OTHER_DATA))).read()) - self.assertEqual(b"3", self._content(self.origin, MANIFEST)) self.assertEqual(b"streamed", self._content(self.origin, UNKNOWN)) - self.assertEqual([], self.created) + with self.assertRaises(FileExistsError): + file_io.write_file(OTHER_DATA, "again") def test_filesystem_reads_follow_the_route(self): for file_io, value in ((self.origin, 1), (self.cluster, 2)): diff --git a/paimon-python/pypaimon/tests/io_cache_routing_test.py b/paimon-python/pypaimon/tests/io_cache_routing_test.py index 14811b07c25d..eb9e30ef109e 100644 --- a/paimon-python/pypaimon/tests/io_cache_routing_test.py +++ b/paimon-python/pypaimon/tests/io_cache_routing_test.py @@ -101,7 +101,7 @@ def test_create(self): self.assertIsNotNone(IoCacheRouting.create(self._options(**{"io-cache.enabled": " TRUE "}))) for overrides in ({"io-cache.enabled": "false"}, {"io-cache.enabled": "yes"}, {"io-cache.policy": "none"}, {"io-cache.policy": "read,none"}, - {"io-cache.policy": "write,exists"}, {"io-cache.targets": ""}, + {"io-cache.policy": "exists,prefetch"}, {"io-cache.targets": ""}, {"io-cache.targets": "Bad_Name,1x"}, {"dlf.oss-endpoint": "oss-cn-hangzhou.aliyuncs.com"}): with self.subTest(overrides=overrides): @@ -110,12 +110,16 @@ def test_create(self): del options["io-cache.enabled"] self.assertIsNone(IoCacheRouting.create(options)) - def test_policy_routes_only_read_and_meta(self): + def test_policy_routes_read_meta_and_write(self): routing = IoCacheRouting.create(self._options(**{"io-cache.policy": " Meta , READ ,prefetch,write"})) + routed = (Op.READ, Op.META, Op.EXISTS, Op.WRITE, Op.TWO_PHASE_WRITE) for op in Op: with self.subTest(op=op): - expected = "default" if op in (Op.READ, Op.META, Op.EXISTS) else None + expected = "default" if op in routed else None self.assertEqual(expected, routing.route(op, self.DATA)) + write_only = IoCacheRouting.create(self._options(**{"io-cache.policy": "write"})) + self.assertEqual("default", write_only.route(Op.WRITE, self.DATA)) + self.assertIsNone(write_only.route(Op.READ, self.DATA)) read_only = IoCacheRouting.create(self._options(**{"io-cache.policy": "read"})) self.assertIsNone(read_only.route(Op.META, self.DATA)) self.assertIsNone(read_only.route(Op.EXISTS, self.DATA)) diff --git a/paimon-python/pypaimon/tests/resources/io_cache/routing.json b/paimon-python/pypaimon/tests/resources/io_cache/routing.json index 0614fff87ee9..afd4b88eb836 100644 --- a/paimon-python/pypaimon/tests/resources/io_cache/routing.json +++ b/paimon-python/pypaimon/tests/resources/io_cache/routing.json @@ -33,7 +33,22 @@ {"name": "single: disabled keeps fs.oss.endpoint for everything", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.enabled": "false", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": "http://cache.example.com", "io-cache.policy": "meta,read", "io-cache.whitelist": "meta,data"}, "op": "exists", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "expect": "http://cache.example.com"}, {"name": "single: no policy means no routing", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": "http://cache.example.com", "io-cache.whitelist": "meta,data"}, "op": "exists", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "expect": "http://cache.example.com"}, {"name": "single: policy none means no routing", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": "http://cache.example.com", "io-cache.policy": "none", "io-cache.whitelist": "meta,data"}, "op": "exists", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "expect": "http://cache.example.com"}, - {"name": "single: write-only policy means no routing", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": "http://cache.example.com", "io-cache.policy": "write", "io-cache.whitelist": "meta,data"}, "op": "exists", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "expect": "http://cache.example.com"}, + {"name": "single: write-only policy keeps status at origin", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": "http://cache.example.com", "io-cache.policy": "write", "io-cache.whitelist": "meta,data"}, "op": "exists", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "expect": "https://oss-cn-hangzhou-internal.aliyuncs.com"}, + {"name": "single: write-only policy routes writes", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": "http://cache.example.com", "io-cache.policy": "write", "io-cache.whitelist": "meta,data"}, "op": "write", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "expect": "http://cache.example.com"}, + {"name": "write: data file write goes to the cache", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": "http://cache.example.com", "io-cache.policy": "meta,read,write", "io-cache.whitelist": "meta,data"}, "op": "write", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "expect": "http://cache.example.com"}, + {"name": "write: two-phase data file write goes to the cache", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": "http://cache.example.com", "io-cache.policy": "meta,read,write", "io-cache.whitelist": "meta,data"}, "op": "two-phase-write", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "expect": "http://cache.example.com"}, + {"name": "write: manifest write goes to the cache", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": "http://cache.example.com", "io-cache.policy": "meta,read,write", "io-cache.whitelist": "meta,data"}, "op": "write", "path": "oss://bkt/db1.db/t1/manifest/manifest-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0", "expect": "http://cache.example.com"}, + {"name": "write: snapshot write stays at origin", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": "http://cache.example.com", "io-cache.policy": "meta,read,write", "io-cache.whitelist": "meta,data"}, "op": "write", "path": "oss://bkt/db1.db/t1/snapshot/snapshot-12", "expect": "https://oss-cn-hangzhou-internal.aliyuncs.com"}, + {"name": "write: hint file write stays at origin", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": "http://cache.example.com", "io-cache.policy": "meta,read,write", "io-cache.whitelist": "meta,data"}, "op": "write", "path": "oss://bkt/db1.db/t1/snapshot/LATEST", "expect": "https://oss-cn-hangzhou-internal.aliyuncs.com"}, + {"name": "write: temp file write stays at origin", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": "http://cache.example.com", "io-cache.policy": "meta,read,write", "io-cache.whitelist": "meta,data"}, "op": "write", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/.data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet.8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b.tmp", "expect": "https://oss-cn-hangzhou-internal.aliyuncs.com"}, + {"name": "write: atomic write stays at origin", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": "http://cache.example.com", "io-cache.policy": "meta,read,write", "io-cache.whitelist": "meta,data"}, "op": "atomic-write", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "expect": "https://oss-cn-hangzhou-internal.aliyuncs.com"}, + {"name": "write: delete stays at origin", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": "http://cache.example.com", "io-cache.policy": "meta,read,write", "io-cache.whitelist": "meta,data"}, "op": "delete", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "expect": "https://oss-cn-hangzhou-internal.aliyuncs.com"}, + {"name": "write: rename stays at origin", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": "http://cache.example.com", "io-cache.policy": "meta,read,write", "io-cache.whitelist": "meta,data"}, "op": "rename", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "expect": "https://oss-cn-hangzhou-internal.aliyuncs.com"}, + {"name": "write: copy stays at origin", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": "http://cache.example.com", "io-cache.policy": "meta,read,write", "io-cache.whitelist": "meta,data"}, "op": "copy", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "expect": "https://oss-cn-hangzhou-internal.aliyuncs.com"}, + {"name": "write: list stays at origin", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": "http://cache.example.com", "io-cache.policy": "meta,read,write", "io-cache.whitelist": "meta,data"}, "op": "list", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "expect": "https://oss-cn-hangzhou-internal.aliyuncs.com"}, + {"name": "write: whitelist filters writes", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": "http://cache.example.com", "io-cache.policy": "meta,read,write", "io-cache.whitelist": "meta"}, "op": "write", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "expect": "https://oss-cn-hangzhou-internal.aliyuncs.com"}, + {"name": "write: data write follows the data route", "options": {"fs.oss.endpoint": "https://accelerator.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.targets": "accel,cluster", "io-cache.target.accel.endpoint": "https://accelerator.example.com", "io-cache.target.accel.region": "cn-hangzhou", "io-cache.target.cluster.endpoint": "http://cluster.example.com", "io-cache.target.cluster.path-style-access": "true", "io-cache.policy": "meta,read,write", "io-cache.whitelist": "*", "io-cache.routes": "meta=accel;data,bucket-index,global-index,file-index=cluster"}, "op": "write", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "expect": "http://cluster.example.com"}, + {"name": "write: manifest write follows the meta route", "options": {"fs.oss.endpoint": "https://accelerator.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.targets": "accel,cluster", "io-cache.target.accel.endpoint": "https://accelerator.example.com", "io-cache.target.accel.region": "cn-hangzhou", "io-cache.target.cluster.endpoint": "http://cluster.example.com", "io-cache.target.cluster.path-style-access": "true", "io-cache.policy": "meta,read,write", "io-cache.whitelist": "*", "io-cache.routes": "meta=accel;data,bucket-index,global-index,file-index=cluster"}, "op": "write", "path": "oss://bkt/db1.db/t1/manifest/manifest-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0", "expect": "https://accelerator.example.com"}, {"name": "single: read-only policy keeps status at origin", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": "http://cache.example.com", "io-cache.policy": "read", "io-cache.whitelist": "meta,data"}, "op": "meta", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "expect": "https://oss-cn-hangzhou-internal.aliyuncs.com"}, {"name": "single: read-only policy still routes reads", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": "http://cache.example.com", "io-cache.policy": "read", "io-cache.whitelist": "meta,data"}, "op": "read", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "expect": "http://cache.example.com"}, {"name": "single: origin defaults to fs.oss.endpoint", "options": {"fs.oss.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.enabled": "true", "io-cache.endpoint": "http://cache.example.com", "io-cache.policy": "meta,read", "io-cache.whitelist": "meta,data"}, "op": "list", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "expect": "https://oss-cn-hangzhou-internal.aliyuncs.com"}, From d92f2ab24ec93ac49b5ea566d6898dca444d3afd Mon Sep 17 00:00:00 2001 From: Sun Dapeng Date: Tue, 6 Oct 2026 23:52:55 +0800 Subject: [PATCH 04/27] [python] Write the io-cache routing test cases inline Replace the JSON test vectors with assertions grouped by topic in io_cache_routing_test.py. --- .../pypaimon/tests/io_cache_file_io_test.py | 38 +-- .../pypaimon/tests/io_cache_routing_test.py | 319 ++++++++++++++++-- .../resources/io_cache/routable-types.json | 69 ---- .../tests/resources/io_cache/routing.json | 105 ------ 4 files changed, 304 insertions(+), 227 deletions(-) delete mode 100644 paimon-python/pypaimon/tests/resources/io_cache/routable-types.json delete mode 100644 paimon-python/pypaimon/tests/resources/io_cache/routing.json diff --git a/paimon-python/pypaimon/tests/io_cache_file_io_test.py b/paimon-python/pypaimon/tests/io_cache_file_io_test.py index 4d4719b9e158..f4138f15c455 100644 --- a/paimon-python/pypaimon/tests/io_cache_file_io_test.py +++ b/paimon-python/pypaimon/tests/io_cache_file_io_test.py @@ -46,7 +46,6 @@ from pypaimon.read.reader.format_pyarrow_reader import FormatPyArrowReader from pypaimon.schema.data_types import AtomicType, DataField from pypaimon.table.row.blob import BlobDescriptor -from pypaimon.tests.io_cache_routing_test import ROUTING from pypaimon.utils.file_type import FileType TABLE = "oss://bkt/db1.db/t1" @@ -485,7 +484,7 @@ def open_append_stream(self, path, metadata): class _RecordingFileIO(FileIO): - """Records every call; the vector test only checks which FileIO was called.""" + """Records every call; the routing tests only check which FileIO was called.""" def __init__(self): self.calls = [] @@ -588,7 +587,7 @@ def to_filesystem_path(self, path): "two-phase-write": {}, "presign": { "create_blob_presigned_url": lambda f, p: f.create_blob_presigned_url( - ROUTING["table_root"], BlobDescriptor(p, 0, 1), timedelta(minutes=1)), + TABLE, BlobDescriptor(p, 0, 1), timedelta(minutes=1)), }, } @@ -627,25 +626,20 @@ def test_write_and_delete_helpers_use_origin(call): assert not file_ios["accel"].calls and not file_ios["cluster"].calls -ROUTED_CALLS = [ - (case, name, call) - for case in ROUTING["cases"] - if IoCacheRouting.create(case["options"]) is not None - for name, call in OP_CALLS[case["op"]].items() -] - - -@pytest.mark.parametrize("case,name,call", ROUTED_CALLS, - ids=["{} / {}".format(case["name"], name) for case, name, _ in ROUTED_CALLS]) -def test_routing_vector_through_file_io(case, name, call): - routing = IoCacheRouting.create(case["options"]) - endpoints = dict(routing.targets(), origin=routing.origin_endpoint()) - assert len(set(endpoints.values())) == len(endpoints) - file_ios = {key: _RecordingFileIO() for key in endpoints} - file_io = IoCacheRoutingFileIO(routing, file_ios["origin"], file_ios.__getitem__) - call(file_io, case["path"]) - called = {key for key, recorder in file_ios.items() if recorder.calls} - assert called == {key for key, endpoint in endpoints.items() if endpoint == case["expect"]} +FILE_IO_CALLS = [(op, name, call) for op, calls in OP_CALLS.items() for name, call in calls.items()] + + +@pytest.mark.parametrize("op,name,call", FILE_IO_CALLS, + ids=["{} / {}".format(op, name) for op, name, _ in FILE_IO_CALLS]) +@pytest.mark.parametrize("policy", ["meta,read", "meta,read,write"]) +def test_each_file_io_call_reaches_its_endpoint(policy, op, name, call): + routed = {"read", "meta", "exists"} | ({"write", "two-phase-write"} if "write" in policy else set()) + routing = _routing(policy=policy) + for path, target in ((DATA, "cluster"), (MANIFEST, "accel"), (SNAPSHOT, "origin"), (UNKNOWN, "origin")): + file_ios = {key: _RecordingFileIO() for key in ("origin", "accel", "cluster")} + call(IoCacheRoutingFileIO(routing, file_ios["origin"], file_ios.__getitem__), path) + called = {key for key, recorder in file_ios.items() if recorder.calls} + assert called == {target if op in routed else "origin"}, path if __name__ == "__main__": diff --git a/paimon-python/pypaimon/tests/io_cache_routing_test.py b/paimon-python/pypaimon/tests/io_cache_routing_test.py index eb9e30ef109e..15a2330038dd 100644 --- a/paimon-python/pypaimon/tests/io_cache_routing_test.py +++ b/paimon-python/pypaimon/tests/io_cache_routing_test.py @@ -15,37 +15,77 @@ # specific language governing permissions and limitations # under the License. -"""Tests for the io-cache routing decision, including the shared routing cases.""" +"""Tests for the io-cache routing decision.""" -import json -import os import unittest -import pytest - from pypaimon.filesystem.io_cache_routing import (IoCacheRouting, Op, data_prefixes, routable_type) from pypaimon.utils.file_type import FileType -_RESOURCES = os.path.join(os.path.dirname(__file__), "resources", "io_cache") - -TYPE_NAMES = { - FileType.META: "meta", - FileType.DATA: "data", - FileType.BUCKET_INDEX: "bucket-index", - FileType.GLOBAL_INDEX: "global-index", - FileType.FILE_INDEX: "file-index", - None: None, -} - - -def load_vectors(name): - with open(os.path.join(_RESOURCES, name), encoding="utf-8") as f: - return json.load(f) - - -ROUTING = load_vectors("routing.json") -ROUTABLE_TYPES = load_vectors("routable-types.json") +UUID = "8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b" +TABLE_ROOT = "oss://bkt/db1.db/t1" + +# paths below are relative to TABLE_ROOT, and {uuid} stands for UUID +DATA_PATH = "dt=1/bucket-0/data-{uuid}-0.parquet" +MANIFEST_PATH = "manifest/manifest-{uuid}-0" +INDEX_PATH = "index/index-{uuid}-0" +GLOBAL_INDEX_PATH = "index/btree-global-index-{uuid}.index" +TEMP_PATH = "dt=1/bucket-0/.data-{uuid}-0.parquet.{uuid}.tmp" +SNAPSHOT_PATH = "snapshot/snapshot-12" +LATEST_PATH = "snapshot/LATEST" + +OSS = "https://oss-cn-hangzhou-internal.aliyuncs.com" +CACHE = "http://cache.example.com" +ACCEL = "https://accelerator.example.com" +CLUSTER = "http://cluster.example.com" +WRITE = "io-cache.policy=meta,read,write" +ORIGIN_OPS = (Op.LIST, Op.DELETE, Op.RENAME, Op.MKDIRS, Op.COPY, Op.ATOMIC_WRITE, Op.PRESIGN) + + +def with_changes(options, changes): + """Applies changes: "key=value" sets an option and "-key" removes it.""" + for change in changes: + if change.startswith("-"): + options.pop(change[1:], None) + else: + key, value = change.split("=", 1) + options[key] = value + return options + + +def single(*changes): + """One cache target, which older clients also get as fs.oss.endpoint.""" + return with_changes({ + "fs.oss.endpoint": CACHE, + "io-cache.enabled": "true", + "io-cache.endpoint": CACHE, + "io-cache.origin.endpoint": OSS, + "io-cache.policy": "meta,read", + "io-cache.whitelist": "meta,data", + }, changes) + + +def multi(*changes): + """Metadata on an accelerator, data and indexes on a cache cluster.""" + return with_changes({ + "fs.oss.endpoint": ACCEL, + "io-cache.enabled": "true", + "io-cache.origin.endpoint": OSS, + "io-cache.targets": "accel,cluster", + "io-cache.target.accel.endpoint": ACCEL, + "io-cache.target.accel.region": "cn-hangzhou", + "io-cache.target.cluster.endpoint": CLUSTER, + "io-cache.target.cluster.path-style-access": "true", + "io-cache.policy": "meta,read", + "io-cache.whitelist": "*", + "io-cache.routes": "meta=accel;data,bucket-index,global-index,file-index=cluster", + }, changes) + + +def resolve(path): + path = path.replace("{uuid}", UUID) + return path if "://" in path else TABLE_ROOT + "/" + path def endpoint(options, op, path): @@ -59,14 +99,231 @@ def endpoint(options, op, path): return routing.origin_endpoint() if name is None else routing.targets()[name] -@pytest.mark.parametrize("case", ROUTING["cases"], ids=[c["name"] for c in ROUTING["cases"]]) -def test_routing_vector(case): - assert endpoint(case["options"], Op(case["op"]), case["path"]) == case["expect"] +class IoCacheRoutingEndpointTest(unittest.TestCase): + + def assert_endpoint(self, options, op, path, expected): + self.assertEqual(expected, endpoint(options, op, resolve(path)), + f"{op.value} {path} with {options}") + + def assert_type(self, path, file_type, *changes): + prefixes = data_prefixes(with_changes({}, changes)) + self.assertEqual(file_type, routable_type(resolve(path), prefixes), f"{path} with {changes}") + + def test_one_cache_target(self): + options = single() + self.assert_endpoint(options, Op.READ, DATA_PATH, CACHE) + self.assert_endpoint(options, Op.META, DATA_PATH, CACHE) + self.assert_endpoint(options, Op.EXISTS, DATA_PATH, CACHE) + self.assert_endpoint(options, Op.READ, MANIFEST_PATH, CACHE) + self.assert_endpoint(options, Op.READ, "manifest/manifest-list-{uuid}-1", CACHE) + self.assert_endpoint(options, Op.READ, "oss://other-bkt/db1.db/t1/" + DATA_PATH, CACHE) + self.assert_endpoint(options, Op.READ, SNAPSHOT_PATH, OSS) + self.assert_endpoint(options, Op.META, SNAPSHOT_PATH, OSS) + self.assert_endpoint(options, Op.EXISTS, SNAPSHOT_PATH, OSS) + self.assert_endpoint(options, Op.READ, LATEST_PATH, OSS) + self.assert_endpoint(options, Op.EXISTS, LATEST_PATH, OSS) + self.assert_endpoint(options, Op.READ, TEMP_PATH, OSS) + self.assert_endpoint(options, Op.READ, "dt=1/bucket-0/000000_0", OSS) + self.assert_endpoint(options, Op.READ, "dls://bkt/db1.db/t1/" + DATA_PATH, OSS) + # the whitelist has no index + self.assert_endpoint(options, Op.READ, INDEX_PATH, OSS) + self.assert_endpoint(options, Op.WRITE, DATA_PATH, OSS) + self.assert_endpoint(options, Op.TWO_PHASE_WRITE, DATA_PATH, OSS) + for op in ORIGIN_OPS: + self.assert_endpoint(options, op, DATA_PATH, OSS) + + def test_policy_and_whitelist(self): + self.assert_endpoint(single("-io-cache.enabled"), Op.READ, DATA_PATH, CACHE) + self.assert_endpoint(single("io-cache.enabled=false"), Op.EXISTS, DATA_PATH, CACHE) + self.assert_endpoint(single("-io-cache.policy"), Op.EXISTS, DATA_PATH, CACHE) + self.assert_endpoint(single("io-cache.policy=none"), Op.EXISTS, DATA_PATH, CACHE) + self.assert_endpoint(single("io-cache.policy=read,NONE"), Op.EXISTS, DATA_PATH, CACHE) + self.assert_endpoint(single("io-cache.policy=thread,metadata"), Op.EXISTS, DATA_PATH, CACHE) + self.assert_endpoint(single("io-cache.policy= READ , Meta "), Op.META, DATA_PATH, CACHE) + self.assert_endpoint(single("io-cache.policy=read,nonetheless"), Op.EXISTS, DATA_PATH, OSS) + + read_only = single("io-cache.policy=read") + self.assert_endpoint(read_only, Op.READ, DATA_PATH, CACHE) + self.assert_endpoint(read_only, Op.META, DATA_PATH, OSS) + self.assert_endpoint(read_only, Op.EXISTS, DATA_PATH, OSS) + write_only = single("io-cache.policy=write") + self.assert_endpoint(write_only, Op.WRITE, DATA_PATH, CACHE) + self.assert_endpoint(write_only, Op.EXISTS, DATA_PATH, OSS) + + self.assert_endpoint(single("-io-cache.whitelist"), Op.READ, INDEX_PATH, CACHE) + self.assert_endpoint(single("io-cache.whitelist=*"), Op.READ, GLOBAL_INDEX_PATH, CACHE) + self.assert_endpoint(single("io-cache.whitelist=*"), Op.READ, DATA_PATH + ".index", CACHE) + + def test_endpoints(self): + self.assert_endpoint(single("io-cache.endpoint= "), Op.READ, DATA_PATH, OSS) + self.assert_endpoint( + single("io-cache.endpoint=", "io-cache.routes=*=default"), Op.READ, DATA_PATH, OSS) + # the origin defaults to fs.oss.endpoint + oss = "fs.oss.endpoint=" + OSS + self.assert_endpoint(single(oss, "-io-cache.origin.endpoint"), Op.LIST, DATA_PATH, OSS) + self.assert_endpoint(single(oss, "io-cache.origin.endpoint= "), Op.LIST, DATA_PATH, OSS) + # a client-side endpoint turns routing off + override = "https://oss-cn-hangzhou.aliyuncs.com" + self.assert_endpoint(single("dlf.oss-endpoint=" + override), Op.READ, DATA_PATH, override) + + def test_write_policy(self): + options = single(WRITE) + self.assert_endpoint(options, Op.WRITE, DATA_PATH, CACHE) + self.assert_endpoint(options, Op.TWO_PHASE_WRITE, DATA_PATH, CACHE) + self.assert_endpoint(options, Op.WRITE, MANIFEST_PATH, CACHE) + self.assert_endpoint(options, Op.WRITE, SNAPSHOT_PATH, OSS) + self.assert_endpoint(options, Op.WRITE, LATEST_PATH, OSS) + self.assert_endpoint(options, Op.WRITE, TEMP_PATH, OSS) + for op in ORIGIN_OPS: + self.assert_endpoint(options, op, DATA_PATH, OSS) + self.assert_endpoint(single(WRITE, "io-cache.whitelist=meta"), Op.WRITE, DATA_PATH, OSS) + + self.assert_endpoint(multi(WRITE), Op.WRITE, DATA_PATH, CLUSTER) + self.assert_endpoint(multi(WRITE), Op.WRITE, MANIFEST_PATH, ACCEL) + + def test_two_cache_targets(self): + options = multi() + self.assert_endpoint(options, Op.READ, MANIFEST_PATH, ACCEL) + self.assert_endpoint(options, Op.META, MANIFEST_PATH, ACCEL) + self.assert_endpoint(options, Op.EXISTS, MANIFEST_PATH, ACCEL) + self.assert_endpoint(options, Op.READ, DATA_PATH, CLUSTER) + self.assert_endpoint(options, Op.EXISTS, DATA_PATH, CLUSTER) + self.assert_endpoint(options, Op.READ, INDEX_PATH, CLUSTER) + self.assert_endpoint(options, Op.READ, GLOBAL_INDEX_PATH, CLUSTER) + self.assert_endpoint(options, Op.READ, SNAPSHOT_PATH, OSS) + self.assert_endpoint(options, Op.WRITE, MANIFEST_PATH, OSS) + self.assert_endpoint(options, Op.TWO_PHASE_WRITE, MANIFEST_PATH, OSS) + for op in ORIGIN_OPS: + self.assert_endpoint(options, op, MANIFEST_PATH, OSS) + self.assert_endpoint(multi("io-cache.whitelist=meta"), Op.READ, DATA_PATH, OSS) - -@pytest.mark.parametrize("case", ROUTABLE_TYPES["cases"], ids=[c["path"] for c in ROUTABLE_TYPES["cases"]]) -def test_routable_type_vector(case): - assert TYPE_NAMES[routable_type(case["path"], data_prefixes(case["options"]))] == case["type"] + def test_targets_and_routes(self): + # without routes the first target with an endpoint takes every type + no_routes = "-io-cache.routes" + self.assert_endpoint(multi(no_routes), Op.READ, DATA_PATH, ACCEL) + self.assert_endpoint(multi(no_routes, "io-cache.endpoint=" + CACHE), Op.READ, DATA_PATH, ACCEL) + self.assert_endpoint( + multi(no_routes, "-io-cache.target.accel.endpoint"), Op.READ, DATA_PATH, CLUSTER) + self.assert_endpoint( + multi(no_routes, "io-cache.target.accel.endpoint= "), Op.READ, DATA_PATH, CLUSTER) + self.assert_endpoint( + multi(no_routes, "io-cache.targets=Bad_Name,cluster"), Op.READ, DATA_PATH, CLUSTER) + self.assert_endpoint( + multi(f"io-cache.target.cluster.endpoint= {CLUSTER} "), Op.READ, DATA_PATH, CLUSTER) + + # a type without a usable route goes to the origin + self.assert_endpoint(multi("io-cache.routes=data=x"), Op.READ, DATA_PATH, OSS) + self.assert_endpoint(multi("-io-cache.target.cluster.endpoint"), Op.READ, DATA_PATH, OSS) + meta_and_data = "io-cache.routes=meta=accel;data=cluster" + self.assert_endpoint(multi(meta_and_data), Op.READ, DATA_PATH + ".index", OSS) + + self.assert_endpoint(multi("io-cache.routes=*=cluster"), Op.READ, MANIFEST_PATH, CLUSTER) + first_wins = "io-cache.routes=data=accel;data,meta=cluster" + self.assert_endpoint(multi(first_wins), Op.READ, DATA_PATH, ACCEL) + malformed = "io-cache.routes==cluster;meta=accel;junk" + self.assert_endpoint(multi(malformed), Op.READ, MANIFEST_PATH, ACCEL) + self.assert_endpoint(multi(malformed), Op.READ, DATA_PATH, OSS) + + def test_data_files_named_like_metadata(self): + manifest = multi("data-file.prefix=manifest-") + self.assert_endpoint(manifest, Op.READ, "dt=1/bucket-0/manifest-{uuid}-0.orc", CLUSTER) + self.assert_endpoint(manifest, Op.READ, "dt=1/bucket-0/7f3a/manifest-{uuid}-0.orc", CLUSTER) + self.assert_endpoint(manifest, Op.READ, MANIFEST_PATH, ACCEL) + self.assert_endpoint(manifest, Op.READ, MANIFEST_PATH + ".avro.sidecar", ACCEL) + snapshot = multi("data-file.prefix=snapshot-") + self.assert_endpoint(snapshot, Op.READ, "dt=1/bucket-0/snapshot-{uuid}-0.orc", CLUSTER) + stat = multi("data-file.prefix=stat-") + self.assert_endpoint(stat, Op.READ, "dt=1/bucket-0/stat-{uuid}-0.orc", CLUSTER) + index = multi("io-cache.routes=data=cluster;bucket-index=accel", "data-file.prefix=index-") + self.assert_endpoint(index, Op.READ, "dt=1/bucket-0/index-{uuid}-0.orc", CLUSTER) + self.assert_endpoint(index, Op.READ, INDEX_PATH, ACCEL) + + custom = "dt=1/bucket-0/custom-{uuid}-0.orc" + self.assert_endpoint(single(), Op.READ, custom, OSS) + self.assert_endpoint(single("data-file.prefix=custom-"), Op.READ, custom, CACHE) + + def test_routable_type(self): + # named by sequence id or rewritten in place + for path in ("snapshot/snapshot-12", + "snapshot/LATEST", + "snapshot/EARLIEST", + "branch/branch-dev/snapshot/LATEST", + "schema/schema-3", + "changelog/changelog-5", + "changelog/LATEST", + "tag/tag-2026-09-30", + "tag/tag-success-file/t1_SUCCESS", + "consumer/consumer-job1", + "service/service-primary-key-lookup", + "dt=1/_SUCCESS", + "oss://bkt/bucket-0/db/t/changelog/changelog-5"): + self.assert_type(path, None) + # temporary or not written by Paimon + for path in ("snapshot/.snapshot-13.{uuid}.tmp", + "dt=1/bucket-0/.data-{uuid}-0.parquet.{uuid}.tmp", + "dt=1/bucket-0/data-{uuid}-0.parquet.tmp-{uuid}", + "dt=1/bucket-0/data-{uuid}-0.parquet.tmp.{uuid}", + "metadata/version-hint.text", + "metadata/v3.metadata.json", + "dt=1/bucket-0/000000_0", + "dt=1/part-00000-1a2b.snappy.parquet", + "dt=1/bucket-0/data-1.parquet", + "dt=1/bucket-0/custom-{uuid}-0.orc", + "README"): + self.assert_type(path, None) + + self.assert_type("manifest/manifest-{uuid}-0", FileType.META) + self.assert_type("manifest/manifest-list-{uuid}-1", FileType.META) + self.assert_type("manifest/index-manifest-{uuid}-0", FileType.META) + self.assert_type("manifest/manifest-{uuid}-0.avro.sidecar", FileType.META) + self.assert_type("statistics/stat-{uuid}-0", FileType.META) + self.assert_type("dt=1/bucket-0/data-{uuid}-0.parquet", FileType.DATA) + self.assert_type("dt=1/bucket-0/changelog-{uuid}-0.parquet", FileType.DATA) + self.assert_type("dt=1/bucket-0/data-{uuid}-0.blob", FileType.DATA) + self.assert_type("dt=1/8b1f7c2e/data-{uuid}-0.parquet", FileType.DATA) + self.assert_type("dt=1/bucket-0/data-{uuid}-0.parquet.index", FileType.FILE_INDEX) + self.assert_type("index/index-{uuid}-0", FileType.BUCKET_INDEX) + self.assert_type("index/btree-global-index-{uuid}.index", FileType.GLOBAL_INDEX) + self.assert_type("index/lumina-global-index-{uuid}.index", FileType.GLOBAL_INDEX) + + def test_routable_type_with_file_prefixes(self): + self.assert_type("dt=1/bucket-0/custom-{uuid}-0.orc", FileType.DATA, "data-file.prefix=custom-") + self.assert_type("dt=1/bucket-0/data-{uuid}-0.orc", FileType.DATA, "data-file.prefix=custom-") + self.assert_type("dt=1/bucket-0/cl-{uuid}-0.orc", FileType.DATA, "changelog-file.prefix=cl-") + self.assert_type("dt=1/bucket-0/ unknown.parquet", None, "data-file.prefix= ") + + # the name decides, so data files may be named like metadata + manifest = "data-file.prefix=manifest-" + self.assert_type("dt=1/bucket-0/manifest-{uuid}-0.orc", FileType.DATA, manifest) + self.assert_type("dt=1/bucket-0/7f3a/manifest-{uuid}-0.orc", FileType.DATA, manifest) + self.assert_type("dt=1/bucket-0/manifest-{uuid}-0.orc.index", FileType.FILE_INDEX, manifest) + self.assert_type("manifest/manifest-{uuid}-0", FileType.META, manifest) + self.assert_type("manifest/manifest-{uuid}-0.avro.sidecar", FileType.META, manifest) + index = "data-file.prefix=index-" + self.assert_type("dt=1/bucket-0/index-{uuid}-0.orc", FileType.DATA, index) + self.assert_type("bucket-postpone/index-{uuid}-0.orc", FileType.DATA, index) + self.assert_type("index/index-{uuid}-0", FileType.BUCKET_INDEX, index) + self.assert_type("dt=1/bucket-0/index-{uuid}-0", FileType.BUCKET_INDEX, index) + stat = "data-file.prefix=stat-" + self.assert_type("dt=1/bucket-0/stat-{uuid}-0.orc", FileType.DATA, stat) + self.assert_type("postpone/stat-{uuid}-0.orc", FileType.DATA, stat) + self.assert_type("statistics/stat-{uuid}-0", FileType.META, stat) + snapshot = "data-file.prefix=snapshot-" + self.assert_type("dt=1/bucket-0/snapshot-{uuid}-0.orc", FileType.DATA, snapshot) + self.assert_type("dt=1/bucket-0/snapshot-{uuid}-0.orc.index", FileType.FILE_INDEX, snapshot) + self.assert_type("dt=1/bucket-0/schema-{uuid}-0.orc", FileType.DATA, "data-file.prefix=schema-") + self.assert_type("dt=1/bucket-0/global-index-{uuid}-0.orc.index", FileType.FILE_INDEX, + "data-file.prefix=global-index-") + + # names rewritten in place never use a cache, whatever the prefix + self.assert_type("dt=1/bucket-0/tag-{uuid}-0.orc", None, "data-file.prefix=tag-") + self.assert_type("dt=1/bucket-0/consumer-{uuid}-0.orc", None, "data-file.prefix=consumer-") + warehouse = "oss://bkt/warehouse/bucket-0/db/t1/" + self.assert_type(warehouse + "tag/tag-file", None, "data-file.prefix=tag-") + self.assert_type(warehouse + "consumer/consumer-file", None, "data-file.prefix=consumer-") + self.assert_type(warehouse + "service/service-file", None, "data-file.prefix=service-") + self.assert_type(warehouse + "metadata/version-hint.text", None, "data-file.prefix=version-") + self.assert_type(warehouse + "branch/branch-feature", None, "data-file.prefix=branch-") class IoCacheRoutingTest(unittest.TestCase): diff --git a/paimon-python/pypaimon/tests/resources/io_cache/routable-types.json b/paimon-python/pypaimon/tests/resources/io_cache/routable-types.json deleted file mode 100644 index bb780c885329..000000000000 --- a/paimon-python/pypaimon/tests/resources/io_cache/routable-types.json +++ /dev/null @@ -1,69 +0,0 @@ -{ - "table_root": "oss://bkt/db1.db/t1", - "cases": [ - {"path": "oss://bkt/db1.db/t1/snapshot/snapshot-12", "options": {}, "type": null, "reason": "snapshot: sequential metadata may be read before it exists"}, - {"path": "oss://bkt/db1.db/t1/snapshot/LATEST", "options": {}, "type": null, "reason": "hint file, rewritten in place"}, - {"path": "oss://bkt/db1.db/t1/snapshot/EARLIEST", "options": {}, "type": null, "reason": "hint file, rewritten in place"}, - {"path": "oss://bkt/db1.db/t1/branch/branch-dev/snapshot/LATEST", "options": {}, "type": null, "reason": "hint file on a branch"}, - {"path": "oss://bkt/db1.db/t1/schema/schema-3", "options": {}, "type": null, "reason": "schema: named by sequence id"}, - {"path": "oss://bkt/db1.db/t1/manifest/manifest-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0", "options": {}, "type": "meta", "reason": "manifest"}, - {"path": "oss://bkt/db1.db/t1/manifest/manifest-list-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-1", "options": {}, "type": "meta", "reason": "manifest list"}, - {"path": "oss://bkt/db1.db/t1/manifest/index-manifest-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0", "options": {}, "type": "meta", "reason": "index manifest; 'manifest' wins over 'index-'"}, - {"path": "oss://bkt/db1.db/t1/manifest/manifest-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.avro.sidecar", "options": {}, "type": "meta", "reason": "manifest sidecar"}, - {"path": "oss://bkt/db1.db/t1/statistics/stat-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0", "options": {}, "type": "meta", "reason": "statistics"}, - {"path": "oss://bkt/db1.db/t1/tag/tag-2026-09-30", "options": {}, "type": null, "reason": "tags can be replaced"}, - {"path": "oss://bkt/db1.db/t1/tag/tag-success-file/t1_SUCCESS", "options": {}, "type": null, "reason": "tag success file"}, - {"path": "oss://bkt/db1.db/t1/consumer/consumer-job1", "options": {}, "type": null, "reason": "consumer progress, rewritten"}, - {"path": "oss://bkt/db1.db/t1/service/service-primary-key-lookup", "options": {}, "type": null, "reason": "service file, rewritten"}, - {"path": "oss://bkt/db1.db/t1/dt=2026-09-30/_SUCCESS", "options": {}, "type": null, "reason": "partition mark-done file"}, - {"path": "oss://bkt/db1.db/t1/changelog/changelog-5", "options": {}, "type": null, "reason": "decoupled changelog metadata, named by sequence id"}, - {"path": "oss://bkt/db1.db/t1/changelog/LATEST", "options": {}, "type": null, "reason": "changelog hint file"}, - {"path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "options": {}, "type": "data", "reason": "data file"}, - {"path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/changelog-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "options": {}, "type": "data", "reason": "changelog data file in a bucket"}, - {"path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.blob", "options": {}, "type": "data", "reason": "blob file"}, - {"path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet.index", "options": {}, "type": "file-index", "reason": "data-file index sidecar"}, - {"path": "oss://bkt/db1.db/t1/index/index-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0", "options": {}, "type": "bucket-index", "reason": "hash index or deletion vectors"}, - {"path": "oss://bkt/db1.db/t1/index/btree-global-index-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b.index", "options": {}, "type": "global-index", "reason": "global btree index"}, - {"path": "oss://bkt/db1.db/t1/index/lumina-global-index-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b.index", "options": {}, "type": "global-index", "reason": "global vector index"}, - {"path": "oss://bkt/db1.db/t1/snapshot/.snapshot-13.8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b.tmp", "options": {}, "type": null, "reason": "temp file of an atomic write"}, - {"path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/.data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet.8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b.tmp", "options": {}, "type": null, "reason": "temp data file"}, - {"path": "oss://bkt/db1.db/t1/metadata/version-hint.text", "options": {}, "type": null, "reason": "Iceberg-compatible metadata hint"}, - {"path": "oss://bkt/db1.db/t1/metadata/v3.metadata.json", "options": {}, "type": null, "reason": "Iceberg-compatible metadata, not a Paimon name"}, - {"path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/000000_0", "options": {}, "type": null, "reason": "migrated Hive file: unknown name"}, - {"path": "oss://bkt/db1.db/t1/dt=2026-09-30/part-00000-1a2b.snappy.parquet", "options": {}, "type": null, "reason": "format-table style name: unknown"}, - {"path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/custom-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.orc", "options": {}, "type": null, "reason": "custom data-file.prefix without the option"}, - {"path": "oss://bkt/db1.db/t1/README", "options": {}, "type": null, "reason": "unknown name"}, - {"path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/manifest-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.orc", "options": {"data-file.prefix": "manifest-"}, "type": "data", "reason": "a data file named like a manifest by data-file.prefix is data"}, - {"path": "oss://bkt/db1.db/t1/manifest/manifest-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0", "options": {"data-file.prefix": "manifest-"}, "type": "meta", "reason": "manifests stay meta with data-file.prefix manifest-"}, - {"path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/index-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.orc", "options": {"data-file.prefix": "index-"}, "type": "data", "reason": "a data file named like a bucket index by data-file.prefix is data"}, - {"path": "oss://bkt/db1.db/t1/index/index-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0", "options": {"data-file.prefix": "index-"}, "type": "bucket-index", "reason": "bucket indexes stay bucket-index with data-file.prefix index-"}, - {"path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/stat-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.orc", "options": {"data-file.prefix": "stat-"}, "type": "data", "reason": "a data file named like statistics by data-file.prefix is data"}, - {"path": "oss://bkt/db1.db/t1/statistics/stat-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0", "options": {"data-file.prefix": "stat-"}, "type": "meta", "reason": "statistics stay meta with data-file.prefix stat-"}, - {"path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/manifest-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.orc.index", "options": {"data-file.prefix": "manifest-"}, "type": "file-index", "reason": "the file index of such a data file stays file-index"}, - {"path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/ unknown.parquet", "options": {"data-file.prefix": " "}, "type": null, "reason": "a whitespace-only prefix does not make unknown files cacheable"}, - {"path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/custom-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.orc", "options": {"data-file.prefix": "custom-"}, "type": "data", "reason": "custom data-file.prefix from the merged options"}, - {"path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/cl-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.orc", "options": {"changelog-file.prefix": "cl-"}, "type": "data", "reason": "custom changelog-file.prefix from the merged options"}, - {"path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.orc", "options": {"data-file.prefix": "custom-"}, "type": "data", "reason": "default prefixes stay routable next to a custom one"}, - {"path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/7f3a/manifest-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.orc", "options": {"data-file.prefix": "manifest-"}, "type": "data", "reason": "an external data path may put an entropy directory between the bucket and the file"}, - {"path": "oss://bkt/db1.db/t1/bucket-postpone/index-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.orc", "options": {"data-file.prefix": "index-"}, "type": "data", "reason": "data file of a postpone bucket"}, - {"path": "oss://bkt/db1.db/t1/postpone/stat-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.orc", "options": {"data-file.prefix": "stat-"}, "type": "data", "reason": "data file of a legacy postpone bucket"}, - {"path": "oss://bkt/db1.db/t1/dt=2026-09-30/8b1f7c2e/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "options": {}, "type": "data", "reason": "the name decides, not the directory"}, - {"path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/snapshot-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.orc", "options": {"data-file.prefix": "snapshot-"}, "type": "data", "reason": "snapshots are named by sequence id, so a uuid name with this prefix is data"}, - {"path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/schema-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.orc", "options": {"data-file.prefix": "schema-"}, "type": "data", "reason": "schemas are named by sequence id, so a uuid name with this prefix is data"}, - {"path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/snapshot-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.orc.index", "options": {"data-file.prefix": "snapshot-"}, "type": "file-index", "reason": "file index of a data file named like a snapshot"}, - {"path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/global-index-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.orc.index", "options": {"data-file.prefix": "global-index-"}, "type": "file-index", "reason": "file index of a data file named like a global index"}, - {"path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/tag-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.orc", "options": {"data-file.prefix": "tag-"}, "type": null, "reason": "names rewritten in place never use a cache, whatever the prefix"}, - {"path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/consumer-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.orc", "options": {"data-file.prefix": "consumer-"}, "type": null, "reason": "names rewritten in place never use a cache, whatever the prefix"}, - {"path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/index-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0", "options": {"data-file.prefix": "index-"}, "type": "bucket-index", "reason": "no extension: a bucket index, not a data file"}, - {"path": "oss://bkt/db1.db/t1/manifest/manifest-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.avro.sidecar", "options": {"data-file.prefix": "manifest-"}, "type": "meta", "reason": "a manifest sidecar is metadata whatever the data prefix"}, - {"path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet.tmp-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b", "options": {}, "type": null, "reason": "temp name with a .tmp- suffix"}, - {"path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet.tmp.8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b", "options": {}, "type": null, "reason": "temp name with a .tmp. infix"}, - {"path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-1.parquet", "options": {}, "type": null, "reason": "not a name Paimon writers produce"}, - {"path": "oss://bkt/bucket-0/db/t/changelog/changelog-5", "options": {}, "type": null, "reason": "decoupled changelog metadata below a bucket-* warehouse directory"}, - {"path": "oss://bkt/warehouse/bucket-0/db/t1/tag/tag-file", "options": {"data-file.prefix": "tag-"}, "type": null, "reason": "tag metadata with a matching data prefix"}, - {"path": "oss://bkt/warehouse/bucket-0/db/t1/consumer/consumer-file", "options": {"data-file.prefix": "consumer-"}, "type": null, "reason": "consumer metadata with a matching data prefix"}, - {"path": "oss://bkt/warehouse/bucket-0/db/t1/service/service-file", "options": {"data-file.prefix": "service-"}, "type": null, "reason": "service metadata with a matching data prefix"}, - {"path": "oss://bkt/warehouse/bucket-0/db/t1/metadata/version-hint.text", "options": {"data-file.prefix": "version-"}, "type": null, "reason": "Iceberg metadata hint with a matching data prefix"}, - {"path": "oss://bkt/warehouse/bucket-0/db/t1/branch/branch-feature", "options": {"data-file.prefix": "branch-"}, "type": null, "reason": "branch directory with a matching data prefix"} - ] -} diff --git a/paimon-python/pypaimon/tests/resources/io_cache/routing.json b/paimon-python/pypaimon/tests/resources/io_cache/routing.json deleted file mode 100644 index afd4b88eb836..000000000000 --- a/paimon-python/pypaimon/tests/resources/io_cache/routing.json +++ /dev/null @@ -1,105 +0,0 @@ -{ - "table_root": "oss://bkt/db1.db/t1", - "cases": [ - {"name": "multi: a data file named like a manifest goes to the data target", "options": {"fs.oss.endpoint": "https://accelerator.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.targets": "accel,cluster", "io-cache.target.accel.endpoint": "https://accelerator.example.com", "io-cache.target.accel.region": "cn-hangzhou", "io-cache.target.cluster.endpoint": "http://cluster.example.com", "io-cache.target.cluster.path-style-access": "true", "io-cache.policy": "meta,read", "io-cache.whitelist": "*", "io-cache.routes": "meta=accel;data,bucket-index,global-index,file-index=cluster", "data-file.prefix": "manifest-"}, "op": "read", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/manifest-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.orc", "expect": "http://cluster.example.com"}, - {"name": "multi: manifests keep the meta target with data-file.prefix manifest-", "options": {"fs.oss.endpoint": "https://accelerator.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.targets": "accel,cluster", "io-cache.target.accel.endpoint": "https://accelerator.example.com", "io-cache.target.accel.region": "cn-hangzhou", "io-cache.target.cluster.endpoint": "http://cluster.example.com", "io-cache.target.cluster.path-style-access": "true", "io-cache.policy": "meta,read", "io-cache.whitelist": "*", "io-cache.routes": "meta=accel;data,bucket-index,global-index,file-index=cluster", "data-file.prefix": "manifest-"}, "op": "read", "path": "oss://bkt/db1.db/t1/manifest/manifest-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0", "expect": "https://accelerator.example.com"}, - {"name": "multi: a data file named like a bucket index goes to the data target", "options": {"fs.oss.endpoint": "https://accelerator.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.targets": "accel,cluster", "io-cache.target.accel.endpoint": "https://accelerator.example.com", "io-cache.target.accel.region": "cn-hangzhou", "io-cache.target.cluster.endpoint": "http://cluster.example.com", "io-cache.target.cluster.path-style-access": "true", "io-cache.policy": "meta,read", "io-cache.whitelist": "*", "io-cache.routes": "data=cluster;bucket-index=accel", "data-file.prefix": "index-"}, "op": "read", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/index-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.orc", "expect": "http://cluster.example.com"}, - {"name": "multi: bucket indexes keep their target with data-file.prefix index-", "options": {"fs.oss.endpoint": "https://accelerator.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.targets": "accel,cluster", "io-cache.target.accel.endpoint": "https://accelerator.example.com", "io-cache.target.accel.region": "cn-hangzhou", "io-cache.target.cluster.endpoint": "http://cluster.example.com", "io-cache.target.cluster.path-style-access": "true", "io-cache.policy": "meta,read", "io-cache.whitelist": "*", "io-cache.routes": "data=cluster;bucket-index=accel", "data-file.prefix": "index-"}, "op": "read", "path": "oss://bkt/db1.db/t1/index/index-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0", "expect": "https://accelerator.example.com"}, - {"name": "multi: a data file named like a snapshot goes to the data target", "options": {"fs.oss.endpoint": "https://accelerator.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.targets": "accel,cluster", "io-cache.target.accel.endpoint": "https://accelerator.example.com", "io-cache.target.accel.region": "cn-hangzhou", "io-cache.target.cluster.endpoint": "http://cluster.example.com", "io-cache.target.cluster.path-style-access": "true", "io-cache.policy": "meta,read", "io-cache.whitelist": "*", "io-cache.routes": "meta=accel;data,bucket-index,global-index,file-index=cluster", "data-file.prefix": "snapshot-"}, "op": "read", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/snapshot-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.orc", "expect": "http://cluster.example.com"}, - {"name": "multi: a data file below an entropy directory goes to the data target", "options": {"fs.oss.endpoint": "https://accelerator.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.targets": "accel,cluster", "io-cache.target.accel.endpoint": "https://accelerator.example.com", "io-cache.target.accel.region": "cn-hangzhou", "io-cache.target.cluster.endpoint": "http://cluster.example.com", "io-cache.target.cluster.path-style-access": "true", "io-cache.policy": "meta,read", "io-cache.whitelist": "*", "io-cache.routes": "meta=accel;data,bucket-index,global-index,file-index=cluster", "data-file.prefix": "manifest-"}, "op": "read", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/7f3a/manifest-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.orc", "expect": "http://cluster.example.com"}, - {"name": "multi: a manifest sidecar keeps the meta target with data-file.prefix manifest-", "options": {"fs.oss.endpoint": "https://accelerator.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.targets": "accel,cluster", "io-cache.target.accel.endpoint": "https://accelerator.example.com", "io-cache.target.accel.region": "cn-hangzhou", "io-cache.target.cluster.endpoint": "http://cluster.example.com", "io-cache.target.cluster.path-style-access": "true", "io-cache.policy": "meta,read", "io-cache.whitelist": "*", "io-cache.routes": "meta=accel;data,bucket-index,global-index,file-index=cluster", "data-file.prefix": "manifest-"}, "op": "read", "path": "oss://bkt/db1.db/t1/manifest/manifest-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.avro.sidecar", "expect": "https://accelerator.example.com"}, - {"name": "multi: a data file named like statistics goes to the data target", "options": {"fs.oss.endpoint": "https://accelerator.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.targets": "accel,cluster", "io-cache.target.accel.endpoint": "https://accelerator.example.com", "io-cache.target.accel.region": "cn-hangzhou", "io-cache.target.cluster.endpoint": "http://cluster.example.com", "io-cache.target.cluster.path-style-access": "true", "io-cache.policy": "meta,read", "io-cache.whitelist": "*", "io-cache.routes": "meta=accel;data,bucket-index,global-index,file-index=cluster", "data-file.prefix": "stat-"}, "op": "read", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/stat-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.orc", "expect": "http://cluster.example.com"}, - {"name": "single: a dls path is not routed", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": "http://cache.example.com", "io-cache.policy": "meta,read", "io-cache.whitelist": "meta,data"}, "op": "read", "path": "dls://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "expect": "https://oss-cn-hangzhou-internal.aliyuncs.com"}, - {"name": "policy: trimmed case-insensitive tokens", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": "http://cache.example.com", "io-cache.policy": " READ , Meta ", "io-cache.whitelist": "meta,data"}, "op": "meta", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "expect": "http://cache.example.com"}, - {"name": "policy: none overrides read", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": "http://cache.example.com", "io-cache.policy": "read,NONE", "io-cache.whitelist": "meta,data"}, "op": "exists", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "expect": "http://cache.example.com"}, - {"name": "policy: substrings are not operations", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": "http://cache.example.com", "io-cache.policy": "thread,metadata", "io-cache.whitelist": "meta,data"}, "op": "exists", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "expect": "http://cache.example.com"}, - {"name": "policy: unknown tokens do not disable read", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": "http://cache.example.com", "io-cache.policy": "read,nonetheless", "io-cache.whitelist": "meta,data"}, "op": "exists", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "expect": "https://oss-cn-hangzhou-internal.aliyuncs.com"}, - {"name": "single: whitespace endpoint is missing", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": " ", "io-cache.policy": "meta,read", "io-cache.whitelist": "meta,data"}, "op": "read", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "expect": "https://oss-cn-hangzhou-internal.aliyuncs.com"}, - {"name": "multi: whitespace endpoint is skipped", "options": {"fs.oss.endpoint": "https://accelerator.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.targets": "accel,cluster", "io-cache.target.accel.endpoint": " ", "io-cache.target.accel.region": "cn-hangzhou", "io-cache.target.cluster.endpoint": "http://cluster.example.com", "io-cache.target.cluster.path-style-access": "true", "io-cache.policy": "meta,read", "io-cache.whitelist": "*"}, "op": "read", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "expect": "http://cluster.example.com"}, - {"name": "multi: target endpoint is trimmed", "options": {"fs.oss.endpoint": "https://accelerator.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.targets": "accel,cluster", "io-cache.target.accel.endpoint": "https://accelerator.example.com", "io-cache.target.accel.region": "cn-hangzhou", "io-cache.target.cluster.endpoint": " http://cluster.example.com ", "io-cache.target.cluster.path-style-access": "true", "io-cache.policy": "meta,read", "io-cache.whitelist": "*", "io-cache.routes": "meta=accel;data,bucket-index,global-index,file-index=cluster"}, "op": "read", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "expect": "http://cluster.example.com"}, - {"name": "single: whitespace origin uses legacy endpoint", "options": {"fs.oss.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": " ", "io-cache.endpoint": "http://cache.example.com", "io-cache.policy": "meta,read", "io-cache.whitelist": "meta,data"}, "op": "list", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "expect": "https://oss-cn-hangzhou-internal.aliyuncs.com"}, - {"name": "single: data read goes to the cache", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": "http://cache.example.com", "io-cache.policy": "meta,read", "io-cache.whitelist": "meta,data"}, "op": "read", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "expect": "http://cache.example.com"}, - {"name": "single: data status goes to the cache", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": "http://cache.example.com", "io-cache.policy": "meta,read", "io-cache.whitelist": "meta,data"}, "op": "meta", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "expect": "http://cache.example.com"}, - {"name": "single: manifest read goes to the cache", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": "http://cache.example.com", "io-cache.policy": "meta,read", "io-cache.whitelist": "meta,data"}, "op": "read", "path": "oss://bkt/db1.db/t1/manifest/manifest-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0", "expect": "http://cache.example.com"}, - {"name": "single: manifest list read goes to the cache", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": "http://cache.example.com", "io-cache.policy": "meta,read", "io-cache.whitelist": "meta,data"}, "op": "read", "path": "oss://bkt/db1.db/t1/manifest/manifest-list-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-1", "expect": "http://cache.example.com"}, - {"name": "single: snapshot read stays at origin", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": "http://cache.example.com", "io-cache.policy": "meta,read", "io-cache.whitelist": "meta,data"}, "op": "read", "path": "oss://bkt/db1.db/t1/snapshot/snapshot-12", "expect": "https://oss-cn-hangzhou-internal.aliyuncs.com"}, - {"name": "single: snapshot status stays at origin", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": "http://cache.example.com", "io-cache.policy": "meta,read", "io-cache.whitelist": "meta,data"}, "op": "meta", "path": "oss://bkt/db1.db/t1/snapshot/snapshot-12", "expect": "https://oss-cn-hangzhou-internal.aliyuncs.com"}, - {"name": "single: hint file stays at origin", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": "http://cache.example.com", "io-cache.policy": "meta,read", "io-cache.whitelist": "meta,data"}, "op": "read", "path": "oss://bkt/db1.db/t1/snapshot/LATEST", "expect": "https://oss-cn-hangzhou-internal.aliyuncs.com"}, - {"name": "single: index type outside the whitelist stays at origin", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": "http://cache.example.com", "io-cache.policy": "meta,read", "io-cache.whitelist": "meta,data"}, "op": "read", "path": "oss://bkt/db1.db/t1/index/index-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0", "expect": "https://oss-cn-hangzhou-internal.aliyuncs.com"}, - {"name": "single: unknown name stays at origin", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": "http://cache.example.com", "io-cache.policy": "meta,read", "io-cache.whitelist": "meta,data"}, "op": "read", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/000000_0", "expect": "https://oss-cn-hangzhou-internal.aliyuncs.com"}, - {"name": "single: temp file stays at origin", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": "http://cache.example.com", "io-cache.policy": "meta,read", "io-cache.whitelist": "meta,data"}, "op": "read", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/.data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet.8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b.tmp", "expect": "https://oss-cn-hangzhou-internal.aliyuncs.com"}, - {"name": "single: a path in another bucket is routed like the table's", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": "http://cache.example.com", "io-cache.policy": "meta,read", "io-cache.whitelist": "meta,data"}, "op": "read", "path": "oss://other-bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "expect": "http://cache.example.com"}, - {"name": "single: disabled by default", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": "http://cache.example.com", "io-cache.policy": "meta,read", "io-cache.whitelist": "meta,data"}, "op": "read", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "expect": "http://cache.example.com"}, - {"name": "single: disabled keeps fs.oss.endpoint for everything", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.enabled": "false", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": "http://cache.example.com", "io-cache.policy": "meta,read", "io-cache.whitelist": "meta,data"}, "op": "exists", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "expect": "http://cache.example.com"}, - {"name": "single: no policy means no routing", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": "http://cache.example.com", "io-cache.whitelist": "meta,data"}, "op": "exists", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "expect": "http://cache.example.com"}, - {"name": "single: policy none means no routing", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": "http://cache.example.com", "io-cache.policy": "none", "io-cache.whitelist": "meta,data"}, "op": "exists", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "expect": "http://cache.example.com"}, - {"name": "single: write-only policy keeps status at origin", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": "http://cache.example.com", "io-cache.policy": "write", "io-cache.whitelist": "meta,data"}, "op": "exists", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "expect": "https://oss-cn-hangzhou-internal.aliyuncs.com"}, - {"name": "single: write-only policy routes writes", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": "http://cache.example.com", "io-cache.policy": "write", "io-cache.whitelist": "meta,data"}, "op": "write", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "expect": "http://cache.example.com"}, - {"name": "write: data file write goes to the cache", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": "http://cache.example.com", "io-cache.policy": "meta,read,write", "io-cache.whitelist": "meta,data"}, "op": "write", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "expect": "http://cache.example.com"}, - {"name": "write: two-phase data file write goes to the cache", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": "http://cache.example.com", "io-cache.policy": "meta,read,write", "io-cache.whitelist": "meta,data"}, "op": "two-phase-write", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "expect": "http://cache.example.com"}, - {"name": "write: manifest write goes to the cache", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": "http://cache.example.com", "io-cache.policy": "meta,read,write", "io-cache.whitelist": "meta,data"}, "op": "write", "path": "oss://bkt/db1.db/t1/manifest/manifest-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0", "expect": "http://cache.example.com"}, - {"name": "write: snapshot write stays at origin", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": "http://cache.example.com", "io-cache.policy": "meta,read,write", "io-cache.whitelist": "meta,data"}, "op": "write", "path": "oss://bkt/db1.db/t1/snapshot/snapshot-12", "expect": "https://oss-cn-hangzhou-internal.aliyuncs.com"}, - {"name": "write: hint file write stays at origin", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": "http://cache.example.com", "io-cache.policy": "meta,read,write", "io-cache.whitelist": "meta,data"}, "op": "write", "path": "oss://bkt/db1.db/t1/snapshot/LATEST", "expect": "https://oss-cn-hangzhou-internal.aliyuncs.com"}, - {"name": "write: temp file write stays at origin", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": "http://cache.example.com", "io-cache.policy": "meta,read,write", "io-cache.whitelist": "meta,data"}, "op": "write", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/.data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet.8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b.tmp", "expect": "https://oss-cn-hangzhou-internal.aliyuncs.com"}, - {"name": "write: atomic write stays at origin", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": "http://cache.example.com", "io-cache.policy": "meta,read,write", "io-cache.whitelist": "meta,data"}, "op": "atomic-write", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "expect": "https://oss-cn-hangzhou-internal.aliyuncs.com"}, - {"name": "write: delete stays at origin", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": "http://cache.example.com", "io-cache.policy": "meta,read,write", "io-cache.whitelist": "meta,data"}, "op": "delete", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "expect": "https://oss-cn-hangzhou-internal.aliyuncs.com"}, - {"name": "write: rename stays at origin", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": "http://cache.example.com", "io-cache.policy": "meta,read,write", "io-cache.whitelist": "meta,data"}, "op": "rename", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "expect": "https://oss-cn-hangzhou-internal.aliyuncs.com"}, - {"name": "write: copy stays at origin", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": "http://cache.example.com", "io-cache.policy": "meta,read,write", "io-cache.whitelist": "meta,data"}, "op": "copy", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "expect": "https://oss-cn-hangzhou-internal.aliyuncs.com"}, - {"name": "write: list stays at origin", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": "http://cache.example.com", "io-cache.policy": "meta,read,write", "io-cache.whitelist": "meta,data"}, "op": "list", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "expect": "https://oss-cn-hangzhou-internal.aliyuncs.com"}, - {"name": "write: whitelist filters writes", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": "http://cache.example.com", "io-cache.policy": "meta,read,write", "io-cache.whitelist": "meta"}, "op": "write", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "expect": "https://oss-cn-hangzhou-internal.aliyuncs.com"}, - {"name": "write: data write follows the data route", "options": {"fs.oss.endpoint": "https://accelerator.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.targets": "accel,cluster", "io-cache.target.accel.endpoint": "https://accelerator.example.com", "io-cache.target.accel.region": "cn-hangzhou", "io-cache.target.cluster.endpoint": "http://cluster.example.com", "io-cache.target.cluster.path-style-access": "true", "io-cache.policy": "meta,read,write", "io-cache.whitelist": "*", "io-cache.routes": "meta=accel;data,bucket-index,global-index,file-index=cluster"}, "op": "write", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "expect": "http://cluster.example.com"}, - {"name": "write: manifest write follows the meta route", "options": {"fs.oss.endpoint": "https://accelerator.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.targets": "accel,cluster", "io-cache.target.accel.endpoint": "https://accelerator.example.com", "io-cache.target.accel.region": "cn-hangzhou", "io-cache.target.cluster.endpoint": "http://cluster.example.com", "io-cache.target.cluster.path-style-access": "true", "io-cache.policy": "meta,read,write", "io-cache.whitelist": "*", "io-cache.routes": "meta=accel;data,bucket-index,global-index,file-index=cluster"}, "op": "write", "path": "oss://bkt/db1.db/t1/manifest/manifest-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0", "expect": "https://accelerator.example.com"}, - {"name": "single: read-only policy keeps status at origin", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": "http://cache.example.com", "io-cache.policy": "read", "io-cache.whitelist": "meta,data"}, "op": "meta", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "expect": "https://oss-cn-hangzhou-internal.aliyuncs.com"}, - {"name": "single: read-only policy still routes reads", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": "http://cache.example.com", "io-cache.policy": "read", "io-cache.whitelist": "meta,data"}, "op": "read", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "expect": "http://cache.example.com"}, - {"name": "single: origin defaults to fs.oss.endpoint", "options": {"fs.oss.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.enabled": "true", "io-cache.endpoint": "http://cache.example.com", "io-cache.policy": "meta,read", "io-cache.whitelist": "meta,data"}, "op": "list", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "expect": "https://oss-cn-hangzhou-internal.aliyuncs.com"}, - {"name": "single: client endpoint override wins", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": "http://cache.example.com", "io-cache.policy": "meta,read", "io-cache.whitelist": "meta,data", "dlf.oss-endpoint": "https://oss-cn-hangzhou.aliyuncs.com"}, "op": "read", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "expect": "https://oss-cn-hangzhou.aliyuncs.com"}, - {"name": "single: absent whitelist allows every routable type", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": "http://cache.example.com", "io-cache.policy": "meta,read"}, "op": "read", "path": "oss://bkt/db1.db/t1/index/index-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0", "expect": "http://cache.example.com"}, - {"name": "single: star whitelist allows every routable type", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": "http://cache.example.com", "io-cache.policy": "meta,read", "io-cache.whitelist": "*"}, "op": "read", "path": "oss://bkt/db1.db/t1/index/btree-global-index-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b.index", "expect": "http://cache.example.com"}, - {"name": "single: file index sidecar with star whitelist", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": "http://cache.example.com", "io-cache.policy": "meta,read", "io-cache.whitelist": "*"}, "op": "read", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet.index", "expect": "http://cache.example.com"}, - {"name": "single: custom data prefix without the option stays at origin", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": "http://cache.example.com", "io-cache.policy": "meta,read", "io-cache.whitelist": "meta,data"}, "op": "read", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/custom-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.orc", "expect": "https://oss-cn-hangzhou-internal.aliyuncs.com"}, - {"name": "single: custom data prefix from options is routed", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": "http://cache.example.com", "io-cache.policy": "meta,read", "io-cache.whitelist": "meta,data", "data-file.prefix": "custom-"}, "op": "read", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/custom-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.orc", "expect": "http://cache.example.com"}, - {"name": "single: empty cache endpoint sends routed reads to origin", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": "", "io-cache.policy": "meta,read", "io-cache.whitelist": "meta,data", "io-cache.routes": "*=default"}, "op": "read", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "expect": "https://oss-cn-hangzhou-internal.aliyuncs.com"}, - {"name": "multi: manifest read goes to the accelerator", "options": {"fs.oss.endpoint": "https://accelerator.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.targets": "accel,cluster", "io-cache.target.accel.endpoint": "https://accelerator.example.com", "io-cache.target.accel.region": "cn-hangzhou", "io-cache.target.cluster.endpoint": "http://cluster.example.com", "io-cache.target.cluster.path-style-access": "true", "io-cache.policy": "meta,read", "io-cache.whitelist": "*", "io-cache.routes": "meta=accel;data,bucket-index,global-index,file-index=cluster"}, "op": "read", "path": "oss://bkt/db1.db/t1/manifest/manifest-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0", "expect": "https://accelerator.example.com"}, - {"name": "multi: manifest status goes to the accelerator", "options": {"fs.oss.endpoint": "https://accelerator.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.targets": "accel,cluster", "io-cache.target.accel.endpoint": "https://accelerator.example.com", "io-cache.target.accel.region": "cn-hangzhou", "io-cache.target.cluster.endpoint": "http://cluster.example.com", "io-cache.target.cluster.path-style-access": "true", "io-cache.policy": "meta,read", "io-cache.whitelist": "*", "io-cache.routes": "meta=accel;data,bucket-index,global-index,file-index=cluster"}, "op": "meta", "path": "oss://bkt/db1.db/t1/manifest/manifest-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0", "expect": "https://accelerator.example.com"}, - {"name": "multi: data read goes to the cluster", "options": {"fs.oss.endpoint": "https://accelerator.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.targets": "accel,cluster", "io-cache.target.accel.endpoint": "https://accelerator.example.com", "io-cache.target.accel.region": "cn-hangzhou", "io-cache.target.cluster.endpoint": "http://cluster.example.com", "io-cache.target.cluster.path-style-access": "true", "io-cache.policy": "meta,read", "io-cache.whitelist": "*", "io-cache.routes": "meta=accel;data,bucket-index,global-index,file-index=cluster"}, "op": "read", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "expect": "http://cluster.example.com"}, - {"name": "multi: deletion vectors go to the cluster", "options": {"fs.oss.endpoint": "https://accelerator.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.targets": "accel,cluster", "io-cache.target.accel.endpoint": "https://accelerator.example.com", "io-cache.target.accel.region": "cn-hangzhou", "io-cache.target.cluster.endpoint": "http://cluster.example.com", "io-cache.target.cluster.path-style-access": "true", "io-cache.policy": "meta,read", "io-cache.whitelist": "*", "io-cache.routes": "meta=accel;data,bucket-index,global-index,file-index=cluster"}, "op": "read", "path": "oss://bkt/db1.db/t1/index/index-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0", "expect": "http://cluster.example.com"}, - {"name": "multi: global index goes to the cluster", "options": {"fs.oss.endpoint": "https://accelerator.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.targets": "accel,cluster", "io-cache.target.accel.endpoint": "https://accelerator.example.com", "io-cache.target.accel.region": "cn-hangzhou", "io-cache.target.cluster.endpoint": "http://cluster.example.com", "io-cache.target.cluster.path-style-access": "true", "io-cache.policy": "meta,read", "io-cache.whitelist": "*", "io-cache.routes": "meta=accel;data,bucket-index,global-index,file-index=cluster"}, "op": "read", "path": "oss://bkt/db1.db/t1/index/btree-global-index-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b.index", "expect": "http://cluster.example.com"}, - {"name": "multi: manifest existence goes to the accelerator", "options": {"fs.oss.endpoint": "https://accelerator.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.targets": "accel,cluster", "io-cache.target.accel.endpoint": "https://accelerator.example.com", "io-cache.target.accel.region": "cn-hangzhou", "io-cache.target.cluster.endpoint": "http://cluster.example.com", "io-cache.target.cluster.path-style-access": "true", "io-cache.policy": "meta,read", "io-cache.whitelist": "*", "io-cache.routes": "meta=accel;data,bucket-index,global-index,file-index=cluster"}, "op": "exists", "path": "oss://bkt/db1.db/t1/manifest/manifest-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0", "expect": "https://accelerator.example.com"}, - {"name": "multi: data existence goes to the cluster", "options": {"fs.oss.endpoint": "https://accelerator.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.targets": "accel,cluster", "io-cache.target.accel.endpoint": "https://accelerator.example.com", "io-cache.target.accel.region": "cn-hangzhou", "io-cache.target.cluster.endpoint": "http://cluster.example.com", "io-cache.target.cluster.path-style-access": "true", "io-cache.policy": "meta,read", "io-cache.whitelist": "*", "io-cache.routes": "meta=accel;data,bucket-index,global-index,file-index=cluster"}, "op": "exists", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "expect": "http://cluster.example.com"}, - {"name": "single: data existence goes to the cache", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": "http://cache.example.com", "io-cache.policy": "meta,read", "io-cache.whitelist": "meta,data"}, "op": "exists", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "expect": "http://cache.example.com"}, - {"name": "single: snapshot existence stays at origin", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": "http://cache.example.com", "io-cache.policy": "meta,read", "io-cache.whitelist": "meta,data"}, "op": "exists", "path": "oss://bkt/db1.db/t1/snapshot/snapshot-12", "expect": "https://oss-cn-hangzhou-internal.aliyuncs.com"}, - {"name": "single: hint file existence stays at origin", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": "http://cache.example.com", "io-cache.policy": "meta,read", "io-cache.whitelist": "meta,data"}, "op": "exists", "path": "oss://bkt/db1.db/t1/snapshot/LATEST", "expect": "https://oss-cn-hangzhou-internal.aliyuncs.com"}, - {"name": "single: read-only policy keeps existence at origin", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": "http://cache.example.com", "io-cache.policy": "read", "io-cache.whitelist": "meta,data"}, "op": "exists", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "expect": "https://oss-cn-hangzhou-internal.aliyuncs.com"}, - {"name": "multi: snapshot stays at origin", "options": {"fs.oss.endpoint": "https://accelerator.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.targets": "accel,cluster", "io-cache.target.accel.endpoint": "https://accelerator.example.com", "io-cache.target.accel.region": "cn-hangzhou", "io-cache.target.cluster.endpoint": "http://cluster.example.com", "io-cache.target.cluster.path-style-access": "true", "io-cache.policy": "meta,read", "io-cache.whitelist": "*", "io-cache.routes": "meta=accel;data,bucket-index,global-index,file-index=cluster"}, "op": "read", "path": "oss://bkt/db1.db/t1/snapshot/snapshot-12", "expect": "https://oss-cn-hangzhou-internal.aliyuncs.com"}, - {"name": "multi: without routes the first usable target takes all", "options": {"fs.oss.endpoint": "https://accelerator.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.targets": "accel,cluster", "io-cache.target.accel.endpoint": "https://accelerator.example.com", "io-cache.target.accel.region": "cn-hangzhou", "io-cache.target.cluster.endpoint": "http://cluster.example.com", "io-cache.target.cluster.path-style-access": "true", "io-cache.policy": "meta,read", "io-cache.whitelist": "*"}, "op": "read", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "expect": "https://accelerator.example.com"}, - {"name": "multi: route to an undeclared target stays at origin", "options": {"fs.oss.endpoint": "https://accelerator.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.targets": "accel,cluster", "io-cache.target.accel.endpoint": "https://accelerator.example.com", "io-cache.target.accel.region": "cn-hangzhou", "io-cache.target.cluster.endpoint": "http://cluster.example.com", "io-cache.target.cluster.path-style-access": "true", "io-cache.policy": "meta,read", "io-cache.whitelist": "*", "io-cache.routes": "data=x"}, "op": "read", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "expect": "https://oss-cn-hangzhou-internal.aliyuncs.com"}, - {"name": "multi: type without a matching route stays at origin", "options": {"fs.oss.endpoint": "https://accelerator.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.targets": "accel,cluster", "io-cache.target.accel.endpoint": "https://accelerator.example.com", "io-cache.target.accel.region": "cn-hangzhou", "io-cache.target.cluster.endpoint": "http://cluster.example.com", "io-cache.target.cluster.path-style-access": "true", "io-cache.policy": "meta,read", "io-cache.whitelist": "*", "io-cache.routes": "meta=accel;data=cluster"}, "op": "read", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet.index", "expect": "https://oss-cn-hangzhou-internal.aliyuncs.com"}, - {"name": "multi: targets win over the single-target shorthand", "options": {"fs.oss.endpoint": "https://accelerator.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.targets": "accel,cluster", "io-cache.target.accel.endpoint": "https://accelerator.example.com", "io-cache.target.accel.region": "cn-hangzhou", "io-cache.target.cluster.endpoint": "http://cluster.example.com", "io-cache.target.cluster.path-style-access": "true", "io-cache.policy": "meta,read", "io-cache.whitelist": "*", "io-cache.endpoint": "http://cache.example.com"}, "op": "read", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "expect": "https://accelerator.example.com"}, - {"name": "multi: target without endpoint is skipped when routes are absent", "options": {"fs.oss.endpoint": "https://accelerator.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.targets": "accel,cluster", "io-cache.target.accel.region": "cn-hangzhou", "io-cache.target.cluster.endpoint": "http://cluster.example.com", "io-cache.target.cluster.path-style-access": "true", "io-cache.policy": "meta,read", "io-cache.whitelist": "*"}, "op": "read", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "expect": "http://cluster.example.com"}, - {"name": "multi: route to a target without endpoint stays at origin", "options": {"fs.oss.endpoint": "https://accelerator.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.targets": "accel,cluster", "io-cache.target.accel.endpoint": "https://accelerator.example.com", "io-cache.target.accel.region": "cn-hangzhou", "io-cache.target.cluster.path-style-access": "true", "io-cache.policy": "meta,read", "io-cache.whitelist": "*", "io-cache.routes": "meta=accel;data,bucket-index,global-index,file-index=cluster"}, "op": "read", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "expect": "https://oss-cn-hangzhou-internal.aliyuncs.com"}, - {"name": "multi: invalid target names are ignored", "options": {"fs.oss.endpoint": "https://accelerator.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.targets": "Bad_Name,cluster", "io-cache.target.accel.endpoint": "https://accelerator.example.com", "io-cache.target.accel.region": "cn-hangzhou", "io-cache.target.cluster.endpoint": "http://cluster.example.com", "io-cache.target.cluster.path-style-access": "true", "io-cache.policy": "meta,read", "io-cache.whitelist": "*"}, "op": "read", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "expect": "http://cluster.example.com"}, - {"name": "multi: star route sends every type to one target", "options": {"fs.oss.endpoint": "https://accelerator.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.targets": "accel,cluster", "io-cache.target.accel.endpoint": "https://accelerator.example.com", "io-cache.target.accel.region": "cn-hangzhou", "io-cache.target.cluster.endpoint": "http://cluster.example.com", "io-cache.target.cluster.path-style-access": "true", "io-cache.policy": "meta,read", "io-cache.whitelist": "*", "io-cache.routes": "*=cluster"}, "op": "read", "path": "oss://bkt/db1.db/t1/manifest/manifest-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0", "expect": "http://cluster.example.com"}, - {"name": "multi: first matching route wins", "options": {"fs.oss.endpoint": "https://accelerator.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.targets": "accel,cluster", "io-cache.target.accel.endpoint": "https://accelerator.example.com", "io-cache.target.accel.region": "cn-hangzhou", "io-cache.target.cluster.endpoint": "http://cluster.example.com", "io-cache.target.cluster.path-style-access": "true", "io-cache.policy": "meta,read", "io-cache.whitelist": "*", "io-cache.routes": "data=accel;data,meta=cluster"}, "op": "read", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "expect": "https://accelerator.example.com"}, - {"name": "multi: malformed routes are skipped", "options": {"fs.oss.endpoint": "https://accelerator.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.targets": "accel,cluster", "io-cache.target.accel.endpoint": "https://accelerator.example.com", "io-cache.target.accel.region": "cn-hangzhou", "io-cache.target.cluster.endpoint": "http://cluster.example.com", "io-cache.target.cluster.path-style-access": "true", "io-cache.policy": "meta,read", "io-cache.whitelist": "*", "io-cache.routes": "=cluster;meta=accel;junk"}, "op": "read", "path": "oss://bkt/db1.db/t1/manifest/manifest-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0", "expect": "https://accelerator.example.com"}, - {"name": "multi: malformed routes do not match other types", "options": {"fs.oss.endpoint": "https://accelerator.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.targets": "accel,cluster", "io-cache.target.accel.endpoint": "https://accelerator.example.com", "io-cache.target.accel.region": "cn-hangzhou", "io-cache.target.cluster.endpoint": "http://cluster.example.com", "io-cache.target.cluster.path-style-access": "true", "io-cache.policy": "meta,read", "io-cache.whitelist": "*", "io-cache.routes": "=cluster;meta=accel;junk"}, "op": "read", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "expect": "https://oss-cn-hangzhou-internal.aliyuncs.com"}, - {"name": "multi: whitelist filters before routes", "options": {"fs.oss.endpoint": "https://accelerator.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.targets": "accel,cluster", "io-cache.target.accel.endpoint": "https://accelerator.example.com", "io-cache.target.accel.region": "cn-hangzhou", "io-cache.target.cluster.endpoint": "http://cluster.example.com", "io-cache.target.cluster.path-style-access": "true", "io-cache.policy": "meta,read", "io-cache.whitelist": "meta", "io-cache.routes": "meta=accel;data,bucket-index,global-index,file-index=cluster"}, "op": "read", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "expect": "https://oss-cn-hangzhou-internal.aliyuncs.com"}, - {"name": "single: write stays at origin", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": "http://cache.example.com", "io-cache.policy": "meta,read", "io-cache.whitelist": "meta,data"}, "op": "write", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "expect": "https://oss-cn-hangzhou-internal.aliyuncs.com"}, - {"name": "multi: write stays at origin", "options": {"fs.oss.endpoint": "https://accelerator.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.targets": "accel,cluster", "io-cache.target.accel.endpoint": "https://accelerator.example.com", "io-cache.target.accel.region": "cn-hangzhou", "io-cache.target.cluster.endpoint": "http://cluster.example.com", "io-cache.target.cluster.path-style-access": "true", "io-cache.policy": "meta,read", "io-cache.whitelist": "*", "io-cache.routes": "meta=accel;data,bucket-index,global-index,file-index=cluster"}, "op": "write", "path": "oss://bkt/db1.db/t1/manifest/manifest-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0", "expect": "https://oss-cn-hangzhou-internal.aliyuncs.com"}, - {"name": "single: list stays at origin", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": "http://cache.example.com", "io-cache.policy": "meta,read", "io-cache.whitelist": "meta,data"}, "op": "list", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "expect": "https://oss-cn-hangzhou-internal.aliyuncs.com"}, - {"name": "multi: list stays at origin", "options": {"fs.oss.endpoint": "https://accelerator.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.targets": "accel,cluster", "io-cache.target.accel.endpoint": "https://accelerator.example.com", "io-cache.target.accel.region": "cn-hangzhou", "io-cache.target.cluster.endpoint": "http://cluster.example.com", "io-cache.target.cluster.path-style-access": "true", "io-cache.policy": "meta,read", "io-cache.whitelist": "*", "io-cache.routes": "meta=accel;data,bucket-index,global-index,file-index=cluster"}, "op": "list", "path": "oss://bkt/db1.db/t1/manifest/manifest-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0", "expect": "https://oss-cn-hangzhou-internal.aliyuncs.com"}, - {"name": "single: delete stays at origin", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": "http://cache.example.com", "io-cache.policy": "meta,read", "io-cache.whitelist": "meta,data"}, "op": "delete", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "expect": "https://oss-cn-hangzhou-internal.aliyuncs.com"}, - {"name": "multi: delete stays at origin", "options": {"fs.oss.endpoint": "https://accelerator.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.targets": "accel,cluster", "io-cache.target.accel.endpoint": "https://accelerator.example.com", "io-cache.target.accel.region": "cn-hangzhou", "io-cache.target.cluster.endpoint": "http://cluster.example.com", "io-cache.target.cluster.path-style-access": "true", "io-cache.policy": "meta,read", "io-cache.whitelist": "*", "io-cache.routes": "meta=accel;data,bucket-index,global-index,file-index=cluster"}, "op": "delete", "path": "oss://bkt/db1.db/t1/manifest/manifest-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0", "expect": "https://oss-cn-hangzhou-internal.aliyuncs.com"}, - {"name": "single: rename stays at origin", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": "http://cache.example.com", "io-cache.policy": "meta,read", "io-cache.whitelist": "meta,data"}, "op": "rename", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "expect": "https://oss-cn-hangzhou-internal.aliyuncs.com"}, - {"name": "multi: rename stays at origin", "options": {"fs.oss.endpoint": "https://accelerator.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.targets": "accel,cluster", "io-cache.target.accel.endpoint": "https://accelerator.example.com", "io-cache.target.accel.region": "cn-hangzhou", "io-cache.target.cluster.endpoint": "http://cluster.example.com", "io-cache.target.cluster.path-style-access": "true", "io-cache.policy": "meta,read", "io-cache.whitelist": "*", "io-cache.routes": "meta=accel;data,bucket-index,global-index,file-index=cluster"}, "op": "rename", "path": "oss://bkt/db1.db/t1/manifest/manifest-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0", "expect": "https://oss-cn-hangzhou-internal.aliyuncs.com"}, - {"name": "single: mkdirs stays at origin", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": "http://cache.example.com", "io-cache.policy": "meta,read", "io-cache.whitelist": "meta,data"}, "op": "mkdirs", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "expect": "https://oss-cn-hangzhou-internal.aliyuncs.com"}, - {"name": "multi: mkdirs stays at origin", "options": {"fs.oss.endpoint": "https://accelerator.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.targets": "accel,cluster", "io-cache.target.accel.endpoint": "https://accelerator.example.com", "io-cache.target.accel.region": "cn-hangzhou", "io-cache.target.cluster.endpoint": "http://cluster.example.com", "io-cache.target.cluster.path-style-access": "true", "io-cache.policy": "meta,read", "io-cache.whitelist": "*", "io-cache.routes": "meta=accel;data,bucket-index,global-index,file-index=cluster"}, "op": "mkdirs", "path": "oss://bkt/db1.db/t1/manifest/manifest-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0", "expect": "https://oss-cn-hangzhou-internal.aliyuncs.com"}, - {"name": "single: copy stays at origin", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": "http://cache.example.com", "io-cache.policy": "meta,read", "io-cache.whitelist": "meta,data"}, "op": "copy", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "expect": "https://oss-cn-hangzhou-internal.aliyuncs.com"}, - {"name": "multi: copy stays at origin", "options": {"fs.oss.endpoint": "https://accelerator.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.targets": "accel,cluster", "io-cache.target.accel.endpoint": "https://accelerator.example.com", "io-cache.target.accel.region": "cn-hangzhou", "io-cache.target.cluster.endpoint": "http://cluster.example.com", "io-cache.target.cluster.path-style-access": "true", "io-cache.policy": "meta,read", "io-cache.whitelist": "*", "io-cache.routes": "meta=accel;data,bucket-index,global-index,file-index=cluster"}, "op": "copy", "path": "oss://bkt/db1.db/t1/manifest/manifest-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0", "expect": "https://oss-cn-hangzhou-internal.aliyuncs.com"}, - {"name": "single: atomic-write stays at origin", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": "http://cache.example.com", "io-cache.policy": "meta,read", "io-cache.whitelist": "meta,data"}, "op": "atomic-write", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "expect": "https://oss-cn-hangzhou-internal.aliyuncs.com"}, - {"name": "multi: atomic-write stays at origin", "options": {"fs.oss.endpoint": "https://accelerator.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.targets": "accel,cluster", "io-cache.target.accel.endpoint": "https://accelerator.example.com", "io-cache.target.accel.region": "cn-hangzhou", "io-cache.target.cluster.endpoint": "http://cluster.example.com", "io-cache.target.cluster.path-style-access": "true", "io-cache.policy": "meta,read", "io-cache.whitelist": "*", "io-cache.routes": "meta=accel;data,bucket-index,global-index,file-index=cluster"}, "op": "atomic-write", "path": "oss://bkt/db1.db/t1/manifest/manifest-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0", "expect": "https://oss-cn-hangzhou-internal.aliyuncs.com"}, - {"name": "single: two-phase-write stays at origin", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": "http://cache.example.com", "io-cache.policy": "meta,read", "io-cache.whitelist": "meta,data"}, "op": "two-phase-write", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "expect": "https://oss-cn-hangzhou-internal.aliyuncs.com"}, - {"name": "multi: two-phase-write stays at origin", "options": {"fs.oss.endpoint": "https://accelerator.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.targets": "accel,cluster", "io-cache.target.accel.endpoint": "https://accelerator.example.com", "io-cache.target.accel.region": "cn-hangzhou", "io-cache.target.cluster.endpoint": "http://cluster.example.com", "io-cache.target.cluster.path-style-access": "true", "io-cache.policy": "meta,read", "io-cache.whitelist": "*", "io-cache.routes": "meta=accel;data,bucket-index,global-index,file-index=cluster"}, "op": "two-phase-write", "path": "oss://bkt/db1.db/t1/manifest/manifest-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0", "expect": "https://oss-cn-hangzhou-internal.aliyuncs.com"}, - {"name": "single: presign stays at origin", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": "http://cache.example.com", "io-cache.policy": "meta,read", "io-cache.whitelist": "meta,data"}, "op": "presign", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "expect": "https://oss-cn-hangzhou-internal.aliyuncs.com"}, - {"name": "multi: presign stays at origin", "options": {"fs.oss.endpoint": "https://accelerator.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.targets": "accel,cluster", "io-cache.target.accel.endpoint": "https://accelerator.example.com", "io-cache.target.accel.region": "cn-hangzhou", "io-cache.target.cluster.endpoint": "http://cluster.example.com", "io-cache.target.cluster.path-style-access": "true", "io-cache.policy": "meta,read", "io-cache.whitelist": "*", "io-cache.routes": "meta=accel;data,bucket-index,global-index,file-index=cluster"}, "op": "presign", "path": "oss://bkt/db1.db/t1/manifest/manifest-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0", "expect": "https://oss-cn-hangzhou-internal.aliyuncs.com"} - ] -} From 26fc70998b5bd0b917ecba96961240467ee6b947 Mon Sep 17 00:00:00 2001 From: Sun Dapeng Date: Wed, 7 Oct 2026 15:07:30 +0800 Subject: [PATCH 05/27] [python] Route exists to a target only with exists in io-cache.policy exists answers whether a file is still there, and a cache that keeps the metadata of deleted objects answers yes after the file was deleted on OSS. meta now covers file status only; exists and exists_batch use a target only when io-cache.policy lists exists, as in the Java client. --- .../pypaimon/filesystem/io_cache_routing.py | 10 ++++----- .../pypaimon/tests/io_cache_file_io_test.py | 8 +++++-- .../pypaimon/tests/io_cache_routing_test.py | 21 ++++++++++++++----- 3 files changed, 26 insertions(+), 13 deletions(-) diff --git a/paimon-python/pypaimon/filesystem/io_cache_routing.py b/paimon-python/pypaimon/filesystem/io_cache_routing.py index a8b38c807182..95339adb8896 100644 --- a/paimon-python/pypaimon/filesystem/io_cache_routing.py +++ b/paimon-python/pypaimon/filesystem/io_cache_routing.py @@ -63,11 +63,11 @@ def _flag(value) -> bool: def _parse_policy(value) -> FrozenSet[Op]: - # read, meta and write can use a target; none turns it off + # read, meta, exists and write can use a target; none turns it off tokens = {token.strip().lower() for token in str(value or "").split(",")} if "none" in tokens: return frozenset() - return frozenset(op for op in (Op.READ, Op.META, Op.WRITE) if op.value in tokens) + return frozenset(op for op in (Op.READ, Op.META, Op.EXISTS, Op.WRITE) if op.value in tokens) def _parse_routes(value: str) -> List[Tuple[set, str]]: @@ -192,10 +192,8 @@ def origin_endpoint(self) -> Optional[str]: def route(self, op: Op, path: str) -> Optional[str]: """Name of the target of one request, or None when it goes to origin.""" - # An existence check is a file status lookup, a two-phase write is a write. - if op is Op.EXISTS: - op = Op.META - elif op is Op.TWO_PHASE_WRITE: + # A two-phase write is a write; exists has its own token, as a cache may keep deleted files. + if op is Op.TWO_PHASE_WRITE: op = Op.WRITE # Only OSS paths can use cache targets. if op not in self._policy or not path.startswith("oss://"): diff --git a/paimon-python/pypaimon/tests/io_cache_file_io_test.py b/paimon-python/pypaimon/tests/io_cache_file_io_test.py index f4138f15c455..88f29cac75b7 100644 --- a/paimon-python/pypaimon/tests/io_cache_file_io_test.py +++ b/paimon-python/pypaimon/tests/io_cache_file_io_test.py @@ -217,6 +217,9 @@ def test_exists_follows_the_route(self): self._put(self.origin, OTHER_DATA, b"origin") self._put(self.origin, SNAPSHOT, b"origin") self._put(self.origin, TABLE + "/manifest/manifest-list-" + UUID + "-0", b"origin") + # Without the exists token the origin, which has neither file, answers. + self.assertFalse(self.file_io.exists(DATA)) + self.file_io = self._file_io(_routing(policy="meta,read,exists")) self.assertTrue(self.file_io.exists(DATA)) # OTHER_DATA is routed to the cluster, which does not have it self.assertFalse(self.file_io.exists(OTHER_DATA)) @@ -631,9 +634,10 @@ def test_write_and_delete_helpers_use_origin(call): @pytest.mark.parametrize("op,name,call", FILE_IO_CALLS, ids=["{} / {}".format(op, name) for op, name, _ in FILE_IO_CALLS]) -@pytest.mark.parametrize("policy", ["meta,read", "meta,read,write"]) +@pytest.mark.parametrize("policy", ["meta,read", "meta,read,write", "meta,read,exists"]) def test_each_file_io_call_reaches_its_endpoint(policy, op, name, call): - routed = {"read", "meta", "exists"} | ({"write", "two-phase-write"} if "write" in policy else set()) + tokens = set(policy.split(",")) + routed = {"read", "meta"} | ({"write", "two-phase-write"} if "write" in tokens else set()) | (tokens & {"exists"}) routing = _routing(policy=policy) for path, target in ((DATA, "cluster"), (MANIFEST, "accel"), (SNAPSHOT, "origin"), (UNKNOWN, "origin")): file_ios = {key: _RecordingFileIO() for key in ("origin", "accel", "cluster")} diff --git a/paimon-python/pypaimon/tests/io_cache_routing_test.py b/paimon-python/pypaimon/tests/io_cache_routing_test.py index 15a2330038dd..66df94420139 100644 --- a/paimon-python/pypaimon/tests/io_cache_routing_test.py +++ b/paimon-python/pypaimon/tests/io_cache_routing_test.py @@ -40,6 +40,7 @@ ACCEL = "https://accelerator.example.com" CLUSTER = "http://cluster.example.com" WRITE = "io-cache.policy=meta,read,write" +EXISTS = "io-cache.policy=meta,read,exists" ORIGIN_OPS = (Op.LIST, Op.DELETE, Op.RENAME, Op.MKDIRS, Op.COPY, Op.ATOMIC_WRITE, Op.PRESIGN) @@ -113,7 +114,10 @@ def test_one_cache_target(self): options = single() self.assert_endpoint(options, Op.READ, DATA_PATH, CACHE) self.assert_endpoint(options, Op.META, DATA_PATH, CACHE) - self.assert_endpoint(options, Op.EXISTS, DATA_PATH, CACHE) + # exists asks whether a file is still there, so it needs its own policy token + self.assert_endpoint(options, Op.EXISTS, DATA_PATH, OSS) + self.assert_endpoint(single(EXISTS), Op.EXISTS, DATA_PATH, CACHE) + self.assert_endpoint(single(EXISTS), Op.EXISTS, SNAPSHOT_PATH, OSS) self.assert_endpoint(options, Op.READ, MANIFEST_PATH, CACHE) self.assert_endpoint(options, Op.READ, "manifest/manifest-list-{uuid}-1", CACHE) self.assert_endpoint(options, Op.READ, "oss://other-bkt/db1.db/t1/" + DATA_PATH, CACHE) @@ -149,6 +153,9 @@ def test_policy_and_whitelist(self): write_only = single("io-cache.policy=write") self.assert_endpoint(write_only, Op.WRITE, DATA_PATH, CACHE) self.assert_endpoint(write_only, Op.EXISTS, DATA_PATH, OSS) + exists_only = single("io-cache.policy=exists") + self.assert_endpoint(exists_only, Op.EXISTS, DATA_PATH, CACHE) + self.assert_endpoint(exists_only, Op.META, DATA_PATH, OSS) self.assert_endpoint(single("-io-cache.whitelist"), Op.READ, INDEX_PATH, CACHE) self.assert_endpoint(single("io-cache.whitelist=*"), Op.READ, GLOBAL_INDEX_PATH, CACHE) @@ -185,9 +192,10 @@ def test_two_cache_targets(self): options = multi() self.assert_endpoint(options, Op.READ, MANIFEST_PATH, ACCEL) self.assert_endpoint(options, Op.META, MANIFEST_PATH, ACCEL) - self.assert_endpoint(options, Op.EXISTS, MANIFEST_PATH, ACCEL) + self.assert_endpoint(options, Op.EXISTS, MANIFEST_PATH, OSS) + self.assert_endpoint(multi(EXISTS), Op.EXISTS, MANIFEST_PATH, ACCEL) self.assert_endpoint(options, Op.READ, DATA_PATH, CLUSTER) - self.assert_endpoint(options, Op.EXISTS, DATA_PATH, CLUSTER) + self.assert_endpoint(multi(EXISTS), Op.EXISTS, DATA_PATH, CLUSTER) self.assert_endpoint(options, Op.READ, INDEX_PATH, CLUSTER) self.assert_endpoint(options, Op.READ, GLOBAL_INDEX_PATH, CLUSTER) self.assert_endpoint(options, Op.READ, SNAPSHOT_PATH, OSS) @@ -358,7 +366,7 @@ def test_create(self): self.assertIsNotNone(IoCacheRouting.create(self._options(**{"io-cache.enabled": " TRUE "}))) for overrides in ({"io-cache.enabled": "false"}, {"io-cache.enabled": "yes"}, {"io-cache.policy": "none"}, {"io-cache.policy": "read,none"}, - {"io-cache.policy": "exists,prefetch"}, {"io-cache.targets": ""}, + {"io-cache.policy": "prefetch,metadata"}, {"io-cache.targets": ""}, {"io-cache.targets": "Bad_Name,1x"}, {"dlf.oss-endpoint": "oss-cn-hangzhou.aliyuncs.com"}): with self.subTest(overrides=overrides): @@ -369,7 +377,7 @@ def test_create(self): def test_policy_routes_read_meta_and_write(self): routing = IoCacheRouting.create(self._options(**{"io-cache.policy": " Meta , READ ,prefetch,write"})) - routed = (Op.READ, Op.META, Op.EXISTS, Op.WRITE, Op.TWO_PHASE_WRITE) + routed = (Op.READ, Op.META, Op.WRITE, Op.TWO_PHASE_WRITE) for op in Op: with self.subTest(op=op): expected = "default" if op in routed else None @@ -377,6 +385,9 @@ def test_policy_routes_read_meta_and_write(self): write_only = IoCacheRouting.create(self._options(**{"io-cache.policy": "write"})) self.assertEqual("default", write_only.route(Op.WRITE, self.DATA)) self.assertIsNone(write_only.route(Op.READ, self.DATA)) + exists_only = IoCacheRouting.create(self._options(**{"io-cache.policy": "exists"})) + self.assertEqual("default", exists_only.route(Op.EXISTS, self.DATA)) + self.assertIsNone(exists_only.route(Op.META, self.DATA)) read_only = IoCacheRouting.create(self._options(**{"io-cache.policy": "read"})) self.assertIsNone(read_only.route(Op.META, self.DATA)) self.assertIsNone(read_only.route(Op.EXISTS, self.DATA)) From da72147920e4ce03a5b86c1ffd80abd6e5ffbb9d Mon Sep 17 00:00:00 2001 From: Sun Dapeng Date: Wed, 7 Oct 2026 19:41:48 +0800 Subject: [PATCH 06/27] [python] Route metadata and index files only when named as Paimon writes them The broad FileType classifier also matched names such as manifest.parquet, stat-2024.parquet or foo.index, which a Format Table may hold as ordinary files and replace in place, so a routed read could return the old file from a cache. A metadata or index file now uses a target only under the exact name Paimon writes, as in the Java and Rust clients; every other name goes to the origin. --- .../pypaimon/filesystem/io_cache_routing.py | 16 +++++++++++++--- .../pypaimon/tests/io_cache_routing_test.py | 17 +++++++++++++++++ 2 files changed, 30 insertions(+), 3 deletions(-) diff --git a/paimon-python/pypaimon/filesystem/io_cache_routing.py b/paimon-python/pypaimon/filesystem/io_cache_routing.py index 95339adb8896..fe17d4ce1c39 100644 --- a/paimon-python/pypaimon/filesystem/io_cache_routing.py +++ b/paimon-python/pypaimon/filesystem/io_cache_routing.py @@ -34,9 +34,17 @@ IO_CACHE_TARGET_PREFIX = "io-cache.target." _TARGET_NAME = re.compile(r"[a-z][a-z0-9-]*") +_UUID = r"[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}" # Data files are named {prefix}{uuid}-{count}.{extension}; this matches what follows the prefix. -_DATA_FILE_SUFFIX = re.compile( - r"[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}-[0-9]+\..+") +_DATA_FILE_SUFFIX = re.compile(_UUID + r"-[0-9]+\..+") +# Other types are routed only under the names Paimon writes; Format Table files may be replaced in place. +_PAIMON_FILE_NAMES = { + FileType.META: re.compile( + r"(manifest|manifest-list|index-manifest|stat)-" + _UUID + r"-[0-9]+|manifest-" + _UUID + + r"-[0-9]+\.avro\.sidecar"), + FileType.BUCKET_INDEX: re.compile(r"index-" + _UUID + r"-[0-9]+"), + FileType.GLOBAL_INDEX: re.compile(r"[a-z0-9_-]+-global-index-" + _UUID + r"\.index"), +} _MANIFEST_SIDECAR_SUFFIX = ".avro.sidecar" # JindoSDK cache client keys; the origin option set must not talk to a cache. _ORIGIN_DROPPED_PREFIXES = ("fs.oss.dlf-cache.", "fs.jindocache.") @@ -119,7 +127,9 @@ def routable_type(path: str, prefixes: Tuple[str, ...]) -> Optional[FileType]: name.startswith("changelog-") and parent == "changelog"): return None file_type = FileType.classify(path) - return None if file_type == FileType.DATA else file_type + # data files and their file indexes are recognized by _is_data_file_name + pattern = _PAIMON_FILE_NAMES.get(file_type) + return file_type if pattern is not None and pattern.fullmatch(name) else None def _is_data_file_name(name: str, prefixes: Tuple[str, ...]) -> bool: diff --git a/paimon-python/pypaimon/tests/io_cache_routing_test.py b/paimon-python/pypaimon/tests/io_cache_routing_test.py index 66df94420139..7e6ee6f37fe4 100644 --- a/paimon-python/pypaimon/tests/io_cache_routing_test.py +++ b/paimon-python/pypaimon/tests/io_cache_routing_test.py @@ -131,6 +131,9 @@ def test_one_cache_target(self): self.assert_endpoint(options, Op.READ, "dls://bkt/db1.db/t1/" + DATA_PATH, OSS) # the whitelist has no index self.assert_endpoint(options, Op.READ, INDEX_PATH, OSS) + # a Format Table file named like a manifest may be replaced in place + self.assert_endpoint(options, Op.READ, "review_external/manifest.parquet", OSS) + self.assert_endpoint(options, Op.META, "review_external/manifest.parquet", OSS) self.assert_endpoint(options, Op.WRITE, DATA_PATH, OSS) self.assert_endpoint(options, Op.TWO_PHASE_WRITE, DATA_PATH, OSS) for op in ORIGIN_OPS: @@ -279,6 +282,20 @@ def test_routable_type(self): "dt=1/bucket-0/custom-{uuid}-0.orc", "README"): self.assert_type(path, None) + # Format Table files named like metadata or indexes, which may be replaced in place + for path in ("dt=1/manifest.parquet", + "dt=1/stat-2024.parquet", + "dt=1/index-a.csv", + "dt=1/foo.index", + "review_external/manifest.parquet", + "manifest/manifest-old", + "manifest/manifest-old.avro.sidecar", + "manifest/manifest-list-{uuid}-1.avro.sidecar", + "statistics/stat-old", + "index/index-old", + "index/my-global-index.index", + "index/btree-global-index-{uuid}-0.index"): + self.assert_type(path, None) self.assert_type("manifest/manifest-{uuid}-0", FileType.META) self.assert_type("manifest/manifest-list-{uuid}-1", FileType.META) From b449c5e266da8b4d1a573fa4601fc592a100e826 Mon Sep 17 00:00:00 2001 From: Sun Dapeng Date: Thu, 8 Oct 2026 16:19:36 +0800 Subject: [PATCH 07/27] [python] Route the changelog manifests paimon-rust writes paimon-rust's TableCommit names input-changelog manifests manifest-{uuid}-changelog-{count}, optionally with .avro.sidecar. They are written once under a unique name like every other manifest, so the metadata name rule now accepts the -changelog infix for manifests (not for manifest lists), matching paimon-rust and the Java client. --- paimon-python/pypaimon/filesystem/io_cache_routing.py | 7 ++++--- paimon-python/pypaimon/tests/io_cache_routing_test.py | 4 ++++ 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/paimon-python/pypaimon/filesystem/io_cache_routing.py b/paimon-python/pypaimon/filesystem/io_cache_routing.py index fe17d4ce1c39..e1be04bfb401 100644 --- a/paimon-python/pypaimon/filesystem/io_cache_routing.py +++ b/paimon-python/pypaimon/filesystem/io_cache_routing.py @@ -37,11 +37,12 @@ _UUID = r"[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}" # Data files are named {prefix}{uuid}-{count}.{extension}; this matches what follows the prefix. _DATA_FILE_SUFFIX = re.compile(_UUID + r"-[0-9]+\..+") -# Other types are routed only under the names Paimon writes; Format Table files may be replaced in place. +# Other types are routed only under the names Paimon writes (paimon-rust adds -changelog to changelog +# manifests); Format Table files may be replaced in place. _PAIMON_FILE_NAMES = { FileType.META: re.compile( - r"(manifest|manifest-list|index-manifest|stat)-" + _UUID + r"-[0-9]+|manifest-" + _UUID - + r"-[0-9]+\.avro\.sidecar"), + r"(manifest-list|index-manifest|stat)-" + _UUID + r"-[0-9]+|manifest-" + _UUID + + r"(-changelog)?-[0-9]+(\.avro\.sidecar)?"), FileType.BUCKET_INDEX: re.compile(r"index-" + _UUID + r"-[0-9]+"), FileType.GLOBAL_INDEX: re.compile(r"[a-z0-9_-]+-global-index-" + _UUID + r"\.index"), } diff --git a/paimon-python/pypaimon/tests/io_cache_routing_test.py b/paimon-python/pypaimon/tests/io_cache_routing_test.py index 7e6ee6f37fe4..389cf1e48cb6 100644 --- a/paimon-python/pypaimon/tests/io_cache_routing_test.py +++ b/paimon-python/pypaimon/tests/io_cache_routing_test.py @@ -291,6 +291,8 @@ def test_routable_type(self): "manifest/manifest-old", "manifest/manifest-old.avro.sidecar", "manifest/manifest-list-{uuid}-1.avro.sidecar", + "manifest/manifest-list-{uuid}-changelog-1", + "manifest/manifest-{uuid}-changelog", "statistics/stat-old", "index/index-old", "index/my-global-index.index", @@ -301,6 +303,8 @@ def test_routable_type(self): self.assert_type("manifest/manifest-list-{uuid}-1", FileType.META) self.assert_type("manifest/index-manifest-{uuid}-0", FileType.META) self.assert_type("manifest/manifest-{uuid}-0.avro.sidecar", FileType.META) + self.assert_type("manifest/manifest-{uuid}-changelog-0", FileType.META) + self.assert_type("manifest/manifest-{uuid}-changelog-0.avro.sidecar", FileType.META) self.assert_type("statistics/stat-{uuid}-0", FileType.META) self.assert_type("dt=1/bucket-0/data-{uuid}-0.parquet", FileType.DATA) self.assert_type("dt=1/bucket-0/changelog-{uuid}-0.parquet", FileType.DATA) From 8e65f00259baa5e9a5d8fd14ec3372134e7efd3e Mon Sep 17 00:00:00 2001 From: Sun Dapeng Date: Tue, 13 Oct 2026 01:51:02 +0800 Subject: [PATCH 08/27] [python] Give Lance and Vortex the origin's options under io-cache routing Lance and Vortex open files with their own storage clients, which take the endpoint as a virtual-hosted HTTPS address and cannot reach an HTTP, IP or path-style io-cache target. They now get the options of the origin FileIO, as without routing, and only Paimon's own FileIO calls use targets. --- .../pypaimon/filesystem/io_cache_file_io.py | 4 -- .../read/reader/format_lance_reader.py | 2 +- .../read/reader/format_vortex_reader.py | 2 +- .../pypaimon/read/reader/lance_utils.py | 17 ++----- .../pypaimon/read/reader/vortex_utils.py | 17 ++----- .../pypaimon/tests/lance_utils_test.py | 50 +++++++++---------- 6 files changed, 33 insertions(+), 59 deletions(-) diff --git a/paimon-python/pypaimon/filesystem/io_cache_file_io.py b/paimon-python/pypaimon/filesystem/io_cache_file_io.py index 094c56cac265..acbd034e9a77 100644 --- a/paimon-python/pypaimon/filesystem/io_cache_file_io.py +++ b/paimon-python/pypaimon/filesystem/io_cache_file_io.py @@ -69,10 +69,6 @@ def _file_io(self, name: Optional[str]) -> FileIO: def _call(self, op: Op, path: str, action): return action(self._file_io(self._route(op, path))) - def read_file_io(self, path: str) -> FileIO: - """FileIO for native readers that open files using their own storage client.""" - return self._file_io(self._route(Op.READ, path)) - def _call_batch(self, op: Op, paths: List[str], action) -> list: """Runs action(file_io, paths) once per routed group; results follow the order of paths.""" results = [None] * len(paths) diff --git a/paimon-python/pypaimon/read/reader/format_lance_reader.py b/paimon-python/pypaimon/read/reader/format_lance_reader.py index a827245805e0..67edfbd02530 100644 --- a/paimon-python/pypaimon/read/reader/format_lance_reader.py +++ b/paimon-python/pypaimon/read/reader/format_lance_reader.py @@ -45,7 +45,7 @@ def __init__(self, file_io: FileIO, file_path: str, read_fields: List[DataField] self._read_field_names = [f.name for f in read_fields] - file_path_for_lance, storage_options = to_lance_specified(file_io, file_path, is_read=True) + file_path_for_lance, storage_options = to_lance_specified(file_io, file_path) # Read file metadata (footer only) to get actual schema file_schema_names = set( diff --git a/paimon-python/pypaimon/read/reader/format_vortex_reader.py b/paimon-python/pypaimon/read/reader/format_vortex_reader.py index 3787717209da..fcd50a24f005 100644 --- a/paimon-python/pypaimon/read/reader/format_vortex_reader.py +++ b/paimon-python/pypaimon/read/reader/format_vortex_reader.py @@ -42,7 +42,7 @@ def __init__(self, file_io: FileIO, file_path: str, read_fields: List[DataField] import vortex from pypaimon.read.reader.vortex_utils import to_vortex_specified - file_path_for_vortex, store_kwargs = to_vortex_specified(file_io, file_path, is_read=True) + file_path_for_vortex, store_kwargs = to_vortex_specified(file_io, file_path) if store_kwargs: from vortex import store diff --git a/paimon-python/pypaimon/read/reader/lance_utils.py b/paimon-python/pypaimon/read/reader/lance_utils.py index b593b15ea670..1c4df400e612 100644 --- a/paimon-python/pypaimon/read/reader/lance_utils.py +++ b/paimon-python/pypaimon/read/reader/lance_utils.py @@ -21,11 +21,9 @@ from pypaimon.common.file_io import FileIO from pypaimon.common.options.config import OssOptions -from pypaimon.filesystem.io_cache_file_io import IoCacheRoutingFileIO -def to_lance_specified(file_io: FileIO, file_path: str, - is_read: bool = False) -> Tuple[str, Optional[Dict[str, str]]]: +def to_lance_specified(file_io: FileIO, file_path: str) -> Tuple[str, Optional[Dict[str, str]]]: """Convert path and extract storage options for Lance format.""" # For RESTTokenFileIO, get underlying FileIO which already has latest token merged # This follows Java implementation: ((RESTTokenFileIO) fileIO).fileIO() @@ -34,9 +32,6 @@ def to_lance_specified(file_io: FileIO, file_path: str, # Call file_io() to get underlying FileIO with latest token # This ensures token is refreshed and merged with catalog options file_io = file_io.file_io() - - if is_read and isinstance(file_io, IoCacheRoutingFileIO): - file_io = file_io.read_file_io(file_path) # Now get properties from the underlying FileIO (which has latest token) if hasattr(file_io, 'get_merged_properties'): @@ -67,13 +62,9 @@ def to_lance_specified(file_io: FileIO, file_path: str, storage_options[key] = value endpoint = properties.get(OssOptions.OSS_ENDPOINT) - path_style = properties.get(OssOptions.OSS_SECOND_LEVEL_DOMAIN_ENABLE) if endpoint: - parsed_endpoint = urlparse(endpoint if '://' in endpoint else 'https://' + endpoint) - host = parsed_endpoint.netloc if path_style else f"{bucket}.{parsed_endpoint.netloc}" - storage_options['endpoint'] = f"{parsed_endpoint.scheme}://{host}" - if parsed_endpoint.scheme == 'http': - storage_options['allow_http'] = 'true' + endpoint_clean = endpoint.replace('http://', '').replace('https://', '') + storage_options['endpoint'] = f"https://{bucket}.{endpoint_clean}" if properties.contains(OssOptions.OSS_ACCESS_KEY_ID): storage_options['access_key_id'] = properties.get(OssOptions.OSS_ACCESS_KEY_ID) @@ -87,7 +78,7 @@ def to_lance_specified(file_io: FileIO, file_path: str, if properties.contains(OssOptions.OSS_ENDPOINT): storage_options['oss_endpoint'] = properties.get(OssOptions.OSS_ENDPOINT) - storage_options['virtual_hosted_style_request'] = 'false' if path_style else 'true' + storage_options['virtual_hosted_style_request'] = 'true' if bucket and path: file_path_for_lance = f"oss://{bucket}/{path}" diff --git a/paimon-python/pypaimon/read/reader/vortex_utils.py b/paimon-python/pypaimon/read/reader/vortex_utils.py index 5f4fb0d554a8..d66682898f9a 100644 --- a/paimon-python/pypaimon/read/reader/vortex_utils.py +++ b/paimon-python/pypaimon/read/reader/vortex_utils.py @@ -21,11 +21,9 @@ from pypaimon.common.file_io import FileIO from pypaimon.common.options.config import OssOptions -from pypaimon.filesystem.io_cache_file_io import IoCacheRoutingFileIO -def to_vortex_specified(file_io: FileIO, file_path: str, - is_read: bool = False) -> Tuple[str, Optional[Dict[str, str]]]: +def to_vortex_specified(file_io: FileIO, file_path: str) -> Tuple[str, Optional[Dict[str, str]]]: """Convert path and extract storage options for Vortex store.from_url(). Returns (url, store_kwargs) where store_kwargs can be passed as @@ -35,9 +33,6 @@ def to_vortex_specified(file_io: FileIO, file_path: str, if hasattr(file_io, 'file_io'): file_io = file_io.file_io() - if is_read and isinstance(file_io, IoCacheRoutingFileIO): - file_io = file_io.read_file_io(file_path) - if hasattr(file_io, 'get_merged_properties'): properties = file_io.get_merged_properties() else: @@ -59,19 +54,13 @@ def to_vortex_specified(file_io: FileIO, file_path: str, if scheme == 'oss' and properties: parsed = urlparse(file_path) bucket = parsed.netloc - endpoint = properties.get(OssOptions.OSS_ENDPOINT) - parsed_endpoint = urlparse(endpoint if '://' in endpoint else 'https://' + endpoint) - path_style = properties.get(OssOptions.OSS_SECOND_LEVEL_DOMAIN_ENABLE) - host = parsed_endpoint.netloc if path_style else f"{bucket}.{parsed_endpoint.netloc}" store_kwargs = { - 'endpoint': f"{parsed_endpoint.scheme}://{host}", + 'endpoint': f"https://{bucket}.{properties.get(OssOptions.OSS_ENDPOINT)}", 'access_key_id': properties.get(OssOptions.OSS_ACCESS_KEY_ID), 'secret_access_key': properties.get(OssOptions.OSS_ACCESS_KEY_SECRET), - 'virtual_hosted_style_request': 'false' if path_style else 'true', + 'virtual_hosted_style_request': 'true', } - if parsed_endpoint.scheme == 'http': - store_kwargs['allow_http'] = 'true' if properties.contains(OssOptions.OSS_SECURITY_TOKEN): store_kwargs['session_token'] = properties.get(OssOptions.OSS_SECURITY_TOKEN) diff --git a/paimon-python/pypaimon/tests/lance_utils_test.py b/paimon-python/pypaimon/tests/lance_utils_test.py index c66b259a36a6..35b0fc893c24 100644 --- a/paimon-python/pypaimon/tests/lance_utils_test.py +++ b/paimon-python/pypaimon/tests/lance_utils_test.py @@ -23,36 +23,10 @@ from pypaimon.filesystem.io_cache_file_io import IoCacheRoutingFileIO from pypaimon.filesystem.io_cache_routing import IoCacheRouting from pypaimon.read.reader.lance_utils import to_lance_specified -from pypaimon.read.reader.vortex_utils import to_vortex_specified class LanceUtilsTest(unittest.TestCase): - def test_native_reads_use_target_addressing_and_writes_use_origin(self): - path = "oss://bkt/t/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-1.lance" - options = { - "fs.oss.endpoint": "https://origin.example.com", - "fs.oss.accessKeyId": "ak", - "fs.oss.accessKeySecret": "sk", - "io-cache.enabled": "true", - "io-cache.policy": "read,meta", - "io-cache.endpoint": "http://cache.example.com:8080", - "io-cache.target.default.path-style-access": "true", - } - routing = IoCacheRouting.create(options) - origin = FileIO.get(path, Options(routing.origin_options(options))) - file_io = IoCacheRoutingFileIO(routing, origin, lambda name: FileIO.get( - path, Options(routing.target_options(options, name)))) - for convert in (to_lance_specified, to_vortex_specified): - with self.subTest(convert=convert.__name__): - _, read = convert(file_io, path, is_read=True) - self.assertEqual("http://cache.example.com:8080", read["endpoint"]) - self.assertEqual("false", read["virtual_hosted_style_request"]) - self.assertEqual("true", read["allow_http"]) - _, write = convert(file_io, path) - self.assertEqual("https://bkt.origin.example.com", write["endpoint"]) - self.assertEqual("true", write["virtual_hosted_style_request"]) - def test_oss_url_bucket_extraction_correctness(self): file_path = "oss://test-bucket/db-name.db/table-name/bucket-0/data.lance" @@ -78,6 +52,30 @@ def test_oss_url_bucket_extraction_correctness(self): self.assertTrue('fs.oss.accessKeyId' in storage_options) self.assertTrue('fs.oss.accessKeySecret' in storage_options) + def test_io_cache_routing_hands_lance_the_origin(self): + # Lance builds its own client from these options, so it reads from origin, never a target. + file_path = "oss://bkt/db.db/t/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.lance" + options = { + OssOptions.OSS_ENDPOINT.key(): "accel.example.com", + OssOptions.OSS_ACCESS_KEY_ID.key(): "ak", + OssOptions.OSS_ACCESS_KEY_SECRET.key(): "sk", + "io-cache.enabled": "true", + "io-cache.origin.endpoint": "origin.example.com", + "io-cache.endpoint": "http://127.0.0.1:8080", + "io-cache.target.default.path-style-access": "true", + "io-cache.policy": "meta,read", + } + routing = IoCacheRouting.create(options) + origin = FileIO.get(file_path, Options(routing.origin_options(options))) + created = [] + file_io = IoCacheRoutingFileIO(routing, origin, lambda name: created.append(name)) + + _, storage_options = to_lance_specified(file_io, file_path) + + self.assertEqual("https://bkt.origin.example.com", storage_options['endpoint']) + self.assertEqual('true', storage_options['virtual_hosted_style_request']) + self.assertEqual([], created) + def test_oss_url_with_security_token(self): file_path = "oss://my-bucket/path/to/file.lance" From bf0c925930fa50a2f0a7703a403fee82b0830a19 Mon Sep 17 00:00:00 2001 From: Sun Dapeng Date: Tue, 13 Oct 2026 01:51:02 +0800 Subject: [PATCH 09/27] [python] Recreate io-cache target state in a forked child A child forked while another thread of the parent was creating a target inherited the held lock and waited on it forever. The routing FileIO now notices the new pid and starts with an empty target map and a new lock. --- .../pypaimon/filesystem/io_cache_file_io.py | 7 ++++ .../pypaimon/tests/io_cache_file_io_test.py | 32 +++++++++++++++++++ 2 files changed, 39 insertions(+) diff --git a/paimon-python/pypaimon/filesystem/io_cache_file_io.py b/paimon-python/pypaimon/filesystem/io_cache_file_io.py index acbd034e9a77..3beab19e50ba 100644 --- a/paimon-python/pypaimon/filesystem/io_cache_file_io.py +++ b/paimon-python/pypaimon/filesystem/io_cache_file_io.py @@ -17,6 +17,7 @@ """FileIO that sends each request to an io-cache target or the origin endpoint.""" +import os import threading from typing import Callable, Dict, List, Optional @@ -38,6 +39,7 @@ def __init__(self, routing: IoCacheRouting, origin: FileIO, self._create_target = create_target self._targets: Dict[str, FileIO] = {} self._targets_lock = threading.Lock() + self._pid = os.getpid() self._filesystem = pafs.PyFileSystem(_IoCacheRoutingHandler(self)) @property @@ -56,6 +58,11 @@ def _file_io(self, name: Optional[str]) -> FileIO: """Origin, or the FileIO of a target, created on first use.""" if name is None: return self._origin + if self._pid != os.getpid(): + # A forked child inherits the lock in whatever state a parent thread left it. + self._targets = {} + self._targets_lock = threading.Lock() + self._pid = os.getpid() file_io = self._targets.get(name) if file_io is not None: return file_io diff --git a/paimon-python/pypaimon/tests/io_cache_file_io_test.py b/paimon-python/pypaimon/tests/io_cache_file_io_test.py index 88f29cac75b7..800614115169 100644 --- a/paimon-python/pypaimon/tests/io_cache_file_io_test.py +++ b/paimon-python/pypaimon/tests/io_cache_file_io_test.py @@ -21,9 +21,11 @@ import gzip import inspect import io +import multiprocessing import os import shutil import tempfile +import threading import unittest from datetime import timedelta from pathlib import Path @@ -196,6 +198,36 @@ def test_targets_are_created_on_first_use(self): self.file_io.new_input_stream(MANIFEST) self.assertEqual(["cluster", "accel"], self.created) + @unittest.skipUnless(hasattr(os, "fork"), "needs fork") + def test_forked_child_does_not_wait_on_the_parents_target_lock(self): + self._put_all(DATA) + parent_pid = os.getpid() + creating, release = threading.Event(), threading.Event() + + def create_target(name): + # the parent stays inside the lock until released; a forked child must not wait for it + if os.getpid() == parent_pid: + creating.set() + release.wait(30) + return self.cluster + + file_io = IoCacheRoutingFileIO(_routing(), self.origin, create_target) + parent = threading.Thread(target=lambda: file_io.new_input_stream(DATA).read()) + parent.start() + try: + self.assertTrue(creating.wait(10)) + child = multiprocessing.get_context("fork").Process( + target=lambda: os._exit(0 if file_io.new_input_stream(DATA).read() == b"cluster" else 1)) + child.start() + child.join(10) + if child.exitcode is None: + child.kill() + child.join() + self.assertEqual(0, child.exitcode) + finally: + release.set() + parent.join(10) + def test_reads_without_read_policy_use_origin(self): self._put_all(DATA) file_io = self._file_io(_routing(policy="meta")) From dd9e89228bd8590d5f9982c7dee978be9e5ed261 Mon Sep 17 00:00:00 2001 From: Sun Dapeng Date: Tue, 13 Oct 2026 02:01:47 +0800 Subject: [PATCH 10/27] [python] Keep path-style OSS addressing after unpickling A PyArrowFileIO restored from a pickle, as in a Ray worker, recomputed whether the bucket goes into the endpoint without looking at path-style addressing, so with PyArrow below 16 it put the bucket in front of a path-style io-cache target again. --- paimon-python/pypaimon/filesystem/pyarrow_file_io.py | 5 ++--- paimon-python/pypaimon/tests/file_io_test.py | 6 +++++- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/paimon-python/pypaimon/filesystem/pyarrow_file_io.py b/paimon-python/pypaimon/filesystem/pyarrow_file_io.py index b4b689174995..90636dddf18b 100644 --- a/paimon-python/pypaimon/filesystem/pyarrow_file_io.py +++ b/paimon-python/pypaimon/filesystem/pyarrow_file_io.py @@ -63,11 +63,10 @@ class PyArrowFileIO(FileIO): def __init__(self, path: str, catalog_options: Options): self.properties = catalog_options self.logger = logging.getLogger(__name__) - self._set_pyarrow_version() scheme, netloc, _ = self.parse_location(path) # Path-style addressing sends bucket/key to the endpoint itself, so the bucket stays out of it. self._oss_path_style = scheme == "oss" and self.properties.get(OssOptions.OSS_SECOND_LEVEL_DOMAIN_ENABLE) - self._oss_bucket_in_endpoint = not self._pyarrow_gte_16 and not self._oss_path_style + self._set_pyarrow_version() self.uri_reader_factory = UriReaderFactory(catalog_options) self._is_oss = scheme in {"oss"} self._is_s3 = scheme in {"s3", "s3a", "s3n"} @@ -120,7 +119,7 @@ def _set_pyarrow_version(self): # goes into endpoint_override, so keys must omit it (init + path share # this flag so they can't drift). self._pyarrow_gte_16 = parse(pyarrow.__version__) >= parse("16.0.0") - self._oss_bucket_in_endpoint = not self._pyarrow_gte_16 + self._oss_bucket_in_endpoint = not self._pyarrow_gte_16 and not getattr(self, "_oss_path_style", False) def _uses_s3_compatibility(self) -> bool: return (not self._use_jindo diff --git a/paimon-python/pypaimon/tests/file_io_test.py b/paimon-python/pypaimon/tests/file_io_test.py index dbd6cacf4ee9..9c7e0a8232a2 100644 --- a/paimon-python/pypaimon/tests/file_io_test.py +++ b/paimon-python/pypaimon/tests/file_io_test.py @@ -16,6 +16,7 @@ # under the License. import os +import pickle import shutil import tempfile import unittest @@ -181,10 +182,13 @@ def test_legacy_oss_path_style_keeps_bucket_out_of_endpoint(self): patch('pypaimon.filesystem.pyarrow_file_io.pafs.AwsStandardS3RetryStrategy', create=True), \ patch('pypaimon.filesystem.pyarrow_file_io.pafs.S3FileSystem') as s3_file_system: oss_io = OssFileIO("oss://test-bucket/warehouse", options) - kwargs = s3_file_system.call_args[1] + kwargs = s3_file_system.call_args[1] + # unpickling (e.g. in a Ray worker) must keep path-style addressing + restored = pickle.loads(pickle.dumps(oss_io)) self.assertEqual('10.0.0.1:8080', kwargs['endpoint_override']) self.assertEqual(virtual, kwargs.get('force_virtual_addressing')) self.assertFalse(oss_io._oss_bucket_in_endpoint) + self.assertFalse(restored._oss_bucket_in_endpoint) oss_io = OssFileIO("oss://test-bucket/warehouse", options) self.assertEqual("test-bucket/warehouse/data-1.parquet", oss_io.to_filesystem_path("oss://test-bucket/warehouse/data-1.parquet")) From 2bd427a35c8b40b60efb3be1ea62acd0b4370dd3 Mon Sep 17 00:00:00 2001 From: Sun Dapeng Date: Sun, 25 Oct 2026 01:41:10 +0800 Subject: [PATCH 11/27] [python] Give Vortex the host of a URL origin endpoint under io-cache routing Under io-cache routing the origin FileIO takes io-cache.origin.endpoint, which may be a URL, so Vortex, which reads with its own client from the origin, got https://bucket.https://host. It now gets the bucket in front of the host. The new test pins that Vortex reads the origin, not a path-style IP target, through RESTTokenFileIO; it replaces the Lance test that went away when upstream dropped the Lance file format. --- .../pypaimon/filesystem/vortex_utils.py | 6 ++++- .../tests/rest/rest_token_file_io_test.py | 23 +++++++++++++++++++ 2 files changed, 28 insertions(+), 1 deletion(-) diff --git a/paimon-python/pypaimon/filesystem/vortex_utils.py b/paimon-python/pypaimon/filesystem/vortex_utils.py index d66682898f9a..fea8cc7e2a98 100644 --- a/paimon-python/pypaimon/filesystem/vortex_utils.py +++ b/paimon-python/pypaimon/filesystem/vortex_utils.py @@ -54,9 +54,13 @@ def to_vortex_specified(file_io: FileIO, file_path: str) -> Tuple[str, Optional[ if scheme == 'oss' and properties: parsed = urlparse(file_path) bucket = parsed.netloc + endpoint = properties.get(OssOptions.OSS_ENDPOINT) + if endpoint and '://' in endpoint: + # The origin endpoint of io-cache routing can be a URL; the bucket goes before its host. + endpoint = endpoint.split('://', 1)[1].rstrip('/') store_kwargs = { - 'endpoint': f"https://{bucket}.{properties.get(OssOptions.OSS_ENDPOINT)}", + 'endpoint': f"https://{bucket}.{endpoint}", 'access_key_id': properties.get(OssOptions.OSS_ACCESS_KEY_ID), 'secret_access_key': properties.get(OssOptions.OSS_ACCESS_KEY_SECRET), 'virtual_hosted_style_request': 'true', diff --git a/paimon-python/pypaimon/tests/rest/rest_token_file_io_test.py b/paimon-python/pypaimon/tests/rest/rest_token_file_io_test.py index 2ff82967b485..286f8f026db8 100644 --- a/paimon-python/pypaimon/tests/rest/rest_token_file_io_test.py +++ b/paimon-python/pypaimon/tests/rest/rest_token_file_io_test.py @@ -40,6 +40,7 @@ from pypaimon.filesystem.io_cache_file_io import IoCacheRoutingFileIO from pypaimon.filesystem.io_cache_routing import Op from pypaimon.filesystem.local_file_io import LocalFileIO +from pypaimon.filesystem.vortex_utils import to_vortex_specified from pypaimon.table.row.blob import BlobDescriptor _FILE_IO_GET = 'pypaimon.catalog.rest.rest_token_file_io.FileIO.get' @@ -1198,6 +1199,28 @@ def test_dlf_oss_endpoint_turns_io_cache_routing_off(self): self.assertNotIsInstance(resolved, IoCacheRoutingFileIO) self.assertEqual([override], [options[OssOptions.OSS_ENDPOINT.key()] for options in created]) + def test_io_cache_routing_hands_vortex_the_origin(self): + # Vortex builds its own virtual-hosted client, so it reads the origin even with a path-style IP target. + token = dict(self._IO_CACHE_TOKEN, **{ + OssOptions.OSS_ACCESS_KEY_ID.key(): "ak", + OssOptions.OSS_ACCESS_KEY_SECRET.key(): "sk", + "io-cache.endpoint": "http://127.0.0.1:8080", + "io-cache.target.default.path-style-access": "true", + }) + path = self._TABLE + "/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.vortex" + file_io = RESTTokenFileIO(self.identifier, self._TABLE, Options({})) + file_io.token = RESTToken(token, int(time.time() * 1000) + 7_200_000) + with patch.object(RESTTokenFileIO, 'try_to_refresh_token'): + url, store_kwargs = to_vortex_specified(file_io, path) + routed = file_io.file_io() + + self.assertIsInstance(routed, IoCacheRoutingFileIO) + self.assertEqual("default", routed._routing.route(Op.READ, path)) + self.assertEqual({}, routed._targets) + self.assertEqual(path.replace("oss://", "s3://"), url) + self.assertEqual("https://bkt.oss-cn-hangzhou-internal.aliyuncs.com", store_kwargs['endpoint']) + self.assertEqual('true', store_kwargs['virtual_hosted_style_request']) + if __name__ == '__main__': unittest.main() From 880d98f08a9e8af78b1c64024074a1a9681225bc Mon Sep 17 00:00:00 2001 From: Sun Dapeng Date: Sun, 25 Oct 2026 18:36:52 +0800 Subject: [PATCH 12/27] [python] Create the parent directories of routed writes on origin With write in io-cache.policy, a routed write ran entirely on the target FileIO. On the PyArrow S3 backend (fs.oss.impl=legacy, also the fallback without pyjindosdk), PyArrowFileIO.new_output_stream first checks the parent directory and creates it, so the target received HEAD and LIST probes of directories, HeadBucket and directory-marker PUTs: requests for keys that are not routable file types. JindoSDK skips that step, which is why only the PyArrow backend showed it. new_output_stream now runs its parent directory step through create_parent_dirs, a no-op by default on FileIO. The routing FileIO runs that step on origin before a routed write, so origin sees the same directory requests as without routing, and target FileIOs open the object without it. The pyarrow filesystem streams of the routing FileIO are unchanged: they never had a parent directory step. --- paimon-python/pypaimon/common/file_io.py | 4 +++ .../pypaimon/filesystem/io_cache_file_io.py | 25 +++++++++---- .../pypaimon/filesystem/pyarrow_file_io.py | 10 ++++-- .../pypaimon/tests/io_cache_file_io_test.py | 36 +++++++++++++++++++ 4 files changed, 67 insertions(+), 8 deletions(-) diff --git a/paimon-python/pypaimon/common/file_io.py b/paimon-python/pypaimon/common/file_io.py index c2d132f7acb6..c64f178f3c89 100644 --- a/paimon-python/pypaimon/common/file_io.py +++ b/paimon-python/pypaimon/common/file_io.py @@ -100,6 +100,10 @@ def new_input_stream(self, path: str): def new_output_stream(self, path: str): pass + def create_parent_dirs(self, path: str): + """Runs only the parent directory step of new_output_stream(path); a no-op unless a FileIO splits it out.""" + pass + @abstractmethod def get_file_status(self, path: str): pass diff --git a/paimon-python/pypaimon/filesystem/io_cache_file_io.py b/paimon-python/pypaimon/filesystem/io_cache_file_io.py index f6e422f0c033..8e3ed35feef6 100644 --- a/paimon-python/pypaimon/filesystem/io_cache_file_io.py +++ b/paimon-python/pypaimon/filesystem/io_cache_file_io.py @@ -26,6 +26,7 @@ from pypaimon.common.file_io import FileIO from pypaimon.filesystem.io_cache_routing import IoCacheRouting, Op +from pypaimon.filesystem.pyarrow_file_io import PyArrowFileIO class IoCacheRoutingFileIO(FileIO): @@ -70,12 +71,21 @@ def _file_io(self, name: Optional[str]) -> FileIO: file_io = self._targets.get(name) if file_io is None: file_io = self._create_target(name) + if isinstance(file_io, PyArrowFileIO): + file_io.creates_parent_dirs = False self._targets[name] = file_io return file_io def _call(self, op: Op, path: str, action): return action(self._file_io(self._route(op, path))) + def _write(self, path: str, action): + name = self._route(Op.WRITE, path) + if name is not None: + # A target takes only the object; its parent directories are created on origin, as unrouted. + self._origin.create_parent_dirs(path) + return action(self._file_io(name)) + def _call_batch(self, op: Op, paths: List[str], action) -> list: """Runs action(file_io, paths) once per routed group; results follow the order of paths.""" results = [None] * len(paths) @@ -99,7 +109,10 @@ def get_file_size(self, path: str) -> int: return self._call(Op.META, path, lambda io: io.get_file_size(path)) def new_output_stream(self, path: str): - return self._call(Op.WRITE, path, lambda io: io.new_output_stream(path)) + return self._write(path, lambda io: io.new_output_stream(path)) + + def create_parent_dirs(self, path: str): + return self._origin.create_parent_dirs(path) def exists(self, path: str) -> bool: return self._call(Op.EXISTS, path, lambda io: io.exists(path)) @@ -135,10 +148,10 @@ def write_file(self, path: str, content: str, overwrite: bool = False): # The existence check before a write runs on origin, so no cache sees the file before it exists. if not overwrite and self._origin.exists(path): raise FileExistsError(f"File {path} already exists and overwrite=False") - return self._call(Op.WRITE, path, lambda io: io.write_file(path, content, True)) + return self._write(path, lambda io: io.write_file(path, content, True)) def overwrite_file_utf8(self, path: str, content: str): - return self._call(Op.WRITE, path, lambda io: io.overwrite_file_utf8(path, content)) + return self._write(path, lambda io: io.overwrite_file_utf8(path, content)) # Helpers that delete, or check right after a delete, run on origin, probes included. @@ -166,15 +179,15 @@ def to_filesystem_path(self, path: str) -> str: def write_parquet(self, path: str, data, compression: str = 'zstd', zstd_level: int = 1, **kwargs): - return self._call(Op.WRITE, path, lambda io: io.write_parquet(path, data, compression, zstd_level, **kwargs)) + return self._write(path, lambda io: io.write_parquet(path, data, compression, zstd_level, **kwargs)) def write_orc(self, path: str, data, compression: str = 'zstd', zstd_level: int = 1, **kwargs): - return self._call(Op.WRITE, path, lambda io: io.write_orc(path, data, compression, zstd_level, **kwargs)) + return self._write(path, lambda io: io.write_orc(path, data, compression, zstd_level, **kwargs)) def write_avro(self, path: str, data, avro_schema=None, compression: str = 'zstd', zstd_level: int = 1, **kwargs): - return self._call(Op.WRITE, path, lambda io: io.write_avro( + return self._write(path, lambda io: io.write_avro( path, data, avro_schema, compression, zstd_level, **kwargs)) def close(self): diff --git a/paimon-python/pypaimon/filesystem/pyarrow_file_io.py b/paimon-python/pypaimon/filesystem/pyarrow_file_io.py index 562c0dd9e543..2a620db32ad1 100644 --- a/paimon-python/pypaimon/filesystem/pyarrow_file_io.py +++ b/paimon-python/pypaimon/filesystem/pyarrow_file_io.py @@ -57,6 +57,9 @@ class LegacyOssDirectoryListingError(RuntimeError): class PyArrowFileIO(FileIO): + # False on an io-cache target, whose writes get their parent directories on origin. + creates_parent_dirs = True + def __init__(self, path: str, catalog_options: Options): self.properties = catalog_options self.logger = logging.getLogger(__name__) @@ -450,6 +453,11 @@ def new_input_stream(self, path: str): return self.filesystem.open_input_file(path_str) def new_output_stream(self, path: str): + if self.creates_parent_dirs: + self.create_parent_dirs(path) + return self.filesystem.open_output_stream(self.to_filesystem_path(path)) + + def create_parent_dirs(self, path: str): path_str = self.to_filesystem_path(path) if self._use_jindo: @@ -468,8 +476,6 @@ def new_output_stream(self, path: str): if str(parent_dir) and not self.exists(str(parent_dir)): self.mkdirs(str(parent_dir)) - return self.filesystem.open_output_stream(path_str) - def _get_file_info(self, path_str: str): try: file_infos = self.filesystem.get_file_info([path_str]) diff --git a/paimon-python/pypaimon/tests/io_cache_file_io_test.py b/paimon-python/pypaimon/tests/io_cache_file_io_test.py index ddc9d0ea1f3b..2a65fafd63f1 100644 --- a/paimon-python/pypaimon/tests/io_cache_file_io_test.py +++ b/paimon-python/pypaimon/tests/io_cache_file_io_test.py @@ -29,6 +29,7 @@ import unittest from datetime import timedelta from pathlib import Path +from unittest import mock from urllib.parse import urlparse import pyarrow as pa @@ -39,6 +40,7 @@ from pyarrow._fs import FileSystemHandler from pypaimon.common.file_io import FileIO +from pypaimon.common.options import Options from pypaimon.execution.python.read.reader.format_avro_reader import FormatAvroReader from pypaimon.execution.python.read.reader.format_pyarrow_reader import FormatPyArrowReader from pypaimon.filesystem.caching_file_io import (CachingFileIO, @@ -46,6 +48,7 @@ from pypaimon.filesystem.io_cache_file_io import IoCacheRoutingFileIO from pypaimon.filesystem.io_cache_routing import IoCacheRouting from pypaimon.filesystem.local_file_io import LocalFileIO +from pypaimon.filesystem.oss_file_io import OssFileIO from pypaimon.schema.data_types import AtomicType, DataField from pypaimon.table.row.blob import BlobDescriptor from pypaimon.utils.file_type import FileType @@ -678,5 +681,38 @@ def test_each_file_io_call_reaches_its_endpoint(policy, op, name, call): assert called == {target if op in routed else "origin"}, path +class _MissingPathsHandler(_RecordingHandler): + """Every path is missing, so a writer probes and creates the parent directory.""" + + def get_file_info(self, paths): + return self._record("get_file_info", [pafs.FileInfo(path, pafs.FileType.NotFound) for path in paths]) + + +def _legacy_oss_file_io(calls): + options = Options({"fs.oss.impl": "legacy", "fs.oss.endpoint": "oss-cn-test.example.com"}) + with mock.patch.object(OssFileIO, "_initialize_oss_fs", + return_value=pafs.PyFileSystem(_MissingPathsHandler(calls))): + return OssFileIO(TABLE, options) + + +@pytest.mark.parametrize("write", [ + lambda f, p: f.new_output_stream(p).close(), + lambda f, p: f.write_parquet(p, pa.table({"a": [1]})), + lambda f, p: f.write_orc(p, pa.table({"a": [1]})), + lambda f, p: f.write_avro(p, pa.table({"a": [1]})), + lambda f, p: f.write_file(p, "x"), + lambda f, p: f.overwrite_file_utf8(p, "x"), +], ids=["new_output_stream", "write_parquet", "write_orc", "write_avro", "write_file", "overwrite_file_utf8"]) +def test_routed_writes_create_parent_directories_on_origin(write): + unrouted, origin, target = [], [], [] + write(_legacy_oss_file_io(unrouted), DATA) + targets = {"accel": _legacy_oss_file_io([]), "cluster": _legacy_oss_file_io(target)} + write(IoCacheRoutingFileIO(_routing(policy="meta,read,write"), _legacy_oss_file_io(origin), + targets.__getitem__), DATA) + assert "create_dir" in unrouted + assert target == ["open_output_stream"] + assert origin == [call for call in unrouted if call != "open_output_stream"] + + if __name__ == "__main__": unittest.main() From 56570683b4e4cb5b57f486fc832b38b01e9e8758 Mon Sep 17 00:00:00 2001 From: Sun Dapeng Date: Tue, 13 Oct 2026 09:40:18 +0800 Subject: [PATCH 13/27] [python] Recognize postpone and paimon-rust managed BLOB file names in io-cache routing The postpone writer names its files {prefix}-u-{commit user}-s-{write id}-w-{uuid}-{count}.{extension}, which did not match {prefix}{uuid}-{count}.{extension}, so their reads and writes always used the origin. paimon-rust names its managed BLOB packs {prefix}{uuid}.managed.blob, without the count Java writes. Both are written once under a unique name and are now data files, with the .index and .blobref files named after them, so all three clients route the same names (apache/paimon-rust#1014). test_whitelist also pins that io-cache.whitelist takes only the contract's file types: a local-cache selector such as structured-data is ignored, as paimon-rust does since 11c8613. --- .../pypaimon/filesystem/io_cache_routing.py | 5 ++-- .../pypaimon/tests/io_cache_routing_test.py | 23 +++++++++++++++++++ 2 files changed, 26 insertions(+), 2 deletions(-) diff --git a/paimon-python/pypaimon/filesystem/io_cache_routing.py b/paimon-python/pypaimon/filesystem/io_cache_routing.py index e1be04bfb401..451e89b19836 100644 --- a/paimon-python/pypaimon/filesystem/io_cache_routing.py +++ b/paimon-python/pypaimon/filesystem/io_cache_routing.py @@ -35,8 +35,9 @@ _TARGET_NAME = re.compile(r"[a-z][a-z0-9-]*") _UUID = r"[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}" -# Data files are named {prefix}{uuid}-{count}.{extension}; this matches what follows the prefix. -_DATA_FILE_SUFFIX = re.compile(_UUID + r"-[0-9]+\..+") +# What follows a data file prefix: {uuid}-{count}.{extension}, also after the postpone writer's +# -u-{commit user}-s-{write id}-w-, or a paimon-rust BLOB pack {uuid}.managed.blob. +_DATA_FILE_SUFFIX = re.compile(r"(?:-u-.+-s-[0-9]+-w-)?" + _UUID + r"-[0-9]+\..+|" + _UUID + r"\.managed\.blob") # Other types are routed only under the names Paimon writes (paimon-rust adds -changelog to changelog # manifests); Format Table files may be replaced in place. _PAIMON_FILE_NAMES = { diff --git a/paimon-python/pypaimon/tests/io_cache_routing_test.py b/paimon-python/pypaimon/tests/io_cache_routing_test.py index 389cf1e48cb6..f0dd87134a09 100644 --- a/paimon-python/pypaimon/tests/io_cache_routing_test.py +++ b/paimon-python/pypaimon/tests/io_cache_routing_test.py @@ -280,6 +280,12 @@ def test_routable_type(self): "dt=1/part-00000-1a2b.snappy.parquet", "dt=1/bucket-0/data-1.parquet", "dt=1/bucket-0/custom-{uuid}-0.orc", + "bucket-postpone/data--u-user-s-x-w-{uuid}-0.parquet", + "bucket-postpone/data--u--s-1-w-{uuid}-0.parquet", + "bucket-postpone/custom--u-user-s-1-w-{uuid}-0.parquet", + "dt=1/bucket-0/data-{uuid}.blob", + "dt=1/bucket-0/data-{uuid}.managed.blob.{uuid}.tmp", + "dt=1/bucket-0/data-old.managed.blob", "README"): self.assert_type(path, None) # Format Table files named like metadata or indexes, which may be replaced in place @@ -311,12 +317,24 @@ def test_routable_type(self): self.assert_type("dt=1/bucket-0/data-{uuid}-0.blob", FileType.DATA) self.assert_type("dt=1/8b1f7c2e/data-{uuid}-0.parquet", FileType.DATA) self.assert_type("dt=1/bucket-0/data-{uuid}-0.parquet.index", FileType.FILE_INDEX) + self.assert_type("dt=1/bucket-0/data-{uuid}-0.parquet.blobref", FileType.DATA) + self.assert_type("dt=1/bucket-0/data-{uuid}-0.managed.blob", FileType.DATA) + self.assert_type("dt=1/bucket-0/data-{uuid}.managed.blob", FileType.DATA) + # the postpone writer puts its commit user and write id before {uuid}-{count} + for postpone in ("bucket-postpone/data--u-io-cache-user-s-7-w-{uuid}-0.parquet", + "dt=1/bucket-postpone/data--u-{uuid}-s-2147483647-w-{uuid}-12.orc", + "bucket-postpone/changelog--u-user-s-0-w-{uuid}-0.parquet"): + self.assert_type(postpone, FileType.DATA) + self.assert_type(postpone + ".index", FileType.FILE_INDEX) + self.assert_type(postpone + ".blobref", FileType.DATA) self.assert_type("index/index-{uuid}-0", FileType.BUCKET_INDEX) self.assert_type("index/btree-global-index-{uuid}.index", FileType.GLOBAL_INDEX) self.assert_type("index/lumina-global-index-{uuid}.index", FileType.GLOBAL_INDEX) def test_routable_type_with_file_prefixes(self): self.assert_type("dt=1/bucket-0/custom-{uuid}-0.orc", FileType.DATA, "data-file.prefix=custom-") + self.assert_type("bucket-postpone/custom--u-user-s-1-w-{uuid}-0.orc", FileType.DATA, + "data-file.prefix=custom-") self.assert_type("dt=1/bucket-0/data-{uuid}-0.orc", FileType.DATA, "data-file.prefix=custom-") self.assert_type("dt=1/bucket-0/cl-{uuid}-0.orc", FileType.DATA, "changelog-file.prefix=cl-") self.assert_type("dt=1/bucket-0/ unknown.parquet", None, "data-file.prefix= ") @@ -421,6 +439,11 @@ def test_whitelist(self): self.assertIsNone(routing.route(Op.READ, self.MANIFEST)) self.assertIsNone(IoCacheRouting.create(self._options(**{"io-cache.whitelist": ""})).route( Op.READ, self.DATA)) + # Only the contract's types count; local-cache selectors such as structured-data are ignored. + for whitelist, routed in (("structured-data", ()), ("structured-data,meta", (self.MANIFEST,))): + routing = IoCacheRouting.create(self._options(**{"io-cache.whitelist": whitelist})) + for path in (self.DATA, self.MANIFEST): + self.assertEqual("default" if path in routed else None, routing.route(Op.READ, path), whitelist) def test_targets_and_routes(self): routing = IoCacheRouting.create(self._multi(**{ From 7045d6838773e6a31fe1221fcdda9e52f37bcaa6 Mon Sep 17 00:00:00 2001 From: Sun Dapeng Date: Mon, 26 Oct 2026 01:13:02 +0800 Subject: [PATCH 14/27] [python] Clean up failed routed writes on origin PyArrowFileIO.write_parquet and write_orc delete the file when the write fails. When io-cache routing sends the write to a target, that cleanup ran on the target's PyArrowFileIO, so the target got a DELETE and a HEAD for a path a target should only see PUT, GET and HEAD of finished files, and the origin, which unrouted holds the cleanup, got nothing. A target PyArrowFileIO now skips the cleanup (cleans_up_failed_writes, like creates_parent_dirs), and the routing FileIO runs delete_quietly on origin after a routed write_parquet or write_orc fails, then re-raises. Unrouted writes are unchanged. test_failed_routed_writes_are_cleaned_up_on_origin fails both writes on a recording backend: the target only sees the upload, origin and the unrouted FileIO each clean up the path once. It fails without the fix. --- .../pypaimon/filesystem/io_cache_file_io.py | 24 ++++++++++---- .../pypaimon/filesystem/pyarrow_file_io.py | 8 +++-- .../pypaimon/tests/io_cache_file_io_test.py | 32 +++++++++++++++++++ 3 files changed, 55 insertions(+), 9 deletions(-) diff --git a/paimon-python/pypaimon/filesystem/io_cache_file_io.py b/paimon-python/pypaimon/filesystem/io_cache_file_io.py index 8e3ed35feef6..b19624bf547f 100644 --- a/paimon-python/pypaimon/filesystem/io_cache_file_io.py +++ b/paimon-python/pypaimon/filesystem/io_cache_file_io.py @@ -73,18 +73,26 @@ def _file_io(self, name: Optional[str]) -> FileIO: file_io = self._create_target(name) if isinstance(file_io, PyArrowFileIO): file_io.creates_parent_dirs = False + file_io.cleans_up_failed_writes = False self._targets[name] = file_io return file_io def _call(self, op: Op, path: str, action): return action(self._file_io(self._route(op, path))) - def _write(self, path: str, action): + def _write(self, path: str, action, cleans_up: bool = False): name = self._route(Op.WRITE, path) - if name is not None: - # A target takes only the object; its parent directories are created on origin, as unrouted. - self._origin.create_parent_dirs(path) - return action(self._file_io(name)) + if name is None: + return action(self._origin) + # A target takes only the object; its parent directories are created on origin, as unrouted. + self._origin.create_parent_dirs(path) + try: + return action(self._file_io(name)) + except Exception: + # The cleanup the origin FileIO would do after a failed write also runs on origin. + if cleans_up: + self._origin.delete_quietly(path) + raise def _call_batch(self, op: Op, paths: List[str], action) -> list: """Runs action(file_io, paths) once per routed group; results follow the order of paths.""" @@ -179,11 +187,13 @@ def to_filesystem_path(self, path: str) -> str: def write_parquet(self, path: str, data, compression: str = 'zstd', zstd_level: int = 1, **kwargs): - return self._write(path, lambda io: io.write_parquet(path, data, compression, zstd_level, **kwargs)) + return self._write( + path, lambda io: io.write_parquet(path, data, compression, zstd_level, **kwargs), True) def write_orc(self, path: str, data, compression: str = 'zstd', zstd_level: int = 1, **kwargs): - return self._write(path, lambda io: io.write_orc(path, data, compression, zstd_level, **kwargs)) + return self._write( + path, lambda io: io.write_orc(path, data, compression, zstd_level, **kwargs), True) def write_avro(self, path: str, data, avro_schema=None, compression: str = 'zstd', zstd_level: int = 1, **kwargs): diff --git a/paimon-python/pypaimon/filesystem/pyarrow_file_io.py b/paimon-python/pypaimon/filesystem/pyarrow_file_io.py index 2a620db32ad1..342d08011d8e 100644 --- a/paimon-python/pypaimon/filesystem/pyarrow_file_io.py +++ b/paimon-python/pypaimon/filesystem/pyarrow_file_io.py @@ -59,6 +59,8 @@ class LegacyOssDirectoryListingError(RuntimeError): class PyArrowFileIO(FileIO): # False on an io-cache target, whose writes get their parent directories on origin. creates_parent_dirs = True + # False on an io-cache target, whose failed writes are cleaned up on origin. + cleans_up_failed_writes = True def __init__(self, path: str, catalog_options: Options): self.properties = catalog_options @@ -921,7 +923,8 @@ def write_parquet(self, path: str, data: pyarrow.Table, compression: str = 'zstd pq.write_table(data, output_stream, compression=compression, **kwargs) except Exception as e: - self.delete_quietly(path) + if self.cleans_up_failed_writes: + self.delete_quietly(path) raise RuntimeError(f"Failed to write Parquet file {path}: {e}") from e def write_orc(self, path: str, data: pyarrow.Table, compression: str = 'zstd', @@ -951,7 +954,8 @@ def write_orc(self, path: str, data: pyarrow.Table, compression: str = 'zstd', ) except Exception as e: - self.delete_quietly(path) + if self.cleans_up_failed_writes: + self.delete_quietly(path) raise RuntimeError(f"Failed to write ORC file {path}: {e}") from e def write_avro( diff --git a/paimon-python/pypaimon/tests/io_cache_file_io_test.py b/paimon-python/pypaimon/tests/io_cache_file_io_test.py index 2a65fafd63f1..ce6ca2f81d31 100644 --- a/paimon-python/pypaimon/tests/io_cache_file_io_test.py +++ b/paimon-python/pypaimon/tests/io_cache_file_io_test.py @@ -714,5 +714,37 @@ def test_routed_writes_create_parent_directories_on_origin(write): assert origin == [call for call in unrouted if call != "open_output_stream"] +class _FailingWritesHandler(_MissingPathsHandler): + """Every path is missing and every upload fails.""" + + def open_output_stream(self, path, metadata): + self._record("open_output_stream") + raise OSError("upload failed") + + +@pytest.mark.parametrize("write", [ + lambda f, p: f.write_parquet(p, pa.table({"a": [1]})), + lambda f, p: f.write_orc(p, pa.table({"a": [1]})), +], ids=["write_parquet", "write_orc"]) +def test_failed_routed_writes_are_cleaned_up_on_origin(write): + def file_io(calls): + options = Options({"fs.oss.impl": "legacy", "fs.oss.endpoint": "oss-cn-test.example.com"}) + with mock.patch.object(OssFileIO, "_initialize_oss_fs", + return_value=pafs.PyFileSystem(_FailingWritesHandler(calls))): + file_io = OssFileIO(TABLE, options) + file_io.delete_quietly = mock.Mock() + return file_io + + unrouted, origin, target_calls = file_io([]), file_io([]), [] + with pytest.raises(RuntimeError): + write(unrouted, DATA) + targets = {"accel": file_io([]), "cluster": file_io(target_calls)} + with pytest.raises(RuntimeError): + write(IoCacheRoutingFileIO(_routing(policy="meta,read,write"), origin, targets.__getitem__), DATA) + unrouted.delete_quietly.assert_called_once_with(DATA) + origin.delete_quietly.assert_called_once_with(DATA) + targets["cluster"].delete_quietly.assert_not_called() + assert target_calls == ["open_output_stream"] + if __name__ == "__main__": unittest.main() From 3b1bdd985483da81e7d483cf13a1507f085e8319 Mon Sep 17 00:00:00 2001 From: Sun Dapeng Date: Mon, 26 Oct 2026 01:14:32 +0800 Subject: [PATCH 15/27] [python] Give each table its view of io-cache routing io-cache routing only knew the data file prefixes in the token and applied the same rules to every table the FileIO served. apache/paimon-rust#1014 review found gaps that this client has as well: - Other writers may replace Format Table files in place under the same name, and manage an Object Table's objects, so a target that caches them can serve stale content. - A table whose schema sets data-file.prefix or changelog-file.prefix writes data files the router did not recognize, so they always used the origin. Files written before such an option changed keep the old prefix, and a branch can set its own. FileIO gets two views with no-op defaults. origin_only() sends no request to a target; FormatTable, ObjectTable and IcebergTable (written by Iceberg writers) take it. for_table(path, prefixes, branch) is taken by FileStoreTable.__init__, which every load, copy, time travel and branch copy goes through, so no way of getting a table bypasses it. The view routes the prefixes of the table's schema, and once a name that may be a data file matches none of them, the prefixes of every schema of the branches it reads, listed from origin at most once. A copy keeps the prefixes and branches of the table it was made from: a branch made from a tag reads files main wrote. If the history cannot be read, the files it would have named stay on origin, which only loses io-cache. RESTTokenFileIO views keep the token and caches and apply the view to the FileIO cached for the token. A REST catalog may keep a branch's schema without a schema directory in storage, so a REST view also asks the catalog for the schema of each branch it reads. CachingFileIO wraps the view of its delegate and shares its cache. An IoCacheRoutingFileIO view shares the origin and targets of its base and closes nothing. Tests: io_cache_table_routing_test writes and reads real tables through a FileSystemCatalog whose FileIO routes to a target over the same storage; after ALTER TABLE changes data-file.prefix, a fresh load reads the older files through the target, and a branch copy reads the files written with the branch's prefix through it. rest_table_file_io_view_test checks the view every kind of REST table holds; the RESTTokenFileIO test checks that a branch view takes the prefix the catalog keeps; the routing and FileIO tests check that the history is only read for unknown data file names, once, from origin, that a failure leaves files on origin, and that a forked child does not wait on a listing the parent has in progress. The table tests fail without the change. --- .../catalog/rest/rest_token_file_io.py | 46 ++++- paimon-python/pypaimon/common/file_io.py | 9 + .../pypaimon/filesystem/caching_file_io.py | 10 ++ .../pypaimon/filesystem/io_cache_file_io.py | 106 +++++++++++- .../pypaimon/filesystem/io_cache_routing.py | 30 +++- .../pypaimon/table/file_store_table.py | 7 +- .../pypaimon/table/format/format_table.py | 3 +- .../pypaimon/table/iceberg/iceberg_table.py | 3 +- .../pypaimon/table/object/object_table.py | 3 +- .../pypaimon/tests/caching_file_io_test.py | 18 ++ .../pypaimon/tests/io_cache_file_io_test.py | 77 +++++++++ .../pypaimon/tests/io_cache_routing_test.py | 18 ++ .../tests/io_cache_table_routing_test.py | 163 ++++++++++++++++++ .../rest/rest_table_file_io_view_test.py | 65 +++++++ .../tests/rest/rest_token_file_io_test.py | 49 ++++++ 15 files changed, 591 insertions(+), 16 deletions(-) create mode 100644 paimon-python/pypaimon/tests/io_cache_table_routing_test.py create mode 100644 paimon-python/pypaimon/tests/rest/rest_table_file_io_view_test.py diff --git a/paimon-python/pypaimon/catalog/rest/rest_token_file_io.py b/paimon-python/pypaimon/catalog/rest/rest_token_file_io.py index 5e9f8fa8ba65..0ea89ea8e7db 100644 --- a/paimon-python/pypaimon/catalog/rest/rest_token_file_io.py +++ b/paimon-python/pypaimon/catalog/rest/rest_token_file_io.py @@ -36,8 +36,10 @@ from pypaimon.common.options import Options from pypaimon.common.options.config import CatalogOptions, OssOptions from pypaimon.common.uri_reader import UriReaderFactory -from pypaimon.filesystem.io_cache_file_io import IoCacheRoutingFileIO -from pypaimon.filesystem.io_cache_routing import IoCacheRouting +from pypaimon.filesystem.io_cache_file_io import (IoCacheRoutingFileIO, + table_view_spec) +from pypaimon.filesystem.io_cache_routing import (IoCacheRouting, + schema_file_prefixes) class RESTTokenFileIO(FileIO): @@ -68,6 +70,10 @@ class RESTTokenFileIO(FileIO): "fds": None, "next_scan": 0.0} _SHORT_TOKEN_STATS = {"count": 0, "next_log": 0.0} + # The table view each FileIO is built for, see FileIO.origin_only and for_table. + _origin_only = False + _table: Optional[tuple] = None + def __init__(self, identifier: Identifier, path: str, catalog_options: Optional[Union[dict, Options]] = None): self.identifier = identifier @@ -129,8 +135,42 @@ def _after_fork_in_child(cls): for file_io in list(cls._INSTANCES): file_io._token_lock = threading.Lock() + def origin_only(self) -> FileIO: + return self if self._origin_only else self._with_view(True, None) + + def for_table(self, table_path: str, file_prefixes, branch: str) -> FileIO: + if self._origin_only: + return self + table = table_view_spec(self._table, table_path, file_prefixes, branch) + return self if table == self._table else self._with_view(False, table) + + def _with_view(self, origin_only: bool, table) -> 'RESTTokenFileIO': + # Same token and caches; each FileIO it returns is the view of the cached one. + view = RESTTokenFileIO.__new__(RESTTokenFileIO) + view.__setstate__(self.__getstate__()) + view.api_instance = self.api_instance + view._origin_only = origin_only + view._table = table + return view + + def _table_view(self, file_io: FileIO) -> FileIO: + if self._origin_only: + return file_io.origin_only() + if self._table is None or not isinstance(file_io, IoCacheRoutingFileIO): + return file_io + return file_io.table_view(self._table, self._catalog_schema_prefixes) + + def _catalog_schema_prefixes(self, branch: str): + # A REST catalog may keep a branch's schema without a schema directory in storage. + if self.api_instance is None: + self.api_instance = RESTApi(self.properties, False) + identifier = Identifier.create(self.identifier.get_database_name(), self.identifier.get_table_name(), + branch=None if branch == "main" else branch) + schema = self.api_instance.get_table(identifier).schema + return schema_file_prefixes(schema.options) if schema is not None else [] + def file_io(self) -> FileIO: - return self._file_io_with_token()[0] + return self._table_view(self._file_io_with_token()[0]) def _file_io_with_token(self, minimum_validity_millis: int = 0): with self._token_lock: diff --git a/paimon-python/pypaimon/common/file_io.py b/paimon-python/pypaimon/common/file_io.py index c64f178f3c89..ddd60b1561c0 100644 --- a/paimon-python/pypaimon/common/file_io.py +++ b/paimon-python/pypaimon/common/file_io.py @@ -92,6 +92,15 @@ class FileIO(ABC): File IO interface to read and write files. """ + def origin_only(self) -> 'FileIO': + """A view that sends no request to an io-cache target, for files other writers may replace.""" + return self + + def for_table(self, table_path: str, file_prefixes: List[str], branch: str) -> 'FileIO': + """A view for one Paimon table: io-cache targets serve only its files under table_path, named + with these prefixes or those of the branch's schemas.""" + return self + @abstractmethod def new_input_stream(self, path: str): pass diff --git a/paimon-python/pypaimon/filesystem/caching_file_io.py b/paimon-python/pypaimon/filesystem/caching_file_io.py index 58a1a9bd12d0..f8b526d954e9 100644 --- a/paimon-python/pypaimon/filesystem/caching_file_io.py +++ b/paimon-python/pypaimon/filesystem/caching_file_io.py @@ -414,6 +414,16 @@ def __getstate__(self): # Python 3.10 otherwise finds the delegate's hook through __getattr__. return self.__dict__.copy() + def origin_only(self) -> FileIO: + return self._with_delegate(self._delegate.origin_only()) + + def for_table(self, table_path: str, file_prefixes, branch: str) -> FileIO: + return self._with_delegate(self._delegate.for_table(table_path, file_prefixes, branch)) + + def _with_delegate(self, view: FileIO) -> FileIO: + # A view shares this wrapper's cache. + return self if view is self._delegate else CachingFileIO(view, self._cache, self._whitelist) + # Fallback caps when local-cache.max-size is unset (memory shares the heap). _DEFAULT_MEMORY_CACHE_MAX_SIZE = 256 * 1024 * 1024 _DEFAULT_DISK_CACHE_MAX_SIZE = 10 * 1024 * 1024 * 1024 diff --git a/paimon-python/pypaimon/filesystem/io_cache_file_io.py b/paimon-python/pypaimon/filesystem/io_cache_file_io.py index b19624bf547f..1c11513b590c 100644 --- a/paimon-python/pypaimon/filesystem/io_cache_file_io.py +++ b/paimon-python/pypaimon/filesystem/io_cache_file_io.py @@ -17,15 +17,16 @@ """FileIO that sends each request to an io-cache target or the origin endpoint.""" +import logging import os import threading -from typing import Callable, Dict, List, Optional +from typing import Callable, Dict, Iterable, List, Optional, Tuple import pyarrow.fs as pafs from pyarrow._fs import FileSystemHandler from pypaimon.common.file_io import FileIO -from pypaimon.filesystem.io_cache_routing import IoCacheRouting, Op +from pypaimon.filesystem.io_cache_routing import IoCacheRouting, Op, schema_file_prefixes from pypaimon.filesystem.pyarrow_file_io import PyArrowFileIO @@ -42,6 +43,41 @@ def __init__(self, routing: IoCacheRouting, origin: FileIO, self._targets_lock = threading.Lock() self._pid = os.getpid() self._filesystem = pafs.PyFileSystem(_IoCacheRoutingHandler(self)) + self._base = self + self._table: Optional[Tuple[str, Tuple[str, ...], Tuple[str, ...]]] = None + self._views: Dict[tuple, 'IoCacheRoutingFileIO'] = {} + + def origin_only(self) -> FileIO: + return self._origin + + def for_table(self, table_path: str, file_prefixes: Iterable[str], branch: str) -> FileIO: + return self._base.table_view(table_view_spec(self._table, table_path, file_prefixes, branch)) + + def table_view(self, table, catalog_prefixes=None) -> 'IoCacheRoutingFileIO': + """The view of this FileIO for table = (path, prefixes, branches), see table_view_spec; + catalog_prefixes(branch) adds the prefixes a catalog keeps outside the schema directory.""" + view = self._views.get(table) + if view is not None: + return view + self._check_pid() + with self._targets_lock: + view = self._views.get(table) + if view is None: + table_path, prefixes, branches = table + history = _SchemaPrefixHistory(self._origin, table_path, branches, catalog_prefixes) + # The view takes its targets from this FileIO. + view = IoCacheRoutingFileIO( + self._routing.for_table(prefixes, history), self._origin, self._file_io) + view._base, view._table = self, table + self._views[table] = view + return view + + def _check_pid(self): + if self._pid != os.getpid(): + # A forked child inherits the lock in whatever state a parent thread left it. + self._targets = {} + self._targets_lock = threading.Lock() + self._pid = os.getpid() @property def properties(self): @@ -59,11 +95,7 @@ def _file_io(self, name: Optional[str]) -> FileIO: """Origin, or the FileIO of a target, created on first use.""" if name is None: return self._origin - if self._pid != os.getpid(): - # A forked child inherits the lock in whatever state a parent thread left it. - self._targets = {} - self._targets_lock = threading.Lock() - self._pid = os.getpid() + self._check_pid() file_io = self._targets.get(name) if file_io is not None: return file_io @@ -201,6 +233,9 @@ def write_avro(self, path: str, data, avro_schema=None, path, data, avro_schema, compression, zstd_level, **kwargs)) def close(self): + if self._base is not self: + # A table view shares the FileIOs of its base, which closes them. + return with self._targets_lock: targets = list(self._targets.values()) self._targets.clear() @@ -217,6 +252,63 @@ def __getattr__(self, name): return getattr(origin, name) +class _SchemaPrefixHistory: + """Data file prefixes of every schema of a table's branches, read from origin at most once.""" + + def __init__(self, origin: FileIO, table_path: str, branches: Tuple[str, ...], + catalog_prefixes: Optional[Callable[[str], Iterable[str]]] = None): + self._origin = origin + self._table_path = table_path + self._branches = branches + self._catalog_prefixes = catalog_prefixes + self._prefixes: Optional[Tuple[str, ...]] = None + self._lock = threading.Lock() + self._pid = os.getpid() + + def __call__(self) -> Tuple[str, ...]: + if self._prefixes is not None: + return self._prefixes + if self._pid != os.getpid(): + self._lock = threading.Lock() + self._pid = os.getpid() + with self._lock: + if self._prefixes is None: + self._prefixes = self._load() + return self._prefixes + + def _load(self) -> Tuple[str, ...]: + prefixes = [] + for branch in self._branches: + prefixes += self._quietly(self._schema_directory_prefixes, branch) + if self._catalog_prefixes is not None: + prefixes += self._quietly(self._catalog_prefixes, branch) + return tuple(dict.fromkeys(prefixes)) + + def _schema_directory_prefixes(self, branch: str) -> List[str]: + from pypaimon.schema.schema_manager import SchemaManager + schemas = SchemaManager(self._origin, self._table_path, branch).list_all() + return [prefix for schema in schemas for prefix in schema_file_prefixes(schema.options)] + + def _quietly(self, load: Callable[[str], Iterable[str]], branch: str) -> List[str]: + try: + return list(load(branch)) + except Exception: + # Files with a prefix missed here are read from origin, which only loses io-cache. + logging.getLogger(__name__).warning( + "Failed to load schema file prefixes of %s (branch %s) for io-cache routing", + self._table_path, branch, exc_info=True) + return [] + + +def table_view_spec(table, table_path: str, file_prefixes: Iterable[str], branch: str): + """(path, prefixes, branches) of a table view; a copy of the same table keeps those it had.""" + prefixes, branches = tuple(dict.fromkeys(file_prefixes)), (branch,) + if table is not None and table[0] == table_path: + prefixes = table[1] + tuple(p for p in prefixes if p not in table[1]) + branches = table[2] + tuple(b for b in branches if b not in table[2]) + return table_path, prefixes, branches + + class _IoCacheRoutingHandler(FileSystemHandler): """Sends each pyarrow filesystem call to the filesystem of origin or of a target FileIO.""" diff --git a/paimon-python/pypaimon/filesystem/io_cache_routing.py b/paimon-python/pypaimon/filesystem/io_cache_routing.py index 451e89b19836..f8821e974525 100644 --- a/paimon-python/pypaimon/filesystem/io_cache_routing.py +++ b/paimon-python/pypaimon/filesystem/io_cache_routing.py @@ -17,10 +17,11 @@ """Chooses the endpoint of each FileIO request from the io-cache.* keys of a table token.""" +import copy import os import re from enum import Enum -from typing import Dict, FrozenSet, List, Mapping, NamedTuple, Optional, Tuple +from typing import Callable, Dict, FrozenSet, Iterable, List, Mapping, NamedTuple, Optional, Tuple from pypaimon.common.options.config import CatalogOptions, OssOptions from pypaimon.utils.file_type import FileType @@ -38,6 +39,8 @@ # What follows a data file prefix: {uuid}-{count}.{extension}, also after the postpone writer's # -u-{commit user}-s-{write id}-w-, or a paimon-rust BLOB pack {uuid}.managed.blob. _DATA_FILE_SUFFIX = re.compile(r"(?:-u-.+-s-[0-9]+-w-)?" + _UUID + r"-[0-9]+\..+|" + _UUID + r"\.managed\.blob") +# A name that may be a data file under a prefix this routing has not seen yet. +_DATA_FILE_NAME = re.compile(r".+?(?:" + _DATA_FILE_SUFFIX.pattern + r")") # Other types are routed only under the names Paimon writes (paimon-rust adds -changelog to changelog # manifests); Format Table files may be replaced in place. _PAIMON_FILE_NAMES = { @@ -105,6 +108,12 @@ def _declared_targets(options: Mapping[str, str]) -> Dict[str, Optional[str]]: return endpoints +def schema_file_prefixes(options) -> List[str]: + """The data-file.prefix and changelog-file.prefix a table schema sets.""" + values = [options.get(key) for key in ("data-file.prefix", "changelog-file.prefix")] + return [value for value in values if value and value.strip()] + + def data_prefixes(options: Mapping[str, str]) -> Tuple[str, ...]: """Data file name prefixes: data- and changelog-, plus the table's own prefixes.""" prefixes = ["data-", "changelog-"] @@ -115,6 +124,10 @@ def data_prefixes(options: Mapping[str, str]) -> Tuple[str, ...]: return tuple(prefixes) +def _new_prefixes(prefixes: Iterable[str], known: Tuple[str, ...]) -> Tuple[str, ...]: + return tuple(p for p in dict.fromkeys(prefixes) if p and p.strip() and p not in known) + + def routable_type(path: str, prefixes: Tuple[str, ...]) -> Optional[FileType]: """The type a cache may serve, or None when the file must be read from the origin.""" path = path.rstrip("/") @@ -185,6 +198,7 @@ def __init__(self, options: Mapping[str, str], policy: FrozenSet[Op], routes = options.get("io-cache.routes") self._routes = None if routes is None else _parse_routes(str(routes)) self._prefixes = data_prefixes(options) + self._history: Optional[Callable[[], Iterable[str]]] = None @staticmethod def create(options: Mapping[str, str]) -> Optional['IoCacheRouting']: @@ -195,6 +209,15 @@ def create(options: Mapping[str, str]) -> Optional['IoCacheRouting']: endpoints = _declared_targets(options) return IoCacheRouting(options, policy, endpoints) if policy and endpoints else None + def for_table(self, prefixes: Iterable[str], + history: Optional[Callable[[], Iterable[str]]] = None) -> 'IoCacheRouting': + """This routing for the files of one table, also named with these prefixes, and with those + history() returns once a name that may be a data file matches no other.""" + routing = copy.copy(self) + routing._prefixes = self._prefixes + _new_prefixes(prefixes, self._prefixes) + routing._history = history + return routing + def targets(self) -> Dict[str, str]: """Endpoints of the targets that have one, by name, in declaration order.""" return {name: target.endpoint for name, target in self._targets.items()} @@ -211,6 +234,11 @@ def route(self, op: Op, path: str) -> Optional[str]: if op not in self._policy or not path.startswith("oss://"): return None file_type = routable_type(path, self._prefixes) + if file_type is None and self._history is not None and _DATA_FILE_NAME.fullmatch( + os.path.basename(path.rstrip("/"))): + # Files written before a prefix option changed keep their names. + known = self._prefixes + file_type = routable_type(path, known + _new_prefixes(self._history(), known)) if file_type is None or file_type not in self._whitelist: return None if self._routes is None: diff --git a/paimon-python/pypaimon/table/file_store_table.py b/paimon-python/pypaimon/table/file_store_table.py index 9b190f5fe3fd..fd1a88d41bd6 100644 --- a/paimon-python/pypaimon/table/file_store_table.py +++ b/paimon-python/pypaimon/table/file_store_table.py @@ -22,6 +22,7 @@ from pypaimon.common.identifier import Identifier from pypaimon.common.options.core_options import CoreOptions from pypaimon.common.options.options import Options +from pypaimon.filesystem.io_cache_routing import schema_file_prefixes from pypaimon.read.read_builder import ReadBuilder from pypaimon.read.stream_read_builder import StreamReadBuilder from pypaimon.schema.schema_manager import SchemaManager @@ -34,9 +35,11 @@ class FileStoreTable(Table): def __init__(self, file_io: FileIO, identifier: Identifier, table_path: str, table_schema: TableSchema, catalog_environment: Optional[CatalogEnvironment] = None): - self.file_io = file_io self.identifier = identifier self.table_path = table_path + # Every table, copy included, tells io-cache routing where its files are and how they are named. + self.file_io = file_io.for_table( + table_path, schema_file_prefixes(table_schema.options), self.current_branch()) self.catalog_environment = catalog_environment or CatalogEnvironment.empty() self.table_schema = table_schema @@ -56,7 +59,7 @@ def __init__(self, file_io: FileIO, identifier: Identifier, table_path: str, self.total_buckets = self.options.bucket() self.schema_manager = SchemaManager( - file_io, table_path, branch=self.current_branch()) + self.file_io, table_path, branch=self.current_branch()) @classmethod def from_path(cls, table_path: str, file_io_options: Optional[dict] = None) -> 'FileStoreTable': diff --git a/paimon-python/pypaimon/table/format/format_table.py b/paimon-python/pypaimon/table/format/format_table.py index 1a1218153d16..3243e39e697a 100644 --- a/paimon-python/pypaimon/table/format/format_table.py +++ b/paimon-python/pypaimon/table/format/format_table.py @@ -54,7 +54,8 @@ def __init__( options: Optional[Dict[str, str]] = None, comment: Optional[str] = None, ): - self.file_io = file_io + # Other writers may replace Format Table files in place. + self.file_io = file_io.origin_only() self.identifier = identifier self._table_schema = table_schema self._location = location.rstrip("/") diff --git a/paimon-python/pypaimon/table/iceberg/iceberg_table.py b/paimon-python/pypaimon/table/iceberg/iceberg_table.py index 8bf17e9544af..74cea8fdd1ba 100644 --- a/paimon-python/pypaimon/table/iceberg/iceberg_table.py +++ b/paimon-python/pypaimon/table/iceberg/iceberg_table.py @@ -40,7 +40,8 @@ def __init__( comment: Optional[str] = None, uuid: Optional[str] = None, ): - self.file_io = file_io + # Iceberg writers own these files. + self.file_io = file_io.origin_only() self.identifier = identifier self._table_schema = table_schema self._location = location.rstrip("/") diff --git a/paimon-python/pypaimon/table/object/object_table.py b/paimon-python/pypaimon/table/object/object_table.py index f875b4d3f6cb..10f1c64ffba0 100644 --- a/paimon-python/pypaimon/table/object/object_table.py +++ b/paimon-python/pypaimon/table/object/object_table.py @@ -45,7 +45,8 @@ def __init__( options: Optional[Dict[str, str]] = None, comment: Optional[str] = None, ): - self.file_io = file_io + # Other writers manage the objects, so they may change under the same name. + self.file_io = file_io.origin_only() self.identifier = identifier self._table_schema = table_schema self._location = location.rstrip("/") diff --git a/paimon-python/pypaimon/tests/caching_file_io_test.py b/paimon-python/pypaimon/tests/caching_file_io_test.py index 88edfbcf4b8e..7963e0630666 100644 --- a/paimon-python/pypaimon/tests/caching_file_io_test.py +++ b/paimon-python/pypaimon/tests/caching_file_io_test.py @@ -659,6 +659,24 @@ def test_write_parquet_when_enable_local_cache(self): self.assertTrue(os.path.exists(out_path)) self.assertEqual(table, pa.parquet.read_table(out_path)) + def test_table_views_wrap_the_view_of_the_delegate(self): + from pypaimon.utils.file_type import FileType + delegate = MagicMock() + origin, table_view = MagicMock(), MagicMock() + delegate.origin_only.return_value = origin + delegate.for_table.side_effect = lambda path, prefixes, branch: table_view if prefixes else delegate + cache = LocalMemoryCacheManager(128, 64) + caching_io = CachingFileIO(delegate, cache, {FileType.DATA}) + + for view, expected in ((caching_io.origin_only(), origin), + (caching_io.for_table("oss://bkt/t", ["custom-"], "main"), table_view)): + self.assertIsInstance(view, CachingFileIO) + self.assertIs(expected, view._delegate) + self.assertIs(cache, view._cache) + self.assertEqual({FileType.DATA}, view._whitelist) + # A delegate without anything to narrow keeps the wrapper. + self.assertIs(caching_io, caching_io.for_table("oss://bkt/t", [], "main")) + def test_file_io_for_data_wraps_cache_when_data_token_enabled(self): from pypaimon.catalog.rest.rest_catalog import RESTCatalog from pypaimon.common.identifier import Identifier diff --git a/paimon-python/pypaimon/tests/io_cache_file_io_test.py b/paimon-python/pypaimon/tests/io_cache_file_io_test.py index ce6ca2f81d31..4d79894756f0 100644 --- a/paimon-python/pypaimon/tests/io_cache_file_io_test.py +++ b/paimon-python/pypaimon/tests/io_cache_file_io_test.py @@ -231,6 +231,37 @@ def create_target(name): release.set() parent.join(10) + def test_forked_child_does_not_wait_on_the_parents_schema_history(self): + older = PARTITION + "/older-" + UUID + "-0.parquet" + self._put_all(older) + parent_pid = os.getpid() + listing, release = threading.Event(), threading.Event() + + def list_all(manager): + # the parent stays inside the history lock until released; a forked child must not wait for it + if os.getpid() == parent_pid: + listing.set() + release.wait(30) + return [mock.Mock(options={"data-file.prefix": "older-"})] + + with mock.patch("pypaimon.schema.schema_manager.SchemaManager.list_all", list_all): + view = self.file_io.for_table(TABLE, [], "main") + parent = threading.Thread(target=lambda: view.new_input_stream(older).read()) + parent.start() + try: + self.assertTrue(listing.wait(10)) + child = multiprocessing.get_context("fork").Process( + target=lambda: os._exit(0 if view.new_input_stream(older).read() == b"cluster" else 1)) + child.start() + child.join(10) + if child.exitcode is None: + child.kill() + child.join() + self.assertEqual(0, child.exitcode) + finally: + release.set() + parent.join(10) + def test_reads_without_read_policy_use_origin(self): self._put_all(DATA) file_io = self._file_io(_routing(policy="meta")) @@ -456,6 +487,52 @@ def test_unrouted_attributes_come_from_origin(self): self.assertIs(self.origin.uri_reader_factory, self.file_io.uri_reader_factory) self.assertFalse(hasattr(self.file_io, "file_io")) + def test_origin_only_view_is_the_origin(self): + self._put_all(DATA, MANIFEST) + self.assertIs(self.origin, self.file_io.origin_only()) + self.assertEqual(b"origin", self.file_io.origin_only().new_input_stream(DATA).read()) + self.assertEqual([], self.created) + + def test_table_views_share_the_targets_and_close_nothing(self): + custom = PARTITION + "/custom-" + UUID + "-0.parquet" + self._put_all(DATA, custom) + view = self.file_io.for_table(TABLE, ["custom-"], "main") + self.assertIs(view, self.file_io.for_table(TABLE, ["custom-"], "main")) + self.assertEqual(b"cluster", view.new_input_stream(custom).read()) + self.assertEqual(b"origin", self.file_io.new_input_stream(custom).read()) + self.assertEqual(["cluster"], self.created) + # A copy of the table keeps the prefixes of the view it was made from. + copy = view.for_table(TABLE, ["dynamic-"], "dev") + self.assertEqual((TABLE, ("custom-", "dynamic-"), ("main", "dev")), copy._table) + view.close() + copy.close() + self.assertEqual((False, False), (self.origin.closed, self.cluster.closed)) + + def test_schema_history_is_read_from_origin_once(self): + older = PARTITION + "/older-" + UUID + "-0.parquet" + self._put_all(older) + listed = [] + schema = mock.Mock(options={"data-file.prefix": "older-"}) + + def list_all(manager): + listed.append((manager.file_io, manager.branch)) + return [schema] + + with mock.patch("pypaimon.schema.schema_manager.SchemaManager.list_all", list_all): + view = self.file_io.for_table(TABLE, [], "dev") + for _ in range(3): + self.assertEqual(b"cluster", view.new_input_stream(older).read()) + self.assertEqual([(self.origin, "dev")], listed) + + def test_schema_history_that_cannot_be_read_leaves_files_on_origin(self): + older = PARTITION + "/older-" + UUID + "-0.parquet" + self._put_all(older) + with mock.patch("pypaimon.schema.schema_manager.SchemaManager.list_all", side_effect=OSError("denied")), \ + self.assertLogs("pypaimon.filesystem.io_cache_file_io", "WARNING"): + view = self.file_io.for_table(TABLE, [], "main") + self.assertEqual(b"origin", view.new_input_stream(older).read()) + self.assertEqual(b"origin", view.new_input_stream(older).read()) + class _RecordingHandler(FileSystemHandler): diff --git a/paimon-python/pypaimon/tests/io_cache_routing_test.py b/paimon-python/pypaimon/tests/io_cache_routing_test.py index f0dd87134a09..bebc93dc834a 100644 --- a/paimon-python/pypaimon/tests/io_cache_routing_test.py +++ b/paimon-python/pypaimon/tests/io_cache_routing_test.py @@ -490,6 +490,24 @@ def test_table_prefixes_add_data_prefixes(self): routing = IoCacheRouting.create(self._options(**{key: "part-"})) self.assertEqual("default", routing.route(Op.READ, path)) + def test_table_view_asks_the_schema_history_only_for_unknown_data_file_names(self): + asked = [] + + def history(): + asked.append(1) + return ["older-", ""] + + view = IoCacheRouting.create(self._options()).for_table(["newer-"], history) + uuid_count = UUID + "-0" + for path in (self.DATA, self.MANIFEST, self.TABLE + "/bucket-0/newer-" + uuid_count + ".parquet"): + self.assertEqual("default", view.route(Op.READ, path), path) + for path in (self.TABLE + "/snapshot/snapshot-3", self.TABLE + "/README"): + self.assertIsNone(view.route(Op.READ, path), path) + self.assertEqual([], asked) + self.assertEqual("default", view.route(Op.READ, self.TABLE + "/bucket-0/older-" + uuid_count + ".orc")) + self.assertIsNone(view.route(Op.READ, self.TABLE + "/bucket-0/other-" + uuid_count + ".orc")) + self.assertEqual(2, len(asked)) + def test_origin_options(self): options = self._options(**{ "fs.oss.dlf-cache.consistent-hash.enabled": "true", diff --git a/paimon-python/pypaimon/tests/io_cache_table_routing_test.py b/paimon-python/pypaimon/tests/io_cache_table_routing_test.py new file mode 100644 index 000000000000..b525d421a32a --- /dev/null +++ b/paimon-python/pypaimon/tests/io_cache_table_routing_test.py @@ -0,0 +1,163 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""io-cache routing of real tables whose target keeps stale copies of objects replaced on origin.""" + +import os +import shutil +import tempfile +import unittest +from pathlib import Path +from unittest import mock +from urllib.parse import urlparse + +import pyarrow as pa + +from pypaimon import CatalogFactory, Schema +from pypaimon.filesystem.io_cache_file_io import IoCacheRoutingFileIO +from pypaimon.filesystem.io_cache_routing import IoCacheRouting +from pypaimon.filesystem.local_file_io import LocalFileIO +from pypaimon.schema.schema_change import SchemaChange + +WAREHOUSE = "oss://bkt/warehouse" + + +class _Endpoint(LocalFileIO): + """One OSS endpoint over a local directory; a cache also serves the copies it retained.""" + + def __init__(self, root, retained=None): + super().__init__() + self.root = root + self.retained = retained + self.requests = [] + + def _to_file(self, path): + parsed = urlparse(path) + if parsed.scheme != "oss": + return super()._to_file(path) + key = Path(parsed.netloc, parsed.path.lstrip("/")) + if self.retained is not None and Path(self.retained, key).exists(): + return Path(self.retained, key) + return Path(self.root, key) + + def new_input_stream(self, path): + self.requests.append(path) + return super().new_input_stream(path) + + def to_filesystem_path(self, path): + # The routing filesystem calls this right before it opens or stats the file. + self.requests.append(path) + return super().to_filesystem_path(path) + + +class IoCacheTableRoutingTest(unittest.TestCase): + """Origin holds the current objects; the target serves what it retained, as a stale cache would.""" + + def setUp(self): + self.temp_dir = tempfile.mkdtemp(prefix="io_cache_table_routing_test_") + storage = os.path.join(self.temp_dir, "storage") + self.origin = _Endpoint(storage) + self.target = _Endpoint(storage, retained=os.path.join(self.temp_dir, "retained")) + self.routing_enabled = True + patcher = mock.patch("pypaimon.catalog.filesystem_catalog.FileIO.get", side_effect=self._file_io) + patcher.start() + self.addCleanup(patcher.stop) + self.catalog = CatalogFactory.create({"warehouse": WAREHOUSE}) + self.catalog.create_database("db", False) + + def tearDown(self): + shutil.rmtree(self.temp_dir, ignore_errors=True) + + def _file_io(self, path, options=None): + if not self.routing_enabled: + return self.origin + routing = IoCacheRouting.create({ + "fs.oss.endpoint": "http://cache", + "io-cache.enabled": "true", + "io-cache.endpoint": "http://cache", + "io-cache.origin.endpoint": "http://origin", + "io-cache.policy": "meta,read", + }) + return IoCacheRoutingFileIO(routing, self.origin, lambda name: self.target) + + def _put(self, path, content, retained=False): + key = urlparse(path) + root = self.target.retained if retained else self.origin.root + file = Path(root, key.netloc, key.path.lstrip("/")) + file.parent.mkdir(parents=True, exist_ok=True) + file.write_bytes(content) + + def _target_reads(self, needle): + return [path for path in self.target.requests if needle in path] + + @staticmethod + def _write(table, data): + write_builder = table.new_batch_write_builder() + writer = write_builder.new_write() + writer.write_arrow(data) + write_builder.new_commit().commit(writer.prepare_commit()) + writer.close() + + @staticmethod + def _read(table): + read_builder = table.new_read_builder() + return read_builder.new_read().to_arrow(read_builder.new_scan().plan().splits()) + + def test_data_files_of_older_schema_prefixes_use_the_target(self): + pa_schema = pa.schema([("id", pa.int32())]) + schema = Schema.from_pyarrow_schema(pa_schema, options={"data-file.prefix": "older-"}) + self.catalog.create_table("db.prefixes", schema, False) + self._write(self.catalog.get_table("db.prefixes"), pa.Table.from_pydict({"id": [1]}, schema=pa_schema)) + self.catalog.alter_table("db.prefixes", [SchemaChange.set_option("data-file.prefix", "newer-")], False) + self._write(self.catalog.get_table("db.prefixes"), pa.Table.from_pydict({"id": [2]}, schema=pa_schema)) + self.target.requests.clear() + + # A fresh load only knows newer- from its schema; older- comes from the schema history. + table = self.catalog.get_table("db.prefixes") + self.assertEqual([1, 2], sorted(self._read(table).column("id").to_pylist())) + self.assertTrue(self._target_reads("/older-")) + self.assertTrue(self._target_reads("/newer-")) + + def test_branch_copies_route_the_data_files_of_the_branch(self): + pa_schema = pa.schema([("id", pa.int32())]) + schema = Schema.from_pyarrow_schema(pa_schema, options={"data-file.prefix": "main-"}) + self.catalog.create_table("db.branched", schema, False) + main = self.catalog.get_table("db.branched") + main.branch_manager().create_branch("dev") + dev = self.catalog.get_table("db.branched$branch_dev") + dev.schema_manager.commit_changes([SchemaChange.set_option("data-file.prefix", "dev-")]) + self._write(self.catalog.get_table("db.branched$branch_dev"), + pa.Table.from_pydict({"id": [1]}, schema=pa_schema)) + self.target.requests.clear() + + copy = main.copy({"branch": "dev"}) + self.assertEqual([1], self._read(copy).column("id").to_pylist()) + self.assertTrue(self._target_reads("/dev-")) + + def test_table_files_named_like_external_objects_still_use_the_target(self): + pa_schema = pa.schema([("id", pa.int32())]) + self.catalog.create_table("db.plain", Schema.from_pyarrow_schema(pa_schema), False) + table = self.catalog.get_table("db.plain") + self._write(table, pa.Table.from_pydict({"id": [7]}, schema=pa_schema)) + self.target.requests.clear() + + self.assertEqual([7], self._read(table).column("id").to_pylist()) + self.assertTrue(self._target_reads("/db.db/plain/bucket-0/data-")) + + +if __name__ == "__main__": + unittest.main() diff --git a/paimon-python/pypaimon/tests/rest/rest_table_file_io_view_test.py b/paimon-python/pypaimon/tests/rest/rest_table_file_io_view_test.py new file mode 100644 index 000000000000..b6a00532152d --- /dev/null +++ b/paimon-python/pypaimon/tests/rest/rest_table_file_io_view_test.py @@ -0,0 +1,65 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Tests that each table of a REST catalog takes its io-cache view of the token FileIO.""" + +import unittest + +import pyarrow as pa + +from pypaimon import CatalogFactory, Schema +from pypaimon.catalog.rest.rest_token_file_io import RESTTokenFileIO +from pypaimon.tests.rest.rest_base_test import RESTBaseTest + + +class RESTTableFileIOViewTest(RESTBaseTest): + + def setUp(self): + super().setUp() + self.options['data-token.enabled'] = 'true' + self.rest_catalog = CatalogFactory.create(self.options) + + def _table(self, name, options): + schema = Schema.from_pyarrow_schema(pa.schema([("id", pa.int32())]), options=options) + self.rest_catalog.create_table(name, schema, False) + table = self.rest_catalog.get_table(name) + self.assertIsInstance(table.file_io, RESTTokenFileIO) + return table + + def test_tables_take_their_view_of_the_token_file_io(self): + # Other writers may replace Format Table files in place, and manage Object Table objects. + self.assertTrue(self._table("default.format_view", {"type": "format-table", "file.format": "csv"}) + .file_io._origin_only) + self.rest_catalog.create_table("default.object_view", Schema(options={"type": "object-table"}), False) + self.assertTrue(self.rest_catalog.get_table("default.object_view").file_io._origin_only) + + table = self._table("default.custom_view", {"data-file.prefix": "custom-", "changelog-file.prefix": "cl-"}) + path = table.table_path + self.assertEqual((False, (path, ("custom-", "cl-"), ("main",))), + (table.file_io._origin_only, table.file_io._table)) + self.assertIs(table.file_io, table.copy({}).file_io) + self.assertEqual((path, ("custom-", "cl-", "dynamic-"), ("main",)), + table.copy({"data-file.prefix": "dynamic-"}).file_io._table) + # A branch copy also reads the schemas of the branch. + self.assertEqual((path, ("custom-", "cl-"), ("main", "dev")), table.copy({"branch": "dev"}).file_io._table) + + plain = self._table("default.plain_view", {}) + self.assertEqual((plain.table_path, (), ("main",)), plain.file_io._table) + + +if __name__ == '__main__': + unittest.main() diff --git a/paimon-python/pypaimon/tests/rest/rest_token_file_io_test.py b/paimon-python/pypaimon/tests/rest/rest_token_file_io_test.py index 286f8f026db8..d9af36dc186c 100644 --- a/paimon-python/pypaimon/tests/rest/rest_token_file_io_test.py +++ b/paimon-python/pypaimon/tests/rest/rest_token_file_io_test.py @@ -1153,6 +1153,55 @@ def test_io_cache_routing_builds_one_file_io_per_target(self): cluster[OssOptions.OSS_ENDPOINT.key()], cluster["fs.oss.second.level.domain.enable"], cluster[OssOptions.OSS_REGION.key()])) + def _token_file_io(self): + file_io = RESTTokenFileIO(self.identifier, self._TABLE, Options({})) + file_io.token = RESTToken(self._IO_CACHE_TOKEN, int(time.time() * 1000) + 7_200_000) + RESTTokenFileIO._FILE_IO_CACHE.clear() + return file_io + + @staticmethod + def _patched_file_io_get(): + return patch('pypaimon.catalog.rest.rest_token_file_io.FileIO.get', + side_effect=lambda location, options: MagicMock(name=location)) + + def test_table_views_of_the_token_file_io(self): + file_io = self._token_file_io() + custom = self._TABLE + "/bucket-0/custom-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet" + with patch.object(RESTTokenFileIO, 'try_to_refresh_token'), self._patched_file_io_get(): + base = file_io.file_io() + origin_only = file_io.origin_only() + self.assertIsInstance(origin_only, RESTTokenFileIO) + self.assertIs(base._origin, origin_only.file_io()) + self.assertIs(origin_only, origin_only.for_table(self._TABLE, ["custom-"], "main")) + + view = file_io.for_table(self._TABLE, ["custom-"], "main") + self.assertIsInstance(view, RESTTokenFileIO) + self.assertIs(view, view.for_table(self._TABLE, ["custom-"], "main")) + routed = view.file_io() + self.assertIs(routed, view.file_io()) + self.assertEqual("default", routed._routing.route(Op.READ, custom)) + self.assertIsNone(base._routing.route(Op.READ, custom)) + restored = pickle.loads(pickle.dumps(view)) + self.assertEqual((False, (self._TABLE, ("custom-",), ("main",))), (restored._origin_only, restored._table)) + self.assertTrue(pickle.loads(pickle.dumps(origin_only))._origin_only) + + def test_branch_views_take_the_prefixes_the_catalog_keeps_for_the_branch(self): + branch_file = self._TABLE + "/branch/branch-dev/bucket-0/dev-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.orc" + view = self._token_file_io().for_table(self._TABLE, [], "dev") + api = MagicMock() + api.get_table.return_value = MagicMock(schema=MagicMock(options={"data-file.prefix": "dev-"})) + view.api_instance = api + # No schema directory in storage: the REST catalog keeps the branch's schema. + with patch.object(RESTTokenFileIO, 'try_to_refresh_token'), self._patched_file_io_get(), \ + patch('pypaimon.schema.schema_manager.SchemaManager.list_all', return_value=[]): + routing = view.file_io()._routing + self.assertEqual("default", routing.route(Op.READ, branch_file)) + self.assertEqual("default", routing.route(Op.READ, branch_file)) + branch = api.get_table.call_args[0][0] + self.assertEqual(("default", "test_table", "dev"), ( + branch.get_database_name(), branch.get_table_name(), branch.get_branch_name())) + api.get_table.assert_called_once() + def test_io_cache_routing_off_without_target(self): token = dict(self._IO_CACHE_TOKEN) del token["io-cache.endpoint"] From 8a96c5a280f5c548b2962698f3da08714f24d67f Mon Sep 17 00:00:00 2001 From: Sun Dapeng Date: Mon, 26 Oct 2026 01:15:06 +0800 Subject: [PATCH 16/27] [python] Route only the files under the table's own location Routing decided by file name alone, so any URI read through a table's FileIO could reach a target if its name looked like a Paimon file. Row data references objects outside the table: BLOB descriptor fields, descriptors returned in blob-as-descriptor mode, multimodal blob reads. Other writers own those objects and may replace them under the same name. apache/paimon-rust#1014 review reproduced the stale read with an external oss://bkt/import/data-{uuid}-1.png that origin had replaced while the cache kept the old bytes; this client read it through the target the same way. A target now only serves paths under the table's location: the path a RESTTokenFileIO is built for, and the path of the table a view is for. Everything else is read from origin, as without routing. This covers every reader of row-data URIs at once, including ones added later, instead of each of them choosing a FileIO. Files the table writes under its location, branches included, keep their routing; blob views read the upstream table through that table's own FileIO and location. Data files under data-file.external-paths are outside the location and now use the origin, which is where they went without routing. io_cache_table_routing_test reads a data-evolution table whose blob-descriptor-field points at that external object: the routed read returns the origin's bytes and the target sees no request for it, while the table's own data files still go to the target. It returns the stale bytes without the change. The routing and RESTTokenFileIO tests check the location rule for a view and for the FileIO of a token. --- .../catalog/rest/rest_token_file_io.py | 2 +- .../pypaimon/filesystem/io_cache_file_io.py | 2 +- .../pypaimon/filesystem/io_cache_routing.py | 22 ++++++++++++++----- .../pypaimon/tests/io_cache_file_io_test.py | 5 ++++- .../pypaimon/tests/io_cache_routing_test.py | 17 +++++++++++++- .../tests/io_cache_table_routing_test.py | 22 +++++++++++++++++++ .../tests/rest/rest_token_file_io_test.py | 6 +++++ 7 files changed, 66 insertions(+), 10 deletions(-) diff --git a/paimon-python/pypaimon/catalog/rest/rest_token_file_io.py b/paimon-python/pypaimon/catalog/rest/rest_token_file_io.py index 0ea89ea8e7db..d13c260ab78d 100644 --- a/paimon-python/pypaimon/catalog/rest/rest_token_file_io.py +++ b/paimon-python/pypaimon/catalog/rest/rest_token_file_io.py @@ -249,7 +249,7 @@ def _merged_options(self, token: RESTToken) -> Options: def _create_file_io(self, options: Options) -> FileIO: properties = options.to_map() - routing = IoCacheRouting.create(properties) + routing = IoCacheRouting.create(properties, root=self.path) if routing is None or urlparse(self.path).scheme != "oss": return FileIO.get(self.path, options) self.log.info( diff --git a/paimon-python/pypaimon/filesystem/io_cache_file_io.py b/paimon-python/pypaimon/filesystem/io_cache_file_io.py index 1c11513b590c..053938c61473 100644 --- a/paimon-python/pypaimon/filesystem/io_cache_file_io.py +++ b/paimon-python/pypaimon/filesystem/io_cache_file_io.py @@ -67,7 +67,7 @@ def table_view(self, table, catalog_prefixes=None) -> 'IoCacheRoutingFileIO': history = _SchemaPrefixHistory(self._origin, table_path, branches, catalog_prefixes) # The view takes its targets from this FileIO. view = IoCacheRoutingFileIO( - self._routing.for_table(prefixes, history), self._origin, self._file_io) + self._routing.for_table(table_path, prefixes, history), self._origin, self._file_io) view._base, view._table = self, table self._views[table] = view return view diff --git a/paimon-python/pypaimon/filesystem/io_cache_routing.py b/paimon-python/pypaimon/filesystem/io_cache_routing.py index f8821e974525..b955091ac161 100644 --- a/paimon-python/pypaimon/filesystem/io_cache_routing.py +++ b/paimon-python/pypaimon/filesystem/io_cache_routing.py @@ -128,6 +128,10 @@ def _new_prefixes(prefixes: Iterable[str], known: Tuple[str, ...]) -> Tuple[str, return tuple(p for p in dict.fromkeys(prefixes) if p and p.strip() and p not in known) +def _table_root(path: Optional[str]) -> Optional[str]: + return path.rstrip("/") + "/" if path else None + + def routable_type(path: str, prefixes: Tuple[str, ...]) -> Optional[FileType]: """The type a cache may serve, or None when the file must be read from the origin.""" path = path.rstrip("/") @@ -183,7 +187,7 @@ class IoCacheRouting: """Per-request choice between the io-cache targets and the origin endpoint.""" def __init__(self, options: Mapping[str, str], policy: FrozenSet[Op], - endpoints: Dict[str, Optional[str]]): + endpoints: Dict[str, Optional[str]], root: Optional[str] = None): origin = options.get("io-cache.origin.endpoint") self._origin = origin if origin and origin.strip() else options.get(FS_OSS_ENDPOINT) self._policy = policy @@ -198,22 +202,25 @@ def __init__(self, options: Mapping[str, str], policy: FrozenSet[Op], routes = options.get("io-cache.routes") self._routes = None if routes is None else _parse_routes(str(routes)) self._prefixes = data_prefixes(options) + # Only files under the table's own location can use a target; None allows any OSS path. + self._root = _table_root(root) self._history: Optional[Callable[[], Iterable[str]]] = None @staticmethod - def create(options: Mapping[str, str]) -> Optional['IoCacheRouting']: + def create(options: Mapping[str, str], root: Optional[str] = None) -> Optional['IoCacheRouting']: """The routing of the options, or None when requests keep using fs.oss.endpoint.""" if str(options.get(DLF_OSS_ENDPOINT) or "").strip() or not _flag(options.get(IO_CACHE_ENABLED)): return None policy = _parse_policy(options.get("io-cache.policy")) endpoints = _declared_targets(options) - return IoCacheRouting(options, policy, endpoints) if policy and endpoints else None + return IoCacheRouting(options, policy, endpoints, root) if policy and endpoints else None - def for_table(self, prefixes: Iterable[str], + def for_table(self, root: str, prefixes: Iterable[str], history: Optional[Callable[[], Iterable[str]]] = None) -> 'IoCacheRouting': - """This routing for the files of one table, also named with these prefixes, and with those - history() returns once a name that may be a data file matches no other.""" + """This routing for the files of the table at root, also named with these prefixes, and with + those history() returns once a name that may be a data file matches no other.""" routing = copy.copy(self) + routing._root = _table_root(root) routing._prefixes = self._prefixes + _new_prefixes(prefixes, self._prefixes) routing._history = history return routing @@ -233,6 +240,9 @@ def route(self, op: Op, path: str) -> Optional[str]: # Only OSS paths can use cache targets. if op not in self._policy or not path.startswith("oss://"): return None + # Row data may reference objects elsewhere, which other writers own and may replace. + if self._root is not None and not path.startswith(self._root): + return None file_type = routable_type(path, self._prefixes) if file_type is None and self._history is not None and _DATA_FILE_NAME.fullmatch( os.path.basename(path.rstrip("/"))): diff --git a/paimon-python/pypaimon/tests/io_cache_file_io_test.py b/paimon-python/pypaimon/tests/io_cache_file_io_test.py index 4d79894756f0..2a69ab08fcf3 100644 --- a/paimon-python/pypaimon/tests/io_cache_file_io_test.py +++ b/paimon-python/pypaimon/tests/io_cache_file_io_test.py @@ -494,11 +494,14 @@ def test_origin_only_view_is_the_origin(self): self.assertEqual([], self.created) def test_table_views_share_the_targets_and_close_nothing(self): + external = "oss://bkt/import/data-" + UUID + "-0.parquet" custom = PARTITION + "/custom-" + UUID + "-0.parquet" - self._put_all(DATA, custom) + self._put_all(DATA, external, custom) view = self.file_io.for_table(TABLE, ["custom-"], "main") self.assertIs(view, self.file_io.for_table(TABLE, ["custom-"], "main")) self.assertEqual(b"cluster", view.new_input_stream(custom).read()) + self.assertEqual(b"origin", view.new_input_stream(external).read()) + self.assertEqual(b"cluster", self.file_io.new_input_stream(external).read()) self.assertEqual(b"origin", self.file_io.new_input_stream(custom).read()) self.assertEqual(["cluster"], self.created) # A copy of the table keeps the prefixes of the view it was made from. diff --git a/paimon-python/pypaimon/tests/io_cache_routing_test.py b/paimon-python/pypaimon/tests/io_cache_routing_test.py index bebc93dc834a..6c68c7f524d2 100644 --- a/paimon-python/pypaimon/tests/io_cache_routing_test.py +++ b/paimon-python/pypaimon/tests/io_cache_routing_test.py @@ -490,6 +490,21 @@ def test_table_prefixes_add_data_prefixes(self): routing = IoCacheRouting.create(self._options(**{key: "part-"})) self.assertEqual("default", routing.route(Op.READ, path)) + def test_table_view_routes_only_files_under_the_table(self): + external = "oss://bkt/import/data-" + UUID + "-0.parquet" + sibling = "oss://bkt/db1.db/t10/bucket-0/data-" + UUID + "-0.parquet" + routing = IoCacheRouting.create(self._options()) + self.assertEqual("default", routing.route(Op.READ, external)) + for view in (IoCacheRouting.create(self._options(), root=self.TABLE), + routing.for_table(self.TABLE + "/", [])): + with self.subTest(view=view): + self.assertEqual("default", view.route(Op.READ, self.DATA)) + self.assertEqual("default", view.route(Op.META, self.MANIFEST)) + branch_file = self.TABLE + "/branch/branch-dev/bucket-0/data-" + UUID + "-0.orc" + self.assertEqual("default", view.route(Op.READ, branch_file)) + self.assertIsNone(view.route(Op.READ, external)) + self.assertIsNone(view.route(Op.READ, sibling)) + def test_table_view_asks_the_schema_history_only_for_unknown_data_file_names(self): asked = [] @@ -497,7 +512,7 @@ def history(): asked.append(1) return ["older-", ""] - view = IoCacheRouting.create(self._options()).for_table(["newer-"], history) + view = IoCacheRouting.create(self._options()).for_table(self.TABLE, ["newer-"], history) uuid_count = UUID + "-0" for path in (self.DATA, self.MANIFEST, self.TABLE + "/bucket-0/newer-" + uuid_count + ".parquet"): self.assertEqual("default", view.route(Op.READ, path), path) diff --git a/paimon-python/pypaimon/tests/io_cache_table_routing_test.py b/paimon-python/pypaimon/tests/io_cache_table_routing_test.py index b525d421a32a..940477098d97 100644 --- a/paimon-python/pypaimon/tests/io_cache_table_routing_test.py +++ b/paimon-python/pypaimon/tests/io_cache_table_routing_test.py @@ -21,6 +21,7 @@ import shutil import tempfile import unittest +import uuid from pathlib import Path from unittest import mock from urllib.parse import urlparse @@ -32,6 +33,7 @@ from pypaimon.filesystem.io_cache_routing import IoCacheRouting from pypaimon.filesystem.local_file_io import LocalFileIO from pypaimon.schema.schema_change import SchemaChange +from pypaimon.table.row.blob import BlobDescriptor WAREHOUSE = "oss://bkt/warehouse" @@ -117,6 +119,26 @@ def _read(table): read_builder = table.new_read_builder() return read_builder.new_read().to_arrow(read_builder.new_scan().plan().splits()) + def _external_descriptor(self): + # Named like a Paimon data file, outside every table, and replaced on origin after the cache kept it. + uri = "oss://bkt/import/data-" + str(uuid.uuid4()) + "-1.png" + self._put(uri, b"NEW") + self._put(uri, b"OLD", retained=True) + return uri, BlobDescriptor(uri, 0, 3).serialize() + + def test_descriptor_fields_are_read_from_origin(self): + pa_schema = pa.schema([("id", pa.int32()), ("payload", pa.large_binary())]) + schema = Schema.from_pyarrow_schema(pa_schema, options={ + "row-tracking.enabled": "true", "data-evolution.enabled": "true", + "blob-descriptor-field": "payload"}) + self.catalog.create_table("db.descriptors", schema, False) + table = self.catalog.get_table("db.descriptors") + uri, descriptor = self._external_descriptor() + self._write(table, pa.Table.from_pydict({"id": [1], "payload": [descriptor]}, schema=pa_schema)) + + self.assertEqual([b"NEW"], self._read(table).column("payload").to_pylist()) + self.assertEqual([], self._target_reads("/import/")) + def test_data_files_of_older_schema_prefixes_use_the_target(self): pa_schema = pa.schema([("id", pa.int32())]) schema = Schema.from_pyarrow_schema(pa_schema, options={"data-file.prefix": "older-"}) diff --git a/paimon-python/pypaimon/tests/rest/rest_token_file_io_test.py b/paimon-python/pypaimon/tests/rest/rest_token_file_io_test.py index d9af36dc186c..fb63c88f45a4 100644 --- a/paimon-python/pypaimon/tests/rest/rest_token_file_io_test.py +++ b/paimon-python/pypaimon/tests/rest/rest_token_file_io_test.py @@ -1164,6 +1164,12 @@ def _patched_file_io_get(): return patch('pypaimon.catalog.rest.rest_token_file_io.FileIO.get', side_effect=lambda location, options: MagicMock(name=location)) + def test_io_cache_routing_serves_only_the_table_of_the_token(self): + external = "oss://bkt/import/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet" + resolved, _ = self._resolve_file_io(self._IO_CACHE_TOKEN) + self.assertEqual("default", resolved._routing.route(Op.READ, self._DATA)) + self.assertIsNone(resolved._routing.route(Op.READ, external)) + def test_table_views_of_the_token_file_io(self): file_io = self._token_file_io() custom = self._TABLE + "/bucket-0/custom-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet" From 877dc1c9ac112b9734b37f19d43b5b4b205d0c2f Mon Sep 17 00:00:00 2001 From: Sun Dapeng Date: Mon, 26 Oct 2026 01:15:44 +0800 Subject: [PATCH 17/27] [python] Give Daft and PVFS the origin's options under io-cache routing Daft's IOConfig for a REST table and PVFS's ossfs or JindoSDK filesystem are built from the vended token and send every request to fs.oss.endpoint. Under io-cache routing that key is kept for clients that do not route, so it may name the cache: Daft read data files and the objects BLOB descriptors point at through it, and PVFS read snapshots, hints and every other file through it, the requests per-request routing keeps on origin. Like the Lance and Vortex readers, these clients now get the origin's options when the token enables routing: the origin endpoint, without the cache client keys. dlf.oss-endpoint still overrides everything, and an io-cache.enabled in PVFS's catalog options wins over the token, as it does for RESTTokenFileIO. Without routing the options are unchanged. test_single_endpoint_clients_get_the_origin_options and test_daft_gets_the_origin_options check the options both clients get with and without routing; they fail without the change. --- paimon-python/pypaimon/daft/daft_paimon.py | 5 ++- .../pypaimon/filesystem/io_cache_routing.py | 6 +++ paimon-python/pypaimon/filesystem/pvfs.py | 8 +++- .../pypaimon/tests/io_cache_routing_test.py | 37 ++++++++++++++++++- 4 files changed, 53 insertions(+), 3 deletions(-) diff --git a/paimon-python/pypaimon/daft/daft_paimon.py b/paimon-python/pypaimon/daft/daft_paimon.py index ec698349462c..af2a83afb751 100644 --- a/paimon-python/pypaimon/daft/daft_paimon.py +++ b/paimon-python/pypaimon/daft/daft_paimon.py @@ -33,6 +33,8 @@ from typing import TYPE_CHECKING, Any, Dict, Optional, Union from urllib.parse import urlparse +from pypaimon.filesystem.io_cache_routing import single_endpoint_options + if TYPE_CHECKING: import daft @@ -53,7 +55,8 @@ def _enrich_options_with_rest_token( file_io.try_to_refresh_token() if file_io.token is None: return catalog_options - enriched = {**catalog_options, **file_io.token.token} + # Daft reads with its own client, which cannot use io-cache targets per request. + enriched = single_endpoint_options({**catalog_options, **file_io.token.token}) parsed = urlparse(getattr(table, "table_path", "") or "") if parsed.scheme and parsed.netloc: enriched["warehouse"] = f"{parsed.scheme}://{parsed.netloc}" diff --git a/paimon-python/pypaimon/filesystem/io_cache_routing.py b/paimon-python/pypaimon/filesystem/io_cache_routing.py index b955091ac161..5680f3142825 100644 --- a/paimon-python/pypaimon/filesystem/io_cache_routing.py +++ b/paimon-python/pypaimon/filesystem/io_cache_routing.py @@ -175,6 +175,12 @@ def _with_endpoint(options: Dict[str, str], endpoint: Optional[str]) -> Dict[str return options +def single_endpoint_options(options: Mapping[str, str]) -> Dict[str, str]: + """Options for a client that sends every request to one endpoint: the origin's under io-cache routing.""" + routing = IoCacheRouting.create(options) + return dict(options) if routing is None else routing.origin_options(options) + + class CacheTarget(NamedTuple): """A named cache target with its endpoint and addressing settings.""" name: str diff --git a/paimon-python/pypaimon/filesystem/pvfs.py b/paimon-python/pypaimon/filesystem/pvfs.py index 823f3ac3e9fa..793abdac5786 100644 --- a/paimon-python/pypaimon/filesystem/pvfs.py +++ b/paimon-python/pypaimon/filesystem/pvfs.py @@ -43,6 +43,7 @@ JINDO_OSSFS_AVAILABLE, create_jindo_oss_filesystem, ) +from pypaimon.filesystem.io_cache_routing import single_endpoint_options from pypaimon.schema.schema import Schema logger = logging.getLogger(__name__) @@ -887,7 +888,12 @@ def _merge_token_with_catalog_options(self, token: dict) -> dict: dlf_oss_endpoint = self.options.get(CatalogOptions.DLF_OSS_ENDPOINT) if dlf_oss_endpoint and dlf_oss_endpoint.strip(): merged_token[OssOptions.OSS_ENDPOINT.key()] = dlf_oss_endpoint - return merged_token + return merged_token + if self.options.contains(CatalogOptions.IO_CACHE_ENABLED): + merged_token[CatalogOptions.IO_CACHE_ENABLED.key()] = str( + self.options.get(CatalogOptions.IO_CACHE_ENABLED)).lower() + # The OSS filesystem sends every request to one endpoint, so it cannot use io-cache targets. + return single_endpoint_options(merged_token) @staticmethod def _get_storage_type(path: str): diff --git a/paimon-python/pypaimon/tests/io_cache_routing_test.py b/paimon-python/pypaimon/tests/io_cache_routing_test.py index 6c68c7f524d2..f5b81fba4816 100644 --- a/paimon-python/pypaimon/tests/io_cache_routing_test.py +++ b/paimon-python/pypaimon/tests/io_cache_routing_test.py @@ -17,9 +17,10 @@ """Tests for the io-cache routing decision.""" +import sys import unittest -from pypaimon.filesystem.io_cache_routing import (IoCacheRouting, Op, +from pypaimon.filesystem.io_cache_routing import (IoCacheRouting, Op, single_endpoint_options, data_prefixes, routable_type) from pypaimon.utils.file_type import FileType @@ -592,3 +593,37 @@ def test_target_keeps_sdk_settings(self): options = self._options(**{"fs.oss.retry.count": "5"}) target = IoCacheRouting.create(options).target_options(options, "default") self.assertEqual("5", target["fs.oss.retry.count"]) + + _ROUTED_TOKEN_ORIGIN = {"fs.oss.endpoint": "https://origin.example.com", "fs.oss.https.enable": "true"} + + def _assert_origin_options(self, options): + self.assertEqual(self._ROUTED_TOKEN_ORIGIN, {key: options[key] for key in self._ROUTED_TOKEN_ORIGIN}) + self.assertNotIn("fs.oss.dlf-cache.consistent-hash.enabled", options) + + def _routed_token(self): + return self._options(**{"fs.oss.dlf-cache.consistent-hash.enabled": "true"}) + + def test_single_endpoint_clients_get_the_origin_options(self): + from pypaimon.filesystem.pvfs import PaimonVirtualFileSystem + + token = self._routed_token() + self._assert_origin_options(single_endpoint_options(token)) + pvfs = PaimonVirtualFileSystem({}, skip_instance_cache=True) + self._assert_origin_options(pvfs._merge_token_with_catalog_options(token)) + + # Without routing, or with routing turned off by the catalog, they keep fs.oss.endpoint. + unrouted = dict(token, **{"io-cache.enabled": "false"}) + self.assertEqual("http://legacy.example.com", single_endpoint_options(unrouted)["fs.oss.endpoint"]) + for catalog_options, expected in (({"io-cache.enabled": "false"}, "http://legacy.example.com"), + ({"dlf.oss-endpoint": "oss-cn-test.example.com"}, "oss-cn-test.example.com")): + pvfs = PaimonVirtualFileSystem(catalog_options, skip_instance_cache=True) + self.assertEqual(expected, pvfs._merge_token_with_catalog_options(token)["fs.oss.endpoint"]) + + @unittest.skipIf(sys.version_info < (3, 7), "pypaimon.daft needs Python 3.7+") + def test_daft_gets_the_origin_options(self): + from unittest.mock import MagicMock + + from pypaimon.daft.daft_paimon import _enrich_options_with_rest_token + + table = MagicMock(file_io=MagicMock(token=MagicMock(token=self._routed_token())), table_path=self.TABLE) + self._assert_origin_options(_enrich_options_with_rest_token({"metastore": "rest"}, table)) From f5da9ba856184dcd7ab47fb3f4b9bfdd8db92523 Mon Sep 17 00:00:00 2001 From: Sun Dapeng Date: Mon, 26 Oct 2026 01:44:53 +0800 Subject: [PATCH 18/27] [python] Give the native core the origin's options under io-cache routing Opt-in native execution (read.native.enabled and the other native options) reuses the data token Python holds for a REST table and passes it to the paimon-rust core unchanged. Today's core reads no io-cache keys and sends every request to fs.oss.endpoint, which under routing is kept for clients that do not route and may name the cache. Native reads of snapshots, hints and manifests then went to the cache, the requests per-request routing keeps on origin. Like Daft, PVFS, Lance and Vortex, the core now gets the origin's options when the token enables routing: the origin endpoint, without the cache client keys. A dlf.oss-endpoint in the catalog options still turns routing off and leaves the token as it is. test_native_rest_cache_gives_the_core_the_origin_under_io_cache_routing checks the token the core receives with and without dlf.oss-endpoint; it fails without the change. --- .../pypaimon/execution/native/table.py | 5 +++- .../pypaimon/filesystem/io_cache_routing.py | 8 +++-- .../pypaimon/tests/native_plan_test.py | 29 +++++++++++++++++++ 3 files changed, 38 insertions(+), 4 deletions(-) diff --git a/paimon-python/pypaimon/execution/native/table.py b/paimon-python/pypaimon/execution/native/table.py index 251d4cc6f69d..c6684dd92807 100644 --- a/paimon-python/pypaimon/execution/native/table.py +++ b/paimon-python/pypaimon/execution/native/table.py @@ -25,6 +25,7 @@ from pypaimon.common.options.config import CatalogOptions, FuseOptions, OssOptions from pypaimon.common.options.options_utils import OptionsUtils +from pypaimon.filesystem.io_cache_routing import single_endpoint_options def _native_blob_view_supported(table, read_names) -> bool: @@ -201,9 +202,11 @@ def create(): if token is None: return Table.from_rest_response( response, database=database, table=table, rest_options=options) + # The native core cannot use io-cache targets per request, and fs.oss.endpoint may name the cache. return Table.from_rest_response_with_token( response, database=database, table=table, rest_options=options, - data_token=dict(token.token), expires_at_millis=token.expire_at_millis) + data_token=single_endpoint_options(token.token, options), + expires_at_millis=token.expire_at_millis) key = (response, database, table, tuple(sorted(options.items()))) return self.get_or_create(key, create) diff --git a/paimon-python/pypaimon/filesystem/io_cache_routing.py b/paimon-python/pypaimon/filesystem/io_cache_routing.py index 5680f3142825..fe66e9314d70 100644 --- a/paimon-python/pypaimon/filesystem/io_cache_routing.py +++ b/paimon-python/pypaimon/filesystem/io_cache_routing.py @@ -175,9 +175,11 @@ def _with_endpoint(options: Dict[str, str], endpoint: Optional[str]) -> Dict[str return options -def single_endpoint_options(options: Mapping[str, str]) -> Dict[str, str]: - """Options for a client that sends every request to one endpoint: the origin's under io-cache routing.""" - routing = IoCacheRouting.create(options) +def single_endpoint_options(options: Mapping[str, str], + catalog_options: Optional[Mapping[str, str]] = None) -> Dict[str, str]: + """Options for a client that sends every request to one endpoint: the origin's under io-cache routing, + unless catalog_options turn routing off (dlf.oss-endpoint).""" + routing = IoCacheRouting.create({**(catalog_options or {}), **options}) return dict(options) if routing is None else routing.origin_options(options) diff --git a/paimon-python/pypaimon/tests/native_plan_test.py b/paimon-python/pypaimon/tests/native_plan_test.py index f8a35522bf06..108f02c6ccd5 100644 --- a/paimon-python/pypaimon/tests/native_plan_test.py +++ b/paimon-python/pypaimon/tests/native_plan_test.py @@ -1199,6 +1199,35 @@ def test_native_rest_cache_passes_only_existing_token(self): fake_df.Table.from_rest_response.assert_called_once_with( 'response', database='default', table='t', rest_options={}) + def test_native_rest_cache_gives_the_core_the_origin_under_io_cache_routing(self): + import time + + from pypaimon.catalog.rest.rest_token import RESTToken + from pypaimon.execution.native.table import _NativeRestTableCache + + token = RESTToken({ + 'fs.oss.endpoint': 'http://cache.example.com', + 'io-cache.enabled': 'true', + 'io-cache.endpoint': 'http://cache.example.com', + 'io-cache.origin.endpoint': 'https://origin.example.com', + 'io-cache.policy': 'meta,read', + 'fs.oss.dlf-cache.consistent-hash.enabled': 'true', + }, int(time.time() * 1000) + 7_200_000) + fake_df = ModuleType('pypaimon_rust.datafusion') + fake_df.Table = Mock() + with patch.dict(sys.modules, {'pypaimon_rust.datafusion': fake_df}): + _NativeRestTableCache().get('response', 'default', 't', {}, token_loader=lambda: token) + data_token = fake_df.Table.from_rest_response_with_token.call_args[1]['data_token'] + self.assertEqual(('https://origin.example.com', 'true'), + (data_token['fs.oss.endpoint'], data_token['fs.oss.https.enable'])) + self.assertNotIn('fs.oss.dlf-cache.consistent-hash.enabled', data_token) + + # dlf.oss-endpoint turns routing off, so the token keeps the endpoint it was given. + fake_df.Table.reset_mock() + options = {'dlf.oss-endpoint': 'oss-cn-test.example.com'} + _NativeRestTableCache().get('response', 'default', 't', options, token_loader=lambda: token) + self.assertEqual(token.token, fake_df.Table.from_rest_response_with_token.call_args[1]['data_token']) + def test_native_rest_cache_retries_failed_construction(self): from pypaimon.execution.native.table import _NativeRestTableCache From f9524022eae7b676f7faebf4086ffa53f093b3d6 Mon Sep 17 00:00:00 2001 From: Sun Dapeng Date: Mon, 26 Oct 2026 03:17:42 +0800 Subject: [PATCH 19/27] [python] Keep io-cache views through ResolvingFileIO and URI reader factories Two FileIO wrappers dropped the view a table takes of io-cache routing, and two factories built FileIOs that could route URIs no table owns. - ResolvingFileIO (resolving-file-io.enabled) inherited origin_only() and for_table() from FileIO, which return the FileIO itself. A Format or Object Table on it was not origin-only, and no table's location, prefixes or schema history reached the FileIOs it resolves per scheme. Its views now share the resolved FileIOs and apply themselves to each one, as CachingFileIO does for its delegate; a view closes nothing. - As an io-cache origin or target, a ResolvingFileIO did not forward create_parent_dirs, and the target role did not reach the PyArrowFileIOs it resolves, so parent directories and the cleanup of a failed write ran on the target. Both now behave as for a PyArrowFileIO. - UriReaderFactory built from catalog options (the default uri_reader_factory of every FileIO) and the source FileIO of LeRobot and rosbag imports create FileIOs that serve no table. They now read through the origin-only view of those FileIOs, so a URI from row data never reaches a target by its name alone, the rule paimon-rust and the Java client follow. Today only RESTTokenFileIO builds a routing FileIO, and it wraps ResolvingFileIO rather than the other way round, so the first and third points are defensive; the second applies when a REST catalog sets resolving-file-io.enabled with routing on. io_cache_table_routing_test runs every table test again through a FileSystemCatalog with resolving-file-io.enabled whose resolved FileIO routes, and adds a Format Table whose file was replaced on origin and URIs read through a factory and a source FileIO; the routing FileIO test writes through ResolvingFileIO origin and targets. All fail without the change. --- paimon-python/pypaimon/common/uri_reader.py | 3 +- .../pypaimon/filesystem/io_cache_file_io.py | 3 +- .../pypaimon/filesystem/resolving_file_io.py | 31 +++++++++ .../pypaimon/multimodal/source_utils.py | 6 +- .../pypaimon/tests/io_cache_file_io_test.py | 25 ++++++++ .../tests/io_cache_table_routing_test.py | 64 ++++++++++++++++++- 6 files changed, 127 insertions(+), 5 deletions(-) diff --git a/paimon-python/pypaimon/common/uri_reader.py b/paimon-python/pypaimon/common/uri_reader.py index 5548329c2965..4c18f0ff5b5d 100644 --- a/paimon-python/pypaimon/common/uri_reader.py +++ b/paimon-python/pypaimon/common/uri_reader.py @@ -179,7 +179,8 @@ def _new_reader(self, key: UriKey, parsed_uri: ParseResult) -> UriReader: uri_string = parsed_uri.geturl() file_io = FileIO.get(uri_string, self.catalog_options) self._owned_file_ios.append(file_io) - return UriReader.from_file(file_io) + # This FileIO serves no table, so the URIs of row data it reads come from origin. + return UriReader.from_file(file_io.origin_only()) except Exception as e: raise RuntimeError(f"Failed to create reader for URI {parsed_uri.geturl()}") from e diff --git a/paimon-python/pypaimon/filesystem/io_cache_file_io.py b/paimon-python/pypaimon/filesystem/io_cache_file_io.py index 053938c61473..2bfa8b8c479d 100644 --- a/paimon-python/pypaimon/filesystem/io_cache_file_io.py +++ b/paimon-python/pypaimon/filesystem/io_cache_file_io.py @@ -28,6 +28,7 @@ from pypaimon.common.file_io import FileIO from pypaimon.filesystem.io_cache_routing import IoCacheRouting, Op, schema_file_prefixes from pypaimon.filesystem.pyarrow_file_io import PyArrowFileIO +from pypaimon.filesystem.resolving_file_io import ResolvingFileIO class IoCacheRoutingFileIO(FileIO): @@ -103,7 +104,7 @@ def _file_io(self, name: Optional[str]) -> FileIO: file_io = self._targets.get(name) if file_io is None: file_io = self._create_target(name) - if isinstance(file_io, PyArrowFileIO): + if isinstance(file_io, (PyArrowFileIO, ResolvingFileIO)): file_io.creates_parent_dirs = False file_io.cleans_up_failed_writes = False self._targets[name] = file_io diff --git a/paimon-python/pypaimon/filesystem/resolving_file_io.py b/paimon-python/pypaimon/filesystem/resolving_file_io.py index b5fd25bc8846..bc8d336f8ad9 100644 --- a/paimon-python/pypaimon/filesystem/resolving_file_io.py +++ b/paimon-python/pypaimon/filesystem/resolving_file_io.py @@ -15,6 +15,7 @@ # specific language governing permissions and limitations # under the License. +import copy from collections import defaultdict from typing import Dict, List from urllib.parse import urlparse @@ -33,6 +34,12 @@ class ResolvingFileIO(FileIO): This is the Python equivalent of Java's ``ResolvingFileIO``. """ + # Views applied to each resolved FileIO, see origin_only and for_table. + _views: tuple = () + # False on an io-cache target, for the PyArrowFileIOs it resolves. + creates_parent_dirs = True + cleans_up_failed_writes = True + def __init__(self, catalog_options: Options): opts_map = dict(catalog_options.to_map()) opts_map.pop(CatalogOptions.RESOLVING_FILE_IO_ENABLED.key(), None) @@ -49,9 +56,27 @@ def _get_fileio(self, path: str) -> FileIO: fileio = self._fileio_cache.get(cache_key) if fileio is None: fileio = FileIO.get(path, self._options) + from pypaimon.filesystem.pyarrow_file_io import PyArrowFileIO + if isinstance(fileio, PyArrowFileIO): + fileio.creates_parent_dirs = self.creates_parent_dirs + fileio.cleans_up_failed_writes = self.cleans_up_failed_writes self._fileio_cache[cache_key] = fileio + for view in self._views: + fileio = fileio.origin_only() if view[0] == "origin" else fileio.for_table(*view[1:]) return fileio + def origin_only(self) -> FileIO: + return self._with_view(("origin",)) + + def for_table(self, table_path: str, file_prefixes, branch: str) -> FileIO: + return self._with_view(("table", table_path, tuple(file_prefixes), branch)) + + def _with_view(self, view) -> 'ResolvingFileIO': + # A view shares the resolved FileIOs and applies itself to each of them. + resolving = copy.copy(self) + resolving._views = self._views + (view,) + return resolving + @property def properties(self): return self._options @@ -71,6 +96,9 @@ def new_input_stream(self, path: str): def new_output_stream(self, path: str): return self._get_fileio(path).new_output_stream(path) + def create_parent_dirs(self, path: str): + return self._get_fileio(path).create_parent_dirs(path) + def get_file_status(self, path: str): return self._get_fileio(path).get_file_status(path) @@ -134,6 +162,9 @@ def write_mosaic(self, path: str, data, **kwargs): return self._get_fileio(path).write_mosaic(path, data, **kwargs) def close(self): + if self._views: + # A view shares the FileIOs of the ResolvingFileIO it was made from, which closes them. + return first_error = None try: self.uri_reader_factory.close() diff --git a/paimon-python/pypaimon/multimodal/source_utils.py b/paimon-python/pypaimon/multimodal/source_utils.py index 6eb0e2782351..09d988519716 100644 --- a/paimon-python/pypaimon/multimodal/source_utils.py +++ b/paimon-python/pypaimon/multimodal/source_utils.py @@ -29,7 +29,9 @@ class _SourceFileIO: """Resolve external source URIs without using target warehouse options.""" def __init__(self, options): - self._resolver = ResolvingFileIO(options) + self._file_io = ResolvingFileIO(options) + # Source objects belong to no table, so they are read from origin. + self._resolver = self._file_io.origin_only() def _resolve(self, path): file_io = self._resolver._get_fileio(path) @@ -54,7 +56,7 @@ def to_filesystem_path(self, path): return self._resolve(path)[1] def close(self): - self._resolver.close() + self._file_io.close() def _source_path_text(value): diff --git a/paimon-python/pypaimon/tests/io_cache_file_io_test.py b/paimon-python/pypaimon/tests/io_cache_file_io_test.py index 2a69ab08fcf3..d760fc4c5aa4 100644 --- a/paimon-python/pypaimon/tests/io_cache_file_io_test.py +++ b/paimon-python/pypaimon/tests/io_cache_file_io_test.py @@ -49,6 +49,7 @@ from pypaimon.filesystem.io_cache_routing import IoCacheRouting from pypaimon.filesystem.local_file_io import LocalFileIO from pypaimon.filesystem.oss_file_io import OssFileIO +from pypaimon.filesystem.resolving_file_io import ResolvingFileIO from pypaimon.schema.data_types import AtomicType, DataField from pypaimon.table.row.blob import BlobDescriptor from pypaimon.utils.file_type import FileType @@ -794,6 +795,30 @@ def test_routed_writes_create_parent_directories_on_origin(write): assert origin == [call for call in unrouted if call != "open_output_stream"] +@pytest.mark.parametrize("write", [ + lambda f, p: f.new_output_stream(p).close(), + lambda f, p: f.write_parquet(p, pa.table({"a": [1]})), + lambda f, p: f.write_file(p, "x"), +], ids=["new_output_stream", "write_parquet", "write_file"]) +def test_routed_writes_through_resolving_file_ios_create_parent_directories_on_origin(write): + # With resolving-file-io.enabled, origin and targets are ResolvingFileIOs over the OSS FileIOs. + unrouted, origin, target = [], [], [] + write(_legacy_oss_file_io(unrouted), DATA) + calls = {"origin": origin, "accel": [], "cluster": target} + + def resolving(name): + return ResolvingFileIO(Options({"endpoint": name})) + + def resolve(path, options): + return _legacy_oss_file_io(calls[options.to_map()["endpoint"]]) + + with mock.patch.object(FileIO, "get", side_effect=resolve): + write(IoCacheRoutingFileIO(_routing(policy="meta,read,write"), resolving("origin"), resolving), DATA) + assert "create_dir" in unrouted + assert target == ["open_output_stream"] + assert origin == [call for call in unrouted if call != "open_output_stream"] + + class _FailingWritesHandler(_MissingPathsHandler): """Every path is missing and every upload fails.""" diff --git a/paimon-python/pypaimon/tests/io_cache_table_routing_test.py b/paimon-python/pypaimon/tests/io_cache_table_routing_test.py index 940477098d97..88fb9f6dcfeb 100644 --- a/paimon-python/pypaimon/tests/io_cache_table_routing_test.py +++ b/paimon-python/pypaimon/tests/io_cache_table_routing_test.py @@ -26,13 +26,22 @@ from unittest import mock from urllib.parse import urlparse +import io + import pyarrow as pa +import pyarrow.parquet as pq from pypaimon import CatalogFactory, Schema +from pypaimon.common.options import Options +from pypaimon.common.options.config import CatalogOptions +from pypaimon.common.uri_reader import UriReaderFactory from pypaimon.filesystem.io_cache_file_io import IoCacheRoutingFileIO from pypaimon.filesystem.io_cache_routing import IoCacheRouting from pypaimon.filesystem.local_file_io import LocalFileIO +from pypaimon.filesystem.resolving_file_io import ResolvingFileIO +from pypaimon.multimodal.source_utils import _SourceFileIO from pypaimon.schema.schema_change import SchemaChange +from pypaimon.table.format.format_table import Format, FormatTable from pypaimon.table.row.blob import BlobDescriptor WAREHOUSE = "oss://bkt/warehouse" @@ -56,6 +65,16 @@ def _to_file(self, path): return Path(self.retained, key) return Path(self.root, key) + def list_status(self, path): + # Children keep the oss:// form of the listed path, as an OSS listing returns them. + statuses = super().list_status(path) + if urlparse(path).scheme != "oss": + return statuses + children = [self.get_file_status(path.rstrip("/") + "/" + Path(status.path).name) for status in statuses] + for child in children: + child.path = child.original_path + return children + def new_input_stream(self, path): self.requests.append(path) return super().new_input_stream(path) @@ -69,6 +88,8 @@ def to_filesystem_path(self, path): class IoCacheTableRoutingTest(unittest.TestCase): """Origin holds the current objects; the target serves what it retained, as a stale cache would.""" + CATALOG_OPTIONS = {} + def setUp(self): self.temp_dir = tempfile.mkdtemp(prefix="io_cache_table_routing_test_") storage = os.path.join(self.temp_dir, "storage") @@ -78,7 +99,7 @@ def setUp(self): patcher = mock.patch("pypaimon.catalog.filesystem_catalog.FileIO.get", side_effect=self._file_io) patcher.start() self.addCleanup(patcher.stop) - self.catalog = CatalogFactory.create({"warehouse": WAREHOUSE}) + self.catalog = CatalogFactory.create(dict(self.CATALOG_OPTIONS, warehouse=WAREHOUSE)) self.catalog.create_database("db", False) def tearDown(self): @@ -170,6 +191,32 @@ def test_branch_copies_route_the_data_files_of_the_branch(self): self.assertEqual([1], self._read(copy).column("id").to_pylist()) self.assertTrue(self._target_reads("/dev-")) + def test_format_table_files_replaced_on_origin_are_read_from_origin(self): + pa_schema = pa.schema([("id", pa.int32())]) + self.catalog.create_table("db.format_schema", Schema.from_pyarrow_schema(pa_schema), False) + source = self.catalog.get_table("db.format_schema") + location = WAREHOUSE + "/db.db/format_files" + # Another writer replaced a file named like a Paimon data file after the cache kept it. + path = location + "/data-" + str(uuid.uuid4()) + "-1.parquet" + for value, retained in ((2, False), (1, True)): + buffer = io.BytesIO() + pq.write_table(pa.Table.from_pydict({"id": [value]}, schema=pa_schema), buffer) + self._put(path, buffer.getvalue(), retained) + # Rooted at the Format Table's own location, as the FileIO a REST catalog builds for it. + file_io = self.catalog.file_io.for_table(location, [], "main") + table = FormatTable(file_io, source.identifier, source.table_schema, location, Format.parse("parquet")) + + self.assertEqual([2], self._read(table).column("id").to_pylist()) + self.assertEqual([], self._target_reads("/format_files/")) + + def test_uri_readers_that_serve_no_table_read_from_origin(self): + uri, _ = self._external_descriptor() + # Factories and source readers built from catalog options create FileIOs for no table. + self.assertEqual(b"NEW", UriReaderFactory(Options({})).create(uri).new_input_stream(uri).read()) + with _SourceFileIO(Options({})).new_input_stream(uri) as stream: + self.assertEqual(b"NEW", stream.read()) + self.assertEqual([], self._target_reads("/import/")) + def test_table_files_named_like_external_objects_still_use_the_target(self): pa_schema = pa.schema([("id", pa.int32())]) self.catalog.create_table("db.plain", Schema.from_pyarrow_schema(pa_schema), False) @@ -181,5 +228,20 @@ def test_table_files_named_like_external_objects_still_use_the_target(self): self.assertTrue(self._target_reads("/db.db/plain/bucket-0/data-")) +class IoCacheResolvingTableRoutingTest(IoCacheTableRoutingTest): + """The same tables through a ResolvingFileIO, which resolves the routing FileIO of each path.""" + + CATALOG_OPTIONS = {CatalogOptions.RESOLVING_FILE_IO_ENABLED.key(): "true"} + + def _file_io(self, path, options=None): + if options is not None and options.get(CatalogOptions.RESOLVING_FILE_IO_ENABLED): + return ResolvingFileIO(options) + return super()._file_io(path, options) + + def test_tables_take_views_of_the_resolving_file_io(self): + self.catalog.create_table("db.resolved", Schema.from_pyarrow_schema(pa.schema([("id", pa.int32())])), False) + self.assertIsInstance(self.catalog.get_table("db.resolved").file_io, ResolvingFileIO) + + if __name__ == "__main__": unittest.main() From cc415cd544dd7b3ee833db609652211725233cf8 Mon Sep 17 00:00:00 2001 From: Sun Dapeng Date: Mon, 26 Oct 2026 06:06:03 +0800 Subject: [PATCH 20/27] [python] Leave the multimodal source FileIO as upstream builds it The previous commit read external HDF5, rosbag and LeRobot sources through the origin-only view of the ResolvingFileIO that _SourceFileIO builds. That view never changes a request: _SourceFileIO resolves each URI with FileIO.get and the caller's source options, and only RESTTokenFileIO builds a routing FileIO. It did break the multimodal tests, which put their own FileIO doubles in place of ResolvingFileIO and do not implement origin_only(). source_utils.py is back to upstream, and the routing test keeps only the UriReaderFactory case. The HDF5 and rosbag suites pass again with h5py and rosbags installed. --- paimon-python/pypaimon/multimodal/source_utils.py | 6 ++---- paimon-python/pypaimon/tests/io_cache_table_routing_test.py | 5 +---- 2 files changed, 3 insertions(+), 8 deletions(-) diff --git a/paimon-python/pypaimon/multimodal/source_utils.py b/paimon-python/pypaimon/multimodal/source_utils.py index 09d988519716..6eb0e2782351 100644 --- a/paimon-python/pypaimon/multimodal/source_utils.py +++ b/paimon-python/pypaimon/multimodal/source_utils.py @@ -29,9 +29,7 @@ class _SourceFileIO: """Resolve external source URIs without using target warehouse options.""" def __init__(self, options): - self._file_io = ResolvingFileIO(options) - # Source objects belong to no table, so they are read from origin. - self._resolver = self._file_io.origin_only() + self._resolver = ResolvingFileIO(options) def _resolve(self, path): file_io = self._resolver._get_fileio(path) @@ -56,7 +54,7 @@ def to_filesystem_path(self, path): return self._resolve(path)[1] def close(self): - self._file_io.close() + self._resolver.close() def _source_path_text(value): diff --git a/paimon-python/pypaimon/tests/io_cache_table_routing_test.py b/paimon-python/pypaimon/tests/io_cache_table_routing_test.py index 88fb9f6dcfeb..8f1d59f25b53 100644 --- a/paimon-python/pypaimon/tests/io_cache_table_routing_test.py +++ b/paimon-python/pypaimon/tests/io_cache_table_routing_test.py @@ -39,7 +39,6 @@ from pypaimon.filesystem.io_cache_routing import IoCacheRouting from pypaimon.filesystem.local_file_io import LocalFileIO from pypaimon.filesystem.resolving_file_io import ResolvingFileIO -from pypaimon.multimodal.source_utils import _SourceFileIO from pypaimon.schema.schema_change import SchemaChange from pypaimon.table.format.format_table import Format, FormatTable from pypaimon.table.row.blob import BlobDescriptor @@ -211,10 +210,8 @@ def test_format_table_files_replaced_on_origin_are_read_from_origin(self): def test_uri_readers_that_serve_no_table_read_from_origin(self): uri, _ = self._external_descriptor() - # Factories and source readers built from catalog options create FileIOs for no table. + # A factory built from catalog options creates FileIOs for no table. self.assertEqual(b"NEW", UriReaderFactory(Options({})).create(uri).new_input_stream(uri).read()) - with _SourceFileIO(Options({})).new_input_stream(uri) as stream: - self.assertEqual(b"NEW", stream.read()) self.assertEqual([], self._target_reads("/import/")) def test_table_files_named_like_external_objects_still_use_the_target(self): From 52fb0b9c86a65abf7f2901d4e54bbc51025c2b54 Mon Sep 17 00:00:00 2001 From: Sun Dapeng Date: Mon, 26 Oct 2026 20:56:46 +0800 Subject: [PATCH 21/27] [python] Keep the shared REST token as vended and apply each catalog's overrides to it RESTTokenFileIO._TOKEN_CACHE is shared by every instance for the same user and table, but refresh_token stored the token after merging in its own catalog's dlf.oss-endpoint and explicit io-cache.enabled. Another catalog with a different setting then reused that token. Python's own FileIO reapplied its catalog switch, but the native core and Daft merge the token last, so they followed the other catalog: with routing off in one catalog and on in the next, the second sent the native core and Daft to the legacy cache that fs.oss.endpoint names for old clients. The shared cache now keeps the token as the REST server vended it, and each instance applies its own catalog's overrides when it takes a token from the server or the cache. io_cache_routing computes the effective options in one place: with_catalog_overrides (dlf.oss-endpoint and an explicit io-cache.enabled) and effective_options (catalog options merged with the token, which wins, then the overrides). RESTTokenFileIO, Daft's REST options and single_endpoint_options, which gives the native core, Daft and PVFS the origin's options, all use them. io_cache_effective_options_test shares one vended token between two RESTTokenFileIOs whose catalogs set io-cache.enabled differently (false then true, false then unset, true then false) and checks the Python routing, the native data token and the Daft options of each, and that a dlf.oss-endpoint stays with its catalog; it fails without the change. --- .../catalog/rest/rest_token_file_io.py | 46 ++++---- paimon-python/pypaimon/daft/daft_paimon.py | 5 +- .../pypaimon/filesystem/io_cache_routing.py | 29 ++++- .../tests/io_cache_effective_options_test.py | 109 ++++++++++++++++++ .../pypaimon/tests/native_plan_test.py | 5 +- .../tests/rest/rest_token_file_io_test.py | 8 +- 6 files changed, 166 insertions(+), 36 deletions(-) create mode 100644 paimon-python/pypaimon/tests/io_cache_effective_options_test.py diff --git a/paimon-python/pypaimon/catalog/rest/rest_token_file_io.py b/paimon-python/pypaimon/catalog/rest/rest_token_file_io.py index d13c260ab78d..f4b0b316aede 100644 --- a/paimon-python/pypaimon/catalog/rest/rest_token_file_io.py +++ b/paimon-python/pypaimon/catalog/rest/rest_token_file_io.py @@ -29,17 +29,17 @@ from pypaimon.api.auth.bearer import BearTokenAuthProvider from pypaimon.api.auth.dlf_provider import DLFAuthProvider from pypaimon.api.rest_api import RESTApi -from pypaimon.api.rest_util import RESTUtil from pypaimon.catalog.rest.rest_token import RESTToken from pypaimon.common.file_io import FileIO from pypaimon.common.identifier import Identifier from pypaimon.common.options import Options -from pypaimon.common.options.config import CatalogOptions, OssOptions from pypaimon.common.uri_reader import UriReaderFactory from pypaimon.filesystem.io_cache_file_io import (IoCacheRoutingFileIO, table_view_spec) from pypaimon.filesystem.io_cache_routing import (IoCacheRouting, - schema_file_prefixes) + effective_options, + schema_file_prefixes, + with_catalog_overrides) class RESTTokenFileIO(FileIO): @@ -87,6 +87,8 @@ def __init__(self, identifier: Identifier, path: str, self.catalog_options = catalog_options self.properties = self.catalog_options or Options({}) # For compatibility with refresh_token() self.token: Optional[RESTToken] = None + # The token as the REST server vended it, without this catalog's overrides. + self._vended_token: Optional[RESTToken] = None self.api_instance: Optional[RESTApi] = None self.log = logging.getLogger(__name__) self._uri_reader_factory_cache: Optional[UriReaderFactory] = None @@ -242,10 +244,10 @@ def _get_file_io_scope(self) -> str: return self._file_io_scope def _merged_options(self, token: RESTToken) -> Options: - return Options(self._merge_token_with_catalog_options(RESTUtil.merge( - self.catalog_options.to_map() if self.catalog_options else {}, - token.token - ))) + return Options(effective_options(self._catalog_map(), token.token)) + + def _catalog_map(self) -> dict: + return self.catalog_options.to_map() if self.catalog_options else {} def _create_file_io(self, options: Options) -> FileIO: properties = options.to_map() @@ -262,16 +264,12 @@ def _create_file_io(self, options: Options) -> FileIO: def _merge_token_with_catalog_options(self, token: dict) -> dict: """Merge token with catalog options, DLF OSS endpoint should override the standard OSS endpoint.""" - merged_token = dict(token) - if self.catalog_options: - dlf_oss_endpoint = self.catalog_options.get(CatalogOptions.DLF_OSS_ENDPOINT) - if dlf_oss_endpoint and dlf_oss_endpoint.strip(): - merged_token[OssOptions.OSS_ENDPOINT.key()] = dlf_oss_endpoint - # An explicit io-cache.enabled in the catalog options wins over the token, as in Java. - if self.catalog_options.contains(CatalogOptions.IO_CACHE_ENABLED): - merged_token[CatalogOptions.IO_CACHE_ENABLED.key()] = str( - self.catalog_options.get(CatalogOptions.IO_CACHE_ENABLED)).lower() - return merged_token + return with_catalog_overrides(token, self._catalog_map()) + + def _adopt(self, vended: RESTToken): + # The shared token cache keeps the vended token; this instance applies its own catalog's overrides. + self._vended_token = vended + self.token = RESTToken(self._merge_token_with_catalog_options(vended.token), vended.expire_at_millis) def new_input_stream(self, path: str): return self.file_io().new_input_stream(path) @@ -383,7 +381,7 @@ def try_to_refresh_token(self, minimum_validity_millis: int = 0): cached_token = self._get_cached_token(identifier_str) if cached_token and not self._is_token_expired(cached_token, minimum_validity_millis): - self.token = cached_token + self._adopt(cached_token) return global_lock = self._get_global_token_lock(identifier_str) @@ -391,13 +389,13 @@ def try_to_refresh_token(self, minimum_validity_millis: int = 0): with global_lock: cached_token = self._get_cached_token(identifier_str) if cached_token and not self._is_token_expired(cached_token, minimum_validity_millis): - self.token = cached_token + self._adopt(cached_token) return token_to_check = cached_token if cached_token else self.token if token_to_check is None or self._is_token_expired(token_to_check, minimum_validity_millis): self.refresh_token() - self._set_cached_token(identifier_str, self.token) + self._set_cached_token(identifier_str, self._vended_token) def _get_cached_token(self, identifier_str: str) -> Optional[RESTToken]: with self._TOKEN_LOCKS_LOCK: @@ -468,11 +466,9 @@ def refresh_token(self): f"end refresh data token for identifier [{self.identifier}] expiresAtMillis [{response.expires_at_millis}]" ) - merged_token_dict = self._merge_token_with_catalog_options(response.token) - new_token = RESTToken(merged_token_dict, response.expires_at_millis) - self.token = new_token - if self._is_token_expired(new_token): - self._log_short_token(new_token) + self._adopt(RESTToken(response.token, response.expires_at_millis)) + if self._is_token_expired(self.token): + self._log_short_token(self.token) def _log_short_token(self, token: RESTToken): with self._TOKEN_LOCKS_LOCK: diff --git a/paimon-python/pypaimon/daft/daft_paimon.py b/paimon-python/pypaimon/daft/daft_paimon.py index af2a83afb751..7a9710c898cb 100644 --- a/paimon-python/pypaimon/daft/daft_paimon.py +++ b/paimon-python/pypaimon/daft/daft_paimon.py @@ -33,7 +33,8 @@ from typing import TYPE_CHECKING, Any, Dict, Optional, Union from urllib.parse import urlparse -from pypaimon.filesystem.io_cache_routing import single_endpoint_options +from pypaimon.filesystem.io_cache_routing import (effective_options, + single_endpoint_options) if TYPE_CHECKING: import daft @@ -56,7 +57,7 @@ def _enrich_options_with_rest_token( if file_io.token is None: return catalog_options # Daft reads with its own client, which cannot use io-cache targets per request. - enriched = single_endpoint_options({**catalog_options, **file_io.token.token}) + enriched = single_endpoint_options(effective_options(catalog_options, file_io.token.token)) parsed = urlparse(getattr(table, "table_path", "") or "") if parsed.scheme and parsed.netloc: enriched["warehouse"] = f"{parsed.scheme}://{parsed.netloc}" diff --git a/paimon-python/pypaimon/filesystem/io_cache_routing.py b/paimon-python/pypaimon/filesystem/io_cache_routing.py index fe66e9314d70..53045a916494 100644 --- a/paimon-python/pypaimon/filesystem/io_cache_routing.py +++ b/paimon-python/pypaimon/filesystem/io_cache_routing.py @@ -175,12 +175,35 @@ def _with_endpoint(options: Dict[str, str], endpoint: Optional[str]) -> Dict[str return options +def with_catalog_overrides(token: Mapping[str, str], + catalog_options: Optional[Mapping[str, str]]) -> Dict[str, str]: + """A table token as a catalog uses it: the catalog's explicit dlf.oss-endpoint and io-cache.enabled win.""" + result = dict(token) + catalog_options = catalog_options or {} + dlf_oss_endpoint = catalog_options.get(DLF_OSS_ENDPOINT) + if dlf_oss_endpoint is not None and str(dlf_oss_endpoint).strip(): + result[FS_OSS_ENDPOINT] = dlf_oss_endpoint + enabled = catalog_options.get(IO_CACHE_ENABLED) + if enabled is not None: + result[IO_CACHE_ENABLED] = str(enabled).strip().lower() + return result + + +def effective_options(catalog_options: Optional[Mapping[str, str]], + token: Mapping[str, str]) -> Dict[str, str]: + """The options a table token takes effect with: the catalog options merged with the token, which wins, + then the catalog overrides.""" + from pypaimon.api.rest_util import RESTUtil + return with_catalog_overrides(RESTUtil.merge(dict(catalog_options or {}), dict(token)), catalog_options) + + def single_endpoint_options(options: Mapping[str, str], catalog_options: Optional[Mapping[str, str]] = None) -> Dict[str, str]: - """Options for a client that sends every request to one endpoint: the origin's under io-cache routing, - unless catalog_options turn routing off (dlf.oss-endpoint).""" + """Options for a client that sends every request to one endpoint: options with the catalog overrides, and + the origin's endpoint when io-cache routing is on, whether its keys come from options or catalog_options.""" + options = with_catalog_overrides(options, catalog_options) routing = IoCacheRouting.create({**(catalog_options or {}), **options}) - return dict(options) if routing is None else routing.origin_options(options) + return options if routing is None else routing.origin_options(options) class CacheTarget(NamedTuple): diff --git a/paimon-python/pypaimon/tests/io_cache_effective_options_test.py b/paimon-python/pypaimon/tests/io_cache_effective_options_test.py new file mode 100644 index 000000000000..5249875b0100 --- /dev/null +++ b/paimon-python/pypaimon/tests/io_cache_effective_options_test.py @@ -0,0 +1,109 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Every component takes io-cache routing from the effective options: catalog options merged with the token.""" + +import sys +import time +import unittest +from types import ModuleType, SimpleNamespace +from unittest import mock + +from pypaimon.catalog.rest.rest_token_file_io import RESTTokenFileIO +from pypaimon.common.identifier import Identifier +from pypaimon.common.options import Options +from pypaimon.filesystem.io_cache_routing import IoCacheRouting + + +class SharedTokenTest(unittest.TestCase): + """Two catalogs that set io-cache.enabled differently share the token the REST server vended for a table.""" + + TABLE = "oss://bkt/wh/db.db/t" + TOKEN = { + "fs.oss.endpoint": "http://legacy.example.com", + "fs.oss.accessKeyId": "ak", + "fs.oss.accessKeySecret": "sk", + "io-cache.enabled": "true", + "io-cache.endpoint": "http://legacy.example.com", + "io-cache.origin.endpoint": "https://origin.example.com", + "io-cache.policy": "meta,read", + } + + def setUp(self): + self.vended = [] + patcher = mock.patch.object(RESTTokenFileIO, "_build_cache_key", return_value="shared-token-test") + patcher.start() + self.addCleanup(patcher.stop) + self.addCleanup(RESTTokenFileIO._TOKEN_CACHE.pop, "shared-token-test", None) + + def _file_io(self, catalog_options): + file_io = RESTTokenFileIO(Identifier.create("db", "t"), self.TABLE, Options(dict(catalog_options))) + + def load_table_token(identifier): + self.vended.append(identifier) + return SimpleNamespace(token=dict(self.TOKEN), expires_at_millis=int(time.time() * 1000) + 7_200_000) + + file_io.api_instance = SimpleNamespace(load_table_token=load_table_token) + file_io.try_to_refresh_token() + return file_io + + def _native_data_token(self, file_io): + from pypaimon.execution.native.table import _NativeRestTableCache + + fake_df = ModuleType("pypaimon_rust.datafusion") + fake_df.Table = mock.Mock() + with mock.patch.dict(sys.modules, {"pypaimon_rust.datafusion": fake_df}): + _NativeRestTableCache().get("response", "db", "t", file_io.catalog_options.to_map(), + token_loader=file_io._existing_valid_token) + return fake_df.Table.from_rest_response_with_token.call_args[1]["data_token"] + + def _daft_options(self, file_io): + from pypaimon.daft.daft_paimon import _enrich_options_with_rest_token + + catalog_options = dict(file_io.catalog_options.to_map(), metastore="rest") + return _enrich_options_with_rest_token(catalog_options, SimpleNamespace(file_io=file_io, table_path=self.TABLE)) + + def _assert_routes(self, file_io, routed): + endpoint = "https://origin.example.com" if routed else "http://legacy.example.com" + self.assertEqual(routed, IoCacheRouting.create(file_io._merged_options(file_io.token).to_map()) is not None) + self.assertEqual(endpoint, self._native_data_token(file_io)["fs.oss.endpoint"]) + if sys.version_info >= (3, 7): + self.assertEqual(endpoint, self._daft_options(file_io)["fs.oss.endpoint"]) + + def test_each_catalog_applies_its_own_switch_to_a_shared_token(self): + for first, second in (({"io-cache.enabled": "false"}, {"io-cache.enabled": "true"}), + ({"io-cache.enabled": "false"}, {}), + ({"io-cache.enabled": "true"}, {"io-cache.enabled": "false"})): + with self.subTest(first=first, second=second): + RESTTokenFileIO._TOKEN_CACHE.pop("shared-token-test", None) + self.vended.clear() + file_ios = [self._file_io(first), self._file_io(second)] + self.assertEqual(1, len(self.vended)) + self.assertEqual(self.TOKEN, RESTTokenFileIO._TOKEN_CACHE["shared-token-test"].token) + for file_io, catalog_options in zip(file_ios, (first, second)): + self._assert_routes(file_io, catalog_options.get("io-cache.enabled", "true") == "true") + + def test_dlf_oss_endpoint_of_one_catalog_stays_with_it(self): + pinned = self._file_io({"dlf.oss-endpoint": "oss-cn-test.example.com"}) + shared = self._file_io({}) + self.assertEqual("oss-cn-test.example.com", pinned._merged_options(pinned.token).to_map()["fs.oss.endpoint"]) + self.assertEqual("oss-cn-test.example.com", self._native_data_token(pinned)["fs.oss.endpoint"]) + self._assert_routes(shared, True) + + +if __name__ == "__main__": + unittest.main() diff --git a/paimon-python/pypaimon/tests/native_plan_test.py b/paimon-python/pypaimon/tests/native_plan_test.py index 108f02c6ccd5..e9a8217fa29f 100644 --- a/paimon-python/pypaimon/tests/native_plan_test.py +++ b/paimon-python/pypaimon/tests/native_plan_test.py @@ -1222,11 +1222,12 @@ def test_native_rest_cache_gives_the_core_the_origin_under_io_cache_routing(self (data_token['fs.oss.endpoint'], data_token['fs.oss.https.enable'])) self.assertNotIn('fs.oss.dlf-cache.consistent-hash.enabled', data_token) - # dlf.oss-endpoint turns routing off, so the token keeps the endpoint it was given. + # dlf.oss-endpoint turns routing off and replaces the endpoint, as for the Python FileIO. fake_df.Table.reset_mock() options = {'dlf.oss-endpoint': 'oss-cn-test.example.com'} _NativeRestTableCache().get('response', 'default', 't', options, token_loader=lambda: token) - self.assertEqual(token.token, fake_df.Table.from_rest_response_with_token.call_args[1]['data_token']) + self.assertEqual(dict(token.token, **{'fs.oss.endpoint': 'oss-cn-test.example.com'}), + fake_df.Table.from_rest_response_with_token.call_args[1]['data_token']) def test_native_rest_cache_retries_failed_construction(self): from pypaimon.execution.native.table import _NativeRestTableCache diff --git a/paimon-python/pypaimon/tests/rest/rest_token_file_io_test.py b/paimon-python/pypaimon/tests/rest/rest_token_file_io_test.py index fb63c88f45a4..0abf6c817a45 100644 --- a/paimon-python/pypaimon/tests/rest/rest_token_file_io_test.py +++ b/paimon-python/pypaimon/tests/rest/rest_token_file_io_test.py @@ -110,7 +110,7 @@ def refresh(): issued.append(RESTToken( {OssOptions.OSS_SECURITY_TOKEN.key(): "sts-{}".format(len(issued))}, (now[0] + 2 * 3600) * 1000)) - file_io.token = issued[-1] + file_io._adopt(issued[-1]) with patch('pypaimon.catalog.rest.rest_token_file_io.time.time', side_effect=lambda: now[0]), \ @@ -391,7 +391,7 @@ def refresh(file_io): if threading.current_thread() in holders: refreshing.set() release.wait(10) - file_io.token = self._valid_token("refreshed") + file_io._adopt(self._valid_token("refreshed")) self.addCleanup(RESTTokenFileIO._TOKEN_CACHE.pop, 'fork-test', None) with patch.object(RESTTokenFileIO, '_build_cache_key', return_value='fork-test'), \ @@ -484,7 +484,7 @@ def _check_presigned_url_lifetime( validity = timedelta(hours=requested_hours) def refresh(): - file_io.token = new_token + file_io._adopt(new_token) with patch('pypaimon.catalog.rest.rest_token_file_io.time.time', return_value=now), \ patch.object(file_io, '_build_cache_key', return_value='presigning-test'), \ @@ -521,7 +521,7 @@ def test_presigned_url_refreshes_and_resigns_after_materialization(self): validity = timedelta(minutes=30) def refresh(): - file_io.token = new_token + file_io._adopt(new_token) def presign(*_): if now[0] == 1700000000: From 66b450c72c31bc95d2fe13218996e8d06801dfd0 Mon Sep 17 00:00:00 2001 From: Sun Dapeng Date: Mon, 26 Oct 2026 20:58:57 +0800 Subject: [PATCH 22/27] [python] Let PVFS see catalog-only io-cache routing when it picks the origin PVFS backs OSS with one client that sends every request to one endpoint, so it takes the origin's options under io-cache routing. It looked for routing in the vended token alone, after copying only io-cache.enabled from the catalog. When the policy, targets and io-cache.origin.endpoint are set in the catalog and the token carries credentials and the legacy fs.oss.endpoint, routing was not recognized and PVFS read the legacy cache: cat_file returned the old object. PVFS now gives the token and its catalog options to single_endpoint_options, which applies the catalog's dlf.oss-endpoint and io-cache.enabled and looks for routing in both, as the native core and Daft do. io_cache_effective_options_test reads through PVFS.cat_file with the routing keys in the catalog only, with the REST calls and the ossfs constructor replaced at the client boundary; it fails without the change. --- paimon-python/pypaimon/filesystem/pvfs.py | 10 +- .../tests/io_cache_effective_options_test.py | 115 ++++++++++++++++++ 2 files changed, 116 insertions(+), 9 deletions(-) diff --git a/paimon-python/pypaimon/filesystem/pvfs.py b/paimon-python/pypaimon/filesystem/pvfs.py index 793abdac5786..12205bf837d3 100644 --- a/paimon-python/pypaimon/filesystem/pvfs.py +++ b/paimon-python/pypaimon/filesystem/pvfs.py @@ -884,16 +884,8 @@ def _get_filesystem(self, pvfs_table_identifier: PVFSTableIdentifier, storage_ty def _merge_token_with_catalog_options(self, token: dict) -> dict: """Merge token with catalog options, DLF OSS endpoint should override the standard OSS endpoint.""" - merged_token = dict(token) - dlf_oss_endpoint = self.options.get(CatalogOptions.DLF_OSS_ENDPOINT) - if dlf_oss_endpoint and dlf_oss_endpoint.strip(): - merged_token[OssOptions.OSS_ENDPOINT.key()] = dlf_oss_endpoint - return merged_token - if self.options.contains(CatalogOptions.IO_CACHE_ENABLED): - merged_token[CatalogOptions.IO_CACHE_ENABLED.key()] = str( - self.options.get(CatalogOptions.IO_CACHE_ENABLED)).lower() # The OSS filesystem sends every request to one endpoint, so it cannot use io-cache targets. - return single_endpoint_options(merged_token) + return single_endpoint_options(token, self.options.to_map()) @staticmethod def _get_storage_type(path: str): diff --git a/paimon-python/pypaimon/tests/io_cache_effective_options_test.py b/paimon-python/pypaimon/tests/io_cache_effective_options_test.py index 5249875b0100..b879a0d22d8b 100644 --- a/paimon-python/pypaimon/tests/io_cache_effective_options_test.py +++ b/paimon-python/pypaimon/tests/io_cache_effective_options_test.py @@ -18,16 +18,131 @@ """Every component takes io-cache routing from the effective options: catalog options merged with the token.""" import sys +import threading import time import unittest +from http.server import BaseHTTPRequestHandler, HTTPServer +from socketserver import ThreadingMixIn from types import ModuleType, SimpleNamespace from unittest import mock +from urllib.parse import unquote, urlsplit from pypaimon.catalog.rest.rest_token_file_io import RESTTokenFileIO from pypaimon.common.identifier import Identifier from pypaimon.common.options import Options from pypaimon.filesystem.io_cache_routing import IoCacheRouting +OBJECT = "/bkt/raw/cam0/000123.bin" +URI = "oss://bkt/raw/cam0/000123.bin" + + +class _ObjectEndpoint: + """An OSS endpoint for the real PyArrow client: path-style HEAD and GET of fixed objects.""" + + def __init__(self, objects): + self.objects = objects + self.requests = [] + endpoint = self + + class Handler(BaseHTTPRequestHandler): + def log_message(self, *args): + pass + + def do_HEAD(self): + self._serve(False) + + def do_GET(self): + self._serve(True) + + def _serve(self, with_body): + path = unquote(urlsplit(self.path).path) + endpoint.requests.append((self.command, path)) + data = endpoint.objects.get(path) + if data is None: + body = b"NoSuchKey" if with_body else b"" + self.send_response(404) + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + return + start, end, status = 0, len(data), 200 + ranged = self.headers.get("Range") + if ranged and ranged.startswith("bytes="): + first, _, last = ranged[len("bytes="):].partition("-") + start, end, status = int(first), min(len(data), int(last) + 1 if last else len(data)), 206 + self.send_response(status) + self.send_header("Content-Length", str(end - start)) + self.send_header("Content-Type", "application/octet-stream") + self.send_header("ETag", '"etag"') + self.send_header("Last-Modified", "Mon, 26 Oct 2026 00:00:00 GMT") + if status == 206: + self.send_header("Content-Range", "bytes {}-{}/{}".format(start, end - 1, len(data))) + self.end_headers() + if with_body: + self.wfile.write(data[start:end]) + + class Server(ThreadingMixIn, HTTPServer): + daemon_threads = True + + self.server = Server(("127.0.0.1", 0), Handler) + self.url = "http://127.0.0.1:{}".format(self.server.server_port) + self.thread = threading.Thread(target=lambda: self.server.serve_forever(poll_interval=0.01), daemon=True) + self.thread.start() + + def close(self): + self.server.shutdown() + self.server.server_close() + self.thread.join() + + +class EffectiveOptionsTest(unittest.TestCase): + """The origin holds the current object; a legacy cache, still fs.oss.endpoint for old clients, keeps an old one.""" + + def setUp(self): + self.origin = _ObjectEndpoint({OBJECT: b"NEW"}) + self.cache = _ObjectEndpoint({OBJECT: b"OLD"}) + self.addCleanup(self.origin.close) + self.addCleanup(self.cache.close) + + def _routing_options(self): + return { + "fs.oss.impl": "legacy", + "fs.oss.region": "cn-hangzhou", + "fs.oss.accessKeyId": "ak", + "fs.oss.accessKeySecret": "sk", + "fs.oss.second.level.domain.enable": "true", + "fs.oss.endpoint": self.cache.url, + "io-cache.enabled": "true", + "io-cache.endpoint": self.cache.url, + "io-cache.target.default.path-style-access": "true", + "io-cache.origin.endpoint": self.origin.url, + "io-cache.policy": "meta,read", + } + + def test_pvfs_reads_through_the_origin_of_catalog_only_routing(self): + from pypaimon.filesystem.pvfs import PaimonVirtualFileSystem + + # The token carries credentials and the legacy endpoint; routing is configured on the catalog only. + token = {key: value for key, value in self._routing_options().items() if not key.startswith("io-cache.")} + catalog = dict({key: value for key, value in self._routing_options().items() if key.startswith("io-cache.")}, + uri="http://rest.example.com", warehouse="wh", **{"fs.oss.impl": "legacy"}) + rest_api = SimpleNamespace( + get_table=lambda identifier: SimpleNamespace(path="oss://bkt/raw", created_at=0, updated_at=0), + load_table_token=lambda identifier: SimpleNamespace( + token=token, expires_at_millis=int(time.time() * 1000) + 7_200_000)) + objects = {self.cache.url: b"old-object", self.origin.url: b"new-object"} + backends = [] + + def ossfs(options): + backends.append(options.to_map()["fs.oss.endpoint"]) + return SimpleNamespace(cat_file=lambda path, start, end, **kwargs: objects[backends[-1]]) + + pvfs = PaimonVirtualFileSystem(catalog, skip_instance_cache=True) + with mock.patch.object(PaimonVirtualFileSystem, "_PaimonVirtualFileSystem__rest_api", return_value=rest_api), \ + mock.patch.object(PaimonVirtualFileSystem, "_get_ossfs_filesystem", side_effect=ossfs): + self.assertEqual(b"new-object", pvfs.cat_file("pvfs://wh/db/t/cam0/000123.bin")) + self.assertEqual([self.origin.url], backends) + class SharedTokenTest(unittest.TestCase): """Two catalogs that set io-cache.enabled differently share the token the REST server vended for a table.""" From 1484678568f4077995be054766315e2f09eca68a Mon Sep 17 00:00:00 2001 From: Sun Dapeng Date: Mon, 26 Oct 2026 20:59:26 +0800 Subject: [PATCH 23/27] [python] Give FileIOs built from options the origin's endpoint under io-cache routing Only RESTTokenFileIO builds a routing FileIO. Every other FileIO comes from FileIO.get, which for oss:// returns an OssFileIO that sends every request to fs.oss.endpoint. When io-cache routing is configured in the catalog options, fs.oss.endpoint may name the legacy cache that old clients use as a whole, so these FileIOs read from it. That covers the FileIOs a UriReaderFactory builds for row-data and BLOB URIs (its origin_only() returned the OssFileIO itself), the source FileIO of HDF5, LeRobot and rosbag imports, ResolvingFileIO and the FileIO of a filesystem or JDBC catalog. Daft's IOConfig for such catalogs and the native core of a table loaded from a filesystem or JDBC catalog took the same options. FileIO.get now gives an OssFileIO the origin's options when the options it gets enable io-cache routing, and Daft and the native core of those tables get single_endpoint_options. So that the routing FileIO's own origin and target FileIOs, also built by FileIO.get, keep their endpoints, their options no longer carry the io-cache keys they were derived from. The origin_only() call in UriReaderFactory is removed, as it changed nothing. io_cache_effective_options_test serves an old object from the legacy cache and the current one from the origin over HTTP to the real PyArrow client, with routing set in the catalog options: a URI reader, an import source FileIO, a filesystem catalog FileIO and a ResolvingFileIO read the current object without a request to the cache, a target FileIO still reads from its target, and Daft's options name the origin. native_plan_test checks the options the native core gets for a filesystem catalog table. Each fails without the change. The routing table test that read URIs through a FileIO.get replaced by the test harness is removed, as it could not see this path. --- paimon-python/pypaimon/common/file_io.py | 5 +++ paimon-python/pypaimon/common/uri_reader.py | 3 +- paimon-python/pypaimon/daft/daft_paimon.py | 8 ++-- .../pypaimon/execution/native/table.py | 3 +- .../pypaimon/filesystem/io_cache_routing.py | 10 +++-- .../tests/io_cache_effective_options_test.py | 44 +++++++++++++++++++ .../tests/io_cache_table_routing_test.py | 8 ---- .../pypaimon/tests/native_plan_test.py | 21 +++++++++ 8 files changed, 83 insertions(+), 19 deletions(-) diff --git a/paimon-python/pypaimon/common/file_io.py b/paimon-python/pypaimon/common/file_io.py index ddd60b1561c0..cff3a5f68495 100644 --- a/paimon-python/pypaimon/common/file_io.py +++ b/paimon-python/pypaimon/common/file_io.py @@ -696,7 +696,12 @@ def get(path: str, catalog_options: Optional[Options] = None) -> 'FileIO': ) if scheme == "oss": + from pypaimon.filesystem.io_cache_routing import IoCacheRouting from pypaimon.filesystem.oss_file_io import OssFileIO + # This FileIO sends every request to one endpoint, so under io-cache routing it is the origin's. + routing = IoCacheRouting.create(opts.to_map()) + if routing is not None: + opts = Options(routing.origin_options(opts.to_map())) return OssFileIO(path, opts) from pypaimon.filesystem.pyarrow_file_io import PyArrowFileIO diff --git a/paimon-python/pypaimon/common/uri_reader.py b/paimon-python/pypaimon/common/uri_reader.py index 4c18f0ff5b5d..5548329c2965 100644 --- a/paimon-python/pypaimon/common/uri_reader.py +++ b/paimon-python/pypaimon/common/uri_reader.py @@ -179,8 +179,7 @@ def _new_reader(self, key: UriKey, parsed_uri: ParseResult) -> UriReader: uri_string = parsed_uri.geturl() file_io = FileIO.get(uri_string, self.catalog_options) self._owned_file_ios.append(file_io) - # This FileIO serves no table, so the URIs of row data it reads come from origin. - return UriReader.from_file(file_io.origin_only()) + return UriReader.from_file(file_io) except Exception as e: raise RuntimeError(f"Failed to create reader for URI {parsed_uri.geturl()}") from e diff --git a/paimon-python/pypaimon/daft/daft_paimon.py b/paimon-python/pypaimon/daft/daft_paimon.py index 7a9710c898cb..6e22c81410bf 100644 --- a/paimon-python/pypaimon/daft/daft_paimon.py +++ b/paimon-python/pypaimon/daft/daft_paimon.py @@ -48,15 +48,15 @@ def _enrich_options_with_rest_token( ) -> Dict[str, str]: # REST catalogs (DLF) keep OSS STS tokens on table.file_io and the bucket on table.table_path, # not in catalog_options; fold both in so Daft's IOConfig routes to OSS with valid credentials. + # Daft reads with its own client, which cannot use io-cache targets per request. if catalog_options.get("metastore") != "rest": - return catalog_options + return single_endpoint_options(catalog_options) file_io = getattr(table, "file_io", None) if file_io is None or not hasattr(file_io, "try_to_refresh_token"): - return catalog_options + return single_endpoint_options(catalog_options) file_io.try_to_refresh_token() if file_io.token is None: - return catalog_options - # Daft reads with its own client, which cannot use io-cache targets per request. + return single_endpoint_options(catalog_options) enriched = single_endpoint_options(effective_options(catalog_options, file_io.token.token)) parsed = urlparse(getattr(table, "table_path", "") or "") if parsed.scheme and parsed.netloc: diff --git a/paimon-python/pypaimon/execution/native/table.py b/paimon-python/pypaimon/execution/native/table.py index c6684dd92807..c14e2b344856 100644 --- a/paimon-python/pypaimon/execution/native/table.py +++ b/paimon-python/pypaimon/execution/native/table.py @@ -307,11 +307,12 @@ def _load_native_table(table): token_loader=lambda: _rest_data_token(table)) elif file_io_options is not None: from pypaimon_rust.datafusion import Table + # As with a token: the core cannot use io-cache targets per request, so it gets the origin's options. rt = Table.from_resolved_schema( table.table_path, _resolved_schema_json(table), database=table.identifier.get_database_name(), table=table.identifier.get_table_name(), - branch=table.current_branch(), options=file_io_options) + branch=table.current_branch(), options=single_endpoint_options(file_io_options)) else: from pypaimon_rust.datafusion import PaimonCatalog # The Python loader has already resolved /config, including typed diff --git a/paimon-python/pypaimon/filesystem/io_cache_routing.py b/paimon-python/pypaimon/filesystem/io_cache_routing.py index 53045a916494..d31830866290 100644 --- a/paimon-python/pypaimon/filesystem/io_cache_routing.py +++ b/paimon-python/pypaimon/filesystem/io_cache_routing.py @@ -51,8 +51,9 @@ FileType.GLOBAL_INDEX: re.compile(r"[a-z0-9_-]+-global-index-" + _UUID + r"\.index"), } _MANIFEST_SIDECAR_SUFFIX = ".avro.sidecar" +IO_CACHE_PREFIX = "io-cache." # JindoSDK cache client keys; the origin option set must not talk to a cache. -_ORIGIN_DROPPED_PREFIXES = ("fs.oss.dlf-cache.", "fs.jindocache.") +_ORIGIN_DROPPED_PREFIXES = ("fs.oss.dlf-cache.", "fs.jindocache.", IO_CACHE_PREFIX) class Op(Enum): @@ -288,19 +289,20 @@ def route(self, op: Op, path: str) -> Optional[str]: return name if name in self._targets else None def origin_options(self, options: Mapping[str, str]) -> Dict[str, str]: - """Options of the origin FileIO: the origin endpoint, without cache client keys.""" + """Options of the origin FileIO: the origin endpoint, without cache client and io-cache keys.""" result = {key: value for key, value in options.items() if not str(key).startswith(_ORIGIN_DROPPED_PREFIXES)} return _with_endpoint(result, self._origin) def target_options(self, options: Mapping[str, str], name: str) -> Dict[str, str]: - """Options of the FileIO of one target: its endpoint, addressing and region.""" + """Options of the FileIO of one target: its endpoint, addressing and region, without io-cache keys.""" target = self._targets[name] endpoint = target.endpoint # A target endpoint without a scheme means https. if "://" not in endpoint: endpoint = "https://" + endpoint - result = _with_endpoint(dict(options), endpoint) + result = _with_endpoint({key: value for key, value in options.items() + if not str(key).startswith(IO_CACHE_PREFIX)}, endpoint) result[FS_OSS_PATH_STYLE] = "true" if target.path_style_access else "false" if target.region and target.region.strip(): result[FS_OSS_REGION] = target.region diff --git a/paimon-python/pypaimon/tests/io_cache_effective_options_test.py b/paimon-python/pypaimon/tests/io_cache_effective_options_test.py index b879a0d22d8b..fa6350e98067 100644 --- a/paimon-python/pypaimon/tests/io_cache_effective_options_test.py +++ b/paimon-python/pypaimon/tests/io_cache_effective_options_test.py @@ -28,8 +28,11 @@ from urllib.parse import unquote, urlsplit from pypaimon.catalog.rest.rest_token_file_io import RESTTokenFileIO +from pypaimon.common.file_io import FileIO from pypaimon.common.identifier import Identifier from pypaimon.common.options import Options +from pypaimon.common.options.config import CatalogOptions +from pypaimon.common.uri_reader import UriReaderFactory from pypaimon.filesystem.io_cache_routing import IoCacheRouting OBJECT = "/bkt/raw/cam0/000123.bin" @@ -119,6 +122,47 @@ def _routing_options(self): "io-cache.policy": "meta,read", } + def _assert_reads_origin(self, read): + self.assertEqual(b"NEW", read()) + self.assertEqual([], self.cache.requests) + self.assertTrue(self.origin.requests) + + def test_uri_reader_factory_reads_row_data_uris_from_origin(self): + reader = UriReaderFactory(Options(self._routing_options())).create(URI) + self._assert_reads_origin(lambda: reader.new_input_stream(URI).read()) + + def test_source_file_io_reads_import_sources_from_origin(self): + from pypaimon.multimodal.source_utils import _SourceFileIO + + source = _SourceFileIO(Options(self._routing_options())) + self._assert_reads_origin(lambda: source.new_input_stream(URI).read()) + + def test_file_io_of_a_catalog_without_tokens_reads_from_origin(self): + from pypaimon.catalog.filesystem_catalog import FileSystemCatalog + + catalog = FileSystemCatalog(Options(dict(self._routing_options(), warehouse="oss://bkt/wh"))) + self._assert_reads_origin(lambda: catalog.file_io.new_input_stream(URI).read()) + + def test_resolving_file_io_reads_from_origin(self): + options = dict(self._routing_options(), **{CatalogOptions.RESOLVING_FILE_IO_ENABLED.key(): "true"}) + file_io = FileIO.get(URI, Options(options)) + self._assert_reads_origin(lambda: file_io.new_input_stream(URI).read()) + + def test_file_ios_of_routing_targets_keep_their_endpoints(self): + options = self._routing_options() + routing = IoCacheRouting.create(options) + target = FileIO.get(URI, Options(routing.target_options(options, "default"))) + self.assertEqual(b"OLD", target.new_input_stream(URI).read()) + self.assertEqual([], self.origin.requests) + + @unittest.skipIf(sys.version_info < (3, 7), "pypaimon.daft needs Python 3.7+") + def test_daft_reads_catalog_only_routing_from_origin(self): + from pypaimon.daft.daft_paimon import _enrich_options_with_rest_token + + options = dict(self._routing_options(), metastore="filesystem", warehouse="oss://bkt/wh") + enriched = _enrich_options_with_rest_token(options, SimpleNamespace(table_path="oss://bkt/wh/db.db/t")) + self.assertEqual((self.origin.url, "oss://bkt/wh"), (enriched["fs.oss.endpoint"], enriched["warehouse"])) + def test_pvfs_reads_through_the_origin_of_catalog_only_routing(self): from pypaimon.filesystem.pvfs import PaimonVirtualFileSystem diff --git a/paimon-python/pypaimon/tests/io_cache_table_routing_test.py b/paimon-python/pypaimon/tests/io_cache_table_routing_test.py index 8f1d59f25b53..3862c71f669e 100644 --- a/paimon-python/pypaimon/tests/io_cache_table_routing_test.py +++ b/paimon-python/pypaimon/tests/io_cache_table_routing_test.py @@ -32,9 +32,7 @@ import pyarrow.parquet as pq from pypaimon import CatalogFactory, Schema -from pypaimon.common.options import Options from pypaimon.common.options.config import CatalogOptions -from pypaimon.common.uri_reader import UriReaderFactory from pypaimon.filesystem.io_cache_file_io import IoCacheRoutingFileIO from pypaimon.filesystem.io_cache_routing import IoCacheRouting from pypaimon.filesystem.local_file_io import LocalFileIO @@ -208,12 +206,6 @@ def test_format_table_files_replaced_on_origin_are_read_from_origin(self): self.assertEqual([2], self._read(table).column("id").to_pylist()) self.assertEqual([], self._target_reads("/format_files/")) - def test_uri_readers_that_serve_no_table_read_from_origin(self): - uri, _ = self._external_descriptor() - # A factory built from catalog options creates FileIOs for no table. - self.assertEqual(b"NEW", UriReaderFactory(Options({})).create(uri).new_input_stream(uri).read()) - self.assertEqual([], self._target_reads("/import/")) - def test_table_files_named_like_external_objects_still_use_the_target(self): pa_schema = pa.schema([("id", pa.int32())]) self.catalog.create_table("db.plain", Schema.from_pyarrow_schema(pa_schema), False) diff --git a/paimon-python/pypaimon/tests/native_plan_test.py b/paimon-python/pypaimon/tests/native_plan_test.py index e9a8217fa29f..b8ed3a1272ab 100644 --- a/paimon-python/pypaimon/tests/native_plan_test.py +++ b/paimon-python/pypaimon/tests/native_plan_test.py @@ -1229,6 +1229,27 @@ def test_native_rest_cache_gives_the_core_the_origin_under_io_cache_routing(self self.assertEqual(dict(token.token, **{'fs.oss.endpoint': 'oss-cn-test.example.com'}), fake_df.Table.from_rest_response_with_token.call_args[1]['data_token']) + def test_native_table_of_a_filesystem_catalog_gets_the_origin_under_io_cache_routing(self): + from pypaimon.execution.native.table import _load_native_table + + table = _scan(True, Mock()).table + table.table_schema = TableSchema(0, []) + table.catalog_environment.catalog_loader = FileSystemCatalogLoader(CatalogContext.create_from_options(Options({ + 'warehouse': 'oss://bkt/wh', + 'fs.oss.endpoint': 'http://cache.example.com', + 'io-cache.enabled': 'true', + 'io-cache.endpoint': 'http://cache.example.com', + 'io-cache.origin.endpoint': 'https://origin.example.com', + 'io-cache.policy': 'meta,read', + }))) + with patch('pypaimon_rust.datafusion.Table', create=True) as native_table: + native_table.from_resolved_schema.return_value.branch.return_value = 'main' + _load_native_table(table) + options = native_table.from_resolved_schema.call_args[1]['options'] + self.assertEqual(('https://origin.example.com', 'oss://bkt/wh'), + (options['fs.oss.endpoint'], options['warehouse'])) + self.assertNotIn('io-cache.enabled', options) + def test_native_rest_cache_retries_failed_construction(self): from pypaimon.execution.native.table import _NativeRestTableCache From 2c87f98eeb02cc20e0dd3f43eb70e37ed57ef421 Mon Sep 17 00:00:00 2001 From: Sun Dapeng Date: Tue, 27 Oct 2026 00:39:37 +0800 Subject: [PATCH 24/27] [python] Cover the older data file prefixes a tag branch inherits A branch created from a tag after data-file.prefix changed takes files named under the earlier prefix. Creating the branch copies the tag's schemas into it, so its schema history holds that prefix and the files reach the io-cache target. The new case reads such a branch through the catalog and fails when the view carries no schema history. A branch whose schemas exist only in the catalog cannot read such files: the scan loads each data file's schema from the branch's own schema directory and fails before any data request, so its history needs no other source. --- .../tests/io_cache_table_routing_test.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/paimon-python/pypaimon/tests/io_cache_table_routing_test.py b/paimon-python/pypaimon/tests/io_cache_table_routing_test.py index 3862c71f669e..d7ce83cc5ff6 100644 --- a/paimon-python/pypaimon/tests/io_cache_table_routing_test.py +++ b/paimon-python/pypaimon/tests/io_cache_table_routing_test.py @@ -188,6 +188,24 @@ def test_branch_copies_route_the_data_files_of_the_branch(self): self.assertEqual([1], self._read(copy).column("id").to_pylist()) self.assertTrue(self._target_reads("/dev-")) + def test_tag_branches_route_the_files_they_took_under_older_prefixes(self): + pa_schema = pa.schema([("id", pa.int32())]) + schema = Schema.from_pyarrow_schema(pa_schema, options={"data-file.prefix": "old-"}) + self.catalog.create_table("db.tagged", schema, False) + self._write(self.catalog.get_table("db.tagged"), pa.Table.from_pydict({"id": [1]}, schema=pa_schema)) + self.catalog.alter_table("db.tagged", [SchemaChange.set_option("data-file.prefix", "new-")], False) + main = self.catalog.get_table("db.tagged") + self._write(main, pa.Table.from_pydict({"id": [2]}, schema=pa_schema)) + main.create_tag("t1") + main.branch_manager().create_branch("dev", "t1") + self.target.requests.clear() + + # The branch knows new- from its schema; old- comes from the schemas the tag copied into it. + branch = self.catalog.get_table("db.tagged$branch_dev") + self.assertEqual([1, 2], sorted(self._read(branch).column("id").to_pylist())) + self.assertTrue(self._target_reads("/db.db/tagged/bucket-0/old-")) + self.assertTrue(self._target_reads("/db.db/tagged/bucket-0/new-")) + def test_format_table_files_replaced_on_origin_are_read_from_origin(self): pa_schema = pa.schema([("id", pa.int32())]) self.catalog.create_table("db.format_schema", Schema.from_pyarrow_schema(pa_schema), False) From 1253205fea2c1dea55b167d841ae17c73683237a Mon Sep 17 00:00:00 2001 From: Sun Dapeng Date: Tue, 27 Oct 2026 02:13:51 +0800 Subject: [PATCH 25/27] Preserve Daft no-op catalog option identity --- paimon-python/pypaimon/daft/daft_paimon.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/paimon-python/pypaimon/daft/daft_paimon.py b/paimon-python/pypaimon/daft/daft_paimon.py index 6e22c81410bf..83040051255f 100644 --- a/paimon-python/pypaimon/daft/daft_paimon.py +++ b/paimon-python/pypaimon/daft/daft_paimon.py @@ -43,6 +43,12 @@ from pypaimon.table.file_store_table import FileStoreTable +def _single_endpoint_options_or_same(options: Dict[str, str]) -> Dict[str, str]: + """Preserve the legacy no-op identity while stripping routing from single-endpoint clients.""" + converted = single_endpoint_options(options) + return options if converted == options else converted + + def _enrich_options_with_rest_token( catalog_options: Dict[str, str], table: "FileStoreTable" ) -> Dict[str, str]: @@ -50,13 +56,13 @@ def _enrich_options_with_rest_token( # not in catalog_options; fold both in so Daft's IOConfig routes to OSS with valid credentials. # Daft reads with its own client, which cannot use io-cache targets per request. if catalog_options.get("metastore") != "rest": - return single_endpoint_options(catalog_options) + return _single_endpoint_options_or_same(catalog_options) file_io = getattr(table, "file_io", None) if file_io is None or not hasattr(file_io, "try_to_refresh_token"): - return single_endpoint_options(catalog_options) + return _single_endpoint_options_or_same(catalog_options) file_io.try_to_refresh_token() if file_io.token is None: - return single_endpoint_options(catalog_options) + return _single_endpoint_options_or_same(catalog_options) enriched = single_endpoint_options(effective_options(catalog_options, file_io.token.token)) parsed = urlparse(getattr(table, "table_path", "") or "") if parsed.scheme and parsed.netloc: From 50b50313b5a2d71de3b8144a0c58d049707173d1 Mon Sep 17 00:00:00 2001 From: Sun Dapeng Date: Tue, 27 Oct 2026 02:25:41 +0800 Subject: [PATCH 26/27] [python] Preserve Daft option identity without needless conversion --- paimon-python/pypaimon/daft/daft_paimon.py | 21 ++++++++++----------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/paimon-python/pypaimon/daft/daft_paimon.py b/paimon-python/pypaimon/daft/daft_paimon.py index 83040051255f..133da53c6d87 100644 --- a/paimon-python/pypaimon/daft/daft_paimon.py +++ b/paimon-python/pypaimon/daft/daft_paimon.py @@ -33,7 +33,8 @@ from typing import TYPE_CHECKING, Any, Dict, Optional, Union from urllib.parse import urlparse -from pypaimon.filesystem.io_cache_routing import (effective_options, +from pypaimon.filesystem.io_cache_routing import (IoCacheRouting, + effective_options, single_endpoint_options) if TYPE_CHECKING: @@ -43,26 +44,19 @@ from pypaimon.table.file_store_table import FileStoreTable -def _single_endpoint_options_or_same(options: Dict[str, str]) -> Dict[str, str]: - """Preserve the legacy no-op identity while stripping routing from single-endpoint clients.""" - converted = single_endpoint_options(options) - return options if converted == options else converted - - def _enrich_options_with_rest_token( catalog_options: Dict[str, str], table: "FileStoreTable" ) -> Dict[str, str]: # REST catalogs (DLF) keep OSS STS tokens on table.file_io and the bucket on table.table_path, # not in catalog_options; fold both in so Daft's IOConfig routes to OSS with valid credentials. - # Daft reads with its own client, which cannot use io-cache targets per request. if catalog_options.get("metastore") != "rest": - return _single_endpoint_options_or_same(catalog_options) + return _origin_only(catalog_options) file_io = getattr(table, "file_io", None) if file_io is None or not hasattr(file_io, "try_to_refresh_token"): - return _single_endpoint_options_or_same(catalog_options) + return _origin_only(catalog_options) file_io.try_to_refresh_token() if file_io.token is None: - return _single_endpoint_options_or_same(catalog_options) + return _origin_only(catalog_options) enriched = single_endpoint_options(effective_options(catalog_options, file_io.token.token)) parsed = urlparse(getattr(table, "table_path", "") or "") if parsed.scheme and parsed.netloc: @@ -70,6 +64,11 @@ def _enrich_options_with_rest_token( return enriched +def _origin_only(options: Dict[str, str]) -> Dict[str, str]: + # Daft reads with its own client, which cannot use io-cache targets per request. + return options if IoCacheRouting.create(options) is None else single_endpoint_options(options) + + # Time-travel targets are mutually exclusive: at most one may be set. TimeTravelTimestamp = Union[int, str, "datetime.datetime"] From 4b1bd71b98a252d440b67a76de0b17672d6d51dc Mon Sep 17 00:00:00 2001 From: Sun Dapeng Date: Tue, 27 Oct 2026 02:53:21 +0800 Subject: [PATCH 27/27] [python] Keep native REST fallback on the origin --- .../pypaimon/execution/native/table.py | 13 +++++- .../pypaimon/tests/native_plan_test.py | 20 +++++++++ .../tests/native_read_path_completion_test.py | 42 +++++++++++++++++++ 3 files changed, 73 insertions(+), 2 deletions(-) diff --git a/paimon-python/pypaimon/execution/native/table.py b/paimon-python/pypaimon/execution/native/table.py index c14e2b344856..8e3e3349fc55 100644 --- a/paimon-python/pypaimon/execution/native/table.py +++ b/paimon-python/pypaimon/execution/native/table.py @@ -200,8 +200,13 @@ def get(self, response, database, table, options, token_loader=None): def create(): token = token_loader() if token_loader else None if token is None: + # A REST table can reach this path before its data token is available (for + # example a copied branch or a token refresh race). The native core sends + # every OSS request to one endpoint, so routing keys in the catalog must not + # leave its legacy fs.oss.endpoint pointed at a cache target. return Table.from_rest_response( - response, database=database, table=table, rest_options=options) + response, database=database, table=table, + rest_options=single_endpoint_options(options)) # The native core cannot use io-cache targets per request, and fs.oss.endpoint may name the cache. return Table.from_rest_response_with_token( response, database=database, table=table, rest_options=options, @@ -317,7 +322,11 @@ def _load_native_table(table): from pypaimon_rust.datafusion import PaimonCatalog # The Python loader has already resolved /config, including typed # FUSE options. Match RESTCatalogLoader.load() and reuse that context. - catalog = PaimonCatalog(_catalog_options(table), config_required=False) + # This catalog is also a single-endpoint client. Keep its metadata/catalog + # configuration, but make its storage endpoint the origin when catalog-only + # io-cache routing is enabled. + catalog = PaimonCatalog( + single_endpoint_options(_catalog_options(table)), config_required=False) # REST may keep branch schemas in the catalog only. Load the base # environment, then attach the schema/branch already resolved here. rt = catalog.get_table(( diff --git a/paimon-python/pypaimon/tests/native_plan_test.py b/paimon-python/pypaimon/tests/native_plan_test.py index b8ed3a1272ab..901407bb714c 100644 --- a/paimon-python/pypaimon/tests/native_plan_test.py +++ b/paimon-python/pypaimon/tests/native_plan_test.py @@ -1199,6 +1199,26 @@ def test_native_rest_cache_passes_only_existing_token(self): fake_df.Table.from_rest_response.assert_called_once_with( 'response', database='default', table='t', rest_options={}) + # A copied branch can reach the REST fallback before a data token is available. + # The native core is single-endpoint, so catalog-only routing must still select + # the origin instead of the legacy endpoint that names a cache target. + fake_df.Table.reset_mock() + options = { + 'fs.oss.endpoint': 'http://cache.example.com', + 'io-cache.enabled': 'true', + 'io-cache.endpoint': 'http://cache.example.com', + 'io-cache.origin.endpoint': 'https://origin.example.com', + 'io-cache.policy': 'meta,read', + } + cache = _NativeRestTableCache() + cache.get('response', 'default', 't', options, token_loader=lambda: None) + fake_df.Table.from_rest_response.assert_called_once_with( + 'response', database='default', table='t', + rest_options={ + 'fs.oss.endpoint': 'https://origin.example.com', + 'fs.oss.https.enable': 'true', + }) + def test_native_rest_cache_gives_the_core_the_origin_under_io_cache_routing(self): import time diff --git a/paimon-python/pypaimon/tests/native_read_path_completion_test.py b/paimon-python/pypaimon/tests/native_read_path_completion_test.py index 0c302155d909..1c59878fe6b4 100644 --- a/paimon-python/pypaimon/tests/native_read_path_completion_test.py +++ b/paimon-python/pypaimon/tests/native_read_path_completion_test.py @@ -204,6 +204,48 @@ def test_native_catalog_preserves_a_reconstructible_fuse_binding(tmp_path, mode, assert module.PaimonCatalog.call_args[0][0]['fuse.enabled'] == 'true' +def test_native_catalog_fallback_uses_origin_for_catalog_only_io_cache(tmp_path): + """A REST catalog without a data token must not hand a cache endpoint to Rust.""" + from pypaimon.catalog.catalog_context import CatalogContext + from pypaimon.catalog.rest.rest_catalog_loader import RESTCatalogLoader + from pypaimon.common.identifier import Identifier + from pypaimon.common.options.options import Options + from pypaimon.execution.native.table import _native_table + from pypaimon.schema.table_schema import TableSchema + from pypaimon.table.file_store_table import FileStoreTable + + options = Options({ + 'warehouse': 'oss://bucket/warehouse', + 'fs.oss.endpoint': 'http://cache.example.com', + 'io-cache.enabled': 'true', + 'io-cache.endpoint': 'http://cache.example.com', + 'io-cache.origin.endpoint': 'https://origin.example.com', + 'io-cache.policy': 'meta,read', + }) + loader = RESTCatalogLoader(CatalogContext.create_from_options(options)) + identifier = Identifier.create('default', 't') + environment = CatalogEnvironment(identifier=identifier, catalog_loader=loader) + file_io = Mock() + file_io.for_table.return_value = file_io + table = FileStoreTable( + file_io, identifier, 'oss://bucket/warehouse/default/t', TableSchema(), environment) + + module = ModuleType('pypaimon_rust.datafusion') + module.PaimonCatalog = Mock() + native_table = module.PaimonCatalog.return_value.get_table.return_value + native_table.location.return_value = table.table_path + with patch.dict('sys.modules', {'pypaimon_rust.datafusion': module}), \ + patch('pypaimon.execution.native.table._with_native_resolved_schema', + return_value=native_table): + assert _native_table(table) is native_table + + native_options = module.PaimonCatalog.call_args.args[0] + assert native_options['fs.oss.endpoint'] == 'https://origin.example.com' + assert native_options['fs.oss.https.enable'] == 'true' + assert 'io-cache.enabled' not in native_options + assert 'io-cache.endpoint' not in native_options + + @pytest.mark.parametrize('transport', ['same-view', 'copy-before', 'copy', 'pickle']) def test_pinned_rest_view_reuses_core_snapshot_validation(guarded_table, transport): import pickle