diff --git a/paimon-python/pypaimon/catalog/rest/rest_token_file_io.py b/paimon-python/pypaimon/catalog/rest/rest_token_file_io.py index 580b7b99f31d..f4b0b316aede 100644 --- a/paimon-python/pypaimon/catalog/rest/rest_token_file_io.py +++ b/paimon-python/pypaimon/catalog/rest/rest_token_file_io.py @@ -22,19 +22,24 @@ import weakref from datetime import timedelta from typing import Optional, Union +from urllib.parse import urlparse from cachetools import TTLCache from pypaimon.api.auth.bearer import BearTokenAuthProvider from pypaimon.api.auth.dlf_provider import DLFAuthProvider from pypaimon.api.rest_api import RESTApi -from pypaimon.api.rest_util import RESTUtil from pypaimon.catalog.rest.rest_token import RESTToken from pypaimon.common.file_io import FileIO from pypaimon.common.identifier import Identifier from pypaimon.common.options import Options -from pypaimon.common.options.config import CatalogOptions, OssOptions from pypaimon.common.uri_reader import UriReaderFactory +from pypaimon.filesystem.io_cache_file_io import (IoCacheRoutingFileIO, + table_view_spec) +from pypaimon.filesystem.io_cache_routing import (IoCacheRouting, + effective_options, + schema_file_prefixes, + with_catalog_overrides) class RESTTokenFileIO(FileIO): @@ -65,6 +70,10 @@ class RESTTokenFileIO(FileIO): "fds": None, "next_scan": 0.0} _SHORT_TOKEN_STATS = {"count": 0, "next_log": 0.0} + # The table view each FileIO is built for, see FileIO.origin_only and for_table. + _origin_only = False + _table: Optional[tuple] = None + def __init__(self, identifier: Identifier, path: str, catalog_options: Optional[Union[dict, Options]] = None): self.identifier = identifier @@ -78,6 +87,8 @@ def __init__(self, identifier: Identifier, path: str, self.catalog_options = catalog_options self.properties = self.catalog_options or Options({}) # For compatibility with refresh_token() self.token: Optional[RESTToken] = None + # The token as the REST server vended it, without this catalog's overrides. + self._vended_token: Optional[RESTToken] = None self.api_instance: Optional[RESTApi] = None self.log = logging.getLogger(__name__) self._uri_reader_factory_cache: Optional[UriReaderFactory] = None @@ -126,8 +137,42 @@ def _after_fork_in_child(cls): for file_io in list(cls._INSTANCES): file_io._token_lock = threading.Lock() + def origin_only(self) -> FileIO: + return self if self._origin_only else self._with_view(True, None) + + def for_table(self, table_path: str, file_prefixes, branch: str) -> FileIO: + if self._origin_only: + return self + table = table_view_spec(self._table, table_path, file_prefixes, branch) + return self if table == self._table else self._with_view(False, table) + + def _with_view(self, origin_only: bool, table) -> 'RESTTokenFileIO': + # Same token and caches; each FileIO it returns is the view of the cached one. + view = RESTTokenFileIO.__new__(RESTTokenFileIO) + view.__setstate__(self.__getstate__()) + view.api_instance = self.api_instance + view._origin_only = origin_only + view._table = table + return view + + def _table_view(self, file_io: FileIO) -> FileIO: + if self._origin_only: + return file_io.origin_only() + if self._table is None or not isinstance(file_io, IoCacheRoutingFileIO): + return file_io + return file_io.table_view(self._table, self._catalog_schema_prefixes) + + def _catalog_schema_prefixes(self, branch: str): + # A REST catalog may keep a branch's schema without a schema directory in storage. + if self.api_instance is None: + self.api_instance = RESTApi(self.properties, False) + identifier = Identifier.create(self.identifier.get_database_name(), self.identifier.get_table_name(), + branch=None if branch == "main" else branch) + schema = self.api_instance.get_table(identifier).schema + return schema_file_prefixes(schema.options) if schema is not None else [] + def file_io(self) -> FileIO: - return self._file_io_with_token()[0] + return self._table_view(self._file_io_with_token()[0]) def _file_io_with_token(self, minimum_validity_millis: int = 0): with self._token_lock: @@ -151,7 +196,7 @@ def _file_io_with_token(self, minimum_validity_millis: int = 0): cache = self._FILE_IO_CACHE file_io = cache.get(cache_key) if file_io is None: - file_io = FileIO.get(self.path, self._merged_options(token)) + file_io = self._create_file_io(self._merged_options(token)) self._log_file_io_created(cache) # Re-inserting on every access expires idle entries only, like expireAfterAccess. cache[cache_key] = file_io @@ -199,24 +244,32 @@ def _get_file_io_scope(self) -> str: return self._file_io_scope def _merged_options(self, token: RESTToken) -> Options: - merged_properties = RESTUtil.merge( - self.catalog_options.to_map() if self.catalog_options else {}, - token.token - ) - if self.catalog_options: - dlf_oss_endpoint = self.catalog_options.get(CatalogOptions.DLF_OSS_ENDPOINT) - if dlf_oss_endpoint and dlf_oss_endpoint.strip(): - merged_properties[OssOptions.OSS_ENDPOINT.key()] = dlf_oss_endpoint - return Options(merged_properties) + return Options(effective_options(self._catalog_map(), token.token)) + + def _catalog_map(self) -> dict: + return self.catalog_options.to_map() if self.catalog_options else {} + + def _create_file_io(self, options: Options) -> FileIO: + properties = options.to_map() + routing = IoCacheRouting.create(properties, root=self.path) + if routing is None or urlparse(self.path).scheme != "oss": + return FileIO.get(self.path, options) + self.log.info( + "io-cache routing enabled for [%s], targets %s, origin endpoint %s", + self.identifier, routing.targets(), routing.origin_endpoint()) + origin = FileIO.get(self.path, Options(routing.origin_options(properties))) + return IoCacheRoutingFileIO( + routing, origin, + lambda name: FileIO.get(self.path, Options(routing.target_options(properties, name)))) def _merge_token_with_catalog_options(self, token: dict) -> dict: """Merge token with catalog options, DLF OSS endpoint should override the standard OSS endpoint.""" - merged_token = dict(token) - if self.catalog_options: - dlf_oss_endpoint = self.catalog_options.get(CatalogOptions.DLF_OSS_ENDPOINT) - if dlf_oss_endpoint and dlf_oss_endpoint.strip(): - merged_token[OssOptions.OSS_ENDPOINT.key()] = dlf_oss_endpoint - return merged_token + return with_catalog_overrides(token, self._catalog_map()) + + def _adopt(self, vended: RESTToken): + # The shared token cache keeps the vended token; this instance applies its own catalog's overrides. + self._vended_token = vended + self.token = RESTToken(self._merge_token_with_catalog_options(vended.token), vended.expire_at_millis) def new_input_stream(self, path: str): return self.file_io().new_input_stream(path) @@ -328,7 +381,7 @@ def try_to_refresh_token(self, minimum_validity_millis: int = 0): cached_token = self._get_cached_token(identifier_str) if cached_token and not self._is_token_expired(cached_token, minimum_validity_millis): - self.token = cached_token + self._adopt(cached_token) return global_lock = self._get_global_token_lock(identifier_str) @@ -336,13 +389,13 @@ def try_to_refresh_token(self, minimum_validity_millis: int = 0): with global_lock: cached_token = self._get_cached_token(identifier_str) if cached_token and not self._is_token_expired(cached_token, minimum_validity_millis): - self.token = cached_token + self._adopt(cached_token) return token_to_check = cached_token if cached_token else self.token if token_to_check is None or self._is_token_expired(token_to_check, minimum_validity_millis): self.refresh_token() - self._set_cached_token(identifier_str, self.token) + self._set_cached_token(identifier_str, self._vended_token) def _get_cached_token(self, identifier_str: str) -> Optional[RESTToken]: with self._TOKEN_LOCKS_LOCK: @@ -413,11 +466,9 @@ def refresh_token(self): f"end refresh data token for identifier [{self.identifier}] expiresAtMillis [{response.expires_at_millis}]" ) - merged_token_dict = self._merge_token_with_catalog_options(response.token) - new_token = RESTToken(merged_token_dict, response.expires_at_millis) - self.token = new_token - if self._is_token_expired(new_token): - self._log_short_token(new_token) + self._adopt(RESTToken(response.token, response.expires_at_millis)) + if self._is_token_expired(self.token): + self._log_short_token(self.token) def _log_short_token(self, token: RESTToken): with self._TOKEN_LOCKS_LOCK: diff --git a/paimon-python/pypaimon/common/file_io.py b/paimon-python/pypaimon/common/file_io.py index c2d132f7acb6..cff3a5f68495 100644 --- a/paimon-python/pypaimon/common/file_io.py +++ b/paimon-python/pypaimon/common/file_io.py @@ -92,6 +92,15 @@ class FileIO(ABC): File IO interface to read and write files. """ + def origin_only(self) -> 'FileIO': + """A view that sends no request to an io-cache target, for files other writers may replace.""" + return self + + def for_table(self, table_path: str, file_prefixes: List[str], branch: str) -> 'FileIO': + """A view for one Paimon table: io-cache targets serve only its files under table_path, named + with these prefixes or those of the branch's schemas.""" + return self + @abstractmethod def new_input_stream(self, path: str): pass @@ -100,6 +109,10 @@ def new_input_stream(self, path: str): def new_output_stream(self, path: str): pass + def create_parent_dirs(self, path: str): + """Runs only the parent directory step of new_output_stream(path); a no-op unless a FileIO splits it out.""" + pass + @abstractmethod def get_file_status(self, path: str): pass @@ -683,7 +696,12 @@ def get(path: str, catalog_options: Optional[Options] = None) -> 'FileIO': ) if scheme == "oss": + from pypaimon.filesystem.io_cache_routing import IoCacheRouting from pypaimon.filesystem.oss_file_io import OssFileIO + # This FileIO sends every request to one endpoint, so under io-cache routing it is the origin's. + routing = IoCacheRouting.create(opts.to_map()) + if routing is not None: + opts = Options(routing.origin_options(opts.to_map())) return OssFileIO(path, opts) from pypaimon.filesystem.pyarrow_file_io import PyArrowFileIO diff --git a/paimon-python/pypaimon/common/options/config.py b/paimon-python/pypaimon/common/options/config.py index bac72d4a4cb6..829593b28bec 100644 --- a/paimon-python/pypaimon/common/options/config.py +++ b/paimon-python/pypaimon/common/options/config.py @@ -33,6 +33,10 @@ class OssOptions: OSS_ENDPOINT = ConfigOptions.key("fs.oss.endpoint").string_type().no_default_value().with_description( "OSS endpoint") OSS_REGION = ConfigOptions.key("fs.oss.region").string_type().no_default_value().with_description("OSS region") + OSS_SECOND_LEVEL_DOMAIN_ENABLE = ConfigOptions.key( + "fs.oss.second.level.domain.enable").boolean_type().default_value(False).with_description( + "Path-style addressing (endpoint/bucket/key) instead of bucket.endpoint, " + "for IP or non-wildcard endpoints") OSS_SSE_METHOD = ConfigOptions.key( "fs.oss.server-side-encryption").string_type().no_default_value().with_description( "OSS atomic metadata encryption method: AES256, KMS or SM4") @@ -129,6 +133,9 @@ class CatalogOptions: "DLF token file path") DLF_OSS_ENDPOINT = ConfigOptions.key("dlf.oss-endpoint").string_type().no_default_value().with_description( "DLF OSS endpoint") + IO_CACHE_ENABLED = ConfigOptions.key("io-cache.enabled").boolean_type().default_value(False).with_description( + "Route reads to the io-cache targets vended with the table token; " + "a value set in the catalog options overrides the token") DLF_TOKEN_LOADER = ConfigOptions.key("dlf.token-loader").string_type().no_default_value().with_description( "DLF token loader") DLF_TOKEN_ECS_ROLE_NAME = ConfigOptions.key( diff --git a/paimon-python/pypaimon/daft/daft_paimon.py b/paimon-python/pypaimon/daft/daft_paimon.py index ec698349462c..133da53c6d87 100644 --- a/paimon-python/pypaimon/daft/daft_paimon.py +++ b/paimon-python/pypaimon/daft/daft_paimon.py @@ -33,6 +33,10 @@ from typing import TYPE_CHECKING, Any, Dict, Optional, Union from urllib.parse import urlparse +from pypaimon.filesystem.io_cache_routing import (IoCacheRouting, + effective_options, + single_endpoint_options) + if TYPE_CHECKING: import daft @@ -46,20 +50,25 @@ def _enrich_options_with_rest_token( # REST catalogs (DLF) keep OSS STS tokens on table.file_io and the bucket on table.table_path, # not in catalog_options; fold both in so Daft's IOConfig routes to OSS with valid credentials. if catalog_options.get("metastore") != "rest": - return catalog_options + return _origin_only(catalog_options) file_io = getattr(table, "file_io", None) if file_io is None or not hasattr(file_io, "try_to_refresh_token"): - return catalog_options + return _origin_only(catalog_options) file_io.try_to_refresh_token() if file_io.token is None: - return catalog_options - enriched = {**catalog_options, **file_io.token.token} + return _origin_only(catalog_options) + enriched = single_endpoint_options(effective_options(catalog_options, file_io.token.token)) parsed = urlparse(getattr(table, "table_path", "") or "") if parsed.scheme and parsed.netloc: enriched["warehouse"] = f"{parsed.scheme}://{parsed.netloc}" return enriched +def _origin_only(options: Dict[str, str]) -> Dict[str, str]: + # Daft reads with its own client, which cannot use io-cache targets per request. + return options if IoCacheRouting.create(options) is None else single_endpoint_options(options) + + # Time-travel targets are mutually exclusive: at most one may be set. TimeTravelTimestamp = Union[int, str, "datetime.datetime"] diff --git a/paimon-python/pypaimon/execution/native/table.py b/paimon-python/pypaimon/execution/native/table.py index 251d4cc6f69d..8e3e3349fc55 100644 --- a/paimon-python/pypaimon/execution/native/table.py +++ b/paimon-python/pypaimon/execution/native/table.py @@ -25,6 +25,7 @@ from pypaimon.common.options.config import CatalogOptions, FuseOptions, OssOptions from pypaimon.common.options.options_utils import OptionsUtils +from pypaimon.filesystem.io_cache_routing import single_endpoint_options def _native_blob_view_supported(table, read_names) -> bool: @@ -199,11 +200,18 @@ def get(self, response, database, table, options, token_loader=None): def create(): token = token_loader() if token_loader else None if token is None: + # A REST table can reach this path before its data token is available (for + # example a copied branch or a token refresh race). The native core sends + # every OSS request to one endpoint, so routing keys in the catalog must not + # leave its legacy fs.oss.endpoint pointed at a cache target. return Table.from_rest_response( - response, database=database, table=table, rest_options=options) + response, database=database, table=table, + rest_options=single_endpoint_options(options)) + # The native core cannot use io-cache targets per request, and fs.oss.endpoint may name the cache. return Table.from_rest_response_with_token( response, database=database, table=table, rest_options=options, - data_token=dict(token.token), expires_at_millis=token.expire_at_millis) + data_token=single_endpoint_options(token.token, options), + expires_at_millis=token.expire_at_millis) key = (response, database, table, tuple(sorted(options.items()))) return self.get_or_create(key, create) @@ -304,16 +312,21 @@ def _load_native_table(table): token_loader=lambda: _rest_data_token(table)) elif file_io_options is not None: from pypaimon_rust.datafusion import Table + # As with a token: the core cannot use io-cache targets per request, so it gets the origin's options. rt = Table.from_resolved_schema( table.table_path, _resolved_schema_json(table), database=table.identifier.get_database_name(), table=table.identifier.get_table_name(), - branch=table.current_branch(), options=file_io_options) + branch=table.current_branch(), options=single_endpoint_options(file_io_options)) else: from pypaimon_rust.datafusion import PaimonCatalog # The Python loader has already resolved /config, including typed # FUSE options. Match RESTCatalogLoader.load() and reuse that context. - catalog = PaimonCatalog(_catalog_options(table), config_required=False) + # This catalog is also a single-endpoint client. Keep its metadata/catalog + # configuration, but make its storage endpoint the origin when catalog-only + # io-cache routing is enabled. + catalog = PaimonCatalog( + single_endpoint_options(_catalog_options(table)), config_required=False) # REST may keep branch schemas in the catalog only. Load the base # environment, then attach the schema/branch already resolved here. rt = catalog.get_table(( diff --git a/paimon-python/pypaimon/filesystem/caching_file_io.py b/paimon-python/pypaimon/filesystem/caching_file_io.py index 58a1a9bd12d0..f8b526d954e9 100644 --- a/paimon-python/pypaimon/filesystem/caching_file_io.py +++ b/paimon-python/pypaimon/filesystem/caching_file_io.py @@ -414,6 +414,16 @@ def __getstate__(self): # Python 3.10 otherwise finds the delegate's hook through __getattr__. return self.__dict__.copy() + def origin_only(self) -> FileIO: + return self._with_delegate(self._delegate.origin_only()) + + def for_table(self, table_path: str, file_prefixes, branch: str) -> FileIO: + return self._with_delegate(self._delegate.for_table(table_path, file_prefixes, branch)) + + def _with_delegate(self, view: FileIO) -> FileIO: + # A view shares this wrapper's cache. + return self if view is self._delegate else CachingFileIO(view, self._cache, self._whitelist) + # Fallback caps when local-cache.max-size is unset (memory shares the heap). _DEFAULT_MEMORY_CACHE_MAX_SIZE = 256 * 1024 * 1024 _DEFAULT_DISK_CACHE_MAX_SIZE = 10 * 1024 * 1024 * 1024 diff --git a/paimon-python/pypaimon/filesystem/io_cache_file_io.py b/paimon-python/pypaimon/filesystem/io_cache_file_io.py new file mode 100644 index 000000000000..2bfa8b8c479d --- /dev/null +++ b/paimon-python/pypaimon/filesystem/io_cache_file_io.py @@ -0,0 +1,411 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""FileIO that sends each request to an io-cache target or the origin endpoint.""" + +import logging +import os +import threading +from typing import Callable, Dict, Iterable, List, Optional, Tuple + +import pyarrow.fs as pafs +from pyarrow._fs import FileSystemHandler + +from pypaimon.common.file_io import FileIO +from pypaimon.filesystem.io_cache_routing import IoCacheRouting, Op, schema_file_prefixes +from pypaimon.filesystem.pyarrow_file_io import PyArrowFileIO +from pypaimon.filesystem.resolving_file_io import ResolvingFileIO + + +class IoCacheRoutingFileIO(FileIO): + """Sends each call to origin or to a target FileIO, all built for the table path. + Calls and attributes that are not routed go to origin.""" + + def __init__(self, routing: IoCacheRouting, origin: FileIO, + create_target: Callable[[str], FileIO]): + self._routing = routing + self._origin = origin + self._create_target = create_target + self._targets: Dict[str, FileIO] = {} + self._targets_lock = threading.Lock() + self._pid = os.getpid() + self._filesystem = pafs.PyFileSystem(_IoCacheRoutingHandler(self)) + self._base = self + self._table: Optional[Tuple[str, Tuple[str, ...], Tuple[str, ...]]] = None + self._views: Dict[tuple, 'IoCacheRoutingFileIO'] = {} + + def origin_only(self) -> FileIO: + return self._origin + + def for_table(self, table_path: str, file_prefixes: Iterable[str], branch: str) -> FileIO: + return self._base.table_view(table_view_spec(self._table, table_path, file_prefixes, branch)) + + def table_view(self, table, catalog_prefixes=None) -> 'IoCacheRoutingFileIO': + """The view of this FileIO for table = (path, prefixes, branches), see table_view_spec; + catalog_prefixes(branch) adds the prefixes a catalog keeps outside the schema directory.""" + view = self._views.get(table) + if view is not None: + return view + self._check_pid() + with self._targets_lock: + view = self._views.get(table) + if view is None: + table_path, prefixes, branches = table + history = _SchemaPrefixHistory(self._origin, table_path, branches, catalog_prefixes) + # The view takes its targets from this FileIO. + view = IoCacheRoutingFileIO( + self._routing.for_table(table_path, prefixes, history), self._origin, self._file_io) + view._base, view._table = self, table + self._views[table] = view + return view + + def _check_pid(self): + if self._pid != os.getpid(): + # A forked child inherits the lock in whatever state a parent thread left it. + self._targets = {} + self._targets_lock = threading.Lock() + self._pid = os.getpid() + + @property + def properties(self): + return self._origin.properties + + @property + def filesystem(self): + """Filesystem for readers; it takes the URIs returned by to_filesystem_path.""" + return self._filesystem + + def _route(self, op: Op, path: str) -> Optional[str]: + return self._routing.route(op, path) + + def _file_io(self, name: Optional[str]) -> FileIO: + """Origin, or the FileIO of a target, created on first use.""" + if name is None: + return self._origin + self._check_pid() + file_io = self._targets.get(name) + if file_io is not None: + return file_io + with self._targets_lock: + file_io = self._targets.get(name) + if file_io is None: + file_io = self._create_target(name) + if isinstance(file_io, (PyArrowFileIO, ResolvingFileIO)): + file_io.creates_parent_dirs = False + file_io.cleans_up_failed_writes = False + self._targets[name] = file_io + return file_io + + def _call(self, op: Op, path: str, action): + return action(self._file_io(self._route(op, path))) + + def _write(self, path: str, action, cleans_up: bool = False): + name = self._route(Op.WRITE, path) + if name is None: + return action(self._origin) + # A target takes only the object; its parent directories are created on origin, as unrouted. + self._origin.create_parent_dirs(path) + try: + return action(self._file_io(name)) + except Exception: + # The cleanup the origin FileIO would do after a failed write also runs on origin. + if cleans_up: + self._origin.delete_quietly(path) + raise + + def _call_batch(self, op: Op, paths: List[str], action) -> list: + """Runs action(file_io, paths) once per routed group; results follow the order of paths.""" + results = [None] * len(paths) + groups: Dict[Optional[str], List[int]] = {} + for index, path in enumerate(paths): + groups.setdefault(self._route(op, path), []).append(index) + for name, indices in groups.items(): + group = [paths[index] for index in indices] + values = action(self._file_io(name), group) + for index, value in zip(indices, values): + results[index] = value + return results + + def new_input_stream(self, path: str): + return self._call(Op.READ, path, lambda io: io.new_input_stream(path)) + + def get_file_status(self, path: str): + return self._call(Op.META, path, lambda io: io.get_file_status(path)) + + def get_file_size(self, path: str) -> int: + return self._call(Op.META, path, lambda io: io.get_file_size(path)) + + def new_output_stream(self, path: str): + return self._write(path, lambda io: io.new_output_stream(path)) + + def create_parent_dirs(self, path: str): + return self._origin.create_parent_dirs(path) + + def exists(self, path: str) -> bool: + return self._call(Op.EXISTS, path, lambda io: io.exists(path)) + + def exists_batch(self, paths: List[str]) -> Dict[str, bool]: + paths = list(paths) + found = self._call_batch( + Op.EXISTS, paths, lambda io, group: [io.exists_batch(group)[p] for p in group]) + return dict(zip(paths, found)) + + def is_dir(self, path: str) -> bool: + return self._origin.is_dir(path) + + def list_status(self, path: str): + return self._origin.list_status(path) + + def delete(self, path: str, recursive: bool = False) -> bool: + return self._origin.delete(path, recursive) + + def mkdirs(self, path: str) -> bool: + return self._origin.mkdirs(path) + + def rename(self, src: str, dst: str) -> bool: + return self._origin.rename(src, dst) + + def copy_file(self, source_path: str, target_path: str, overwrite: bool = False): + return self._origin.copy_file(source_path, target_path, overwrite) + + def try_to_write_atomic(self, path: str, content: str) -> bool: + return self._origin.try_to_write_atomic(path, content) + + def write_file(self, path: str, content: str, overwrite: bool = False): + # The existence check before a write runs on origin, so no cache sees the file before it exists. + if not overwrite and self._origin.exists(path): + raise FileExistsError(f"File {path} already exists and overwrite=False") + return self._write(path, lambda io: io.write_file(path, content, True)) + + def overwrite_file_utf8(self, path: str, content: str): + return self._write(path, lambda io: io.overwrite_file_utf8(path, content)) + + # Helpers that delete, or check right after a delete, run on origin, probes included. + + def delete_quietly(self, path: str): + return self._origin.delete_quietly(path) + + def delete_files_quietly(self, files: List[str]): + return self._origin.delete_files_quietly(files) + + def delete_directory_quietly(self, directory: str): + return self._origin.delete_directory_quietly(directory) + + def check_or_mkdirs(self, path: str): + return self._origin.check_or_mkdirs(path) + + def copy_files(self, source_directory: str, target_directory: str, overwrite: bool = False): + return self._origin.copy_files(source_directory, target_directory, overwrite) + + def create_blob_presigned_url(self, table_root, descriptor, validity) -> str: + return self._origin.create_blob_presigned_url(table_root, descriptor, validity) + + def to_filesystem_path(self, path: str) -> str: + # The routing filesystem needs the full path, it converts it for the chosen filesystem. + return path + + def write_parquet(self, path: str, data, compression: str = 'zstd', + zstd_level: int = 1, **kwargs): + return self._write( + path, lambda io: io.write_parquet(path, data, compression, zstd_level, **kwargs), True) + + def write_orc(self, path: str, data, compression: str = 'zstd', + zstd_level: int = 1, **kwargs): + return self._write( + path, lambda io: io.write_orc(path, data, compression, zstd_level, **kwargs), True) + + def write_avro(self, path: str, data, avro_schema=None, + compression: str = 'zstd', zstd_level: int = 1, **kwargs): + return self._write(path, lambda io: io.write_avro( + path, data, avro_schema, compression, zstd_level, **kwargs)) + + def close(self): + if self._base is not self: + # A table view shares the FileIOs of its base, which closes them. + return + with self._targets_lock: + targets = list(self._targets.values()) + self._targets.clear() + try: + self._origin.close() + finally: + for target in targets: + target.close() + + def __getattr__(self, name): + origin = self.__dict__.get('_origin') + if origin is None: + raise AttributeError(name) + return getattr(origin, name) + + +class _SchemaPrefixHistory: + """Data file prefixes of every schema of a table's branches, read from origin at most once.""" + + def __init__(self, origin: FileIO, table_path: str, branches: Tuple[str, ...], + catalog_prefixes: Optional[Callable[[str], Iterable[str]]] = None): + self._origin = origin + self._table_path = table_path + self._branches = branches + self._catalog_prefixes = catalog_prefixes + self._prefixes: Optional[Tuple[str, ...]] = None + self._lock = threading.Lock() + self._pid = os.getpid() + + def __call__(self) -> Tuple[str, ...]: + if self._prefixes is not None: + return self._prefixes + if self._pid != os.getpid(): + self._lock = threading.Lock() + self._pid = os.getpid() + with self._lock: + if self._prefixes is None: + self._prefixes = self._load() + return self._prefixes + + def _load(self) -> Tuple[str, ...]: + prefixes = [] + for branch in self._branches: + prefixes += self._quietly(self._schema_directory_prefixes, branch) + if self._catalog_prefixes is not None: + prefixes += self._quietly(self._catalog_prefixes, branch) + return tuple(dict.fromkeys(prefixes)) + + def _schema_directory_prefixes(self, branch: str) -> List[str]: + from pypaimon.schema.schema_manager import SchemaManager + schemas = SchemaManager(self._origin, self._table_path, branch).list_all() + return [prefix for schema in schemas for prefix in schema_file_prefixes(schema.options)] + + def _quietly(self, load: Callable[[str], Iterable[str]], branch: str) -> List[str]: + try: + return list(load(branch)) + except Exception: + # Files with a prefix missed here are read from origin, which only loses io-cache. + logging.getLogger(__name__).warning( + "Failed to load schema file prefixes of %s (branch %s) for io-cache routing", + self._table_path, branch, exc_info=True) + return [] + + +def table_view_spec(table, table_path: str, file_prefixes: Iterable[str], branch: str): + """(path, prefixes, branches) of a table view; a copy of the same table keeps those it had.""" + prefixes, branches = tuple(dict.fromkeys(file_prefixes)), (branch,) + if table is not None and table[0] == table_path: + prefixes = table[1] + tuple(p for p in prefixes if p not in table[1]) + branches = table[2] + tuple(b for b in branches if b not in table[2]) + return table_path, prefixes, branches + + +class _IoCacheRoutingHandler(FileSystemHandler): + """Sends each pyarrow filesystem call to the filesystem of origin or of a target FileIO.""" + + def __init__(self, file_io: IoCacheRoutingFileIO): + self._routing_io = file_io + + def __eq__(self, other): + if isinstance(other, _IoCacheRoutingHandler): + return self._routing_io is other._routing_io + return NotImplemented + + def __ne__(self, other): + if isinstance(other, _IoCacheRoutingHandler): + return self._routing_io is not other._routing_io + return NotImplemented + + def get_type_name(self) -> str: + return "io-cache-routing" + + def normalize_path(self, path: str) -> str: + return path + + def register_file_size(self, path: str, file_size: int): + file_io = self._routing_io._file_io(self._routing_io._route(Op.READ, path)) + handler = getattr(file_io.filesystem, "handler", None) + register = getattr(handler, "register_file_size", None) + if register is not None: + register(file_io.to_filesystem_path(path), file_size) + + def get_file_info(self, paths) -> list: + infos = self._routing_io._call_batch( + Op.META, list(paths), + lambda io, group: io.filesystem.get_file_info([io.to_filesystem_path(path) for path in group])) + return [self._file_info(path, info) for path, info in zip(paths, infos)] + + @staticmethod + def _file_info(path, info): + return pafs.FileInfo(path, info.type, mtime_ns=info.mtime_ns, size=info.size) + + def open_input_file(self, path: str): + return self._routing_io._call( + Op.READ, path, + lambda io: io.filesystem.open_input_file(io.to_filesystem_path(path))) + + def open_input_stream(self, path: str): + # The outer PyFileSystem already applied the requested decompression. + return self._routing_io._call( + Op.READ, path, + lambda io: io.filesystem.open_input_stream(io.to_filesystem_path(path), compression=None)) + + def open_output_stream(self, path: str, metadata): + return self._routing_io._call( + Op.WRITE, path, + lambda io: io.filesystem.open_output_stream( + io.to_filesystem_path(path), compression=None, metadata=metadata)) + + def open_append_stream(self, path: str, metadata): + return self._routing_io._call( + Op.WRITE, path, + lambda io: io.filesystem.open_append_stream( + io.to_filesystem_path(path), compression=None, metadata=metadata)) + + def get_file_info_selector(self, selector) -> list: + origin = self._routing_io._origin + base = origin.to_filesystem_path(selector.base_dir).rstrip("/") + infos = origin.filesystem.get_file_info(pafs.FileSelector( + base, + allow_not_found=selector.allow_not_found, recursive=selector.recursive)) + # Backend paths must stay in the full-URI namespace accepted by this filesystem. + return [self._file_info(selector.base_dir.rstrip("/") + info.path[len(base):], info) + for info in infos] + + def create_dir(self, path: str, recursive: bool): + origin = self._routing_io._origin + origin.filesystem.create_dir(origin.to_filesystem_path(path), recursive=recursive) + + def delete_dir(self, path: str): + origin = self._routing_io._origin + origin.filesystem.delete_dir(origin.to_filesystem_path(path)) + + def delete_dir_contents(self, path: str, missing_dir_ok: bool = False): + origin = self._routing_io._origin + origin.filesystem.delete_dir_contents( + origin.to_filesystem_path(path), missing_dir_ok=missing_dir_ok) + + def delete_root_dir_contents(self): + raise NotImplementedError("io-cache routing filesystem has no root directory") + + def delete_file(self, path: str): + origin = self._routing_io._origin + origin.filesystem.delete_file(origin.to_filesystem_path(path)) + + def move(self, src: str, dest: str): + origin = self._routing_io._origin + origin.filesystem.move(origin.to_filesystem_path(src), origin.to_filesystem_path(dest)) + + def copy_file(self, src: str, dest: str): + origin = self._routing_io._origin + origin.filesystem.copy_file(origin.to_filesystem_path(src), origin.to_filesystem_path(dest)) diff --git a/paimon-python/pypaimon/filesystem/io_cache_routing.py b/paimon-python/pypaimon/filesystem/io_cache_routing.py new file mode 100644 index 000000000000..d31830866290 --- /dev/null +++ b/paimon-python/pypaimon/filesystem/io_cache_routing.py @@ -0,0 +1,309 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Chooses the endpoint of each FileIO request from the io-cache.* keys of a table token.""" + +import copy +import os +import re +from enum import Enum +from typing import Callable, Dict, FrozenSet, Iterable, List, Mapping, NamedTuple, Optional, Tuple + +from pypaimon.common.options.config import CatalogOptions, OssOptions +from pypaimon.utils.file_type import FileType + +FS_OSS_ENDPOINT = OssOptions.OSS_ENDPOINT.key() +FS_OSS_REGION = OssOptions.OSS_REGION.key() +FS_OSS_PATH_STYLE = OssOptions.OSS_SECOND_LEVEL_DOMAIN_ENABLE.key() +FS_OSS_HTTPS_ENABLE = "fs.oss.https.enable" +DLF_OSS_ENDPOINT = CatalogOptions.DLF_OSS_ENDPOINT.key() +IO_CACHE_ENABLED = CatalogOptions.IO_CACHE_ENABLED.key() +IO_CACHE_TARGET_PREFIX = "io-cache.target." + +_TARGET_NAME = re.compile(r"[a-z][a-z0-9-]*") +_UUID = r"[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}" +# What follows a data file prefix: {uuid}-{count}.{extension}, also after the postpone writer's +# -u-{commit user}-s-{write id}-w-, or a paimon-rust BLOB pack {uuid}.managed.blob. +_DATA_FILE_SUFFIX = re.compile(r"(?:-u-.+-s-[0-9]+-w-)?" + _UUID + r"-[0-9]+\..+|" + _UUID + r"\.managed\.blob") +# A name that may be a data file under a prefix this routing has not seen yet. +_DATA_FILE_NAME = re.compile(r".+?(?:" + _DATA_FILE_SUFFIX.pattern + r")") +# Other types are routed only under the names Paimon writes (paimon-rust adds -changelog to changelog +# manifests); Format Table files may be replaced in place. +_PAIMON_FILE_NAMES = { + FileType.META: re.compile( + r"(manifest-list|index-manifest|stat)-" + _UUID + r"-[0-9]+|manifest-" + _UUID + + r"(-changelog)?-[0-9]+(\.avro\.sidecar)?"), + FileType.BUCKET_INDEX: re.compile(r"index-" + _UUID + r"-[0-9]+"), + FileType.GLOBAL_INDEX: re.compile(r"[a-z0-9_-]+-global-index-" + _UUID + r"\.index"), +} +_MANIFEST_SIDECAR_SUFFIX = ".avro.sidecar" +IO_CACHE_PREFIX = "io-cache." +# JindoSDK cache client keys; the origin option set must not talk to a cache. +_ORIGIN_DROPPED_PREFIXES = ("fs.oss.dlf-cache.", "fs.jindocache.", IO_CACHE_PREFIX) + + +class Op(Enum): + """FileIO operations; only READ, META, EXISTS, WRITE and TWO_PHASE_WRITE can go to a target.""" + READ = "read" + META = "meta" + EXISTS = "exists" + WRITE = "write" + LIST = "list" + DELETE = "delete" + RENAME = "rename" + MKDIRS = "mkdirs" + COPY = "copy" + ATOMIC_WRITE = "atomic-write" + TWO_PHASE_WRITE = "two-phase-write" + PRESIGN = "presign" + + +def _flag(value) -> bool: + return value is not None and str(value).strip().lower() == "true" + + +def _parse_policy(value) -> FrozenSet[Op]: + # read, meta, exists and write can use a target; none turns it off + tokens = {token.strip().lower() for token in str(value or "").split(",")} + if "none" in tokens: + return frozenset() + return frozenset(op for op in (Op.READ, Op.META, Op.EXISTS, Op.WRITE) if op.value in tokens) + + +def _parse_routes(value: str) -> List[Tuple[set, str]]: + # "types=target;...": the first rule containing a type picks its target + rules = [] + for rule in value.lower().split(";"): + types, eq, target = rule.partition("=") + types = FileType.parse_whitelist(types) if eq and target.strip() else set() + if types: + rules.append((types, target.strip())) + return rules + + +def _declared_targets(options: Mapping[str, str]) -> Dict[str, Optional[str]]: + # io-cache.targets with io-cache.target..endpoint, else io-cache.endpoint as "default" + names = options.get("io-cache.targets") + if names is None: + endpoint = options.get("io-cache.endpoint") + return {} if endpoint is None else {"default": endpoint} + endpoints = {} + for name in str(names).lower().split(","): + name = name.strip() + if _TARGET_NAME.fullmatch(name) and name not in endpoints: + endpoints[name] = options.get(IO_CACHE_TARGET_PREFIX + name + ".endpoint") + return endpoints + + +def schema_file_prefixes(options) -> List[str]: + """The data-file.prefix and changelog-file.prefix a table schema sets.""" + values = [options.get(key) for key in ("data-file.prefix", "changelog-file.prefix")] + return [value for value in values if value and value.strip()] + + +def data_prefixes(options: Mapping[str, str]) -> Tuple[str, ...]: + """Data file name prefixes: data- and changelog-, plus the table's own prefixes.""" + prefixes = ["data-", "changelog-"] + for key in ("data-file.prefix", "changelog-file.prefix"): + value = options.get(key) + if value and value.strip() and value not in prefixes: + prefixes.append(value) + return tuple(prefixes) + + +def _new_prefixes(prefixes: Iterable[str], known: Tuple[str, ...]) -> Tuple[str, ...]: + return tuple(p for p in dict.fromkeys(prefixes) if p and p.strip() and p not in known) + + +def _table_root(path: Optional[str]) -> Optional[str]: + return path.rstrip("/") + "/" if path else None + + +def routable_type(path: str, prefixes: Tuple[str, ...]) -> Optional[FileType]: + """The type a cache may serve, or None when the file must be read from the origin.""" + path = path.rstrip("/") + name = os.path.basename(path) + if _is_temporary(name) or FileType.is_mutable(path): + return None + if _is_data_file_name(name, prefixes): + return FileType.FILE_INDEX if name.endswith(".index") else FileType.DATA + # Sequential metadata may be read before it exists; cached NotFound would hide new commits. + parent = os.path.basename(os.path.dirname(path)) + if name.startswith(("snapshot-", "schema-")) or ( + name.startswith("changelog-") and parent == "changelog"): + return None + file_type = FileType.classify(path) + # data files and their file indexes are recognized by _is_data_file_name + pattern = _PAIMON_FILE_NAMES.get(file_type) + return file_type if pattern is not None and pattern.fullmatch(name) else None + + +def _is_data_file_name(name: str, prefixes: Tuple[str, ...]) -> bool: + """Manifests, indexes and statistics share the uuid-count shape but have no extension.""" + if name.endswith(_MANIFEST_SIDECAR_SUFFIX): + return False + return any(name.startswith(p) and _DATA_FILE_SUFFIX.fullmatch(name[len(p):]) for p in prefixes) + + +def _is_temporary(name: str) -> bool: + return name.endswith(".tmp") or ".tmp-" in name or ".tmp." in name + + +def _with_endpoint(options: Dict[str, str], endpoint: Optional[str]) -> Dict[str, str]: + if not endpoint: + return options + # JindoSDK drops the scheme of a URL endpoint, so carry it in fs.oss.https.enable. + scheme, separator, host = endpoint.partition("://") + if separator: + scheme = scheme.lower() + options[FS_OSS_HTTPS_ENABLE] = "true" if scheme == "https" else "false" + endpoint = scheme + separator + host + options[FS_OSS_ENDPOINT] = endpoint.rstrip("/") + return options + + +def with_catalog_overrides(token: Mapping[str, str], + catalog_options: Optional[Mapping[str, str]]) -> Dict[str, str]: + """A table token as a catalog uses it: the catalog's explicit dlf.oss-endpoint and io-cache.enabled win.""" + result = dict(token) + catalog_options = catalog_options or {} + dlf_oss_endpoint = catalog_options.get(DLF_OSS_ENDPOINT) + if dlf_oss_endpoint is not None and str(dlf_oss_endpoint).strip(): + result[FS_OSS_ENDPOINT] = dlf_oss_endpoint + enabled = catalog_options.get(IO_CACHE_ENABLED) + if enabled is not None: + result[IO_CACHE_ENABLED] = str(enabled).strip().lower() + return result + + +def effective_options(catalog_options: Optional[Mapping[str, str]], + token: Mapping[str, str]) -> Dict[str, str]: + """The options a table token takes effect with: the catalog options merged with the token, which wins, + then the catalog overrides.""" + from pypaimon.api.rest_util import RESTUtil + return with_catalog_overrides(RESTUtil.merge(dict(catalog_options or {}), dict(token)), catalog_options) + + +def single_endpoint_options(options: Mapping[str, str], + catalog_options: Optional[Mapping[str, str]] = None) -> Dict[str, str]: + """Options for a client that sends every request to one endpoint: options with the catalog overrides, and + the origin's endpoint when io-cache routing is on, whether its keys come from options or catalog_options.""" + options = with_catalog_overrides(options, catalog_options) + routing = IoCacheRouting.create({**(catalog_options or {}), **options}) + return options if routing is None else routing.origin_options(options) + + +class CacheTarget(NamedTuple): + """A named cache target with its endpoint and addressing settings.""" + name: str + endpoint: str + path_style_access: bool + region: Optional[str] + + +class IoCacheRouting: + """Per-request choice between the io-cache targets and the origin endpoint.""" + + def __init__(self, options: Mapping[str, str], policy: FrozenSet[Op], + endpoints: Dict[str, Optional[str]], root: Optional[str] = None): + origin = options.get("io-cache.origin.endpoint") + self._origin = origin if origin and origin.strip() else options.get(FS_OSS_ENDPOINT) + self._policy = policy + self._targets = { + name: CacheTarget(name, endpoint.strip(), + _flag(options.get(IO_CACHE_TARGET_PREFIX + name + ".path-style-access")), + options.get(IO_CACHE_TARGET_PREFIX + name + ".region")) + for name, endpoint in endpoints.items() if endpoint and endpoint.strip()} + whitelist = options.get("io-cache.whitelist") + self._whitelist = set(FileType) if whitelist is None else FileType.parse_whitelist( + str(whitelist).lower()) + routes = options.get("io-cache.routes") + self._routes = None if routes is None else _parse_routes(str(routes)) + self._prefixes = data_prefixes(options) + # Only files under the table's own location can use a target; None allows any OSS path. + self._root = _table_root(root) + self._history: Optional[Callable[[], Iterable[str]]] = None + + @staticmethod + def create(options: Mapping[str, str], root: Optional[str] = None) -> Optional['IoCacheRouting']: + """The routing of the options, or None when requests keep using fs.oss.endpoint.""" + if str(options.get(DLF_OSS_ENDPOINT) or "").strip() or not _flag(options.get(IO_CACHE_ENABLED)): + return None + policy = _parse_policy(options.get("io-cache.policy")) + endpoints = _declared_targets(options) + return IoCacheRouting(options, policy, endpoints, root) if policy and endpoints else None + + def for_table(self, root: str, prefixes: Iterable[str], + history: Optional[Callable[[], Iterable[str]]] = None) -> 'IoCacheRouting': + """This routing for the files of the table at root, also named with these prefixes, and with + those history() returns once a name that may be a data file matches no other.""" + routing = copy.copy(self) + routing._root = _table_root(root) + routing._prefixes = self._prefixes + _new_prefixes(prefixes, self._prefixes) + routing._history = history + return routing + + def targets(self) -> Dict[str, str]: + """Endpoints of the targets that have one, by name, in declaration order.""" + return {name: target.endpoint for name, target in self._targets.items()} + + def origin_endpoint(self) -> Optional[str]: + return self._origin + + def route(self, op: Op, path: str) -> Optional[str]: + """Name of the target of one request, or None when it goes to origin.""" + # A two-phase write is a write; exists has its own token, as a cache may keep deleted files. + if op is Op.TWO_PHASE_WRITE: + op = Op.WRITE + # Only OSS paths can use cache targets. + if op not in self._policy or not path.startswith("oss://"): + return None + # Row data may reference objects elsewhere, which other writers own and may replace. + if self._root is not None and not path.startswith(self._root): + return None + file_type = routable_type(path, self._prefixes) + if file_type is None and self._history is not None and _DATA_FILE_NAME.fullmatch( + os.path.basename(path.rstrip("/"))): + # Files written before a prefix option changed keep their names. + known = self._prefixes + file_type = routable_type(path, known + _new_prefixes(self._history(), known)) + if file_type is None or file_type not in self._whitelist: + return None + if self._routes is None: + return next(iter(self._targets), None) + name = next((target for types, target in self._routes if file_type in types), None) + return name if name in self._targets else None + + def origin_options(self, options: Mapping[str, str]) -> Dict[str, str]: + """Options of the origin FileIO: the origin endpoint, without cache client and io-cache keys.""" + result = {key: value for key, value in options.items() + if not str(key).startswith(_ORIGIN_DROPPED_PREFIXES)} + return _with_endpoint(result, self._origin) + + def target_options(self, options: Mapping[str, str], name: str) -> Dict[str, str]: + """Options of the FileIO of one target: its endpoint, addressing and region, without io-cache keys.""" + target = self._targets[name] + endpoint = target.endpoint + # A target endpoint without a scheme means https. + if "://" not in endpoint: + endpoint = "https://" + endpoint + result = _with_endpoint({key: value for key, value in options.items() + if not str(key).startswith(IO_CACHE_PREFIX)}, endpoint) + result[FS_OSS_PATH_STYLE] = "true" if target.path_style_access else "false" + if target.region and target.region.strip(): + result[FS_OSS_REGION] = target.region + return result diff --git a/paimon-python/pypaimon/filesystem/pvfs.py b/paimon-python/pypaimon/filesystem/pvfs.py index 823f3ac3e9fa..12205bf837d3 100644 --- a/paimon-python/pypaimon/filesystem/pvfs.py +++ b/paimon-python/pypaimon/filesystem/pvfs.py @@ -43,6 +43,7 @@ JINDO_OSSFS_AVAILABLE, create_jindo_oss_filesystem, ) +from pypaimon.filesystem.io_cache_routing import single_endpoint_options from pypaimon.schema.schema import Schema logger = logging.getLogger(__name__) @@ -883,11 +884,8 @@ def _get_filesystem(self, pvfs_table_identifier: PVFSTableIdentifier, storage_ty def _merge_token_with_catalog_options(self, token: dict) -> dict: """Merge token with catalog options, DLF OSS endpoint should override the standard OSS endpoint.""" - merged_token = dict(token) - dlf_oss_endpoint = self.options.get(CatalogOptions.DLF_OSS_ENDPOINT) - if dlf_oss_endpoint and dlf_oss_endpoint.strip(): - merged_token[OssOptions.OSS_ENDPOINT.key()] = dlf_oss_endpoint - return merged_token + # The OSS filesystem sends every request to one endpoint, so it cannot use io-cache targets. + return single_endpoint_options(token, self.options.to_map()) @staticmethod def _get_storage_type(path: str): diff --git a/paimon-python/pypaimon/filesystem/pyarrow_file_io.py b/paimon-python/pypaimon/filesystem/pyarrow_file_io.py index 6ce9e2e0f7c5..342d08011d8e 100644 --- a/paimon-python/pypaimon/filesystem/pyarrow_file_io.py +++ b/paimon-python/pypaimon/filesystem/pyarrow_file_io.py @@ -57,11 +57,18 @@ class LegacyOssDirectoryListingError(RuntimeError): class PyArrowFileIO(FileIO): + # False on an io-cache target, whose writes get their parent directories on origin. + creates_parent_dirs = True + # False on an io-cache target, whose failed writes are cleaned up on origin. + cleans_up_failed_writes = True + def __init__(self, path: str, catalog_options: Options): self.properties = catalog_options self.logger = logging.getLogger(__name__) - self._set_pyarrow_version() scheme, netloc, _ = self.parse_location(path) + # Path-style addressing sends bucket/key to the endpoint itself, so the bucket stays out of it. + self._oss_path_style = scheme == "oss" and self.properties.get(OssOptions.OSS_SECOND_LEVEL_DOMAIN_ENABLE) + self._set_pyarrow_version() self.uri_reader_factory = UriReaderFactory(catalog_options) self._is_oss = scheme in {"oss"} self._is_s3 = scheme in {"s3", "s3a", "s3n"} @@ -114,7 +121,7 @@ def _set_pyarrow_version(self): # goes into endpoint_override, so keys must omit it (init + path share # this flag so they can't drift). self._pyarrow_gte_16 = parse(pyarrow.__version__) >= parse("16.0.0") - self._oss_bucket_in_endpoint = not self._pyarrow_gte_16 + self._oss_bucket_in_endpoint = not self._pyarrow_gte_16 and not getattr(self, "_oss_path_style", False) def _uses_s3_compatibility(self) -> bool: return (not self._use_jindo @@ -265,12 +272,18 @@ def _initialize_oss_fs(self, path) -> FileSystem: "region": self.properties.get(OssOptions.OSS_REGION), } + endpoint = self.properties.get(OssOptions.OSS_ENDPOINT) + if endpoint and '://' in endpoint: + # endpoint_override takes host[:port]; a URL endpoint carries its scheme separately + scheme, _, endpoint = endpoint.partition('://') + client_kwargs['scheme'] = scheme.lower() + endpoint = endpoint.rstrip('/') if not self._oss_bucket_in_endpoint: - client_kwargs['force_virtual_addressing'] = True - client_kwargs['endpoint_override'] = self.properties.get(OssOptions.OSS_ENDPOINT) + if self._pyarrow_gte_16: + client_kwargs['force_virtual_addressing'] = not self._oss_path_style + client_kwargs['endpoint_override'] = endpoint else: - client_kwargs['endpoint_override'] = (self._oss_bucket + "." + - self.properties.get(OssOptions.OSS_ENDPOINT)) + client_kwargs['endpoint_override'] = self._oss_bucket + "." + endpoint retry_config = self._create_s3_retry_config() client_kwargs.update(retry_config) @@ -442,6 +455,11 @@ def new_input_stream(self, path: str): return self.filesystem.open_input_file(path_str) def new_output_stream(self, path: str): + if self.creates_parent_dirs: + self.create_parent_dirs(path) + return self.filesystem.open_output_stream(self.to_filesystem_path(path)) + + def create_parent_dirs(self, path: str): path_str = self.to_filesystem_path(path) if self._use_jindo: @@ -460,8 +478,6 @@ def new_output_stream(self, path: str): if str(parent_dir) and not self.exists(str(parent_dir)): self.mkdirs(str(parent_dir)) - return self.filesystem.open_output_stream(path_str) - def _get_file_info(self, path_str: str): try: file_infos = self.filesystem.get_file_info([path_str]) @@ -907,7 +923,8 @@ def write_parquet(self, path: str, data: pyarrow.Table, compression: str = 'zstd pq.write_table(data, output_stream, compression=compression, **kwargs) except Exception as e: - self.delete_quietly(path) + if self.cleans_up_failed_writes: + self.delete_quietly(path) raise RuntimeError(f"Failed to write Parquet file {path}: {e}") from e def write_orc(self, path: str, data: pyarrow.Table, compression: str = 'zstd', @@ -937,7 +954,8 @@ def write_orc(self, path: str, data: pyarrow.Table, compression: str = 'zstd', ) except Exception as e: - self.delete_quietly(path) + if self.cleans_up_failed_writes: + self.delete_quietly(path) raise RuntimeError(f"Failed to write ORC file {path}: {e}") from e def write_avro( diff --git a/paimon-python/pypaimon/filesystem/resolving_file_io.py b/paimon-python/pypaimon/filesystem/resolving_file_io.py index b5fd25bc8846..bc8d336f8ad9 100644 --- a/paimon-python/pypaimon/filesystem/resolving_file_io.py +++ b/paimon-python/pypaimon/filesystem/resolving_file_io.py @@ -15,6 +15,7 @@ # specific language governing permissions and limitations # under the License. +import copy from collections import defaultdict from typing import Dict, List from urllib.parse import urlparse @@ -33,6 +34,12 @@ class ResolvingFileIO(FileIO): This is the Python equivalent of Java's ``ResolvingFileIO``. """ + # Views applied to each resolved FileIO, see origin_only and for_table. + _views: tuple = () + # False on an io-cache target, for the PyArrowFileIOs it resolves. + creates_parent_dirs = True + cleans_up_failed_writes = True + def __init__(self, catalog_options: Options): opts_map = dict(catalog_options.to_map()) opts_map.pop(CatalogOptions.RESOLVING_FILE_IO_ENABLED.key(), None) @@ -49,9 +56,27 @@ def _get_fileio(self, path: str) -> FileIO: fileio = self._fileio_cache.get(cache_key) if fileio is None: fileio = FileIO.get(path, self._options) + from pypaimon.filesystem.pyarrow_file_io import PyArrowFileIO + if isinstance(fileio, PyArrowFileIO): + fileio.creates_parent_dirs = self.creates_parent_dirs + fileio.cleans_up_failed_writes = self.cleans_up_failed_writes self._fileio_cache[cache_key] = fileio + for view in self._views: + fileio = fileio.origin_only() if view[0] == "origin" else fileio.for_table(*view[1:]) return fileio + def origin_only(self) -> FileIO: + return self._with_view(("origin",)) + + def for_table(self, table_path: str, file_prefixes, branch: str) -> FileIO: + return self._with_view(("table", table_path, tuple(file_prefixes), branch)) + + def _with_view(self, view) -> 'ResolvingFileIO': + # A view shares the resolved FileIOs and applies itself to each of them. + resolving = copy.copy(self) + resolving._views = self._views + (view,) + return resolving + @property def properties(self): return self._options @@ -71,6 +96,9 @@ def new_input_stream(self, path: str): def new_output_stream(self, path: str): return self._get_fileio(path).new_output_stream(path) + def create_parent_dirs(self, path: str): + return self._get_fileio(path).create_parent_dirs(path) + def get_file_status(self, path: str): return self._get_fileio(path).get_file_status(path) @@ -134,6 +162,9 @@ def write_mosaic(self, path: str, data, **kwargs): return self._get_fileio(path).write_mosaic(path, data, **kwargs) def close(self): + if self._views: + # A view shares the FileIOs of the ResolvingFileIO it was made from, which closes them. + return first_error = None try: self.uri_reader_factory.close() diff --git a/paimon-python/pypaimon/filesystem/vortex_utils.py b/paimon-python/pypaimon/filesystem/vortex_utils.py index d66682898f9a..fea8cc7e2a98 100644 --- a/paimon-python/pypaimon/filesystem/vortex_utils.py +++ b/paimon-python/pypaimon/filesystem/vortex_utils.py @@ -54,9 +54,13 @@ def to_vortex_specified(file_io: FileIO, file_path: str) -> Tuple[str, Optional[ if scheme == 'oss' and properties: parsed = urlparse(file_path) bucket = parsed.netloc + endpoint = properties.get(OssOptions.OSS_ENDPOINT) + if endpoint and '://' in endpoint: + # The origin endpoint of io-cache routing can be a URL; the bucket goes before its host. + endpoint = endpoint.split('://', 1)[1].rstrip('/') store_kwargs = { - 'endpoint': f"https://{bucket}.{properties.get(OssOptions.OSS_ENDPOINT)}", + 'endpoint': f"https://{bucket}.{endpoint}", 'access_key_id': properties.get(OssOptions.OSS_ACCESS_KEY_ID), 'secret_access_key': properties.get(OssOptions.OSS_ACCESS_KEY_SECRET), 'virtual_hosted_style_request': 'true', diff --git a/paimon-python/pypaimon/table/file_store_table.py b/paimon-python/pypaimon/table/file_store_table.py index 9b190f5fe3fd..fd1a88d41bd6 100644 --- a/paimon-python/pypaimon/table/file_store_table.py +++ b/paimon-python/pypaimon/table/file_store_table.py @@ -22,6 +22,7 @@ from pypaimon.common.identifier import Identifier from pypaimon.common.options.core_options import CoreOptions from pypaimon.common.options.options import Options +from pypaimon.filesystem.io_cache_routing import schema_file_prefixes from pypaimon.read.read_builder import ReadBuilder from pypaimon.read.stream_read_builder import StreamReadBuilder from pypaimon.schema.schema_manager import SchemaManager @@ -34,9 +35,11 @@ class FileStoreTable(Table): def __init__(self, file_io: FileIO, identifier: Identifier, table_path: str, table_schema: TableSchema, catalog_environment: Optional[CatalogEnvironment] = None): - self.file_io = file_io self.identifier = identifier self.table_path = table_path + # Every table, copy included, tells io-cache routing where its files are and how they are named. + self.file_io = file_io.for_table( + table_path, schema_file_prefixes(table_schema.options), self.current_branch()) self.catalog_environment = catalog_environment or CatalogEnvironment.empty() self.table_schema = table_schema @@ -56,7 +59,7 @@ def __init__(self, file_io: FileIO, identifier: Identifier, table_path: str, self.total_buckets = self.options.bucket() self.schema_manager = SchemaManager( - file_io, table_path, branch=self.current_branch()) + self.file_io, table_path, branch=self.current_branch()) @classmethod def from_path(cls, table_path: str, file_io_options: Optional[dict] = None) -> 'FileStoreTable': diff --git a/paimon-python/pypaimon/table/format/format_table.py b/paimon-python/pypaimon/table/format/format_table.py index 1a1218153d16..3243e39e697a 100644 --- a/paimon-python/pypaimon/table/format/format_table.py +++ b/paimon-python/pypaimon/table/format/format_table.py @@ -54,7 +54,8 @@ def __init__( options: Optional[Dict[str, str]] = None, comment: Optional[str] = None, ): - self.file_io = file_io + # Other writers may replace Format Table files in place. + self.file_io = file_io.origin_only() self.identifier = identifier self._table_schema = table_schema self._location = location.rstrip("/") diff --git a/paimon-python/pypaimon/table/iceberg/iceberg_table.py b/paimon-python/pypaimon/table/iceberg/iceberg_table.py index 8bf17e9544af..74cea8fdd1ba 100644 --- a/paimon-python/pypaimon/table/iceberg/iceberg_table.py +++ b/paimon-python/pypaimon/table/iceberg/iceberg_table.py @@ -40,7 +40,8 @@ def __init__( comment: Optional[str] = None, uuid: Optional[str] = None, ): - self.file_io = file_io + # Iceberg writers own these files. + self.file_io = file_io.origin_only() self.identifier = identifier self._table_schema = table_schema self._location = location.rstrip("/") diff --git a/paimon-python/pypaimon/table/object/object_table.py b/paimon-python/pypaimon/table/object/object_table.py index f875b4d3f6cb..10f1c64ffba0 100644 --- a/paimon-python/pypaimon/table/object/object_table.py +++ b/paimon-python/pypaimon/table/object/object_table.py @@ -45,7 +45,8 @@ def __init__( options: Optional[Dict[str, str]] = None, comment: Optional[str] = None, ): - self.file_io = file_io + # Other writers manage the objects, so they may change under the same name. + self.file_io = file_io.origin_only() self.identifier = identifier self._table_schema = table_schema self._location = location.rstrip("/") diff --git a/paimon-python/pypaimon/tests/caching_file_io_test.py b/paimon-python/pypaimon/tests/caching_file_io_test.py index 88edfbcf4b8e..7963e0630666 100644 --- a/paimon-python/pypaimon/tests/caching_file_io_test.py +++ b/paimon-python/pypaimon/tests/caching_file_io_test.py @@ -659,6 +659,24 @@ def test_write_parquet_when_enable_local_cache(self): self.assertTrue(os.path.exists(out_path)) self.assertEqual(table, pa.parquet.read_table(out_path)) + def test_table_views_wrap_the_view_of_the_delegate(self): + from pypaimon.utils.file_type import FileType + delegate = MagicMock() + origin, table_view = MagicMock(), MagicMock() + delegate.origin_only.return_value = origin + delegate.for_table.side_effect = lambda path, prefixes, branch: table_view if prefixes else delegate + cache = LocalMemoryCacheManager(128, 64) + caching_io = CachingFileIO(delegate, cache, {FileType.DATA}) + + for view, expected in ((caching_io.origin_only(), origin), + (caching_io.for_table("oss://bkt/t", ["custom-"], "main"), table_view)): + self.assertIsInstance(view, CachingFileIO) + self.assertIs(expected, view._delegate) + self.assertIs(cache, view._cache) + self.assertEqual({FileType.DATA}, view._whitelist) + # A delegate without anything to narrow keeps the wrapper. + self.assertIs(caching_io, caching_io.for_table("oss://bkt/t", [], "main")) + def test_file_io_for_data_wraps_cache_when_data_token_enabled(self): from pypaimon.catalog.rest.rest_catalog import RESTCatalog from pypaimon.common.identifier import Identifier diff --git a/paimon-python/pypaimon/tests/file_io_test.py b/paimon-python/pypaimon/tests/file_io_test.py index 53c8e8d29698..9c7e0a8232a2 100644 --- a/paimon-python/pypaimon/tests/file_io_test.py +++ b/paimon-python/pypaimon/tests/file_io_test.py @@ -16,6 +16,7 @@ # under the License. import os +import pickle import shutil import tempfile import unittest @@ -149,6 +150,49 @@ def record_get_file_info(paths): "OSS+PyArrow<16 must pass key only to get_file_info, not bucket/key. Got: %r" % (p,) ) + def test_legacy_oss_url_endpoint_passes_scheme(self): + options = { + OssOptions.OSS_ACCESS_KEY_ID.key(): 'test-key', + OssOptions.OSS_ACCESS_KEY_SECRET.key(): 'test-secret', + OssOptions.OSS_IMPL.key(): 'legacy', + } + cases = (('http://127.0.0.1:9000/', 'http', '127.0.0.1:9000'), + ('HTTPS://oss-cn-hangzhou.aliyuncs.com', 'https', 'oss-cn-hangzhou.aliyuncs.com'), + ('oss-cn-hangzhou.aliyuncs.com', None, 'oss-cn-hangzhou.aliyuncs.com')) + for endpoint, scheme, host in cases: + with patch('pypaimon.filesystem.pyarrow_file_io.pafs.S3FileSystem') as s3_file_system: + oss_io = OssFileIO("oss://test-bucket/warehouse", Options( + {**options, OssOptions.OSS_ENDPOINT.key(): endpoint})) + kwargs = s3_file_system.call_args[1] + self.assertEqual(scheme, kwargs.get('scheme')) + expected = 'test-bucket.' + host if oss_io._oss_bucket_in_endpoint else host + self.assertEqual(expected, kwargs['endpoint_override']) + + def test_legacy_oss_path_style_keeps_bucket_out_of_endpoint(self): + options = Options({ + OssOptions.OSS_ENDPOINT.key(): 'http://10.0.0.1:8080', + OssOptions.OSS_ACCESS_KEY_ID.key(): 'test-key', + OssOptions.OSS_ACCESS_KEY_SECRET.key(): 'test-secret', + OssOptions.OSS_IMPL.key(): 'legacy', + OssOptions.OSS_SECOND_LEVEL_DOMAIN_ENABLE.key(): 'true', + }) + for version, virtual in (('15.0.0', None), ('16.0.0', False)): + # the retry strategy is missing from the old pyarrow some CI jobs install + with patch('pyarrow.__version__', version), \ + patch('pypaimon.filesystem.pyarrow_file_io.pafs.AwsStandardS3RetryStrategy', create=True), \ + patch('pypaimon.filesystem.pyarrow_file_io.pafs.S3FileSystem') as s3_file_system: + oss_io = OssFileIO("oss://test-bucket/warehouse", options) + kwargs = s3_file_system.call_args[1] + # unpickling (e.g. in a Ray worker) must keep path-style addressing + restored = pickle.loads(pickle.dumps(oss_io)) + self.assertEqual('10.0.0.1:8080', kwargs['endpoint_override']) + self.assertEqual(virtual, kwargs.get('force_virtual_addressing')) + self.assertFalse(oss_io._oss_bucket_in_endpoint) + self.assertFalse(restored._oss_bucket_in_endpoint) + oss_io = OssFileIO("oss://test-bucket/warehouse", options) + self.assertEqual("test-bucket/warehouse/data-1.parquet", + oss_io.to_filesystem_path("oss://test-bucket/warehouse/data-1.parquet")) + def test_exists(self): lt7 = _pyarrow_lt_7() with tempfile.TemporaryDirectory(prefix="file_io_nonexistent_") as tmpdir: diff --git a/paimon-python/pypaimon/tests/io_cache_effective_options_test.py b/paimon-python/pypaimon/tests/io_cache_effective_options_test.py new file mode 100644 index 000000000000..fa6350e98067 --- /dev/null +++ b/paimon-python/pypaimon/tests/io_cache_effective_options_test.py @@ -0,0 +1,268 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Every component takes io-cache routing from the effective options: catalog options merged with the token.""" + +import sys +import threading +import time +import unittest +from http.server import BaseHTTPRequestHandler, HTTPServer +from socketserver import ThreadingMixIn +from types import ModuleType, SimpleNamespace +from unittest import mock +from urllib.parse import unquote, urlsplit + +from pypaimon.catalog.rest.rest_token_file_io import RESTTokenFileIO +from pypaimon.common.file_io import FileIO +from pypaimon.common.identifier import Identifier +from pypaimon.common.options import Options +from pypaimon.common.options.config import CatalogOptions +from pypaimon.common.uri_reader import UriReaderFactory +from pypaimon.filesystem.io_cache_routing import IoCacheRouting + +OBJECT = "/bkt/raw/cam0/000123.bin" +URI = "oss://bkt/raw/cam0/000123.bin" + + +class _ObjectEndpoint: + """An OSS endpoint for the real PyArrow client: path-style HEAD and GET of fixed objects.""" + + def __init__(self, objects): + self.objects = objects + self.requests = [] + endpoint = self + + class Handler(BaseHTTPRequestHandler): + def log_message(self, *args): + pass + + def do_HEAD(self): + self._serve(False) + + def do_GET(self): + self._serve(True) + + def _serve(self, with_body): + path = unquote(urlsplit(self.path).path) + endpoint.requests.append((self.command, path)) + data = endpoint.objects.get(path) + if data is None: + body = b"NoSuchKey" if with_body else b"" + self.send_response(404) + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + return + start, end, status = 0, len(data), 200 + ranged = self.headers.get("Range") + if ranged and ranged.startswith("bytes="): + first, _, last = ranged[len("bytes="):].partition("-") + start, end, status = int(first), min(len(data), int(last) + 1 if last else len(data)), 206 + self.send_response(status) + self.send_header("Content-Length", str(end - start)) + self.send_header("Content-Type", "application/octet-stream") + self.send_header("ETag", '"etag"') + self.send_header("Last-Modified", "Mon, 26 Oct 2026 00:00:00 GMT") + if status == 206: + self.send_header("Content-Range", "bytes {}-{}/{}".format(start, end - 1, len(data))) + self.end_headers() + if with_body: + self.wfile.write(data[start:end]) + + class Server(ThreadingMixIn, HTTPServer): + daemon_threads = True + + self.server = Server(("127.0.0.1", 0), Handler) + self.url = "http://127.0.0.1:{}".format(self.server.server_port) + self.thread = threading.Thread(target=lambda: self.server.serve_forever(poll_interval=0.01), daemon=True) + self.thread.start() + + def close(self): + self.server.shutdown() + self.server.server_close() + self.thread.join() + + +class EffectiveOptionsTest(unittest.TestCase): + """The origin holds the current object; a legacy cache, still fs.oss.endpoint for old clients, keeps an old one.""" + + def setUp(self): + self.origin = _ObjectEndpoint({OBJECT: b"NEW"}) + self.cache = _ObjectEndpoint({OBJECT: b"OLD"}) + self.addCleanup(self.origin.close) + self.addCleanup(self.cache.close) + + def _routing_options(self): + return { + "fs.oss.impl": "legacy", + "fs.oss.region": "cn-hangzhou", + "fs.oss.accessKeyId": "ak", + "fs.oss.accessKeySecret": "sk", + "fs.oss.second.level.domain.enable": "true", + "fs.oss.endpoint": self.cache.url, + "io-cache.enabled": "true", + "io-cache.endpoint": self.cache.url, + "io-cache.target.default.path-style-access": "true", + "io-cache.origin.endpoint": self.origin.url, + "io-cache.policy": "meta,read", + } + + def _assert_reads_origin(self, read): + self.assertEqual(b"NEW", read()) + self.assertEqual([], self.cache.requests) + self.assertTrue(self.origin.requests) + + def test_uri_reader_factory_reads_row_data_uris_from_origin(self): + reader = UriReaderFactory(Options(self._routing_options())).create(URI) + self._assert_reads_origin(lambda: reader.new_input_stream(URI).read()) + + def test_source_file_io_reads_import_sources_from_origin(self): + from pypaimon.multimodal.source_utils import _SourceFileIO + + source = _SourceFileIO(Options(self._routing_options())) + self._assert_reads_origin(lambda: source.new_input_stream(URI).read()) + + def test_file_io_of_a_catalog_without_tokens_reads_from_origin(self): + from pypaimon.catalog.filesystem_catalog import FileSystemCatalog + + catalog = FileSystemCatalog(Options(dict(self._routing_options(), warehouse="oss://bkt/wh"))) + self._assert_reads_origin(lambda: catalog.file_io.new_input_stream(URI).read()) + + def test_resolving_file_io_reads_from_origin(self): + options = dict(self._routing_options(), **{CatalogOptions.RESOLVING_FILE_IO_ENABLED.key(): "true"}) + file_io = FileIO.get(URI, Options(options)) + self._assert_reads_origin(lambda: file_io.new_input_stream(URI).read()) + + def test_file_ios_of_routing_targets_keep_their_endpoints(self): + options = self._routing_options() + routing = IoCacheRouting.create(options) + target = FileIO.get(URI, Options(routing.target_options(options, "default"))) + self.assertEqual(b"OLD", target.new_input_stream(URI).read()) + self.assertEqual([], self.origin.requests) + + @unittest.skipIf(sys.version_info < (3, 7), "pypaimon.daft needs Python 3.7+") + def test_daft_reads_catalog_only_routing_from_origin(self): + from pypaimon.daft.daft_paimon import _enrich_options_with_rest_token + + options = dict(self._routing_options(), metastore="filesystem", warehouse="oss://bkt/wh") + enriched = _enrich_options_with_rest_token(options, SimpleNamespace(table_path="oss://bkt/wh/db.db/t")) + self.assertEqual((self.origin.url, "oss://bkt/wh"), (enriched["fs.oss.endpoint"], enriched["warehouse"])) + + def test_pvfs_reads_through_the_origin_of_catalog_only_routing(self): + from pypaimon.filesystem.pvfs import PaimonVirtualFileSystem + + # The token carries credentials and the legacy endpoint; routing is configured on the catalog only. + token = {key: value for key, value in self._routing_options().items() if not key.startswith("io-cache.")} + catalog = dict({key: value for key, value in self._routing_options().items() if key.startswith("io-cache.")}, + uri="http://rest.example.com", warehouse="wh", **{"fs.oss.impl": "legacy"}) + rest_api = SimpleNamespace( + get_table=lambda identifier: SimpleNamespace(path="oss://bkt/raw", created_at=0, updated_at=0), + load_table_token=lambda identifier: SimpleNamespace( + token=token, expires_at_millis=int(time.time() * 1000) + 7_200_000)) + objects = {self.cache.url: b"old-object", self.origin.url: b"new-object"} + backends = [] + + def ossfs(options): + backends.append(options.to_map()["fs.oss.endpoint"]) + return SimpleNamespace(cat_file=lambda path, start, end, **kwargs: objects[backends[-1]]) + + pvfs = PaimonVirtualFileSystem(catalog, skip_instance_cache=True) + with mock.patch.object(PaimonVirtualFileSystem, "_PaimonVirtualFileSystem__rest_api", return_value=rest_api), \ + mock.patch.object(PaimonVirtualFileSystem, "_get_ossfs_filesystem", side_effect=ossfs): + self.assertEqual(b"new-object", pvfs.cat_file("pvfs://wh/db/t/cam0/000123.bin")) + self.assertEqual([self.origin.url], backends) + + +class SharedTokenTest(unittest.TestCase): + """Two catalogs that set io-cache.enabled differently share the token the REST server vended for a table.""" + + TABLE = "oss://bkt/wh/db.db/t" + TOKEN = { + "fs.oss.endpoint": "http://legacy.example.com", + "fs.oss.accessKeyId": "ak", + "fs.oss.accessKeySecret": "sk", + "io-cache.enabled": "true", + "io-cache.endpoint": "http://legacy.example.com", + "io-cache.origin.endpoint": "https://origin.example.com", + "io-cache.policy": "meta,read", + } + + def setUp(self): + self.vended = [] + patcher = mock.patch.object(RESTTokenFileIO, "_build_cache_key", return_value="shared-token-test") + patcher.start() + self.addCleanup(patcher.stop) + self.addCleanup(RESTTokenFileIO._TOKEN_CACHE.pop, "shared-token-test", None) + + def _file_io(self, catalog_options): + file_io = RESTTokenFileIO(Identifier.create("db", "t"), self.TABLE, Options(dict(catalog_options))) + + def load_table_token(identifier): + self.vended.append(identifier) + return SimpleNamespace(token=dict(self.TOKEN), expires_at_millis=int(time.time() * 1000) + 7_200_000) + + file_io.api_instance = SimpleNamespace(load_table_token=load_table_token) + file_io.try_to_refresh_token() + return file_io + + def _native_data_token(self, file_io): + from pypaimon.execution.native.table import _NativeRestTableCache + + fake_df = ModuleType("pypaimon_rust.datafusion") + fake_df.Table = mock.Mock() + with mock.patch.dict(sys.modules, {"pypaimon_rust.datafusion": fake_df}): + _NativeRestTableCache().get("response", "db", "t", file_io.catalog_options.to_map(), + token_loader=file_io._existing_valid_token) + return fake_df.Table.from_rest_response_with_token.call_args[1]["data_token"] + + def _daft_options(self, file_io): + from pypaimon.daft.daft_paimon import _enrich_options_with_rest_token + + catalog_options = dict(file_io.catalog_options.to_map(), metastore="rest") + return _enrich_options_with_rest_token(catalog_options, SimpleNamespace(file_io=file_io, table_path=self.TABLE)) + + def _assert_routes(self, file_io, routed): + endpoint = "https://origin.example.com" if routed else "http://legacy.example.com" + self.assertEqual(routed, IoCacheRouting.create(file_io._merged_options(file_io.token).to_map()) is not None) + self.assertEqual(endpoint, self._native_data_token(file_io)["fs.oss.endpoint"]) + if sys.version_info >= (3, 7): + self.assertEqual(endpoint, self._daft_options(file_io)["fs.oss.endpoint"]) + + def test_each_catalog_applies_its_own_switch_to_a_shared_token(self): + for first, second in (({"io-cache.enabled": "false"}, {"io-cache.enabled": "true"}), + ({"io-cache.enabled": "false"}, {}), + ({"io-cache.enabled": "true"}, {"io-cache.enabled": "false"})): + with self.subTest(first=first, second=second): + RESTTokenFileIO._TOKEN_CACHE.pop("shared-token-test", None) + self.vended.clear() + file_ios = [self._file_io(first), self._file_io(second)] + self.assertEqual(1, len(self.vended)) + self.assertEqual(self.TOKEN, RESTTokenFileIO._TOKEN_CACHE["shared-token-test"].token) + for file_io, catalog_options in zip(file_ios, (first, second)): + self._assert_routes(file_io, catalog_options.get("io-cache.enabled", "true") == "true") + + def test_dlf_oss_endpoint_of_one_catalog_stays_with_it(self): + pinned = self._file_io({"dlf.oss-endpoint": "oss-cn-test.example.com"}) + shared = self._file_io({}) + self.assertEqual("oss-cn-test.example.com", pinned._merged_options(pinned.token).to_map()["fs.oss.endpoint"]) + self.assertEqual("oss-cn-test.example.com", self._native_data_token(pinned)["fs.oss.endpoint"]) + self._assert_routes(shared, True) + + +if __name__ == "__main__": + unittest.main() diff --git a/paimon-python/pypaimon/tests/io_cache_file_io_test.py b/paimon-python/pypaimon/tests/io_cache_file_io_test.py new file mode 100644 index 000000000000..d760fc4c5aa4 --- /dev/null +++ b/paimon-python/pypaimon/tests/io_cache_file_io_test.py @@ -0,0 +1,855 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Tests for IoCacheRoutingFileIO and its pyarrow filesystem.""" + +import contextlib +import gzip +import inspect +import io +import multiprocessing +import os +import shutil +import tempfile +import threading +import unittest +from datetime import timedelta +from pathlib import Path +from unittest import mock +from urllib.parse import urlparse + +import pyarrow as pa +import pyarrow.dataset as ds +import pyarrow.fs as pafs +import pyarrow.parquet as pq +import pytest +from pyarrow._fs import FileSystemHandler + +from pypaimon.common.file_io import FileIO +from pypaimon.common.options import Options +from pypaimon.execution.python.read.reader.format_avro_reader import FormatAvroReader +from pypaimon.execution.python.read.reader.format_pyarrow_reader import FormatPyArrowReader +from pypaimon.filesystem.caching_file_io import (CachingFileIO, + LocalMemoryCacheManager) +from pypaimon.filesystem.io_cache_file_io import IoCacheRoutingFileIO +from pypaimon.filesystem.io_cache_routing import IoCacheRouting +from pypaimon.filesystem.local_file_io import LocalFileIO +from pypaimon.filesystem.oss_file_io import OssFileIO +from pypaimon.filesystem.resolving_file_io import ResolvingFileIO +from pypaimon.schema.data_types import AtomicType, DataField +from pypaimon.table.row.blob import BlobDescriptor +from pypaimon.utils.file_type import FileType + +TABLE = "oss://bkt/db1.db/t1" +UUID = "8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b" +PARTITION = TABLE + "/dt=1/bucket-0" +DATA = PARTITION + "/data-" + UUID + "-0.parquet" +OTHER_DATA = PARTITION + "/data-" + UUID + "-1.parquet" +UNKNOWN = PARTITION + "/part-" + UUID + "-0.parquet" +MANIFEST = TABLE + "/manifest/manifest-" + UUID + "-0" +SNAPSHOT = TABLE + "/snapshot/snapshot-1" +LATEST = TABLE + "/snapshot/LATEST" + + +def _routing(policy="meta,read", whitelist="*", **overrides): + """Manifests go to the accel target, data files to the cluster target.""" + options = { + "fs.oss.endpoint": "http://accel.example.com", + "io-cache.enabled": "true", + "io-cache.origin.endpoint": "https://origin.example.com", + "io-cache.policy": policy, + "io-cache.whitelist": whitelist, + "io-cache.targets": "accel,cluster", + "io-cache.target.accel.endpoint": "http://accel.example.com", + "io-cache.target.cluster.endpoint": "http://10.0.0.1:8080", + "io-cache.routes": "meta=accel;data=cluster", + } + options.update(overrides) + return IoCacheRouting.create(options) + + +class _DirFileIO(LocalFileIO): + """LocalFileIO that keeps oss://bucket/key under root/bucket/key, standing in for one endpoint.""" + + def __init__(self, root): + super().__init__() + self.root = root + self.error = None + self.attempts = 0 + self.closed = False + + def _to_file(self, path): + parsed = urlparse(path) + if parsed.scheme != "oss": + return super()._to_file(path) + return Path(self.root, parsed.netloc, parsed.path.lstrip("/")) + + def _check(self): + self.attempts += 1 + if self.error is not None: + raise self.error + + def new_input_stream(self, path): + self._check() + return super().new_input_stream(path) + + def new_output_stream(self, path): + self._check() + return super().new_output_stream(path) + + def get_file_status(self, path): + self._check() + return super().get_file_status(path) + + def exists(self, path): + self._check() + return super().exists(path) + + def to_filesystem_path(self, path): + # Only the routing filesystem calls this, right before it calls the filesystem. + self._check() + return super().to_filesystem_path(path) + + def close(self): + self.closed = True + + +class IoCacheRoutingFileIOTest(unittest.TestCase): + + def setUp(self): + self.temp_dir = tempfile.mkdtemp(prefix="io_cache_file_io_test_") + self.origin = _DirFileIO(os.path.join(self.temp_dir, "origin")) + self.accel = _DirFileIO(os.path.join(self.temp_dir, "accel")) + self.cluster = _DirFileIO(os.path.join(self.temp_dir, "cluster")) + self.created = [] + self.creation_error = None + self.file_io = self._file_io(_routing()) + + def tearDown(self): + shutil.rmtree(self.temp_dir, ignore_errors=True) + + def _create_target(self, name): + self.created.append(name) + if self.creation_error is not None: + raise self.creation_error + return {"accel": self.accel, "cluster": self.cluster}[name] + + def _file_io(self, routing): + return IoCacheRoutingFileIO(routing, self.origin, self._create_target) + + @staticmethod + def _put(file_io, path, content): + file_io._to_file(path).parent.mkdir(parents=True, exist_ok=True) + file_io._to_file(path).write_bytes(content) + + def _put_all(self, *paths): + for path in paths: + for name, file_io in (("origin", self.origin), ("accel", self.accel), ("cluster", self.cluster)): + self._put(file_io, path, name.encode()) + + @staticmethod + def _content(file_io, path): + file = file_io._to_file(path) + return file.read_bytes() if file.exists() else None + + def _attempts(self): + return self.origin.attempts, self.accel.attempts, self.cluster.attempts + + def test_reads_follow_the_route(self): + self._put_all(DATA, MANIFEST, LATEST, UNKNOWN, SNAPSHOT) + self.assertEqual(b"cluster", self.file_io.new_input_stream(DATA).read()) + self.assertEqual("cluster", self.file_io.read_file_utf8(DATA)) + self.assertEqual(b"lus", self.file_io.read_file_range(DATA, 1, 3)) + self.assertEqual(b"accel", self.file_io.new_input_stream(MANIFEST).read()) + for path in (LATEST, UNKNOWN, SNAPSHOT): + with self.subTest(path=path): + self._put(self.origin, path, b"origin") + self.assertEqual(b"origin", self.file_io.new_input_stream(path).read()) + + def test_single_target_takes_every_routed_type(self): + self._put_all(DATA, MANIFEST) + routing = IoCacheRouting.create({ + "io-cache.enabled": "true", + "io-cache.endpoint": "http://cluster.example.com", + "io-cache.policy": "read", + }) + file_io = IoCacheRoutingFileIO(routing, self.origin, lambda name: {"default": self.cluster}[name]) + self.assertEqual(b"cluster", file_io.new_input_stream(DATA).read()) + self.assertEqual(b"cluster", file_io.filesystem.open_input_file(MANIFEST).read()) + self.assertEqual(6, file_io.get_file_size(DATA)) + + def test_targets_are_created_on_first_use(self): + self._put_all(DATA, MANIFEST, LATEST) + self.file_io.new_input_stream(LATEST) + self.assertEqual([], self.created) + self.file_io.new_input_stream(DATA) + self.file_io.get_file_status(DATA) + self.assertEqual(["cluster"], self.created) + self.file_io.new_input_stream(MANIFEST) + self.assertEqual(["cluster", "accel"], self.created) + + @unittest.skipUnless(hasattr(os, "fork"), "needs fork") + def test_forked_child_does_not_wait_on_the_parents_target_lock(self): + self._put_all(DATA) + parent_pid = os.getpid() + creating, release = threading.Event(), threading.Event() + + def create_target(name): + # the parent stays inside the lock until released; a forked child must not wait for it + if os.getpid() == parent_pid: + creating.set() + release.wait(30) + return self.cluster + + file_io = IoCacheRoutingFileIO(_routing(), self.origin, create_target) + parent = threading.Thread(target=lambda: file_io.new_input_stream(DATA).read()) + parent.start() + try: + self.assertTrue(creating.wait(10)) + child = multiprocessing.get_context("fork").Process( + target=lambda: os._exit(0 if file_io.new_input_stream(DATA).read() == b"cluster" else 1)) + child.start() + child.join(10) + if child.exitcode is None: + child.kill() + child.join() + self.assertEqual(0, child.exitcode) + finally: + release.set() + parent.join(10) + + def test_forked_child_does_not_wait_on_the_parents_schema_history(self): + older = PARTITION + "/older-" + UUID + "-0.parquet" + self._put_all(older) + parent_pid = os.getpid() + listing, release = threading.Event(), threading.Event() + + def list_all(manager): + # the parent stays inside the history lock until released; a forked child must not wait for it + if os.getpid() == parent_pid: + listing.set() + release.wait(30) + return [mock.Mock(options={"data-file.prefix": "older-"})] + + with mock.patch("pypaimon.schema.schema_manager.SchemaManager.list_all", list_all): + view = self.file_io.for_table(TABLE, [], "main") + parent = threading.Thread(target=lambda: view.new_input_stream(older).read()) + parent.start() + try: + self.assertTrue(listing.wait(10)) + child = multiprocessing.get_context("fork").Process( + target=lambda: os._exit(0 if view.new_input_stream(older).read() == b"cluster" else 1)) + child.start() + child.join(10) + if child.exitcode is None: + child.kill() + child.join() + self.assertEqual(0, child.exitcode) + finally: + release.set() + parent.join(10) + + def test_reads_without_read_policy_use_origin(self): + self._put_all(DATA) + file_io = self._file_io(_routing(policy="meta")) + self.assertEqual(b"origin", file_io.new_input_stream(DATA).read()) + self.assertEqual(7, file_io.get_file_size(DATA)) + self.assertEqual(b"origin", self._file_io(_routing(whitelist="meta")).new_input_stream(DATA).read()) + + def test_meta_follows_the_route(self): + self._put(self.cluster, DATA, b"cached!") + self._put(self.accel, MANIFEST, b"m") + self.assertEqual(7, self.file_io.get_file_size(DATA)) + self.assertEqual(1, self.file_io.get_file_status(MANIFEST).size) + with self.assertRaises(FileNotFoundError): + self._file_io(_routing(policy="read")).get_file_size(DATA) + + def test_exists_follows_the_route(self): + self._put(self.cluster, DATA, b"cached!") + self._put(self.accel, MANIFEST, b"m") + self._put(self.origin, OTHER_DATA, b"origin") + self._put(self.origin, SNAPSHOT, b"origin") + self._put(self.origin, TABLE + "/manifest/manifest-list-" + UUID + "-0", b"origin") + # Without the exists token the origin, which has neither file, answers. + self.assertFalse(self.file_io.exists(DATA)) + self.file_io = self._file_io(_routing(policy="meta,read,exists")) + self.assertTrue(self.file_io.exists(DATA)) + # OTHER_DATA is routed to the cluster, which does not have it + self.assertFalse(self.file_io.exists(OTHER_DATA)) + self.assertTrue(self.file_io.exists(SNAPSHOT)) + self.assertEqual({MANIFEST: True, DATA: True, SNAPSHOT: True, OTHER_DATA: False}, + self.file_io.exists_batch([MANIFEST, DATA, SNAPSHOT, OTHER_DATA])) + self.assertTrue(self._file_io(_routing(policy="read")).exists(OTHER_DATA)) + self.assertTrue(self.file_io.is_dir(PARTITION)) + self.assertTrue(self.file_io.is_dir(TABLE + "/manifest")) + + def test_listing_and_mutations_use_origin(self): + self._put_all(DATA, SNAPSHOT) + listed = self.file_io.list_status(PARTITION) + self.assertEqual([str(self.origin._to_file(DATA))], [status.path for status in listed]) + + self.assertTrue(self.file_io.rename(DATA, OTHER_DATA)) + self.assertIsNone(self._content(self.origin, DATA)) + self.assertEqual(b"origin", self._content(self.origin, OTHER_DATA)) + self.assertEqual(b"cluster", self._content(self.cluster, DATA)) + + self.file_io.copy_file(OTHER_DATA, DATA) + self.assertEqual(b"origin", self._content(self.origin, DATA)) + self.assertIsNone(self._content(self.cluster, OTHER_DATA)) + + self.assertTrue(self.file_io.delete(DATA)) + self.assertIsNone(self._content(self.origin, DATA)) + self.assertEqual(b"cluster", self._content(self.cluster, DATA)) + + self.assertTrue(self.file_io.mkdirs(TABLE + "/new-dir")) + self.assertTrue(self.origin._to_file(TABLE + "/new-dir").is_dir()) + self.assertFalse(self.cluster._to_file(TABLE + "/new-dir").exists()) + + self.assertTrue(self.file_io.try_to_write_atomic(TABLE + "/snapshot/snapshot-2", "{}")) + self.assertEqual(b"{}", self._content(self.origin, TABLE + "/snapshot/snapshot-2")) + self.assertEqual([], self.created) + + def test_writes_use_origin_without_write_policy(self): + file_io = self._file_io(_routing(policy="read,meta")) + with file_io.new_output_stream(DATA) as out: + out.write(b"written") + file_io.write_file(MANIFEST, "3") + self.assertEqual(b"written", self._content(self.origin, DATA)) + self.assertEqual(b"3", self._content(self.origin, MANIFEST)) + self.assertEqual([], self.created) + + def test_writes_follow_the_route_with_write_policy(self): + table = pa.table({"a": [1, 2]}) + file_io = self._file_io(_routing(policy="read,meta,write")) + with file_io.new_output_stream(DATA) as out: + out.write(b"written") + file_io.write_parquet(OTHER_DATA, table) + file_io.write_file(MANIFEST, "3") + file_io.write_file(SNAPSHOT, "{}") + self.assertEqual(b"written", self._content(self.cluster, DATA)) + self.assertEqual(table, pq.ParquetFile(str(self.cluster._to_file(OTHER_DATA))).read()) + self.assertEqual(b"3", self._content(self.accel, MANIFEST)) + self.assertEqual(b"{}", self._content(self.origin, SNAPSHOT)) + # write_file checks existence on origin, not on the target + self._put(self.origin, OTHER_DATA, b"origin") + # unknown names are written on origin + with file_io.filesystem.open_output_stream(UNKNOWN) as out: + out.write(b"streamed") + self.assertEqual(b"streamed", self._content(self.origin, UNKNOWN)) + with self.assertRaises(FileExistsError): + file_io.write_file(OTHER_DATA, "again") + + def test_filesystem_reads_follow_the_route(self): + for file_io, value in ((self.origin, 1), (self.cluster, 2)): + sink = pa.BufferOutputStream() + pq.write_table(pa.table({"a": [value]}), sink) + self._put(file_io, DATA, sink.getvalue().to_pybytes()) + filesystem = self.file_io.filesystem + self.assertIs(filesystem, self.file_io.filesystem) + path = self.file_io.to_filesystem_path(DATA) + self.assertEqual([2], pq.read_table(path, filesystem=filesystem).column("a").to_pylist()) + with filesystem.open_input_file(path) as f: + self.assertEqual([2], pq.read_table(f).column("a").to_pylist()) + origin_only = self._file_io(_routing(policy="meta")) + self.assertEqual([1], pq.read_table(DATA, filesystem=origin_only.filesystem).column("a").to_pylist()) + + def test_readers_use_the_routing_filesystem(self): + for file_io, value in ((self.origin, 1), (self.accel, 2), (self.cluster, 3)): + file_io.write_parquet(DATA, pa.table({"a": pa.array([value], pa.int64())})) + file_io.write_avro(MANIFEST, pa.table({"a": pa.array([value], pa.int64())})) + fields = [DataField(0, "a", AtomicType("BIGINT"))] + size = self.cluster._to_file(DATA).stat().st_size + + reader = FormatPyArrowReader(self.file_io, "parquet", DATA, fields, None, file_size=size) + self.assertEqual([3], reader.read_arrow_batch().column(0).to_pylist()) + reader = FormatAvroReader(self.file_io, MANIFEST, ["a"], fields, None) + self.assertEqual([2], reader.read_arrow_batch().column(0).to_pylist()) + reader = FormatPyArrowReader(self._file_io(_routing(policy="meta")), "parquet", DATA, fields, None) + self.assertEqual([1], reader.read_arrow_batch().column(0).to_pylist()) + + def test_filesystem_file_info_follows_the_route(self): + self._put(self.cluster, DATA, b"cached!") + self._put(self.accel, MANIFEST, b"m") + self._put(self.origin, LATEST, b"1") + infos = self.file_io.filesystem.get_file_info([LATEST, DATA, MANIFEST, OTHER_DATA]) + self.assertEqual([pafs.FileType.File, pafs.FileType.File, pafs.FileType.File, pafs.FileType.NotFound], + [info.type for info in infos]) + self.assertEqual([1, 7, 1], [info.size for info in infos[:3]]) + self.assertEqual([LATEST, DATA, MANIFEST, OTHER_DATA], [info.path for info in infos]) + + def test_directory_dataset_keeps_oss_paths_and_routes_reads(self): + for file_io, value in ((self.origin, 1), (self.cluster, 2)): + file_io.write_parquet(DATA, pa.table({"a": [value]})) + dataset = ds.dataset(PARTITION, format="parquet", filesystem=self.file_io.filesystem) + self.assertEqual([2], dataset.to_table().column("a").to_pylist()) + + def test_filesystem_mutations_use_origin(self): + self._put_all(DATA) + filesystem = self.file_io.filesystem + selected = filesystem.get_file_info(pafs.FileSelector(PARTITION)) + self.assertEqual([DATA], [info.path for info in selected]) + filesystem.copy_file(DATA, OTHER_DATA) + self.assertEqual(b"origin", self._content(self.origin, OTHER_DATA)) + filesystem.delete_file(OTHER_DATA) + filesystem.move(DATA, OTHER_DATA) + self.assertEqual(b"origin", self._content(self.origin, OTHER_DATA)) + self.assertEqual(b"cluster", self._content(self.cluster, DATA)) + filesystem.create_dir(TABLE + "/new-dir") + self.assertTrue(self.origin._to_file(TABLE + "/new-dir").is_dir()) + filesystem.delete_dir(TABLE + "/new-dir") + filesystem.delete_dir_contents(PARTITION) + self.assertEqual([], list(self.origin._to_file(PARTITION).iterdir())) + self.assertEqual(b"cluster", self._content(self.cluster, DATA)) + self.assertEqual([], self.created) + + def test_filesystem_streams_are_compressed_once(self): + path = PARTITION + "/data-" + UUID + "-0.json.gz" + self.origin._to_file(path).parent.mkdir(parents=True) + with self.file_io.filesystem.open_output_stream(path) as out: + out.write(b"payload") + self.assertEqual(b"payload", gzip.decompress(self._content(self.origin, path))) + self._put(self.cluster, path, self._content(self.origin, path)) + self.origin._to_file(path).unlink() + with self.file_io.filesystem.open_input_stream(path) as stream: + self.assertEqual(b"payload", stream.read()) + + def test_target_errors_are_raised(self): + self._put(self.origin, DATA, b"origin") + with self.assertRaises(FileNotFoundError): + self.file_io.new_input_stream(DATA) + with self.assertRaises(FileNotFoundError): + self.file_io.get_file_status(DATA) + with self.assertRaises(FileNotFoundError): + self.file_io.filesystem.open_input_file(DATA) + with self.assertRaises(FileNotFoundError): + self.file_io.filesystem.open_input_stream(DATA) + # Target errors propagate without retrying the request on origin. + for error in (OSError("[E1010]HTTP Status: 503 Error Code: SlowDown"), ConnectionRefusedError("refused"), + TimeoutError("timed out"), OSError("[E1010]HTTP Status: 403 Error Code: AccessDenied")): + self.cluster.error = error + with self.assertRaises(type(error)): + self.file_io.new_input_stream(DATA) + with self.assertRaises(type(error)): + self.file_io.filesystem.open_input_file(DATA) + self.assertEqual(0, self.origin.attempts) + + def test_target_that_cannot_be_created_raises(self): + self._put_all(DATA, MANIFEST) + self.creation_error = RuntimeError("cannot connect") + with self.assertRaisesRegex(RuntimeError, "cannot connect"): + self.file_io.new_input_stream(DATA) + self.creation_error = None + self.assertEqual(b"cluster", self.file_io.new_input_stream(DATA).read()) + self.assertEqual(["cluster", "cluster"], self.created) + + def test_register_file_size_reaches_the_routed_filesystem(self): + origin, targets = _RecordingFileIO(), {} + + def create(name): + targets[name] = _RecordingFileIO() + return targets[name] + + file_io = IoCacheRoutingFileIO(_routing(), origin, create) + handler = file_io.filesystem.handler + handler.register_file_size(DATA, 7) + handler.register_file_size(LATEST, 1) + self.assertEqual(["register_file_size"], origin.calls) + self.assertEqual({"cluster": ["register_file_size"]}, {name: io.calls for name, io in targets.items()}) + + def test_close_closes_created_targets(self): + self._put_all(DATA) + self.file_io.new_input_stream(DATA) + self.file_io.close() + self.assertEqual((True, False, True), (self.origin.closed, self.accel.closed, self.cluster.closed)) + + def test_local_block_cache_on_top(self): + self._put_all(DATA) + caching = CachingFileIO(self.file_io, LocalMemoryCacheManager(1 << 20), {FileType.DATA}) + self.assertEqual(b"cluster", caching.new_input_stream(DATA).read()) + self.assertIs(self.file_io.filesystem, caching.filesystem) + self.assertEqual(DATA, caching.to_filesystem_path(DATA)) + + def test_unrouted_attributes_come_from_origin(self): + self.assertIs(self.origin.properties, self.file_io.properties) + self.assertIs(self.origin.uri_reader_factory, self.file_io.uri_reader_factory) + self.assertFalse(hasattr(self.file_io, "file_io")) + + def test_origin_only_view_is_the_origin(self): + self._put_all(DATA, MANIFEST) + self.assertIs(self.origin, self.file_io.origin_only()) + self.assertEqual(b"origin", self.file_io.origin_only().new_input_stream(DATA).read()) + self.assertEqual([], self.created) + + def test_table_views_share_the_targets_and_close_nothing(self): + external = "oss://bkt/import/data-" + UUID + "-0.parquet" + custom = PARTITION + "/custom-" + UUID + "-0.parquet" + self._put_all(DATA, external, custom) + view = self.file_io.for_table(TABLE, ["custom-"], "main") + self.assertIs(view, self.file_io.for_table(TABLE, ["custom-"], "main")) + self.assertEqual(b"cluster", view.new_input_stream(custom).read()) + self.assertEqual(b"origin", view.new_input_stream(external).read()) + self.assertEqual(b"cluster", self.file_io.new_input_stream(external).read()) + self.assertEqual(b"origin", self.file_io.new_input_stream(custom).read()) + self.assertEqual(["cluster"], self.created) + # A copy of the table keeps the prefixes of the view it was made from. + copy = view.for_table(TABLE, ["dynamic-"], "dev") + self.assertEqual((TABLE, ("custom-", "dynamic-"), ("main", "dev")), copy._table) + view.close() + copy.close() + self.assertEqual((False, False), (self.origin.closed, self.cluster.closed)) + + def test_schema_history_is_read_from_origin_once(self): + older = PARTITION + "/older-" + UUID + "-0.parquet" + self._put_all(older) + listed = [] + schema = mock.Mock(options={"data-file.prefix": "older-"}) + + def list_all(manager): + listed.append((manager.file_io, manager.branch)) + return [schema] + + with mock.patch("pypaimon.schema.schema_manager.SchemaManager.list_all", list_all): + view = self.file_io.for_table(TABLE, [], "dev") + for _ in range(3): + self.assertEqual(b"cluster", view.new_input_stream(older).read()) + self.assertEqual([(self.origin, "dev")], listed) + + def test_schema_history_that_cannot_be_read_leaves_files_on_origin(self): + older = PARTITION + "/older-" + UUID + "-0.parquet" + self._put_all(older) + with mock.patch("pypaimon.schema.schema_manager.SchemaManager.list_all", side_effect=OSError("denied")), \ + self.assertLogs("pypaimon.filesystem.io_cache_file_io", "WARNING"): + view = self.file_io.for_table(TABLE, [], "main") + self.assertEqual(b"origin", view.new_input_stream(older).read()) + self.assertEqual(b"origin", view.new_input_stream(older).read()) + + +class _RecordingHandler(FileSystemHandler): + + def __init__(self, calls): + self.calls = calls + + def _record(self, name, result=None): + self.calls.append(name) + return result + + def __eq__(self, other): + return self is other + + def __ne__(self, other): + return self is not other + + def get_type_name(self): + return "recording" + + def normalize_path(self, path): + return path + + def register_file_size(self, path, file_size): + self._record("register_file_size") + + def get_file_info(self, paths): + return self._record("get_file_info", [pafs.FileInfo(path, pafs.FileType.File, size=0) for path in paths]) + + def get_file_info_selector(self, selector): + return self._record("get_file_info_selector", []) + + def create_dir(self, path, recursive): + self._record("create_dir") + + def delete_dir(self, path): + self._record("delete_dir") + + def delete_dir_contents(self, path, missing_dir_ok=False): + self._record("delete_dir_contents") + + def delete_root_dir_contents(self): + self._record("delete_root_dir_contents") + + def delete_file(self, path): + self._record("delete_file") + + def move(self, src, dest): + self._record("move") + + def copy_file(self, src, dest): + self._record("copy_file") + + def open_input_stream(self, path): + return self._record("open_input_stream", pa.BufferReader(b"")) + + def open_input_file(self, path): + return self._record("open_input_file", pa.BufferReader(b"")) + + def open_output_stream(self, path, metadata): + return self._record("open_output_stream", pa.BufferOutputStream()) + + def open_append_stream(self, path, metadata): + return self._record("open_append_stream", pa.BufferOutputStream()) + + +class _RecordingFileIO(FileIO): + """Records every call; the routing tests only check which FileIO was called.""" + + def __init__(self): + self.calls = [] + self.filesystem = pafs.PyFileSystem(_RecordingHandler(self.calls)) + + def _record(self, name, result=None): + self.calls.append(name) + return result + + def new_input_stream(self, path): + return self._record("new_input_stream", io.BytesIO(b"")) + + def new_output_stream(self, path): + return self._record("new_output_stream", io.BytesIO()) + + def get_file_status(self, path): + return self._record("get_file_status", pafs.FileInfo(path, pafs.FileType.File, size=0)) + + def list_status(self, path): + return self._record("list_status", []) + + def exists(self, path): + return self._record("exists", True) + + def exists_batch(self, paths): + return self._record("exists_batch", {path: True for path in paths}) + + def delete(self, path, recursive=False): + return self._record("delete", True) + + def mkdirs(self, path): + return self._record("mkdirs", True) + + def rename(self, src, dst): + return self._record("rename", True) + + def copy_file(self, source_path, target_path, overwrite=False): + self._record("copy_file") + + def try_to_write_atomic(self, path, content): + return self._record("try_to_write_atomic", True) + + def create_blob_presigned_url(self, table_root, descriptor, validity): + return self._record("create_blob_presigned_url", "https://signed") + + def write_parquet(self, path, data, compression='zstd', zstd_level=1, **kwargs): + self._record("write_parquet") + + def to_filesystem_path(self, path): + return path + + +# The FileIO and filesystem calls of each routing operation; pypaimon has no two-phase writer. +OP_CALLS = { + "read": { + "new_input_stream": lambda f, p: f.new_input_stream(p), + "read_file_range": lambda f, p: f.read_file_range(p, 0, 1), + "fs.open_input_file": lambda f, p: f.filesystem.open_input_file(p), + "fs.open_input_stream": lambda f, p: f.filesystem.open_input_stream(p), + }, + "meta": { + "get_file_status": lambda f, p: f.get_file_status(p), + "get_file_size": lambda f, p: f.get_file_size(p), + "fs.get_file_info": lambda f, p: f.filesystem.get_file_info([p]), + }, + "exists": { + "exists": lambda f, p: f.exists(p), + "exists_batch": lambda f, p: f.exists_batch([p]), + }, + "write": { + "new_output_stream": lambda f, p: f.new_output_stream(p), + "write_parquet": lambda f, p: f.write_parquet(p, None), + "fs.open_output_stream": lambda f, p: f.filesystem.open_output_stream(p), + "fs.open_append_stream": lambda f, p: f.filesystem.open_append_stream(p), + }, + "list": { + "list_status": lambda f, p: f.list_status(p), + "fs.get_file_info_selector": lambda f, p: f.filesystem.get_file_info(pafs.FileSelector(p)), + }, + "delete": { + "delete": lambda f, p: f.delete(p), + "fs.delete_file": lambda f, p: f.filesystem.delete_file(p), + "fs.delete_dir": lambda f, p: f.filesystem.delete_dir(p), + }, + "rename": { + "rename": lambda f, p: f.rename(p, p + "-renamed"), + "fs.move": lambda f, p: f.filesystem.move(p, p + "-renamed"), + }, + "mkdirs": { + "mkdirs": lambda f, p: f.mkdirs(p), + "fs.create_dir": lambda f, p: f.filesystem.create_dir(p), + }, + "copy": { + "copy_file": lambda f, p: f.copy_file(p, p + "-copy"), + "fs.copy_file": lambda f, p: f.filesystem.copy_file(p, p + "-copy"), + }, + "atomic-write": { + "try_to_write_atomic": lambda f, p: f.try_to_write_atomic(p, "{}"), + }, + "two-phase-write": {}, + "presign": { + "create_blob_presigned_url": lambda f, p: f.create_blob_presigned_url( + TABLE, BlobDescriptor(p, 0, 1), timedelta(minutes=1)), + }, +} + +# Inherited FileIO methods that only read through new_input_stream, or do no I/O. +INHERITED_READS = { + "read_blobs_concurrent", "read_file_range", "read_file_utf8", "read_overwritten_file_utf8", + "read_ranges_coalesced", "read_ranges_coalesced_views", +} +NO_IO = {"get", "parse_location"} + + +def test_every_file_io_method_is_classified(): + """A FileIO method the routing FileIO neither overrides nor lists here could reach a cache.""" + public = {name for name, value in inspect.getmembers(FileIO) + if not name.startswith("_") and callable(value)} + overridden = {name for name in public if name in IoCacheRoutingFileIO.__dict__} + assert public - overridden - INHERITED_READS - NO_IO == set() + assert not overridden & INHERITED_READS + + +@pytest.mark.parametrize("call", [ + lambda f: f.write_file(DATA, "x"), + lambda f: f.overwrite_file_utf8(DATA, "x"), + lambda f: f.delete_quietly(DATA), + lambda f: f.delete_files_quietly([DATA]), + lambda f: f.check_or_mkdirs(PARTITION), + lambda f: f.copy_files(PARTITION, PARTITION + "-copy"), +]) +def test_write_and_delete_helpers_use_origin(call): + routing = _routing() + file_ios = {key: _RecordingFileIO() for key in ("origin", "accel", "cluster")} + # the recorder says every path exists, so the existence checks may raise + with contextlib.suppress(FileExistsError, ValueError): + call(IoCacheRoutingFileIO(routing, file_ios["origin"], file_ios.__getitem__)) + assert file_ios["origin"].calls + assert not file_ios["accel"].calls and not file_ios["cluster"].calls + + +FILE_IO_CALLS = [(op, name, call) for op, calls in OP_CALLS.items() for name, call in calls.items()] + + +@pytest.mark.parametrize("op,name,call", FILE_IO_CALLS, + ids=["{} / {}".format(op, name) for op, name, _ in FILE_IO_CALLS]) +@pytest.mark.parametrize("policy", ["meta,read", "meta,read,write", "meta,read,exists"]) +def test_each_file_io_call_reaches_its_endpoint(policy, op, name, call): + tokens = set(policy.split(",")) + routed = {"read", "meta"} | ({"write", "two-phase-write"} if "write" in tokens else set()) | (tokens & {"exists"}) + routing = _routing(policy=policy) + for path, target in ((DATA, "cluster"), (MANIFEST, "accel"), (SNAPSHOT, "origin"), (UNKNOWN, "origin")): + file_ios = {key: _RecordingFileIO() for key in ("origin", "accel", "cluster")} + call(IoCacheRoutingFileIO(routing, file_ios["origin"], file_ios.__getitem__), path) + called = {key for key, recorder in file_ios.items() if recorder.calls} + assert called == {target if op in routed else "origin"}, path + + +class _MissingPathsHandler(_RecordingHandler): + """Every path is missing, so a writer probes and creates the parent directory.""" + + def get_file_info(self, paths): + return self._record("get_file_info", [pafs.FileInfo(path, pafs.FileType.NotFound) for path in paths]) + + +def _legacy_oss_file_io(calls): + options = Options({"fs.oss.impl": "legacy", "fs.oss.endpoint": "oss-cn-test.example.com"}) + with mock.patch.object(OssFileIO, "_initialize_oss_fs", + return_value=pafs.PyFileSystem(_MissingPathsHandler(calls))): + return OssFileIO(TABLE, options) + + +@pytest.mark.parametrize("write", [ + lambda f, p: f.new_output_stream(p).close(), + lambda f, p: f.write_parquet(p, pa.table({"a": [1]})), + lambda f, p: f.write_orc(p, pa.table({"a": [1]})), + lambda f, p: f.write_avro(p, pa.table({"a": [1]})), + lambda f, p: f.write_file(p, "x"), + lambda f, p: f.overwrite_file_utf8(p, "x"), +], ids=["new_output_stream", "write_parquet", "write_orc", "write_avro", "write_file", "overwrite_file_utf8"]) +def test_routed_writes_create_parent_directories_on_origin(write): + unrouted, origin, target = [], [], [] + write(_legacy_oss_file_io(unrouted), DATA) + targets = {"accel": _legacy_oss_file_io([]), "cluster": _legacy_oss_file_io(target)} + write(IoCacheRoutingFileIO(_routing(policy="meta,read,write"), _legacy_oss_file_io(origin), + targets.__getitem__), DATA) + assert "create_dir" in unrouted + assert target == ["open_output_stream"] + assert origin == [call for call in unrouted if call != "open_output_stream"] + + +@pytest.mark.parametrize("write", [ + lambda f, p: f.new_output_stream(p).close(), + lambda f, p: f.write_parquet(p, pa.table({"a": [1]})), + lambda f, p: f.write_file(p, "x"), +], ids=["new_output_stream", "write_parquet", "write_file"]) +def test_routed_writes_through_resolving_file_ios_create_parent_directories_on_origin(write): + # With resolving-file-io.enabled, origin and targets are ResolvingFileIOs over the OSS FileIOs. + unrouted, origin, target = [], [], [] + write(_legacy_oss_file_io(unrouted), DATA) + calls = {"origin": origin, "accel": [], "cluster": target} + + def resolving(name): + return ResolvingFileIO(Options({"endpoint": name})) + + def resolve(path, options): + return _legacy_oss_file_io(calls[options.to_map()["endpoint"]]) + + with mock.patch.object(FileIO, "get", side_effect=resolve): + write(IoCacheRoutingFileIO(_routing(policy="meta,read,write"), resolving("origin"), resolving), DATA) + assert "create_dir" in unrouted + assert target == ["open_output_stream"] + assert origin == [call for call in unrouted if call != "open_output_stream"] + + +class _FailingWritesHandler(_MissingPathsHandler): + """Every path is missing and every upload fails.""" + + def open_output_stream(self, path, metadata): + self._record("open_output_stream") + raise OSError("upload failed") + + +@pytest.mark.parametrize("write", [ + lambda f, p: f.write_parquet(p, pa.table({"a": [1]})), + lambda f, p: f.write_orc(p, pa.table({"a": [1]})), +], ids=["write_parquet", "write_orc"]) +def test_failed_routed_writes_are_cleaned_up_on_origin(write): + def file_io(calls): + options = Options({"fs.oss.impl": "legacy", "fs.oss.endpoint": "oss-cn-test.example.com"}) + with mock.patch.object(OssFileIO, "_initialize_oss_fs", + return_value=pafs.PyFileSystem(_FailingWritesHandler(calls))): + file_io = OssFileIO(TABLE, options) + file_io.delete_quietly = mock.Mock() + return file_io + + unrouted, origin, target_calls = file_io([]), file_io([]), [] + with pytest.raises(RuntimeError): + write(unrouted, DATA) + targets = {"accel": file_io([]), "cluster": file_io(target_calls)} + with pytest.raises(RuntimeError): + write(IoCacheRoutingFileIO(_routing(policy="meta,read,write"), origin, targets.__getitem__), DATA) + unrouted.delete_quietly.assert_called_once_with(DATA) + origin.delete_quietly.assert_called_once_with(DATA) + targets["cluster"].delete_quietly.assert_not_called() + assert target_calls == ["open_output_stream"] + +if __name__ == "__main__": + unittest.main() diff --git a/paimon-python/pypaimon/tests/io_cache_routing_test.py b/paimon-python/pypaimon/tests/io_cache_routing_test.py new file mode 100644 index 000000000000..f5b81fba4816 --- /dev/null +++ b/paimon-python/pypaimon/tests/io_cache_routing_test.py @@ -0,0 +1,629 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Tests for the io-cache routing decision.""" + +import sys +import unittest + +from pypaimon.filesystem.io_cache_routing import (IoCacheRouting, Op, single_endpoint_options, + data_prefixes, routable_type) +from pypaimon.utils.file_type import FileType + +UUID = "8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b" +TABLE_ROOT = "oss://bkt/db1.db/t1" + +# paths below are relative to TABLE_ROOT, and {uuid} stands for UUID +DATA_PATH = "dt=1/bucket-0/data-{uuid}-0.parquet" +MANIFEST_PATH = "manifest/manifest-{uuid}-0" +INDEX_PATH = "index/index-{uuid}-0" +GLOBAL_INDEX_PATH = "index/btree-global-index-{uuid}.index" +TEMP_PATH = "dt=1/bucket-0/.data-{uuid}-0.parquet.{uuid}.tmp" +SNAPSHOT_PATH = "snapshot/snapshot-12" +LATEST_PATH = "snapshot/LATEST" + +OSS = "https://oss-cn-hangzhou-internal.aliyuncs.com" +CACHE = "http://cache.example.com" +ACCEL = "https://accelerator.example.com" +CLUSTER = "http://cluster.example.com" +WRITE = "io-cache.policy=meta,read,write" +EXISTS = "io-cache.policy=meta,read,exists" +ORIGIN_OPS = (Op.LIST, Op.DELETE, Op.RENAME, Op.MKDIRS, Op.COPY, Op.ATOMIC_WRITE, Op.PRESIGN) + + +def with_changes(options, changes): + """Applies changes: "key=value" sets an option and "-key" removes it.""" + for change in changes: + if change.startswith("-"): + options.pop(change[1:], None) + else: + key, value = change.split("=", 1) + options[key] = value + return options + + +def single(*changes): + """One cache target, which older clients also get as fs.oss.endpoint.""" + return with_changes({ + "fs.oss.endpoint": CACHE, + "io-cache.enabled": "true", + "io-cache.endpoint": CACHE, + "io-cache.origin.endpoint": OSS, + "io-cache.policy": "meta,read", + "io-cache.whitelist": "meta,data", + }, changes) + + +def multi(*changes): + """Metadata on an accelerator, data and indexes on a cache cluster.""" + return with_changes({ + "fs.oss.endpoint": ACCEL, + "io-cache.enabled": "true", + "io-cache.origin.endpoint": OSS, + "io-cache.targets": "accel,cluster", + "io-cache.target.accel.endpoint": ACCEL, + "io-cache.target.accel.region": "cn-hangzhou", + "io-cache.target.cluster.endpoint": CLUSTER, + "io-cache.target.cluster.path-style-access": "true", + "io-cache.policy": "meta,read", + "io-cache.whitelist": "*", + "io-cache.routes": "meta=accel;data,bucket-index,global-index,file-index=cluster", + }, changes) + + +def resolve(path): + path = path.replace("{uuid}", UUID) + return path if "://" in path else TABLE_ROOT + "/" + path + + +def endpoint(options, op, path): + """The endpoint of one request as a client sees it: override, old endpoint, origin or a target.""" + if options.get("dlf.oss-endpoint"): + return options["dlf.oss-endpoint"] + routing = IoCacheRouting.create(options) + if routing is None: + return options.get("fs.oss.endpoint") + name = routing.route(op, path) + return routing.origin_endpoint() if name is None else routing.targets()[name] + + +class IoCacheRoutingEndpointTest(unittest.TestCase): + + def assert_endpoint(self, options, op, path, expected): + self.assertEqual(expected, endpoint(options, op, resolve(path)), + f"{op.value} {path} with {options}") + + def assert_type(self, path, file_type, *changes): + prefixes = data_prefixes(with_changes({}, changes)) + self.assertEqual(file_type, routable_type(resolve(path), prefixes), f"{path} with {changes}") + + def test_one_cache_target(self): + options = single() + self.assert_endpoint(options, Op.READ, DATA_PATH, CACHE) + self.assert_endpoint(options, Op.META, DATA_PATH, CACHE) + # exists asks whether a file is still there, so it needs its own policy token + self.assert_endpoint(options, Op.EXISTS, DATA_PATH, OSS) + self.assert_endpoint(single(EXISTS), Op.EXISTS, DATA_PATH, CACHE) + self.assert_endpoint(single(EXISTS), Op.EXISTS, SNAPSHOT_PATH, OSS) + self.assert_endpoint(options, Op.READ, MANIFEST_PATH, CACHE) + self.assert_endpoint(options, Op.READ, "manifest/manifest-list-{uuid}-1", CACHE) + self.assert_endpoint(options, Op.READ, "oss://other-bkt/db1.db/t1/" + DATA_PATH, CACHE) + self.assert_endpoint(options, Op.READ, SNAPSHOT_PATH, OSS) + self.assert_endpoint(options, Op.META, SNAPSHOT_PATH, OSS) + self.assert_endpoint(options, Op.EXISTS, SNAPSHOT_PATH, OSS) + self.assert_endpoint(options, Op.READ, LATEST_PATH, OSS) + self.assert_endpoint(options, Op.EXISTS, LATEST_PATH, OSS) + self.assert_endpoint(options, Op.READ, TEMP_PATH, OSS) + self.assert_endpoint(options, Op.READ, "dt=1/bucket-0/000000_0", OSS) + self.assert_endpoint(options, Op.READ, "dls://bkt/db1.db/t1/" + DATA_PATH, OSS) + # the whitelist has no index + self.assert_endpoint(options, Op.READ, INDEX_PATH, OSS) + # a Format Table file named like a manifest may be replaced in place + self.assert_endpoint(options, Op.READ, "review_external/manifest.parquet", OSS) + self.assert_endpoint(options, Op.META, "review_external/manifest.parquet", OSS) + self.assert_endpoint(options, Op.WRITE, DATA_PATH, OSS) + self.assert_endpoint(options, Op.TWO_PHASE_WRITE, DATA_PATH, OSS) + for op in ORIGIN_OPS: + self.assert_endpoint(options, op, DATA_PATH, OSS) + + def test_policy_and_whitelist(self): + self.assert_endpoint(single("-io-cache.enabled"), Op.READ, DATA_PATH, CACHE) + self.assert_endpoint(single("io-cache.enabled=false"), Op.EXISTS, DATA_PATH, CACHE) + self.assert_endpoint(single("-io-cache.policy"), Op.EXISTS, DATA_PATH, CACHE) + self.assert_endpoint(single("io-cache.policy=none"), Op.EXISTS, DATA_PATH, CACHE) + self.assert_endpoint(single("io-cache.policy=read,NONE"), Op.EXISTS, DATA_PATH, CACHE) + self.assert_endpoint(single("io-cache.policy=thread,metadata"), Op.EXISTS, DATA_PATH, CACHE) + self.assert_endpoint(single("io-cache.policy= READ , Meta "), Op.META, DATA_PATH, CACHE) + self.assert_endpoint(single("io-cache.policy=read,nonetheless"), Op.EXISTS, DATA_PATH, OSS) + + read_only = single("io-cache.policy=read") + self.assert_endpoint(read_only, Op.READ, DATA_PATH, CACHE) + self.assert_endpoint(read_only, Op.META, DATA_PATH, OSS) + self.assert_endpoint(read_only, Op.EXISTS, DATA_PATH, OSS) + write_only = single("io-cache.policy=write") + self.assert_endpoint(write_only, Op.WRITE, DATA_PATH, CACHE) + self.assert_endpoint(write_only, Op.EXISTS, DATA_PATH, OSS) + exists_only = single("io-cache.policy=exists") + self.assert_endpoint(exists_only, Op.EXISTS, DATA_PATH, CACHE) + self.assert_endpoint(exists_only, Op.META, DATA_PATH, OSS) + + self.assert_endpoint(single("-io-cache.whitelist"), Op.READ, INDEX_PATH, CACHE) + self.assert_endpoint(single("io-cache.whitelist=*"), Op.READ, GLOBAL_INDEX_PATH, CACHE) + self.assert_endpoint(single("io-cache.whitelist=*"), Op.READ, DATA_PATH + ".index", CACHE) + + def test_endpoints(self): + self.assert_endpoint(single("io-cache.endpoint= "), Op.READ, DATA_PATH, OSS) + self.assert_endpoint( + single("io-cache.endpoint=", "io-cache.routes=*=default"), Op.READ, DATA_PATH, OSS) + # the origin defaults to fs.oss.endpoint + oss = "fs.oss.endpoint=" + OSS + self.assert_endpoint(single(oss, "-io-cache.origin.endpoint"), Op.LIST, DATA_PATH, OSS) + self.assert_endpoint(single(oss, "io-cache.origin.endpoint= "), Op.LIST, DATA_PATH, OSS) + # a client-side endpoint turns routing off + override = "https://oss-cn-hangzhou.aliyuncs.com" + self.assert_endpoint(single("dlf.oss-endpoint=" + override), Op.READ, DATA_PATH, override) + + def test_write_policy(self): + options = single(WRITE) + self.assert_endpoint(options, Op.WRITE, DATA_PATH, CACHE) + self.assert_endpoint(options, Op.TWO_PHASE_WRITE, DATA_PATH, CACHE) + self.assert_endpoint(options, Op.WRITE, MANIFEST_PATH, CACHE) + self.assert_endpoint(options, Op.WRITE, SNAPSHOT_PATH, OSS) + self.assert_endpoint(options, Op.WRITE, LATEST_PATH, OSS) + self.assert_endpoint(options, Op.WRITE, TEMP_PATH, OSS) + for op in ORIGIN_OPS: + self.assert_endpoint(options, op, DATA_PATH, OSS) + self.assert_endpoint(single(WRITE, "io-cache.whitelist=meta"), Op.WRITE, DATA_PATH, OSS) + + self.assert_endpoint(multi(WRITE), Op.WRITE, DATA_PATH, CLUSTER) + self.assert_endpoint(multi(WRITE), Op.WRITE, MANIFEST_PATH, ACCEL) + + def test_two_cache_targets(self): + options = multi() + self.assert_endpoint(options, Op.READ, MANIFEST_PATH, ACCEL) + self.assert_endpoint(options, Op.META, MANIFEST_PATH, ACCEL) + self.assert_endpoint(options, Op.EXISTS, MANIFEST_PATH, OSS) + self.assert_endpoint(multi(EXISTS), Op.EXISTS, MANIFEST_PATH, ACCEL) + self.assert_endpoint(options, Op.READ, DATA_PATH, CLUSTER) + self.assert_endpoint(multi(EXISTS), Op.EXISTS, DATA_PATH, CLUSTER) + self.assert_endpoint(options, Op.READ, INDEX_PATH, CLUSTER) + self.assert_endpoint(options, Op.READ, GLOBAL_INDEX_PATH, CLUSTER) + self.assert_endpoint(options, Op.READ, SNAPSHOT_PATH, OSS) + self.assert_endpoint(options, Op.WRITE, MANIFEST_PATH, OSS) + self.assert_endpoint(options, Op.TWO_PHASE_WRITE, MANIFEST_PATH, OSS) + for op in ORIGIN_OPS: + self.assert_endpoint(options, op, MANIFEST_PATH, OSS) + self.assert_endpoint(multi("io-cache.whitelist=meta"), Op.READ, DATA_PATH, OSS) + + def test_targets_and_routes(self): + # without routes the first target with an endpoint takes every type + no_routes = "-io-cache.routes" + self.assert_endpoint(multi(no_routes), Op.READ, DATA_PATH, ACCEL) + self.assert_endpoint(multi(no_routes, "io-cache.endpoint=" + CACHE), Op.READ, DATA_PATH, ACCEL) + self.assert_endpoint( + multi(no_routes, "-io-cache.target.accel.endpoint"), Op.READ, DATA_PATH, CLUSTER) + self.assert_endpoint( + multi(no_routes, "io-cache.target.accel.endpoint= "), Op.READ, DATA_PATH, CLUSTER) + self.assert_endpoint( + multi(no_routes, "io-cache.targets=Bad_Name,cluster"), Op.READ, DATA_PATH, CLUSTER) + self.assert_endpoint( + multi(f"io-cache.target.cluster.endpoint= {CLUSTER} "), Op.READ, DATA_PATH, CLUSTER) + + # a type without a usable route goes to the origin + self.assert_endpoint(multi("io-cache.routes=data=x"), Op.READ, DATA_PATH, OSS) + self.assert_endpoint(multi("-io-cache.target.cluster.endpoint"), Op.READ, DATA_PATH, OSS) + meta_and_data = "io-cache.routes=meta=accel;data=cluster" + self.assert_endpoint(multi(meta_and_data), Op.READ, DATA_PATH + ".index", OSS) + + self.assert_endpoint(multi("io-cache.routes=*=cluster"), Op.READ, MANIFEST_PATH, CLUSTER) + first_wins = "io-cache.routes=data=accel;data,meta=cluster" + self.assert_endpoint(multi(first_wins), Op.READ, DATA_PATH, ACCEL) + malformed = "io-cache.routes==cluster;meta=accel;junk" + self.assert_endpoint(multi(malformed), Op.READ, MANIFEST_PATH, ACCEL) + self.assert_endpoint(multi(malformed), Op.READ, DATA_PATH, OSS) + + def test_data_files_named_like_metadata(self): + manifest = multi("data-file.prefix=manifest-") + self.assert_endpoint(manifest, Op.READ, "dt=1/bucket-0/manifest-{uuid}-0.orc", CLUSTER) + self.assert_endpoint(manifest, Op.READ, "dt=1/bucket-0/7f3a/manifest-{uuid}-0.orc", CLUSTER) + self.assert_endpoint(manifest, Op.READ, MANIFEST_PATH, ACCEL) + self.assert_endpoint(manifest, Op.READ, MANIFEST_PATH + ".avro.sidecar", ACCEL) + snapshot = multi("data-file.prefix=snapshot-") + self.assert_endpoint(snapshot, Op.READ, "dt=1/bucket-0/snapshot-{uuid}-0.orc", CLUSTER) + stat = multi("data-file.prefix=stat-") + self.assert_endpoint(stat, Op.READ, "dt=1/bucket-0/stat-{uuid}-0.orc", CLUSTER) + index = multi("io-cache.routes=data=cluster;bucket-index=accel", "data-file.prefix=index-") + self.assert_endpoint(index, Op.READ, "dt=1/bucket-0/index-{uuid}-0.orc", CLUSTER) + self.assert_endpoint(index, Op.READ, INDEX_PATH, ACCEL) + + custom = "dt=1/bucket-0/custom-{uuid}-0.orc" + self.assert_endpoint(single(), Op.READ, custom, OSS) + self.assert_endpoint(single("data-file.prefix=custom-"), Op.READ, custom, CACHE) + + def test_routable_type(self): + # named by sequence id or rewritten in place + for path in ("snapshot/snapshot-12", + "snapshot/LATEST", + "snapshot/EARLIEST", + "branch/branch-dev/snapshot/LATEST", + "schema/schema-3", + "changelog/changelog-5", + "changelog/LATEST", + "tag/tag-2026-09-30", + "tag/tag-success-file/t1_SUCCESS", + "consumer/consumer-job1", + "service/service-primary-key-lookup", + "dt=1/_SUCCESS", + "oss://bkt/bucket-0/db/t/changelog/changelog-5"): + self.assert_type(path, None) + # temporary or not written by Paimon + for path in ("snapshot/.snapshot-13.{uuid}.tmp", + "dt=1/bucket-0/.data-{uuid}-0.parquet.{uuid}.tmp", + "dt=1/bucket-0/data-{uuid}-0.parquet.tmp-{uuid}", + "dt=1/bucket-0/data-{uuid}-0.parquet.tmp.{uuid}", + "metadata/version-hint.text", + "metadata/v3.metadata.json", + "dt=1/bucket-0/000000_0", + "dt=1/part-00000-1a2b.snappy.parquet", + "dt=1/bucket-0/data-1.parquet", + "dt=1/bucket-0/custom-{uuid}-0.orc", + "bucket-postpone/data--u-user-s-x-w-{uuid}-0.parquet", + "bucket-postpone/data--u--s-1-w-{uuid}-0.parquet", + "bucket-postpone/custom--u-user-s-1-w-{uuid}-0.parquet", + "dt=1/bucket-0/data-{uuid}.blob", + "dt=1/bucket-0/data-{uuid}.managed.blob.{uuid}.tmp", + "dt=1/bucket-0/data-old.managed.blob", + "README"): + self.assert_type(path, None) + # Format Table files named like metadata or indexes, which may be replaced in place + for path in ("dt=1/manifest.parquet", + "dt=1/stat-2024.parquet", + "dt=1/index-a.csv", + "dt=1/foo.index", + "review_external/manifest.parquet", + "manifest/manifest-old", + "manifest/manifest-old.avro.sidecar", + "manifest/manifest-list-{uuid}-1.avro.sidecar", + "manifest/manifest-list-{uuid}-changelog-1", + "manifest/manifest-{uuid}-changelog", + "statistics/stat-old", + "index/index-old", + "index/my-global-index.index", + "index/btree-global-index-{uuid}-0.index"): + self.assert_type(path, None) + + self.assert_type("manifest/manifest-{uuid}-0", FileType.META) + self.assert_type("manifest/manifest-list-{uuid}-1", FileType.META) + self.assert_type("manifest/index-manifest-{uuid}-0", FileType.META) + self.assert_type("manifest/manifest-{uuid}-0.avro.sidecar", FileType.META) + self.assert_type("manifest/manifest-{uuid}-changelog-0", FileType.META) + self.assert_type("manifest/manifest-{uuid}-changelog-0.avro.sidecar", FileType.META) + self.assert_type("statistics/stat-{uuid}-0", FileType.META) + self.assert_type("dt=1/bucket-0/data-{uuid}-0.parquet", FileType.DATA) + self.assert_type("dt=1/bucket-0/changelog-{uuid}-0.parquet", FileType.DATA) + self.assert_type("dt=1/bucket-0/data-{uuid}-0.blob", FileType.DATA) + self.assert_type("dt=1/8b1f7c2e/data-{uuid}-0.parquet", FileType.DATA) + self.assert_type("dt=1/bucket-0/data-{uuid}-0.parquet.index", FileType.FILE_INDEX) + self.assert_type("dt=1/bucket-0/data-{uuid}-0.parquet.blobref", FileType.DATA) + self.assert_type("dt=1/bucket-0/data-{uuid}-0.managed.blob", FileType.DATA) + self.assert_type("dt=1/bucket-0/data-{uuid}.managed.blob", FileType.DATA) + # the postpone writer puts its commit user and write id before {uuid}-{count} + for postpone in ("bucket-postpone/data--u-io-cache-user-s-7-w-{uuid}-0.parquet", + "dt=1/bucket-postpone/data--u-{uuid}-s-2147483647-w-{uuid}-12.orc", + "bucket-postpone/changelog--u-user-s-0-w-{uuid}-0.parquet"): + self.assert_type(postpone, FileType.DATA) + self.assert_type(postpone + ".index", FileType.FILE_INDEX) + self.assert_type(postpone + ".blobref", FileType.DATA) + self.assert_type("index/index-{uuid}-0", FileType.BUCKET_INDEX) + self.assert_type("index/btree-global-index-{uuid}.index", FileType.GLOBAL_INDEX) + self.assert_type("index/lumina-global-index-{uuid}.index", FileType.GLOBAL_INDEX) + + def test_routable_type_with_file_prefixes(self): + self.assert_type("dt=1/bucket-0/custom-{uuid}-0.orc", FileType.DATA, "data-file.prefix=custom-") + self.assert_type("bucket-postpone/custom--u-user-s-1-w-{uuid}-0.orc", FileType.DATA, + "data-file.prefix=custom-") + self.assert_type("dt=1/bucket-0/data-{uuid}-0.orc", FileType.DATA, "data-file.prefix=custom-") + self.assert_type("dt=1/bucket-0/cl-{uuid}-0.orc", FileType.DATA, "changelog-file.prefix=cl-") + self.assert_type("dt=1/bucket-0/ unknown.parquet", None, "data-file.prefix= ") + + # the name decides, so data files may be named like metadata + manifest = "data-file.prefix=manifest-" + self.assert_type("dt=1/bucket-0/manifest-{uuid}-0.orc", FileType.DATA, manifest) + self.assert_type("dt=1/bucket-0/7f3a/manifest-{uuid}-0.orc", FileType.DATA, manifest) + self.assert_type("dt=1/bucket-0/manifest-{uuid}-0.orc.index", FileType.FILE_INDEX, manifest) + self.assert_type("manifest/manifest-{uuid}-0", FileType.META, manifest) + self.assert_type("manifest/manifest-{uuid}-0.avro.sidecar", FileType.META, manifest) + index = "data-file.prefix=index-" + self.assert_type("dt=1/bucket-0/index-{uuid}-0.orc", FileType.DATA, index) + self.assert_type("bucket-postpone/index-{uuid}-0.orc", FileType.DATA, index) + self.assert_type("index/index-{uuid}-0", FileType.BUCKET_INDEX, index) + self.assert_type("dt=1/bucket-0/index-{uuid}-0", FileType.BUCKET_INDEX, index) + stat = "data-file.prefix=stat-" + self.assert_type("dt=1/bucket-0/stat-{uuid}-0.orc", FileType.DATA, stat) + self.assert_type("postpone/stat-{uuid}-0.orc", FileType.DATA, stat) + self.assert_type("statistics/stat-{uuid}-0", FileType.META, stat) + snapshot = "data-file.prefix=snapshot-" + self.assert_type("dt=1/bucket-0/snapshot-{uuid}-0.orc", FileType.DATA, snapshot) + self.assert_type("dt=1/bucket-0/snapshot-{uuid}-0.orc.index", FileType.FILE_INDEX, snapshot) + self.assert_type("dt=1/bucket-0/schema-{uuid}-0.orc", FileType.DATA, "data-file.prefix=schema-") + self.assert_type("dt=1/bucket-0/global-index-{uuid}-0.orc.index", FileType.FILE_INDEX, + "data-file.prefix=global-index-") + + # names rewritten in place never use a cache, whatever the prefix + self.assert_type("dt=1/bucket-0/tag-{uuid}-0.orc", None, "data-file.prefix=tag-") + self.assert_type("dt=1/bucket-0/consumer-{uuid}-0.orc", None, "data-file.prefix=consumer-") + warehouse = "oss://bkt/warehouse/bucket-0/db/t1/" + self.assert_type(warehouse + "tag/tag-file", None, "data-file.prefix=tag-") + self.assert_type(warehouse + "consumer/consumer-file", None, "data-file.prefix=consumer-") + self.assert_type(warehouse + "service/service-file", None, "data-file.prefix=service-") + self.assert_type(warehouse + "metadata/version-hint.text", None, "data-file.prefix=version-") + self.assert_type(warehouse + "branch/branch-feature", None, "data-file.prefix=branch-") + + +class IoCacheRoutingTest(unittest.TestCase): + + TABLE = "oss://bkt/db1.db/t1" + DATA = TABLE + "/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet" + MANIFEST = TABLE + "/manifest/manifest-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0" + + def _options(self, **overrides): + options = { + "fs.oss.endpoint": "http://legacy.example.com", + "io-cache.enabled": "true", + "io-cache.endpoint": "http://cache.example.com", + "io-cache.origin.endpoint": "https://origin.example.com", + "io-cache.policy": "meta,read", + } + options.update(overrides) + return options + + def _multi(self, **overrides): + options = self._options(**{ + "io-cache.targets": "accel,cluster", + "io-cache.target.accel.endpoint": "http://accel.example.com", + "io-cache.target.cluster.endpoint": "http://10.0.0.1:8080", + "io-cache.routes": "meta=accel;data=cluster", + }) + options.update(overrides) + return options + + def test_create(self): + self.assertIsNotNone(IoCacheRouting.create(self._options())) + self.assertIsNotNone(IoCacheRouting.create(self._options(**{"io-cache.enabled": " TRUE "}))) + for overrides in ({"io-cache.enabled": "false"}, {"io-cache.enabled": "yes"}, + {"io-cache.policy": "none"}, {"io-cache.policy": "read,none"}, + {"io-cache.policy": "prefetch,metadata"}, {"io-cache.targets": ""}, + {"io-cache.targets": "Bad_Name,1x"}, + {"dlf.oss-endpoint": "oss-cn-hangzhou.aliyuncs.com"}): + with self.subTest(overrides=overrides): + self.assertIsNone(IoCacheRouting.create(self._options(**overrides))) + options = self._options() + del options["io-cache.enabled"] + self.assertIsNone(IoCacheRouting.create(options)) + + def test_policy_routes_read_meta_and_write(self): + routing = IoCacheRouting.create(self._options(**{"io-cache.policy": " Meta , READ ,prefetch,write"})) + routed = (Op.READ, Op.META, Op.WRITE, Op.TWO_PHASE_WRITE) + for op in Op: + with self.subTest(op=op): + expected = "default" if op in routed else None + self.assertEqual(expected, routing.route(op, self.DATA)) + write_only = IoCacheRouting.create(self._options(**{"io-cache.policy": "write"})) + self.assertEqual("default", write_only.route(Op.WRITE, self.DATA)) + self.assertIsNone(write_only.route(Op.READ, self.DATA)) + exists_only = IoCacheRouting.create(self._options(**{"io-cache.policy": "exists"})) + self.assertEqual("default", exists_only.route(Op.EXISTS, self.DATA)) + self.assertIsNone(exists_only.route(Op.META, self.DATA)) + read_only = IoCacheRouting.create(self._options(**{"io-cache.policy": "read"})) + self.assertIsNone(read_only.route(Op.META, self.DATA)) + self.assertIsNone(read_only.route(Op.EXISTS, self.DATA)) + + def test_whitelist(self): + index = self.TABLE + "/index/btree-global-index-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b.index" + routing = IoCacheRouting.create(self._options(**{"io-cache.whitelist": "Global-Index,DATA,video"})) + self.assertEqual("default", routing.route(Op.READ, index)) + self.assertEqual("default", routing.route(Op.READ, self.DATA)) + self.assertIsNone(routing.route(Op.READ, self.MANIFEST)) + self.assertIsNone(IoCacheRouting.create(self._options(**{"io-cache.whitelist": ""})).route( + Op.READ, self.DATA)) + # Only the contract's types count; local-cache selectors such as structured-data are ignored. + for whitelist, routed in (("structured-data", ()), ("structured-data,meta", (self.MANIFEST,))): + routing = IoCacheRouting.create(self._options(**{"io-cache.whitelist": whitelist})) + for path in (self.DATA, self.MANIFEST): + self.assertEqual("default" if path in routed else None, routing.route(Op.READ, path), whitelist) + + def test_targets_and_routes(self): + routing = IoCacheRouting.create(self._multi(**{ + "io-cache.targets": " Accel ,cluster,1x,accel,c_d,missing,", + "io-cache.routes": " Meta = ACCEL ;; data= ; =x ; video=y ; data ; *=cluster", + })) + self.assertEqual({"accel": "http://accel.example.com", "cluster": "http://10.0.0.1:8080"}, + routing.targets()) + self.assertEqual("accel", routing.route(Op.META, self.MANIFEST)) + self.assertEqual("cluster", routing.route(Op.READ, self.DATA)) + to_missing = IoCacheRouting.create(self._multi(**{"io-cache.routes": "data=missing"})) + self.assertIsNone(to_missing.route(Op.READ, self.DATA)) + + def test_first_target_with_an_endpoint_without_routes(self): + options = self._multi() + del options["io-cache.routes"] + del options["io-cache.target.accel.endpoint"] + routing = IoCacheRouting.create(options) + self.assertEqual({"cluster": "http://10.0.0.1:8080"}, routing.targets()) + self.assertEqual("cluster", routing.route(Op.READ, self.MANIFEST)) + + def test_target_without_endpoint_keeps_routing_on_origin(self): + routing = IoCacheRouting.create(self._options(**{"io-cache.endpoint": ""})) + self.assertEqual({}, routing.targets()) + self.assertIsNone(routing.route(Op.READ, self.DATA)) + self.assertEqual("https://origin.example.com", endpoint( + self._options(**{"io-cache.endpoint": ""}), Op.READ, self.DATA)) + + def test_origin_defaults_to_fs_oss_endpoint(self): + options = self._options() + del options["io-cache.origin.endpoint"] + self.assertEqual("http://legacy.example.com", IoCacheRouting.create(options).origin_endpoint()) + + def test_directory_with_trailing_slash_uses_its_name(self): + routing = IoCacheRouting.create(self._options()) + self.assertIsNone(routing.route(Op.META, self.TABLE + "/tag/tag-1/")) + self.assertEqual("default", routing.route(Op.META, self.MANIFEST + "/")) + self.assertIsNone(routing.route(Op.META, self.TABLE + "/snapshot/snapshot-1/")) + + def test_table_prefixes_add_data_prefixes(self): + path = self.TABLE + "/bucket-0/part-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet" + self.assertIsNone(IoCacheRouting.create(self._options()).route(Op.READ, path)) + for key in ("data-file.prefix", "changelog-file.prefix"): + routing = IoCacheRouting.create(self._options(**{key: "part-"})) + self.assertEqual("default", routing.route(Op.READ, path)) + + def test_table_view_routes_only_files_under_the_table(self): + external = "oss://bkt/import/data-" + UUID + "-0.parquet" + sibling = "oss://bkt/db1.db/t10/bucket-0/data-" + UUID + "-0.parquet" + routing = IoCacheRouting.create(self._options()) + self.assertEqual("default", routing.route(Op.READ, external)) + for view in (IoCacheRouting.create(self._options(), root=self.TABLE), + routing.for_table(self.TABLE + "/", [])): + with self.subTest(view=view): + self.assertEqual("default", view.route(Op.READ, self.DATA)) + self.assertEqual("default", view.route(Op.META, self.MANIFEST)) + branch_file = self.TABLE + "/branch/branch-dev/bucket-0/data-" + UUID + "-0.orc" + self.assertEqual("default", view.route(Op.READ, branch_file)) + self.assertIsNone(view.route(Op.READ, external)) + self.assertIsNone(view.route(Op.READ, sibling)) + + def test_table_view_asks_the_schema_history_only_for_unknown_data_file_names(self): + asked = [] + + def history(): + asked.append(1) + return ["older-", ""] + + view = IoCacheRouting.create(self._options()).for_table(self.TABLE, ["newer-"], history) + uuid_count = UUID + "-0" + for path in (self.DATA, self.MANIFEST, self.TABLE + "/bucket-0/newer-" + uuid_count + ".parquet"): + self.assertEqual("default", view.route(Op.READ, path), path) + for path in (self.TABLE + "/snapshot/snapshot-3", self.TABLE + "/README"): + self.assertIsNone(view.route(Op.READ, path), path) + self.assertEqual([], asked) + self.assertEqual("default", view.route(Op.READ, self.TABLE + "/bucket-0/older-" + uuid_count + ".orc")) + self.assertIsNone(view.route(Op.READ, self.TABLE + "/bucket-0/other-" + uuid_count + ".orc")) + self.assertEqual(2, len(asked)) + + def test_origin_options(self): + options = self._options(**{ + "fs.oss.dlf-cache.consistent-hash.enabled": "true", + "fs.jindocache.namespace.rpc.address": "10.0.0.1:8101", + "fs.oss.accessKeyId": "ak", + "fs.oss.retry.count": "5", + }) + origin = IoCacheRouting.create(options).origin_options(options) + self.assertEqual("https://origin.example.com", origin["fs.oss.endpoint"]) + self.assertEqual("true", origin["fs.oss.https.enable"]) + self.assertEqual("ak", origin["fs.oss.accessKeyId"]) + self.assertEqual("5", origin["fs.oss.retry.count"]) + self.assertNotIn("fs.oss.dlf-cache.consistent-hash.enabled", origin) + self.assertNotIn("fs.jindocache.namespace.rpc.address", origin) + self.assertNotIn("fs.oss.second.level.domain.enable", origin) + self.assertEqual("http://legacy.example.com", options["fs.oss.endpoint"]) + + def test_target_options(self): + options = self._multi(**{ + "io-cache.target.accel.endpoint": "HTTP://accel.example.com:8080/", + "io-cache.target.cluster.endpoint": "10.0.0.1:8443", + "io-cache.target.cluster.path-style-access": "true", + "io-cache.target.cluster.region": "cn-shanghai", + "fs.oss.region": "cn-hangzhou", + "fs.oss.https.enable": "true", + "fs.oss.second.level.domain.enable": "true", + "fs.oss.dlf-cache.consistent-hash.enabled": "true", + }) + routing = IoCacheRouting.create(options) + accel = routing.target_options(options, "accel") + self.assertEqual("http://accel.example.com:8080", accel["fs.oss.endpoint"]) + self.assertEqual("false", accel["fs.oss.https.enable"]) + self.assertEqual("false", accel["fs.oss.second.level.domain.enable"]) + self.assertEqual("cn-hangzhou", accel["fs.oss.region"]) + self.assertEqual("true", accel["fs.oss.dlf-cache.consistent-hash.enabled"]) + cluster = routing.target_options(options, "cluster") + self.assertEqual("https://10.0.0.1:8443", cluster["fs.oss.endpoint"]) + self.assertEqual("true", cluster["fs.oss.https.enable"]) + self.assertEqual("true", cluster["fs.oss.second.level.domain.enable"]) + self.assertEqual("cn-shanghai", cluster["fs.oss.region"]) + self.assertEqual("cn-hangzhou", options["fs.oss.region"]) + + def test_blank_region_and_file_prefix_are_ignored(self): + options = self._options(**{ + "io-cache.target.default.region": " ", + "fs.oss.region": "cn-hangzhou", + "data-file.prefix": " ", + }) + routing = IoCacheRouting.create(options) + self.assertEqual("cn-hangzhou", routing.target_options(options, "default")["fs.oss.region"]) + self.assertEqual(("data-", "changelog-"), data_prefixes(options)) + + def test_default_target_options(self): + options = self._options(**{ + "io-cache.endpoint": "cache.example.com", + "io-cache.target.default.path-style-access": "true", + "io-cache.target.default.region": "cn-beijing", + "fs.oss.https.enable": "false", + }) + default = IoCacheRouting.create(options).target_options(options, "default") + self.assertEqual("https://cache.example.com", default["fs.oss.endpoint"]) + self.assertEqual("true", default["fs.oss.https.enable"]) + self.assertEqual("true", default["fs.oss.second.level.domain.enable"]) + self.assertEqual("cn-beijing", default["fs.oss.region"]) + + def test_target_keeps_sdk_settings(self): + options = self._options(**{"fs.oss.retry.count": "5"}) + target = IoCacheRouting.create(options).target_options(options, "default") + self.assertEqual("5", target["fs.oss.retry.count"]) + + _ROUTED_TOKEN_ORIGIN = {"fs.oss.endpoint": "https://origin.example.com", "fs.oss.https.enable": "true"} + + def _assert_origin_options(self, options): + self.assertEqual(self._ROUTED_TOKEN_ORIGIN, {key: options[key] for key in self._ROUTED_TOKEN_ORIGIN}) + self.assertNotIn("fs.oss.dlf-cache.consistent-hash.enabled", options) + + def _routed_token(self): + return self._options(**{"fs.oss.dlf-cache.consistent-hash.enabled": "true"}) + + def test_single_endpoint_clients_get_the_origin_options(self): + from pypaimon.filesystem.pvfs import PaimonVirtualFileSystem + + token = self._routed_token() + self._assert_origin_options(single_endpoint_options(token)) + pvfs = PaimonVirtualFileSystem({}, skip_instance_cache=True) + self._assert_origin_options(pvfs._merge_token_with_catalog_options(token)) + + # Without routing, or with routing turned off by the catalog, they keep fs.oss.endpoint. + unrouted = dict(token, **{"io-cache.enabled": "false"}) + self.assertEqual("http://legacy.example.com", single_endpoint_options(unrouted)["fs.oss.endpoint"]) + for catalog_options, expected in (({"io-cache.enabled": "false"}, "http://legacy.example.com"), + ({"dlf.oss-endpoint": "oss-cn-test.example.com"}, "oss-cn-test.example.com")): + pvfs = PaimonVirtualFileSystem(catalog_options, skip_instance_cache=True) + self.assertEqual(expected, pvfs._merge_token_with_catalog_options(token)["fs.oss.endpoint"]) + + @unittest.skipIf(sys.version_info < (3, 7), "pypaimon.daft needs Python 3.7+") + def test_daft_gets_the_origin_options(self): + from unittest.mock import MagicMock + + from pypaimon.daft.daft_paimon import _enrich_options_with_rest_token + + table = MagicMock(file_io=MagicMock(token=MagicMock(token=self._routed_token())), table_path=self.TABLE) + self._assert_origin_options(_enrich_options_with_rest_token({"metastore": "rest"}, table)) diff --git a/paimon-python/pypaimon/tests/io_cache_table_routing_test.py b/paimon-python/pypaimon/tests/io_cache_table_routing_test.py new file mode 100644 index 000000000000..d7ce83cc5ff6 --- /dev/null +++ b/paimon-python/pypaimon/tests/io_cache_table_routing_test.py @@ -0,0 +1,254 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""io-cache routing of real tables whose target keeps stale copies of objects replaced on origin.""" + +import os +import shutil +import tempfile +import unittest +import uuid +from pathlib import Path +from unittest import mock +from urllib.parse import urlparse + +import io + +import pyarrow as pa +import pyarrow.parquet as pq + +from pypaimon import CatalogFactory, Schema +from pypaimon.common.options.config import CatalogOptions +from pypaimon.filesystem.io_cache_file_io import IoCacheRoutingFileIO +from pypaimon.filesystem.io_cache_routing import IoCacheRouting +from pypaimon.filesystem.local_file_io import LocalFileIO +from pypaimon.filesystem.resolving_file_io import ResolvingFileIO +from pypaimon.schema.schema_change import SchemaChange +from pypaimon.table.format.format_table import Format, FormatTable +from pypaimon.table.row.blob import BlobDescriptor + +WAREHOUSE = "oss://bkt/warehouse" + + +class _Endpoint(LocalFileIO): + """One OSS endpoint over a local directory; a cache also serves the copies it retained.""" + + def __init__(self, root, retained=None): + super().__init__() + self.root = root + self.retained = retained + self.requests = [] + + def _to_file(self, path): + parsed = urlparse(path) + if parsed.scheme != "oss": + return super()._to_file(path) + key = Path(parsed.netloc, parsed.path.lstrip("/")) + if self.retained is not None and Path(self.retained, key).exists(): + return Path(self.retained, key) + return Path(self.root, key) + + def list_status(self, path): + # Children keep the oss:// form of the listed path, as an OSS listing returns them. + statuses = super().list_status(path) + if urlparse(path).scheme != "oss": + return statuses + children = [self.get_file_status(path.rstrip("/") + "/" + Path(status.path).name) for status in statuses] + for child in children: + child.path = child.original_path + return children + + def new_input_stream(self, path): + self.requests.append(path) + return super().new_input_stream(path) + + def to_filesystem_path(self, path): + # The routing filesystem calls this right before it opens or stats the file. + self.requests.append(path) + return super().to_filesystem_path(path) + + +class IoCacheTableRoutingTest(unittest.TestCase): + """Origin holds the current objects; the target serves what it retained, as a stale cache would.""" + + CATALOG_OPTIONS = {} + + def setUp(self): + self.temp_dir = tempfile.mkdtemp(prefix="io_cache_table_routing_test_") + storage = os.path.join(self.temp_dir, "storage") + self.origin = _Endpoint(storage) + self.target = _Endpoint(storage, retained=os.path.join(self.temp_dir, "retained")) + self.routing_enabled = True + patcher = mock.patch("pypaimon.catalog.filesystem_catalog.FileIO.get", side_effect=self._file_io) + patcher.start() + self.addCleanup(patcher.stop) + self.catalog = CatalogFactory.create(dict(self.CATALOG_OPTIONS, warehouse=WAREHOUSE)) + self.catalog.create_database("db", False) + + def tearDown(self): + shutil.rmtree(self.temp_dir, ignore_errors=True) + + def _file_io(self, path, options=None): + if not self.routing_enabled: + return self.origin + routing = IoCacheRouting.create({ + "fs.oss.endpoint": "http://cache", + "io-cache.enabled": "true", + "io-cache.endpoint": "http://cache", + "io-cache.origin.endpoint": "http://origin", + "io-cache.policy": "meta,read", + }) + return IoCacheRoutingFileIO(routing, self.origin, lambda name: self.target) + + def _put(self, path, content, retained=False): + key = urlparse(path) + root = self.target.retained if retained else self.origin.root + file = Path(root, key.netloc, key.path.lstrip("/")) + file.parent.mkdir(parents=True, exist_ok=True) + file.write_bytes(content) + + def _target_reads(self, needle): + return [path for path in self.target.requests if needle in path] + + @staticmethod + def _write(table, data): + write_builder = table.new_batch_write_builder() + writer = write_builder.new_write() + writer.write_arrow(data) + write_builder.new_commit().commit(writer.prepare_commit()) + writer.close() + + @staticmethod + def _read(table): + read_builder = table.new_read_builder() + return read_builder.new_read().to_arrow(read_builder.new_scan().plan().splits()) + + def _external_descriptor(self): + # Named like a Paimon data file, outside every table, and replaced on origin after the cache kept it. + uri = "oss://bkt/import/data-" + str(uuid.uuid4()) + "-1.png" + self._put(uri, b"NEW") + self._put(uri, b"OLD", retained=True) + return uri, BlobDescriptor(uri, 0, 3).serialize() + + def test_descriptor_fields_are_read_from_origin(self): + pa_schema = pa.schema([("id", pa.int32()), ("payload", pa.large_binary())]) + schema = Schema.from_pyarrow_schema(pa_schema, options={ + "row-tracking.enabled": "true", "data-evolution.enabled": "true", + "blob-descriptor-field": "payload"}) + self.catalog.create_table("db.descriptors", schema, False) + table = self.catalog.get_table("db.descriptors") + uri, descriptor = self._external_descriptor() + self._write(table, pa.Table.from_pydict({"id": [1], "payload": [descriptor]}, schema=pa_schema)) + + self.assertEqual([b"NEW"], self._read(table).column("payload").to_pylist()) + self.assertEqual([], self._target_reads("/import/")) + + def test_data_files_of_older_schema_prefixes_use_the_target(self): + pa_schema = pa.schema([("id", pa.int32())]) + schema = Schema.from_pyarrow_schema(pa_schema, options={"data-file.prefix": "older-"}) + self.catalog.create_table("db.prefixes", schema, False) + self._write(self.catalog.get_table("db.prefixes"), pa.Table.from_pydict({"id": [1]}, schema=pa_schema)) + self.catalog.alter_table("db.prefixes", [SchemaChange.set_option("data-file.prefix", "newer-")], False) + self._write(self.catalog.get_table("db.prefixes"), pa.Table.from_pydict({"id": [2]}, schema=pa_schema)) + self.target.requests.clear() + + # A fresh load only knows newer- from its schema; older- comes from the schema history. + table = self.catalog.get_table("db.prefixes") + self.assertEqual([1, 2], sorted(self._read(table).column("id").to_pylist())) + self.assertTrue(self._target_reads("/older-")) + self.assertTrue(self._target_reads("/newer-")) + + def test_branch_copies_route_the_data_files_of_the_branch(self): + pa_schema = pa.schema([("id", pa.int32())]) + schema = Schema.from_pyarrow_schema(pa_schema, options={"data-file.prefix": "main-"}) + self.catalog.create_table("db.branched", schema, False) + main = self.catalog.get_table("db.branched") + main.branch_manager().create_branch("dev") + dev = self.catalog.get_table("db.branched$branch_dev") + dev.schema_manager.commit_changes([SchemaChange.set_option("data-file.prefix", "dev-")]) + self._write(self.catalog.get_table("db.branched$branch_dev"), + pa.Table.from_pydict({"id": [1]}, schema=pa_schema)) + self.target.requests.clear() + + copy = main.copy({"branch": "dev"}) + self.assertEqual([1], self._read(copy).column("id").to_pylist()) + self.assertTrue(self._target_reads("/dev-")) + + def test_tag_branches_route_the_files_they_took_under_older_prefixes(self): + pa_schema = pa.schema([("id", pa.int32())]) + schema = Schema.from_pyarrow_schema(pa_schema, options={"data-file.prefix": "old-"}) + self.catalog.create_table("db.tagged", schema, False) + self._write(self.catalog.get_table("db.tagged"), pa.Table.from_pydict({"id": [1]}, schema=pa_schema)) + self.catalog.alter_table("db.tagged", [SchemaChange.set_option("data-file.prefix", "new-")], False) + main = self.catalog.get_table("db.tagged") + self._write(main, pa.Table.from_pydict({"id": [2]}, schema=pa_schema)) + main.create_tag("t1") + main.branch_manager().create_branch("dev", "t1") + self.target.requests.clear() + + # The branch knows new- from its schema; old- comes from the schemas the tag copied into it. + branch = self.catalog.get_table("db.tagged$branch_dev") + self.assertEqual([1, 2], sorted(self._read(branch).column("id").to_pylist())) + self.assertTrue(self._target_reads("/db.db/tagged/bucket-0/old-")) + self.assertTrue(self._target_reads("/db.db/tagged/bucket-0/new-")) + + def test_format_table_files_replaced_on_origin_are_read_from_origin(self): + pa_schema = pa.schema([("id", pa.int32())]) + self.catalog.create_table("db.format_schema", Schema.from_pyarrow_schema(pa_schema), False) + source = self.catalog.get_table("db.format_schema") + location = WAREHOUSE + "/db.db/format_files" + # Another writer replaced a file named like a Paimon data file after the cache kept it. + path = location + "/data-" + str(uuid.uuid4()) + "-1.parquet" + for value, retained in ((2, False), (1, True)): + buffer = io.BytesIO() + pq.write_table(pa.Table.from_pydict({"id": [value]}, schema=pa_schema), buffer) + self._put(path, buffer.getvalue(), retained) + # Rooted at the Format Table's own location, as the FileIO a REST catalog builds for it. + file_io = self.catalog.file_io.for_table(location, [], "main") + table = FormatTable(file_io, source.identifier, source.table_schema, location, Format.parse("parquet")) + + self.assertEqual([2], self._read(table).column("id").to_pylist()) + self.assertEqual([], self._target_reads("/format_files/")) + + def test_table_files_named_like_external_objects_still_use_the_target(self): + pa_schema = pa.schema([("id", pa.int32())]) + self.catalog.create_table("db.plain", Schema.from_pyarrow_schema(pa_schema), False) + table = self.catalog.get_table("db.plain") + self._write(table, pa.Table.from_pydict({"id": [7]}, schema=pa_schema)) + self.target.requests.clear() + + self.assertEqual([7], self._read(table).column("id").to_pylist()) + self.assertTrue(self._target_reads("/db.db/plain/bucket-0/data-")) + + +class IoCacheResolvingTableRoutingTest(IoCacheTableRoutingTest): + """The same tables through a ResolvingFileIO, which resolves the routing FileIO of each path.""" + + CATALOG_OPTIONS = {CatalogOptions.RESOLVING_FILE_IO_ENABLED.key(): "true"} + + def _file_io(self, path, options=None): + if options is not None and options.get(CatalogOptions.RESOLVING_FILE_IO_ENABLED): + return ResolvingFileIO(options) + return super()._file_io(path, options) + + def test_tables_take_views_of_the_resolving_file_io(self): + self.catalog.create_table("db.resolved", Schema.from_pyarrow_schema(pa.schema([("id", pa.int32())])), False) + self.assertIsInstance(self.catalog.get_table("db.resolved").file_io, ResolvingFileIO) + + +if __name__ == "__main__": + unittest.main() diff --git a/paimon-python/pypaimon/tests/native_plan_test.py b/paimon-python/pypaimon/tests/native_plan_test.py index f8a35522bf06..901407bb714c 100644 --- a/paimon-python/pypaimon/tests/native_plan_test.py +++ b/paimon-python/pypaimon/tests/native_plan_test.py @@ -1199,6 +1199,77 @@ def test_native_rest_cache_passes_only_existing_token(self): fake_df.Table.from_rest_response.assert_called_once_with( 'response', database='default', table='t', rest_options={}) + # A copied branch can reach the REST fallback before a data token is available. + # The native core is single-endpoint, so catalog-only routing must still select + # the origin instead of the legacy endpoint that names a cache target. + fake_df.Table.reset_mock() + options = { + 'fs.oss.endpoint': 'http://cache.example.com', + 'io-cache.enabled': 'true', + 'io-cache.endpoint': 'http://cache.example.com', + 'io-cache.origin.endpoint': 'https://origin.example.com', + 'io-cache.policy': 'meta,read', + } + cache = _NativeRestTableCache() + cache.get('response', 'default', 't', options, token_loader=lambda: None) + fake_df.Table.from_rest_response.assert_called_once_with( + 'response', database='default', table='t', + rest_options={ + 'fs.oss.endpoint': 'https://origin.example.com', + 'fs.oss.https.enable': 'true', + }) + + def test_native_rest_cache_gives_the_core_the_origin_under_io_cache_routing(self): + import time + + from pypaimon.catalog.rest.rest_token import RESTToken + from pypaimon.execution.native.table import _NativeRestTableCache + + token = RESTToken({ + 'fs.oss.endpoint': 'http://cache.example.com', + 'io-cache.enabled': 'true', + 'io-cache.endpoint': 'http://cache.example.com', + 'io-cache.origin.endpoint': 'https://origin.example.com', + 'io-cache.policy': 'meta,read', + 'fs.oss.dlf-cache.consistent-hash.enabled': 'true', + }, int(time.time() * 1000) + 7_200_000) + fake_df = ModuleType('pypaimon_rust.datafusion') + fake_df.Table = Mock() + with patch.dict(sys.modules, {'pypaimon_rust.datafusion': fake_df}): + _NativeRestTableCache().get('response', 'default', 't', {}, token_loader=lambda: token) + data_token = fake_df.Table.from_rest_response_with_token.call_args[1]['data_token'] + self.assertEqual(('https://origin.example.com', 'true'), + (data_token['fs.oss.endpoint'], data_token['fs.oss.https.enable'])) + self.assertNotIn('fs.oss.dlf-cache.consistent-hash.enabled', data_token) + + # dlf.oss-endpoint turns routing off and replaces the endpoint, as for the Python FileIO. + fake_df.Table.reset_mock() + options = {'dlf.oss-endpoint': 'oss-cn-test.example.com'} + _NativeRestTableCache().get('response', 'default', 't', options, token_loader=lambda: token) + self.assertEqual(dict(token.token, **{'fs.oss.endpoint': 'oss-cn-test.example.com'}), + fake_df.Table.from_rest_response_with_token.call_args[1]['data_token']) + + def test_native_table_of_a_filesystem_catalog_gets_the_origin_under_io_cache_routing(self): + from pypaimon.execution.native.table import _load_native_table + + table = _scan(True, Mock()).table + table.table_schema = TableSchema(0, []) + table.catalog_environment.catalog_loader = FileSystemCatalogLoader(CatalogContext.create_from_options(Options({ + 'warehouse': 'oss://bkt/wh', + 'fs.oss.endpoint': 'http://cache.example.com', + 'io-cache.enabled': 'true', + 'io-cache.endpoint': 'http://cache.example.com', + 'io-cache.origin.endpoint': 'https://origin.example.com', + 'io-cache.policy': 'meta,read', + }))) + with patch('pypaimon_rust.datafusion.Table', create=True) as native_table: + native_table.from_resolved_schema.return_value.branch.return_value = 'main' + _load_native_table(table) + options = native_table.from_resolved_schema.call_args[1]['options'] + self.assertEqual(('https://origin.example.com', 'oss://bkt/wh'), + (options['fs.oss.endpoint'], options['warehouse'])) + self.assertNotIn('io-cache.enabled', options) + def test_native_rest_cache_retries_failed_construction(self): from pypaimon.execution.native.table import _NativeRestTableCache diff --git a/paimon-python/pypaimon/tests/native_read_path_completion_test.py b/paimon-python/pypaimon/tests/native_read_path_completion_test.py index 0c302155d909..1c59878fe6b4 100644 --- a/paimon-python/pypaimon/tests/native_read_path_completion_test.py +++ b/paimon-python/pypaimon/tests/native_read_path_completion_test.py @@ -204,6 +204,48 @@ def test_native_catalog_preserves_a_reconstructible_fuse_binding(tmp_path, mode, assert module.PaimonCatalog.call_args[0][0]['fuse.enabled'] == 'true' +def test_native_catalog_fallback_uses_origin_for_catalog_only_io_cache(tmp_path): + """A REST catalog without a data token must not hand a cache endpoint to Rust.""" + from pypaimon.catalog.catalog_context import CatalogContext + from pypaimon.catalog.rest.rest_catalog_loader import RESTCatalogLoader + from pypaimon.common.identifier import Identifier + from pypaimon.common.options.options import Options + from pypaimon.execution.native.table import _native_table + from pypaimon.schema.table_schema import TableSchema + from pypaimon.table.file_store_table import FileStoreTable + + options = Options({ + 'warehouse': 'oss://bucket/warehouse', + 'fs.oss.endpoint': 'http://cache.example.com', + 'io-cache.enabled': 'true', + 'io-cache.endpoint': 'http://cache.example.com', + 'io-cache.origin.endpoint': 'https://origin.example.com', + 'io-cache.policy': 'meta,read', + }) + loader = RESTCatalogLoader(CatalogContext.create_from_options(options)) + identifier = Identifier.create('default', 't') + environment = CatalogEnvironment(identifier=identifier, catalog_loader=loader) + file_io = Mock() + file_io.for_table.return_value = file_io + table = FileStoreTable( + file_io, identifier, 'oss://bucket/warehouse/default/t', TableSchema(), environment) + + module = ModuleType('pypaimon_rust.datafusion') + module.PaimonCatalog = Mock() + native_table = module.PaimonCatalog.return_value.get_table.return_value + native_table.location.return_value = table.table_path + with patch.dict('sys.modules', {'pypaimon_rust.datafusion': module}), \ + patch('pypaimon.execution.native.table._with_native_resolved_schema', + return_value=native_table): + assert _native_table(table) is native_table + + native_options = module.PaimonCatalog.call_args.args[0] + assert native_options['fs.oss.endpoint'] == 'https://origin.example.com' + assert native_options['fs.oss.https.enable'] == 'true' + assert 'io-cache.enabled' not in native_options + assert 'io-cache.endpoint' not in native_options + + @pytest.mark.parametrize('transport', ['same-view', 'copy-before', 'copy', 'pickle']) def test_pinned_rest_view_reuses_core_snapshot_validation(guarded_table, transport): import pickle diff --git a/paimon-python/pypaimon/tests/rest/rest_table_file_io_view_test.py b/paimon-python/pypaimon/tests/rest/rest_table_file_io_view_test.py new file mode 100644 index 000000000000..b6a00532152d --- /dev/null +++ b/paimon-python/pypaimon/tests/rest/rest_table_file_io_view_test.py @@ -0,0 +1,65 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Tests that each table of a REST catalog takes its io-cache view of the token FileIO.""" + +import unittest + +import pyarrow as pa + +from pypaimon import CatalogFactory, Schema +from pypaimon.catalog.rest.rest_token_file_io import RESTTokenFileIO +from pypaimon.tests.rest.rest_base_test import RESTBaseTest + + +class RESTTableFileIOViewTest(RESTBaseTest): + + def setUp(self): + super().setUp() + self.options['data-token.enabled'] = 'true' + self.rest_catalog = CatalogFactory.create(self.options) + + def _table(self, name, options): + schema = Schema.from_pyarrow_schema(pa.schema([("id", pa.int32())]), options=options) + self.rest_catalog.create_table(name, schema, False) + table = self.rest_catalog.get_table(name) + self.assertIsInstance(table.file_io, RESTTokenFileIO) + return table + + def test_tables_take_their_view_of_the_token_file_io(self): + # Other writers may replace Format Table files in place, and manage Object Table objects. + self.assertTrue(self._table("default.format_view", {"type": "format-table", "file.format": "csv"}) + .file_io._origin_only) + self.rest_catalog.create_table("default.object_view", Schema(options={"type": "object-table"}), False) + self.assertTrue(self.rest_catalog.get_table("default.object_view").file_io._origin_only) + + table = self._table("default.custom_view", {"data-file.prefix": "custom-", "changelog-file.prefix": "cl-"}) + path = table.table_path + self.assertEqual((False, (path, ("custom-", "cl-"), ("main",))), + (table.file_io._origin_only, table.file_io._table)) + self.assertIs(table.file_io, table.copy({}).file_io) + self.assertEqual((path, ("custom-", "cl-", "dynamic-"), ("main",)), + table.copy({"data-file.prefix": "dynamic-"}).file_io._table) + # A branch copy also reads the schemas of the branch. + self.assertEqual((path, ("custom-", "cl-"), ("main", "dev")), table.copy({"branch": "dev"}).file_io._table) + + plain = self._table("default.plain_view", {}) + self.assertEqual((plain.table_path, (), ("main",)), plain.file_io._table) + + +if __name__ == '__main__': + unittest.main() diff --git a/paimon-python/pypaimon/tests/rest/rest_token_file_io_test.py b/paimon-python/pypaimon/tests/rest/rest_token_file_io_test.py index ad76d12d03d6..0abf6c817a45 100644 --- a/paimon-python/pypaimon/tests/rest/rest_token_file_io_test.py +++ b/paimon-python/pypaimon/tests/rest/rest_token_file_io_test.py @@ -37,7 +37,10 @@ from pypaimon.common.identifier import Identifier from pypaimon.common.options import Options from pypaimon.common.options.config import CatalogOptions, OssOptions +from pypaimon.filesystem.io_cache_file_io import IoCacheRoutingFileIO +from pypaimon.filesystem.io_cache_routing import Op from pypaimon.filesystem.local_file_io import LocalFileIO +from pypaimon.filesystem.vortex_utils import to_vortex_specified from pypaimon.table.row.blob import BlobDescriptor _FILE_IO_GET = 'pypaimon.catalog.rest.rest_token_file_io.FileIO.get' @@ -107,7 +110,7 @@ def refresh(): issued.append(RESTToken( {OssOptions.OSS_SECURITY_TOKEN.key(): "sts-{}".format(len(issued))}, (now[0] + 2 * 3600) * 1000)) - file_io.token = issued[-1] + file_io._adopt(issued[-1]) with patch('pypaimon.catalog.rest.rest_token_file_io.time.time', side_effect=lambda: now[0]), \ @@ -388,7 +391,7 @@ def refresh(file_io): if threading.current_thread() in holders: refreshing.set() release.wait(10) - file_io.token = self._valid_token("refreshed") + file_io._adopt(self._valid_token("refreshed")) self.addCleanup(RESTTokenFileIO._TOKEN_CACHE.pop, 'fork-test', None) with patch.object(RESTTokenFileIO, '_build_cache_key', return_value='fork-test'), \ @@ -481,7 +484,7 @@ def _check_presigned_url_lifetime( validity = timedelta(hours=requested_hours) def refresh(): - file_io.token = new_token + file_io._adopt(new_token) with patch('pypaimon.catalog.rest.rest_token_file_io.time.time', return_value=now), \ patch.object(file_io, '_build_cache_key', return_value='presigning-test'), \ @@ -518,7 +521,7 @@ def test_presigned_url_refreshes_and_resigns_after_materialization(self): validity = timedelta(minutes=30) def refresh(): - file_io.token = new_token + file_io._adopt(new_token) def presign(*_): if now[0] == 1700000000: @@ -1088,6 +1091,191 @@ def test_refresh_token_keeps_normal_identifier(self): called_identifier = mock_api.load_table_token.call_args[0][0] self.assertEqual(called_identifier.get_object_name(), "my_table") + _TABLE = "oss://bkt/db1.db/t1" + _DATA = _TABLE + "/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet" + _IO_CACHE_TOKEN = { + OssOptions.OSS_ENDPOINT.key(): "http://cache.example.com", + "io-cache.enabled": "true", + "io-cache.endpoint": "http://cache.example.com", + "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", + "io-cache.policy": "meta,read", + "fs.oss.dlf-cache.consistent-hash.enabled": "true", + } + + def _resolve_file_io(self, token, catalog_options=None, path=_TABLE, targets=()): + """Returns the FileIO built for the token and the options of every FileIO.get call, + after creating the given targets.""" + file_io = RESTTokenFileIO(self.identifier, path, Options(catalog_options or {})) + file_io.token = RESTToken(token, int(time.time() * 1000) + 7_200_000) + # The FileIO cache is process-wide; a token equal to the previous call's must not reuse its FileIO. + RESTTokenFileIO._FILE_IO_CACHE.clear() + with patch.object(RESTTokenFileIO, 'try_to_refresh_token'), \ + patch('pypaimon.catalog.rest.rest_token_file_io.FileIO.get') as get_io: + get_io.side_effect = lambda location, options: MagicMock(name=location) + resolved = file_io.file_io() + for target in targets: + resolved._file_io(target) + for call in get_io.call_args_list: + self.assertEqual(path, call[0][0]) + return resolved, [call[0][1].to_map() for call in get_io.call_args_list] + + def test_io_cache_routing_builds_origin_and_target_file_io(self): + resolved, created = self._resolve_file_io(self._IO_CACHE_TOKEN) + self.assertIsInstance(resolved, IoCacheRoutingFileIO) + self.assertEqual(1, len(created)) + origin = created[0] + self.assertEqual("https://oss-cn-hangzhou-internal.aliyuncs.com", origin[OssOptions.OSS_ENDPOINT.key()]) + self.assertEqual("true", origin["fs.oss.https.enable"]) + self.assertNotIn("fs.oss.dlf-cache.consistent-hash.enabled", origin) + self.assertEqual("default", resolved._routing.route(Op.READ, self._DATA)) + + _, (_, target) = self._resolve_file_io(self._IO_CACHE_TOKEN, targets=["default"]) + self.assertEqual("http://cache.example.com", target[OssOptions.OSS_ENDPOINT.key()]) + self.assertEqual("false", target["fs.oss.https.enable"]) + self.assertEqual("true", target["fs.oss.dlf-cache.consistent-hash.enabled"]) + + def test_io_cache_routing_builds_one_file_io_per_target(self): + token = dict(self._IO_CACHE_TOKEN, **{ + "io-cache.targets": "accel,cluster", + "io-cache.target.accel.endpoint": "https://accel.example.com", + "io-cache.target.cluster.endpoint": "http://10.0.0.1:8080", + "io-cache.target.cluster.path-style-access": "true", + "io-cache.target.cluster.region": "cn-shanghai", + "io-cache.routes": "meta=accel;data=cluster", + OssOptions.OSS_REGION.key(): "cn-hangzhou", + }) + resolved, (_, accel, cluster) = self._resolve_file_io(token, targets=["accel", "cluster"]) + self.assertEqual("cluster", resolved._routing.route(Op.READ, self._DATA)) + self.assertEqual(("https://accel.example.com", "false", "cn-hangzhou"), ( + accel[OssOptions.OSS_ENDPOINT.key()], accel["fs.oss.second.level.domain.enable"], + accel[OssOptions.OSS_REGION.key()])) + self.assertEqual(("http://10.0.0.1:8080", "true", "cn-shanghai"), ( + cluster[OssOptions.OSS_ENDPOINT.key()], cluster["fs.oss.second.level.domain.enable"], + cluster[OssOptions.OSS_REGION.key()])) + + def _token_file_io(self): + file_io = RESTTokenFileIO(self.identifier, self._TABLE, Options({})) + file_io.token = RESTToken(self._IO_CACHE_TOKEN, int(time.time() * 1000) + 7_200_000) + RESTTokenFileIO._FILE_IO_CACHE.clear() + return file_io + + @staticmethod + def _patched_file_io_get(): + return patch('pypaimon.catalog.rest.rest_token_file_io.FileIO.get', + side_effect=lambda location, options: MagicMock(name=location)) + + def test_io_cache_routing_serves_only_the_table_of_the_token(self): + external = "oss://bkt/import/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet" + resolved, _ = self._resolve_file_io(self._IO_CACHE_TOKEN) + self.assertEqual("default", resolved._routing.route(Op.READ, self._DATA)) + self.assertIsNone(resolved._routing.route(Op.READ, external)) + + def test_table_views_of_the_token_file_io(self): + file_io = self._token_file_io() + custom = self._TABLE + "/bucket-0/custom-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet" + with patch.object(RESTTokenFileIO, 'try_to_refresh_token'), self._patched_file_io_get(): + base = file_io.file_io() + origin_only = file_io.origin_only() + self.assertIsInstance(origin_only, RESTTokenFileIO) + self.assertIs(base._origin, origin_only.file_io()) + self.assertIs(origin_only, origin_only.for_table(self._TABLE, ["custom-"], "main")) + + view = file_io.for_table(self._TABLE, ["custom-"], "main") + self.assertIsInstance(view, RESTTokenFileIO) + self.assertIs(view, view.for_table(self._TABLE, ["custom-"], "main")) + routed = view.file_io() + self.assertIs(routed, view.file_io()) + self.assertEqual("default", routed._routing.route(Op.READ, custom)) + self.assertIsNone(base._routing.route(Op.READ, custom)) + restored = pickle.loads(pickle.dumps(view)) + self.assertEqual((False, (self._TABLE, ("custom-",), ("main",))), (restored._origin_only, restored._table)) + self.assertTrue(pickle.loads(pickle.dumps(origin_only))._origin_only) + + def test_branch_views_take_the_prefixes_the_catalog_keeps_for_the_branch(self): + branch_file = self._TABLE + "/branch/branch-dev/bucket-0/dev-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.orc" + view = self._token_file_io().for_table(self._TABLE, [], "dev") + api = MagicMock() + api.get_table.return_value = MagicMock(schema=MagicMock(options={"data-file.prefix": "dev-"})) + view.api_instance = api + # No schema directory in storage: the REST catalog keeps the branch's schema. + with patch.object(RESTTokenFileIO, 'try_to_refresh_token'), self._patched_file_io_get(), \ + patch('pypaimon.schema.schema_manager.SchemaManager.list_all', return_value=[]): + routing = view.file_io()._routing + self.assertEqual("default", routing.route(Op.READ, branch_file)) + self.assertEqual("default", routing.route(Op.READ, branch_file)) + branch = api.get_table.call_args[0][0] + self.assertEqual(("default", "test_table", "dev"), ( + branch.get_database_name(), branch.get_table_name(), branch.get_branch_name())) + api.get_table.assert_called_once() + + def test_io_cache_routing_off_without_target(self): + token = dict(self._IO_CACHE_TOKEN) + del token["io-cache.endpoint"] + resolved, created = self._resolve_file_io(token) + self.assertNotIsInstance(resolved, IoCacheRoutingFileIO) + self.assertEqual([token], [{key: options[key] for key in token} for options in created]) + + def test_io_cache_routing_off_without_enabled(self): + token = dict(self._IO_CACHE_TOKEN) + del token["io-cache.enabled"] + resolved, created = self._resolve_file_io(token) + self.assertNotIsInstance(resolved, IoCacheRoutingFileIO) + self.assertEqual(["http://cache.example.com"], [options[OssOptions.OSS_ENDPOINT.key()] for options in created]) + + def test_catalog_io_cache_enabled_wins_over_token(self): + enabled = CatalogOptions.IO_CACHE_ENABLED.key() + resolved, created = self._resolve_file_io(self._IO_CACHE_TOKEN, {enabled: "False"}) + self.assertNotIsInstance(resolved, IoCacheRoutingFileIO) + self.assertEqual("false", created[0][enabled]) + + token = dict(self._IO_CACHE_TOKEN) + del token[enabled] + resolved, _ = self._resolve_file_io(token, {enabled: "TRUE"}) + self.assertIsInstance(resolved, IoCacheRoutingFileIO) + + def test_merge_token_applies_catalog_io_cache_enabled(self): + enabled = CatalogOptions.IO_CACHE_ENABLED.key() + token = {enabled: "true", "io-cache.endpoint": "http://cache.example.com"} + for catalog_options, expected in (({}, "true"), ({enabled: False}, "false"), ({enabled: "TRUE"}, "true")): + with self.subTest(catalog_options=catalog_options): + file_io = RESTTokenFileIO(self.identifier, self._TABLE, Options(catalog_options)) + self.assertEqual(expected, file_io._merge_token_with_catalog_options(token)[enabled]) + self.assertEqual("true", token[enabled]) + + def test_io_cache_routing_needs_an_oss_table(self): + resolved, created = self._resolve_file_io(self._IO_CACHE_TOKEN, path=self.warehouse_path) + self.assertNotIsInstance(resolved, IoCacheRoutingFileIO) + self.assertEqual(1, len(created)) + + def test_dlf_oss_endpoint_turns_io_cache_routing_off(self): + override = "oss-cn-hangzhou.aliyuncs.com" + resolved, created = self._resolve_file_io( + self._IO_CACHE_TOKEN, {CatalogOptions.DLF_OSS_ENDPOINT.key(): override}) + self.assertNotIsInstance(resolved, IoCacheRoutingFileIO) + self.assertEqual([override], [options[OssOptions.OSS_ENDPOINT.key()] for options in created]) + + def test_io_cache_routing_hands_vortex_the_origin(self): + # Vortex builds its own virtual-hosted client, so it reads the origin even with a path-style IP target. + token = dict(self._IO_CACHE_TOKEN, **{ + OssOptions.OSS_ACCESS_KEY_ID.key(): "ak", + OssOptions.OSS_ACCESS_KEY_SECRET.key(): "sk", + "io-cache.endpoint": "http://127.0.0.1:8080", + "io-cache.target.default.path-style-access": "true", + }) + path = self._TABLE + "/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.vortex" + file_io = RESTTokenFileIO(self.identifier, self._TABLE, Options({})) + file_io.token = RESTToken(token, int(time.time() * 1000) + 7_200_000) + with patch.object(RESTTokenFileIO, 'try_to_refresh_token'): + url, store_kwargs = to_vortex_specified(file_io, path) + routed = file_io.file_io() + + self.assertIsInstance(routed, IoCacheRoutingFileIO) + self.assertEqual("default", routed._routing.route(Op.READ, path)) + self.assertEqual({}, routed._targets) + self.assertEqual(path.replace("oss://", "s3://"), url) + self.assertEqual("https://bkt.oss-cn-hangzhou-internal.aliyuncs.com", store_kwargs['endpoint']) + self.assertEqual('true', store_kwargs['virtual_hosted_style_request']) + if __name__ == '__main__': unittest.main()