From ef2a480664e6f0feb6a14b2e971312327033119e Mon Sep 17 00:00:00 2001 From: Sun Dapeng Date: Mon, 5 Oct 2026 13:25:02 +0800 Subject: [PATCH 01/21] [core] Route reads to io-cache targets per file type When the options carry io-cache endpoints, for example vended with a REST table token, the OpenDAL OSS FileIO now reads immutable files through a cache target chosen by file type and keeps every other request on the OSS endpoint. The rules are the same as in the Java Jindo FileIO and pypaimon. - io-cache.endpoint names one target. io-cache.targets with io-cache.target..endpoint (and optional .path-style-access) names several, and io-cache.routes maps file types to them. io-cache.whitelist and io-cache.policy keep their meaning, and io-cache.origin.endpoint names the OSS endpoint when fs.oss.endpoint points at a cache for older clients. io-cache.enabled set in the catalog options overrides the token. - Only read, stat and exists requests use a target. Writes, including OutputFile::exists, listings, deletes, renames, copies and non-oss paths stay on OSS, and a client-side dlf.oss-endpoint turns routing off. - Files rewritten in place (FileType::is_mutable) and metadata named by a sequence id never use a target. - Data files are recognized by name: a data or changelog file prefix, including a custom one, followed by {uuid}-{count}.{extension}. Manifests, indexes and statistics share the {uuid}-{count} shape but have no extension, so a custom prefix such as manifest- cannot mix them up; the manifest sidecar is excluded explicitly. Other names keep their FileType, and names Paimon does not write stay on OSS. - Errors from a target are returned as they are, without retrying on OSS. - Adds fs.oss.path-style-access for the OSS endpoint and documents the options in getting-started. The routing cases in testdata/io_cache are shared with the Java Jindo FileIO and pypaimon. --- .../src/catalog/rest/rest_token_file_io.rs | 165 ++++- crates/paimon/src/io/cache/file_type.rs | 8 +- crates/paimon/src/io/cache/mod.rs | 2 +- crates/paimon/src/io/cache_routing.rs | 628 ++++++++++++++++++ crates/paimon/src/io/file_io.rs | 182 +++-- .../src/io/file_io/cache_routing_tests.rs | 400 +++++++++++ crates/paimon/src/io/mod.rs | 4 + crates/paimon/src/io/oss_test_server.rs | 268 ++++++++ crates/paimon/src/io/storage.rs | 80 ++- crates/paimon/src/io/storage_oss.rs | 216 +++++- crates/paimon/src/table/snapshot_manager.rs | 37 ++ .../testdata/io_cache/routable-types.json | 69 ++ crates/paimon/testdata/io_cache/routing.json | 90 +++ docs/src/getting-started.md | 36 + 14 files changed, 2106 insertions(+), 79 deletions(-) create mode 100644 crates/paimon/src/io/cache_routing.rs create mode 100644 crates/paimon/src/io/file_io/cache_routing_tests.rs create mode 100644 crates/paimon/src/io/oss_test_server.rs create mode 100644 crates/paimon/testdata/io_cache/routable-types.json create mode 100644 crates/paimon/testdata/io_cache/routing.json diff --git a/crates/paimon/src/catalog/rest/rest_token_file_io.rs b/crates/paimon/src/catalog/rest/rest_token_file_io.rs index 446a56eaa..869f4d0fb 100644 --- a/crates/paimon/src/catalog/rest/rest_token_file_io.rs +++ b/crates/paimon/src/catalog/rest/rest_token_file_io.rs @@ -31,7 +31,7 @@ use crate::api::rest_util::RESTUtil; use crate::catalog::Identifier; use crate::common::{CatalogOptions, Options}; use crate::io::cache::LocalCache; -use crate::io::{FileIO, FileIOCacheContext, FileIOProvider}; +use crate::io::{FileIO, FileIOCacheContext, FileIOProvider, RoutedOperator, IO_CACHE_ENABLED}; use crate::Result; use super::rest_token::RESTToken; @@ -166,6 +166,10 @@ impl RESTTokenFileIO { merged.insert(OSS_ENDPOINT.to_string(), dlf_oss_endpoint.clone()); } } + // An explicit client value wins over the token, so users can turn io-cache routing off. + if let Some(enabled) = self.catalog_options.get(IO_CACHE_ENABLED) { + merged.insert(IO_CACHE_ENABLED.to_string(), enabled.clone()); + } merged } } @@ -185,6 +189,13 @@ impl FileIOProvider for RESTTokenFileIO { let (op, relative_path) = file_io.create_static(path)?; Ok((op, relative_path, Some(cache_namespace))) } + + async fn create_routed(&self, path: &str) -> Result<(RoutedOperator, String, Option)> { + let file_io = self.current_file_io().await?; + let namespace = file_io.cache_namespace_for_path(path)?; + let (op, relative_path) = file_io.create_routed_static(path)?; + Ok((op, relative_path, Some(namespace))) + } } #[cfg(test)] @@ -376,4 +387,156 @@ mod tests { assert_eq!(requests.load(Ordering::SeqCst), 2); server.abort(); } + + #[cfg(feature = "storage-oss")] + mod io_cache { + use super::*; + use crate::io::oss_test_server::TestOss; + + async fn fixed_token( + State(token): State>>, + ) -> Json { + let now = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_millis() as i64; + Json(GetTableTokenResponse { + token: token.as_ref().clone(), + expires_at_millis: Some(now + TOKEN_EXPIRATION_SAFE_TIME_MILLIS * 2), + }) + } + + /// A token FileIO for `oss://bkt/db.db/t` whose token vends io-cache routing. + async fn io_cache_file_io( + origin: &TestOss, + cache: &TestOss, + catalog_options: &[(&str, &str)], + ) -> (FileIO, tokio::task::JoinHandle<()>) { + let token = HashMap::from( + [ + ("fs.oss.endpoint", cache.endpoint()), + ("fs.oss.accessKeyId", "ak"), + ("fs.oss.accessKeySecret", "sk"), + ("io-cache.enabled", "true"), + ("io-cache.endpoint", cache.endpoint()), + ("io-cache.target.default.path-style-access", "true"), + ("io-cache.origin.endpoint", origin.endpoint()), + ("io-cache.policy", "meta,read"), + ] + .map(|(k, v)| (k.to_string(), v.to_string())), + ); + let app = Router::new() + .route("/v1/databases/db/tables/t/token", get(fixed_token)) + .with_state(Arc::new(token)); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let address = listener.local_addr().unwrap(); + let server = tokio::spawn(async move { + axum::serve(listener, app).await.unwrap(); + }); + + let mut options = Options::new(); + options.set(CatalogOptions::URI, format!("http://{address}")); + options.set(CatalogOptions::TOKEN_PROVIDER, "bear"); + options.set(CatalogOptions::TOKEN, "test-token"); + options.set("fs.oss.path-style-access", "true"); + for (key, value) in catalog_options { + options.set(*key, *value); + } + let api = Arc::new(RESTApi::new(options.clone(), false).await.unwrap()); + let file_io_cache = file_io_cache(&options); + let token_file_io = Arc::new(RESTTokenFileIO::new( + Identifier::new("db", "t"), + "oss://bkt/db.db/t".to_string(), + options, + api, + None, + file_io_cache, + )); + (token_file_io.build_file_io().await.unwrap(), server) + } + + #[tokio::test] + async fn test_token_file_io_routes_table_files_to_io_cache() { + let origin = TestOss::start().await; + let cache = TestOss::start_sharing(&origin).await; + let (file_io, server) = io_cache_file_io(&origin, &cache, &[]).await; + + let data = + "oss://bkt/db.db/t/bucket-0/data-123e4567-e89b-12d3-a456-426614174000-1.parquet"; + file_io + .new_output(data) + .unwrap() + .write(Bytes::from_static(b"data")) + .await + .unwrap(); + assert!(file_io.exists(data).await.unwrap()); + assert_eq!( + file_io.new_input(data).unwrap().read().await.unwrap(), + "data" + ); + assert_eq!( + cache.take_requests(), + [ + "HEAD bkt/db.db/t/bucket-0/data-123e4567-e89b-12d3-a456-426614174000-1.parquet", + "GET bkt/db.db/t/bucket-0/data-123e4567-e89b-12d3-a456-426614174000-1.parquet" + ] + ); + assert_eq!( + origin.take_requests(), + ["PUT bkt/db.db/t/bucket-0/data-123e4567-e89b-12d3-a456-426614174000-1.parquet"] + ); + + server.abort(); + } + + #[tokio::test] + async fn test_dlf_oss_endpoint_turns_off_io_cache_routing() { + let origin = TestOss::start().await; + let cache = TestOss::start_sharing(&origin).await; + let (file_io, server) = io_cache_file_io( + &origin, + &cache, + &[(CatalogOptions::DLF_OSS_ENDPOINT, origin.endpoint())], + ) + .await; + + let data = + "oss://bkt/db.db/t/bucket-0/data-123e4567-e89b-12d3-a456-426614174000-1.parquet"; + file_io + .new_output(data) + .unwrap() + .write(Bytes::from_static(b"data")) + .await + .unwrap(); + assert!(file_io.exists(data).await.unwrap()); + assert!(cache.take_requests().is_empty()); + assert_eq!(origin.take_requests().len(), 2); + server.abort(); + } + + #[tokio::test] + async fn test_client_io_cache_enabled_wins_over_token() { + let origin = TestOss::start().await; + let cache = TestOss::start_sharing(&origin).await; + let (file_io, server) = + io_cache_file_io(&origin, &cache, &[("io-cache.enabled", "false")]).await; + + // Like clients without routing, every request uses fs.oss.endpoint. + let data = + "oss://bkt/db.db/t/bucket-0/data-123e4567-e89b-12d3-a456-426614174000-1.parquet"; + file_io + .new_output(data) + .unwrap() + .write(Bytes::from_static(b"data")) + .await + .unwrap(); + assert_eq!( + file_io.new_input(data).unwrap().read().await.unwrap(), + "data" + ); + assert_eq!(cache.take_requests().len(), 2); + assert!(origin.take_requests().is_empty()); + server.abort(); + } + } } diff --git a/crates/paimon/src/io/cache/file_type.rs b/crates/paimon/src/io/cache/file_type.rs index e01a6e15e..7858c7412 100644 --- a/crates/paimon/src/io/cache/file_type.rs +++ b/crates/paimon/src/io/cache/file_type.rs @@ -22,7 +22,7 @@ use std::collections::HashSet; const TEMP_FILE_SUFFIX_LEN: usize = 41; #[derive(Clone, Copy, Debug, Eq, Hash, PartialEq)] -pub(super) enum FileType { +pub(crate) enum FileType { Meta, Data, BucketIndex, @@ -40,7 +40,7 @@ impl FileType { ]; /// Mirrors Java `org.apache.paimon.utils.FileType#classify`, including its check order. - pub(super) fn classify(path: &str) -> Self { + pub(crate) fn classify(path: &str) -> Self { let mut segments = path.rsplit('/'); let name = unwrap_temp_file_name(segments.next().unwrap_or(path)); @@ -81,7 +81,7 @@ impl FileType { Self::Data } - pub(super) fn is_mutable(path: &str) -> bool { + pub(crate) fn is_mutable(path: &str) -> bool { let name = path.rsplit('/').next().unwrap_or(path); // Iceberg-compatible `version-hint.text`, `retire-pending` and `v{N}.metadata.json` // (on tag changes) are rewritten in place. @@ -98,7 +98,7 @@ impl FileType { || name.contains(".tmp.") } - pub(super) fn parse_whitelist(value: &str) -> HashSet { + pub(crate) fn parse_whitelist(value: &str) -> HashSet { value .split(',') .flat_map(|name| -> &'static [Self] { diff --git a/crates/paimon/src/io/cache/mod.rs b/crates/paimon/src/io/cache/mod.rs index 061cec191..c6b6f0475 100644 --- a/crates/paimon/src/io/cache/mod.rs +++ b/crates/paimon/src/io/cache/mod.rs @@ -21,7 +21,7 @@ mod memory; mod reader; mod state; -use self::file_type::FileType; +pub(crate) use self::file_type::FileType; use self::memory::MemoryCache; use self::state::{BlockKey, CacheCoordinator, CacheReadToken}; use crate::common::{CatalogOptions, Options}; diff --git a/crates/paimon/src/io/cache_routing.rs b/crates/paimon/src/io/cache_routing.rs new file mode 100644 index 000000000..4302d10ac --- /dev/null +++ b/crates/paimon/src/io/cache_routing.rs @@ -0,0 +1,628 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Per-request choice between io-cache targets and the origin OSS endpoint, following the +//! `io-cache.*` options that a REST catalog may vend with a table token. + +#![cfg_attr(not(feature = "storage-oss"), allow(dead_code))] + +use std::collections::{HashMap, HashSet}; +use std::sync::{Arc, LazyLock}; + +use opendal::Operator; +use regex::Regex; + +use crate::common::CatalogOptions; +use crate::io::cache::FileType; + +/// OSS endpoint, compatible with paimon-java's `fs.oss.endpoint`. +pub(crate) const OSS_ENDPOINT: &str = "fs.oss.endpoint"; +pub(crate) const IO_CACHE_ENABLED: &str = "io-cache.enabled"; +pub(crate) const IO_CACHE_ORIGIN_ENDPOINT: &str = "io-cache.origin.endpoint"; +pub(crate) const IO_CACHE_POLICY: &str = "io-cache.policy"; +pub(crate) const IO_CACHE_WHITELIST: &str = "io-cache.whitelist"; +/// Single-target shorthand for a target named `default`, ignored when `io-cache.targets` is set. +pub(crate) const IO_CACHE_ENDPOINT: &str = "io-cache.endpoint"; +pub(crate) const IO_CACHE_TARGETS: &str = "io-cache.targets"; +/// Prefix of `io-cache.target..{endpoint,path-style-access,region}`. +pub(crate) const IO_CACHE_TARGET_PREFIX: &str = "io-cache.target."; +pub(crate) const IO_CACHE_ROUTES: &str = "io-cache.routes"; + +const DEFAULT_TARGET: &str = "default"; +const DATA_FILE_PREFIX: &str = "data-file.prefix"; +const CHANGELOG_FILE_PREFIX: &str = "changelog-file.prefix"; +const DEFAULT_DATA_PREFIXES: [&str; 2] = ["data-", "changelog-"]; +const MANIFEST_SIDECAR_SUFFIX: &str = ".avro.sidecar"; + +// Data files are named {prefix}{uuid}-{count}.{extension}; this matches what follows the prefix. +static DATA_FILE_SUFFIX: LazyLock = LazyLock::new(|| { + Regex::new(r"^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}-[0-9]+\..+$") + .expect("valid data file pattern") +}); + +/// Operation class of a FileIO request; only `Meta` and `Read` can leave origin. +#[derive(Clone, Copy, Debug, Eq, Hash, PartialEq)] +pub(crate) enum OpClass { + /// File status and existence checks, including the size lookup before a cached read. + Meta, + Read, + /// Writes and their existence checks, listing, deletes, renames, mkdirs, copies. + Origin, +} + +impl OpClass { + fn bit(self) -> u8 { + match self { + Self::Meta => 1, + Self::Read => 2, + Self::Origin => 0, + } + } +} + +/// A set of [`OpClass`]es; `Origin` is never a member. +#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)] +pub(crate) struct OpClasses(u8); + +impl OpClasses { + fn insert(&mut self, op: OpClass) { + self.0 |= op.bit(); + } + + pub(crate) fn contains(self, op: OpClass) -> bool { + self.0 & op.bit() != 0 + } + + pub(crate) fn is_empty(self) -> bool { + self.0 == 0 + } +} + +/// Endpoint chosen for a request. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(crate) enum Target { + /// `fs.oss.endpoint`, what clients without io-cache routing use. + Default, + /// `dlf.oss-endpoint` set on the client. + Override, + /// `io-cache.origin.endpoint`, defaulting to `fs.oss.endpoint`. + Origin, + /// The io-cache target at this index of [`IoCacheRouting::targets`]. + Cache(usize), +} + +/// A declared io-cache target. +#[derive(Debug)] +pub(crate) struct CacheTarget { + pub(crate) name: String, + /// `None` when the endpoint is missing or empty. + pub(crate) endpoint: Option, + pub(crate) path_style_access: bool, +} + +/// The `io-cache.*` keys of merged table-token options. +#[derive(Debug)] +pub(crate) struct IoCacheRouting { + override_endpoint: Option, + default_endpoint: Option, + origin_endpoint: Option, + enabled: bool, + policy: OpClasses, + whitelist: HashSet, + targets: Vec, + /// Ordered `(file types, target name)` rules, when `io-cache.routes` is set. + routes: Option, String)>>, + data_prefixes: Vec, +} + +impl IoCacheRouting { + pub(crate) fn from_props(props: &HashMap) -> Self { + let non_empty = |key: &str| props.get(key).filter(|v| !v.trim().is_empty()).cloned(); + let default_endpoint = non_empty(OSS_ENDPOINT); + Self { + override_endpoint: non_empty(CatalogOptions::DLF_OSS_ENDPOINT), + origin_endpoint: non_empty(IO_CACHE_ORIGIN_ENDPOINT) + .or_else(|| default_endpoint.clone()), + default_endpoint, + enabled: flag(props, IO_CACHE_ENABLED, false), + policy: parse_policy(props.get(IO_CACHE_POLICY).map(String::as_str)), + whitelist: FileType::parse_whitelist( + &props + .get(IO_CACHE_WHITELIST) + .map_or("*".to_string(), |value| value.to_ascii_lowercase()), + ), + targets: parse_targets(props), + routes: props.get(IO_CACHE_ROUTES).map(|value| parse_routes(value)), + data_prefixes: data_prefixes(props), + } + } + + /// Whether requests may go anywhere other than the endpoint of clients without routing. + pub(crate) fn enabled(&self) -> bool { + self.override_endpoint.is_none() && self.has_targets() + } + + fn has_targets(&self) -> bool { + self.enabled && !self.targets.is_empty() && !self.policy.is_empty() + } + + pub(crate) fn targets(&self) -> &[CacheTarget] { + &self.targets + } + + /// Applies the routing rules in order, the first match winning. + pub(crate) fn route(&self, op: OpClass, path: &str) -> Target { + if self.override_endpoint.is_some() { + return Target::Override; + } + if !self.has_targets() { + return Target::Default; + } + // Only OSS paths can use cache targets. + if !self.policy.contains(op) || !path.starts_with("oss://") { + return Target::Origin; + } + let Some(file_type) = + routable_type(path, &self.data_prefixes).filter(|t| self.whitelist.contains(t)) + else { + return Target::Origin; + }; + let chosen = match &self.routes { + Some(routes) => routes + .iter() + .find(|(types, _)| types.contains(&file_type)) + .and_then(|(_, name)| self.targets.iter().position(|t| t.name == *name)), + None => self.targets.iter().position(|t| t.endpoint.is_some()), + }; + match chosen { + Some(index) if self.targets[index].endpoint.is_some() => Target::Cache(index), + _ => Target::Origin, + } + } + + /// The target that [`Self::route`] picks for `path`, and the operation classes it serves. + pub(crate) fn cache_target(&self, path: &str) -> Option<(usize, OpClasses)> { + let mut found = None; + let mut classes = OpClasses::default(); + for op in [OpClass::Meta, OpClass::Read] { + if let Target::Cache(index) = self.route(op, path) { + found = Some(index); + classes.insert(op); + } + } + found.map(|index| (index, classes)) + } + + /// Endpoint of requests that never leave origin. + pub(crate) fn origin(&self) -> Option<&str> { + self.endpoint(self.route(OpClass::Origin, "")) + } + + pub(crate) fn endpoint(&self, target: Target) -> Option<&str> { + match target { + Target::Default => self.default_endpoint.as_deref(), + Target::Override => self.override_endpoint.as_deref(), + Target::Origin => self.origin_endpoint.as_deref(), + Target::Cache(index) => self.targets.get(index)?.endpoint.as_deref(), + } + } +} + +/// The type a cache may serve, or `None` when the file must be read from the origin. +pub(crate) fn routable_type(path: &str, data_prefixes: &[String]) -> Option { + let path = path.trim_end_matches('/'); + let name = path.rsplit('/').next().unwrap_or(path); + // Temp files are renamed away once written, whatever prefix they carry. + let temp = name.ends_with(".tmp") || name.contains(".tmp-") || name.contains(".tmp."); + if temp || FileType::is_mutable(path) { + return None; + } + if is_data_file_name(name, data_prefixes) { + return Some(if name.ends_with(".index") { + FileType::FileIndex + } else { + FileType::Data + }); + } + if is_sequential(path) { + return None; + } + match FileType::classify(path) { + FileType::Data => None, + file_type => Some(file_type), + } +} + +// Manifests, indexes and statistics share the uuid-count shape but have no extension. +fn is_data_file_name(name: &str, data_prefixes: &[String]) -> bool { + !name.ends_with(MANIFEST_SIDECAR_SUFFIX) + && data_prefixes.iter().any(|prefix| { + name.strip_prefix(prefix.as_str()) + .is_some_and(|rest| DATA_FILE_SUFFIX.is_match(rest)) + }) +} + +// Sequential metadata may be read before it exists; cached NotFound would hide new commits. +fn is_sequential(path: &str) -> bool { + let mut segments = path.rsplit('/'); + let name = segments.next().unwrap_or(path); + name.starts_with("snapshot-") + || name.starts_with("schema-") + || (name.starts_with("changelog-") && segments.next() == Some("changelog")) +} + +fn flag(props: &HashMap, key: &str, default: bool) -> bool { + props + .get(key) + .map_or(default, |value| value.trim().eq_ignore_ascii_case("true")) +} + +/// Unknown tokens, including `write`, are ignored, and `none` empties the policy. +fn parse_policy(value: Option<&str>) -> OpClasses { + let mut policy = OpClasses::default(); + for token in value.unwrap_or_default().split(',') { + match token.trim().to_ascii_lowercase().as_str() { + "none" => return OpClasses::default(), + "meta" => policy.insert(OpClass::Meta), + "read" => policy.insert(OpClass::Read), + _ => {} + } + } + policy +} + +/// Declared targets in order; `io-cache.targets` wins over the `io-cache.endpoint` shorthand. +fn parse_targets(props: &HashMap) -> Vec { + // OpenDAL's OSS service signs without a region, so `io-cache.target..region` is ignored. + let target = |name: String, endpoint_key: &str| { + let path_style_key = format!("{IO_CACHE_TARGET_PREFIX}{name}.path-style-access"); + CacheTarget { + endpoint: props + .get(endpoint_key) + .map(|endpoint| endpoint.trim()) + .filter(|endpoint| !endpoint.is_empty()) + .map(String::from), + path_style_access: flag(props, &path_style_key, false), + name, + } + }; + let Some(names) = props.get(IO_CACHE_TARGETS) else { + return match props.get(IO_CACHE_ENDPOINT) { + Some(_) => vec![target(DEFAULT_TARGET.to_string(), IO_CACHE_ENDPOINT)], + None => Vec::new(), + }; + }; + let mut unique: Vec = Vec::new(); + for name in names.split(',') { + let name = name.trim().to_ascii_lowercase(); + if is_valid_target_name(&name) && !unique.contains(&name) { + unique.push(name); + } + } + unique + .into_iter() + .map(|name| { + let endpoint_key = format!("{IO_CACHE_TARGET_PREFIX}{name}.endpoint"); + target(name, &endpoint_key) + }) + .collect() +} + +/// `[a-z][a-z0-9-]*`. +fn is_valid_target_name(name: &str) -> bool { + name.starts_with(|c: char| c.is_ascii_lowercase()) + && name + .chars() + .all(|c| c.is_ascii_lowercase() || c.is_ascii_digit() || c == '-') +} + +/// Rules `types=target` separated by `;`; malformed rules are skipped. +fn parse_routes(value: &str) -> Vec<(HashSet, String)> { + value + .split(';') + .filter_map(|rule| { + let (types, target) = rule.trim().split_once('=')?; + let target = target.trim().to_ascii_lowercase(); + let types = FileType::parse_whitelist(&types.to_ascii_lowercase()); + (!target.is_empty() && !types.is_empty()).then_some((types, target)) + }) + .collect() +} + +/// `data-` and `changelog-`, plus the table's own prefixes when they are in the options. +fn data_prefixes(props: &HashMap) -> Vec { + let mut prefixes: Vec = DEFAULT_DATA_PREFIXES.map(String::from).to_vec(); + for key in [DATA_FILE_PREFIX, CHANGELOG_FILE_PREFIX] { + if let Some(prefix) = props.get(key).filter(|prefix| !prefix.trim().is_empty()) { + if !prefixes.contains(prefix) { + prefixes.push(prefix.clone()); + } + } + } + prefixes +} + +/// Operators serving one path: origin, plus the io-cache target when the path may use one. +#[derive(Clone, Debug)] +pub struct RoutedOperator { + origin: Operator, + target: Option>, +} + +#[derive(Debug)] +struct TargetOperator { + operator: Operator, + classes: OpClasses, +} + +impl RoutedOperator { + pub(crate) fn origin(op: Operator) -> Self { + Self { + origin: op, + target: None, + } + } + + pub(crate) fn with_target(origin: Operator, target: Operator, classes: OpClasses) -> Self { + Self { + origin, + target: (!classes.is_empty()).then(|| { + Arc::new(TargetOperator { + operator: target, + classes, + }) + }), + } + } + + pub(crate) fn without_cache(self) -> Self { + Self::origin(self.origin) + } + + pub(crate) fn origin_operator(&self) -> &Operator { + &self.origin + } + + /// The operator for a request of class `op`: the io-cache target when it serves `op`. + pub(crate) fn operator(&self, op: OpClass) -> &Operator { + match self.target.as_deref() { + Some(target) if target.classes.contains(op) => &target.operator, + _ => &self.origin, + } + } + + #[cfg(test)] + pub(crate) fn target(&self) -> Option<&Operator> { + self.target.as_deref().map(|target| &target.operator) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + use serde::Deserialize; + + #[derive(Deserialize)] + struct Vectors { + cases: Vec, + } + + #[derive(Deserialize)] + struct RoutingCase { + name: String, + options: HashMap, + op: String, + path: String, + expect: String, + } + + #[derive(Deserialize)] + struct RoutableTypeCase { + path: String, + options: HashMap, + #[serde(rename = "type")] + file_type: Option, + } + + fn load(name: &str) -> Vectors { + let path = format!("{}/testdata/io_cache/{name}", env!("CARGO_MANIFEST_DIR")); + serde_json::from_str(&std::fs::read_to_string(&path).unwrap()).unwrap() + } + + /// Maps the operation names of the vectors to classes. + fn op_class(name: &str) -> OpClass { + match name { + "read" => OpClass::Read, + "meta" | "exists" => OpClass::Meta, + "write" | "list" | "delete" | "rename" | "mkdirs" | "copy" | "atomic-write" + | "two-phase-write" | "presign" => OpClass::Origin, + other => panic!("unknown operation {other}"), + } + } + + fn routing(options: &[(&str, &str)]) -> IoCacheRouting { + IoCacheRouting::from_props( + &options + .iter() + .map(|(k, v)| (k.to_string(), v.to_string())) + .collect(), + ) + } + + fn route(routing: &IoCacheRouting, op: OpClass, path: &str) -> Option { + routing.endpoint(routing.route(op, path)).map(String::from) + } + + #[test] + fn test_routing_vectors() { + let vectors: Vectors = load("routing.json"); + assert!(!vectors.cases.is_empty()); + for case in vectors.cases { + let routing = IoCacheRouting::from_props(&case.options); + let target = routing.route(op_class(&case.op), &case.path); + assert_eq!( + routing.endpoint(target), + Some(case.expect.as_str()), + "{}", + case.name + ); + } + } + + #[test] + fn test_routable_type_vectors() { + let vectors: Vectors = load("routable-types.json"); + assert!(!vectors.cases.is_empty()); + for case in vectors.cases { + let expected = case + .file_type + .as_deref() + .and_then(|name| FileType::parse_whitelist(name).into_iter().next()); + assert_eq!( + routable_type(&case.path, &data_prefixes(&case.options)), + expected, + "{}", + case.path + ); + } + } + + #[test] + fn test_rust_temp_names_are_not_routable() { + let prefixes = data_prefixes(&HashMap::new()); + for path in [ + "oss://b/t/snapshot/snapshot-1.tmp-123e4567-e89b-12d3-a456-426614174000", + "oss://b/t/bucket-0/data-123e4567-e89b-12d3-a456-426614174000-1.parquet.tmp-123e4567-e89b-12d3-a456-426614174000", + ] { + assert_eq!(routable_type(path, &prefixes), None, "{path}"); + } + assert_eq!( + routable_type( + "oss://b/t/bucket-0/data-123e4567-e89b-12d3-a456-426614174000-0.parquet/", + &prefixes + ), + Some(FileType::Data) + ); + } + + #[test] + fn test_policy_whitelist_and_flags_are_case_insensitive() { + let routing = routing(&[ + (OSS_ENDPOINT, "origin"), + (IO_CACHE_ENABLED, " TRUE "), + (IO_CACHE_ENDPOINT, "cache"), + (IO_CACHE_POLICY, " READ , Meta , write"), + (IO_CACHE_WHITELIST, "DATA"), + ]); + let data = "oss://b/t/bucket-0/data-123e4567-e89b-12d3-a456-426614174000-1.parquet"; + assert_eq!(route(&routing, OpClass::Read, data).unwrap(), "cache"); + assert_eq!(route(&routing, OpClass::Meta, data).unwrap(), "cache"); + assert_eq!(route(&routing, OpClass::Origin, data).unwrap(), "origin"); + assert_eq!( + route(&routing, OpClass::Read, "oss://b/t/manifest/manifest-1").unwrap(), + "origin" + ); + } + + #[test] + fn test_cache_target_is_shared_by_read_and_meta() { + let routing = routing(&[ + (OSS_ENDPOINT, "origin"), + (IO_CACHE_ENABLED, "true"), + (IO_CACHE_TARGETS, "accel, cluster"), + ("io-cache.target.accel.endpoint", "accel"), + ("io-cache.target.cluster.endpoint", "cluster"), + ("io-cache.target.cluster.path-style-access", "true"), + (IO_CACHE_POLICY, "meta,read"), + (IO_CACHE_ROUTES, "meta=accel;data=cluster"), + ]); + let both = OpClasses(OpClass::Meta.bit() | OpClass::Read.bit()); + assert_eq!( + routing.cache_target("oss://b/t/manifest/manifest-1"), + Some((0, both)) + ); + assert_eq!( + routing.cache_target( + "oss://b/t/bucket-0/data-123e4567-e89b-12d3-a456-426614174000-1.parquet" + ), + Some((1, both)) + ); + assert_eq!(routing.cache_target("oss://b/t/snapshot/snapshot-1"), None); + assert_eq!(routing.cache_target("oss://b/t/index/index-1"), None); + assert!(!routing.targets()[0].path_style_access); + assert!(routing.targets()[1].path_style_access); + } + + #[test] + fn test_shorthand_target_settings() { + let routing = routing(&[ + (OSS_ENDPOINT, "origin"), + (IO_CACHE_ENABLED, "true"), + (IO_CACHE_ENDPOINT, "http://cache"), + ("io-cache.target.default.path-style-access", "true"), + (IO_CACHE_POLICY, "read"), + ]); + let [target] = routing.targets() else { + panic!("expected one target"); + }; + assert_eq!(target.name, DEFAULT_TARGET); + assert_eq!(target.endpoint.as_deref(), Some("http://cache")); + assert!(target.path_style_access); + } + + #[test] + fn test_routing_is_off_without_enabled_targets_or_with_override() { + let base = [ + (OSS_ENDPOINT, "default"), + (IO_CACHE_ORIGIN_ENDPOINT, "origin"), + (IO_CACHE_ENDPOINT, "cache"), + (IO_CACHE_POLICY, "read"), + ]; + let data = "oss://b/t/bucket-0/data-123e4567-e89b-12d3-a456-426614174000-1.parquet"; + + let disabled = routing(&base); + assert!(!disabled.enabled()); + assert_eq!(route(&disabled, OpClass::Read, data).unwrap(), "default"); + assert_eq!(disabled.origin(), Some("default")); + + let enabled = routing(&[&base[..], &[(IO_CACHE_ENABLED, "true")]].concat()); + assert!(enabled.enabled()); + assert_eq!(enabled.origin(), Some("origin")); + + let without_targets = routing(&[ + (OSS_ENDPOINT, "default"), + (IO_CACHE_ENABLED, "true"), + (IO_CACHE_POLICY, "read"), + ]); + assert!(!without_targets.enabled()); + + let overridden = routing( + &[ + &base[..], + &[ + (IO_CACHE_ENABLED, "true"), + (CatalogOptions::DLF_OSS_ENDPOINT, "override"), + ], + ] + .concat(), + ); + assert!(!overridden.enabled()); + assert_eq!(route(&overridden, OpClass::Read, data).unwrap(), "override"); + assert_eq!(overridden.origin(), Some("override")); + } +} diff --git a/crates/paimon/src/io/file_io.rs b/crates/paimon/src/io/file_io.rs index e81af113c..a317fb8d0 100644 --- a/crates/paimon/src/io/file_io.rs +++ b/crates/paimon/src/io/file_io.rs @@ -37,8 +37,11 @@ use tokio_util::compat::FuturesAsyncWriteCompatExt; use url::Url; use super::cache::{CachedFileReader, LocalCache}; +use super::cache_routing::{OpClass, RoutedOperator}; use super::Storage; +#[cfg(all(test, feature = "storage-oss"))] +mod cache_routing_tests; #[cfg(all(test, feature = "storage-memory"))] mod provider_tests; @@ -90,6 +93,16 @@ pub trait FileIOProvider: std::fmt::Debug + Send + Sync + 'static { let (op, relative_path) = self.create(path).await?; Ok((op, relative_path, None)) } + + /// Like [`Self::create`], adding the io-cache endpoint for requests that may use it. + #[doc(hidden)] + async fn create_routed( + &self, + path: &str, + ) -> crate::Result<(RoutedOperator, String, Option)> { + let (op, relative_path, namespace) = self.create_with_cache_namespace(path).await?; + Ok((RoutedOperator::origin(op), relative_path, namespace)) + } } #[derive(Clone, Debug)] @@ -105,6 +118,7 @@ pub struct FileIO { cache: Option>, file_format_metadata_cache: Arc, blob_index_cache: Arc, + origin_only: bool, } pub(crate) const DEFAULT_FILE_FORMAT_METADATA_CACHE_MAX_BYTES: usize = 50 * 1024 * 1024; @@ -302,13 +316,38 @@ impl FileIO { } } - pub(crate) fn create_static(&self, path: &str) -> crate::Result<(Operator, String)> { + /// A view of this FileIO that sends every request to origin, as copies require. + pub(crate) fn origin_only(&self) -> Self { + let mut file_io = self.clone(); + file_io.origin_only = true; + file_io + } + + fn storage(&self) -> crate::Result<&Storage> { let FileIOBackend::Storage(storage) = &self.backend else { return Err(Error::IoUnsupported { message: "A FileIOProvider requires async path resolution".to_string(), }); }; - let (op, relative_path) = storage.create(path)?; + Ok(storage) + } + + /// The operator for requests that always go to origin, and the relative path. + pub(crate) fn create_static(&self, path: &str) -> crate::Result<(Operator, String)> { + let (op, relative_path) = self.storage()?.create(path)?; + Ok((op, relative_path.into_owned())) + } + + pub(crate) fn create_routed_static( + &self, + path: &str, + ) -> crate::Result<(RoutedOperator, String)> { + let (op, relative_path) = self.storage()?.create_routed(path)?; + let op = if self.origin_only { + op.without_cache() + } else { + op + }; Ok((op, relative_path.into_owned())) } @@ -343,13 +382,28 @@ impl FileIO { Ok((op, relative_path, cache_path)) } + async fn create_routed(&self, path: &str) -> crate::Result<(RoutedOperator, String)> { + match &self.backend { + FileIOBackend::Provider(provider) => { + let (op, relative_path, _) = + resolve_provider_routed(provider.as_ref(), path, self.origin_only).await?; + Ok((op, relative_path)) + } + FileIOBackend::Storage(_) => self.create_routed_static(path), + } + } + fn file_source(&self, path: &str) -> crate::Result { match &self.backend { - FileIOBackend::Provider(provider) => Ok(FileSource::Provider(provider.clone())), + FileIOBackend::Provider(provider) => Ok(FileSource::Provider { + provider: provider.clone(), + origin_only: self.origin_only, + }), FileIOBackend::Storage(_) => { - let (op, relative_path) = self.create_static(path)?; + let (op, relative_path) = self.create_routed_static(path)?; let namespace = self.cache_namespace_for_path(path)?; - let cache_path = cache_object_path(&namespace, &op, &relative_path); + let cache_path = + cache_object_path(&namespace, op.origin_operator(), &relative_path); Ok(FileSource::Static { op, relative_path, @@ -459,13 +513,17 @@ impl FileIO { /// /// Reference: pub async fn get_status(&self, path: &str) -> Result { - let (op, relative_path) = self.create(path).await?; - let meta = op.stat(relative_path.as_ref()).await.map_err(|error| { - Error::from_opendal_with_context( - error, - format!("Failed to get file status for '{path}'"), - ) - })?; + let (op, relative_path) = self.create_routed(path).await?; + let meta = op + .operator(OpClass::Meta) + .stat(relative_path.as_ref()) + .await + .map_err(|error| { + Error::from_opendal_with_context( + error, + format!("Failed to get file status for '{path}'"), + ) + })?; Ok(FileStatus { size: meta.content_length(), @@ -600,14 +658,17 @@ impl FileIO { /// /// References: pub async fn exists(&self, path: &str) -> Result { - let (op, relative_path) = self.create(path).await?; + let (op, relative_path) = self.create_routed(path).await?; - op.exists(relative_path.as_ref()).await.map_err(|error| { - Error::from_opendal_with_context( - error, - format!("Failed to check existence of '{path}'"), - ) - }) + op.operator(OpClass::Meta) + .exists(relative_path.as_ref()) + .await + .map_err(|error| { + Error::from_opendal_with_context( + error, + format!("Failed to check existence of '{path}'"), + ) + }) } /// Check if a directory exists. @@ -684,9 +745,10 @@ impl FileIO { /// /// Overwrites dst if it already exists. pub async fn copy_file(&self, src: &str, dst: &str) -> Result<()> { - let input = self.new_input(src)?; + let file_io = self.origin_only(); + let input = file_io.new_input(src)?; let bytes = input.read().await?; - let output = self.new_output(dst)?; + let output = file_io.new_output(dst)?; output.write(bytes).await?; Ok(()) } @@ -696,10 +758,11 @@ impl FileIO { /// rename, as is common for object stores and the in-memory test backend. pub async fn copy_file_streaming(&self, src: &str, dst: &str) -> Result<()> { const CHUNK_SIZE: u64 = 8 * 1024 * 1024; - let input = self.new_input(src)?; + let file_io = self.origin_only(); + let input = file_io.new_input(src)?; let size = input.metadata().await?.size; let reader = input.reader().await?; - let output = self.new_output(dst)?; + let output = file_io.new_output(dst)?; let mut writer = output.writer().await?; let mut position = 0; while position < size { @@ -776,6 +839,21 @@ async fn resolve_provider_with_cache_namespace( Ok((op, relative_path, cache_namespace)) } +async fn resolve_provider_routed( + provider: &dyn FileIOProvider, + path: &str, + origin_only: bool, +) -> crate::Result<(RoutedOperator, String, Option)> { + let (op, relative_path, namespace) = if origin_only { + let (op, relative_path, namespace) = provider.create_with_cache_namespace(path).await?; + (RoutedOperator::origin(op), relative_path, namespace) + } else { + provider.create_routed(path).await? + }; + validate_provider_path(path, &relative_path)?; + Ok((op, relative_path, namespace)) +} + fn validate_provider_path(path: &str, relative_path: &str) -> Result<()> { // Filesystem paths retain their existing separator normalization. Object // keys must not silently resolve to another object through OpenDAL's path @@ -987,6 +1065,7 @@ impl FileIOBuilder { cache, file_format_metadata_cache, blob_index_cache, + origin_only: false, }) } } @@ -1186,21 +1265,28 @@ pub struct FileStatus { #[derive(Clone, Debug)] enum FileSource { Static { - op: Operator, + op: RoutedOperator, relative_path: String, cache_path: String, }, - Provider(Arc), + Provider { + provider: Arc, + origin_only: bool, + }, } impl FileSource { - async fn resolve(&self, path: &str) -> crate::Result<(Operator, String, Option)> { + async fn resolve(&self, path: &str) -> crate::Result<(RoutedOperator, String, Option)> { match self { - Self::Provider(provider) => { - let (op, relative_path, resolved_namespace) = - resolve_provider_with_cache_namespace(provider.as_ref(), path).await?; - let cache_path = resolved_namespace - .map(|namespace| cache_object_path(&namespace, &op, &relative_path)); + Self::Provider { + provider, + origin_only, + } => { + let (op, relative_path, namespace) = + resolve_provider_routed(provider.as_ref(), path, *origin_only).await?; + let cache_path = namespace.map(|namespace| { + cache_object_path(&namespace, op.origin_operator(), &relative_path) + }); Ok((op, relative_path, cache_path)) } Self::Static { @@ -1228,12 +1314,12 @@ impl InputFile { pub async fn exists(&self) -> crate::Result { let (op, relative_path, _) = self.source.resolve(&self.path).await?; - Ok(op.exists(&relative_path).await?) + Ok(op.operator(OpClass::Meta).exists(&relative_path).await?) } pub async fn metadata(&self) -> crate::Result { let (op, relative_path, _) = self.source.resolve(&self.path).await?; - let meta = op.stat(&relative_path).await?; + let meta = op.operator(OpClass::Meta).stat(&relative_path).await?; Ok(FileStatus { size: meta.content_length(), @@ -1248,17 +1334,25 @@ impl InputFile { pub async fn read(&self) -> crate::Result { let (op, relative_path, cache_path) = self.source.resolve(&self.path).await?; let (Some(cache), Some(cache_path)) = (&self.cache, cache_path) else { - return Ok(op.read(&relative_path).await?.to_bytes()); + return Ok(op + .operator(OpClass::Read) + .read(&relative_path) + .await? + .to_bytes()); }; let read_token = cache.read_token(&cache_path); let size = if let Some(size) = cache.file_size(&cache_path, &read_token).await { size } else { - let size = op.stat(&relative_path).await?.content_length(); + let size = op + .operator(OpClass::Meta) + .stat(&relative_path) + .await? + .content_length(); cache.put_file_size(&cache_path, size, &read_token).await; size }; - let delegate = Arc::new(op.reader(&relative_path).await?); + let delegate = Arc::new(op.operator(OpClass::Read).reader(&relative_path).await?); CachedFileReader::new_with_token(delegate, &cache_path, size, cache.clone(), read_token) .read_full() .await @@ -1266,7 +1360,7 @@ impl InputFile { pub async fn reader(&self) -> crate::Result { let (op, relative_path, cache_path) = self.source.resolve(&self.path).await?; - let reader = op.reader(&relative_path).await?; + let reader = op.operator(OpClass::Read).reader(&relative_path).await?; let (Some(cache), Some(cache_path)) = (&self.cache, cache_path.as_ref()) else { return Ok(InputFileReader::Direct { reader, @@ -1279,7 +1373,11 @@ impl InputFile { let size = if let Some(size) = cache.file_size(cache_path, &read_token).await { size } else { - let size = op.stat(&relative_path).await?.content_length(); + let size = op + .operator(OpClass::Meta) + .stat(&relative_path) + .await? + .content_length(); cache.put_file_size(cache_path, size, &read_token).await; size }; @@ -1314,7 +1412,7 @@ impl OutputFile { pub async fn exists(&self) -> crate::Result { let (op, relative_path, _) = self.source.resolve(&self.path).await?; - Ok(op.exists(&relative_path).await?) + Ok(op.origin_operator().exists(&relative_path).await?) } pub fn to_input_file(self) -> InputFile { @@ -1337,7 +1435,8 @@ impl OutputFile { pub async fn writer(&self) -> crate::Result> { let (op, relative_path, cache_path) = self.source.resolve(&self.path).await?; let writer: Box = Box::new( - op.writer_with(&relative_path) + op.origin_operator() + .writer_with(&relative_path) .chunk(8 * 1024 * 1024) .await?, ); @@ -1355,7 +1454,8 @@ impl OutputFile { pub(crate) async fn async_writer(&self) -> crate::Result> { let (op, relative_path, cache_path) = self.source.resolve(&self.path).await?; let writer: Box = Box::new( - op.writer_with(&relative_path) + op.origin_operator() + .writer_with(&relative_path) .chunk(8 * 1024 * 1024) .concurrent(1) .await? diff --git a/crates/paimon/src/io/file_io/cache_routing_tests.rs b/crates/paimon/src/io/file_io/cache_routing_tests.rs new file mode 100644 index 000000000..291fbf6ba --- /dev/null +++ b/crates/paimon/src/io/file_io/cache_routing_tests.rs @@ -0,0 +1,400 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use super::*; +use crate::io::oss_test_server::TestOss; + +const DATA: &str = "oss://bkt/db.db/t/bucket-0/data-123e4567-e89b-12d3-a456-426614174000-1.parquet"; +const DATA_KEY: &str = "bkt/db.db/t/bucket-0/data-123e4567-e89b-12d3-a456-426614174000-1.parquet"; +const MANIFEST: &str = "oss://bkt/db.db/t/manifest/manifest-1"; +const MANIFEST_KEY: &str = "bkt/db.db/t/manifest/manifest-1"; +const INDEX: &str = "oss://bkt/db.db/t/index/index-1"; +const LATEST: &str = "oss://bkt/db.db/t/snapshot/LATEST"; +const SNAPSHOT: &str = "oss://bkt/db.db/t/snapshot/snapshot-1"; +const UNREACHABLE: &str = "http://127.0.0.1:1"; + +fn oss_file_io(props: &[(&str, &str)]) -> FileIO { + FileIOBuilder::new("oss") + .with_props([ + ("fs.oss.accessKeyId", "ak"), + ("fs.oss.accessKeySecret", "sk"), + ("fs.oss.path-style-access", "true"), + ("fs.oss.retry.count", "3"), + ("fs.oss.retry.interval.millisecond", "1"), + ]) + .with_props(props.iter().copied()) + .build() + .unwrap() +} + +/// Cache and origin servers over the same objects, and a FileIO routing to the single target. +async fn routed(policy: &str) -> (TestOss, TestOss, FileIO) { + let origin = TestOss::start().await; + let cache = TestOss::start_sharing(&origin).await; + let file_io = oss_file_io(&[ + ("fs.oss.endpoint", cache.endpoint()), + ("io-cache.enabled", "true"), + ("io-cache.endpoint", cache.endpoint()), + ("io-cache.target.default.path-style-access", "true"), + ("io-cache.origin.endpoint", origin.endpoint()), + ("io-cache.policy", policy), + ]); + (origin, cache, file_io) +} + +/// Origin and two targets over the same objects: manifests use `accel`, data uses `cluster`. +async fn multi_target() -> (TestOss, TestOss, TestOss, FileIO) { + let origin = TestOss::start().await; + let accel = TestOss::start_sharing(&origin).await; + let cluster = TestOss::start_sharing(&origin).await; + let file_io = oss_file_io(&[ + ("fs.oss.endpoint", origin.endpoint()), + ("io-cache.enabled", "true"), + ("io-cache.targets", "accel,cluster"), + ("io-cache.target.accel.endpoint", accel.endpoint()), + ("io-cache.target.accel.path-style-access", "true"), + ("io-cache.target.accel.region", "cn-hangzhou"), + ("io-cache.target.cluster.endpoint", cluster.endpoint()), + ("io-cache.target.cluster.path-style-access", "true"), + ("io-cache.policy", "meta,read"), + ("io-cache.routes", "meta=accel;data=cluster"), + ]); + (origin, accel, cluster, file_io) +} + +async fn write(file_io: &FileIO, path: &str, data: &'static [u8]) { + file_io + .new_output(path) + .unwrap() + .write(Bytes::from_static(data)) + .await + .unwrap(); +} + +async fn read(file_io: &FileIO, path: &str) -> Bytes { + file_io.new_input(path).unwrap().read().await.unwrap() +} + +fn is_not_found(error: crate::Error) -> bool { + let crate::Error::IoUnexpected { source, .. } = error else { + return false; + }; + source.kind() == opendal::ErrorKind::NotFound +} + +#[tokio::test] +async fn test_only_reads_and_status_use_the_target() { + let (origin, cache, file_io) = routed("meta,read").await; + + write(&file_io, DATA, b"0123456789").await; + // A writer checks existence on origin; readers check it like file status. + assert!(file_io.new_output(DATA).unwrap().exists().await.unwrap()); + assert_eq!( + origin.take_requests(), + [format!("PUT {DATA_KEY}"), format!("HEAD {DATA_KEY}")] + ); + assert!(file_io.exists(DATA).await.unwrap()); + let input = file_io.new_input(DATA).unwrap(); + assert!(input.exists().await.unwrap()); + assert_eq!( + cache.take_requests(), + [format!("HEAD {DATA_KEY}"), format!("HEAD {DATA_KEY}")] + ); + + assert_eq!(file_io.get_status(DATA).await.unwrap().size, 10); + assert_eq!(input.metadata().await.unwrap().size, 10); + assert_eq!(input.read().await.unwrap(), "0123456789"); + let reader = input.reader().await.unwrap(); + assert_eq!(reader.read(2..5).await.unwrap(), "234"); + assert_eq!( + cache.take_requests(), + [ + format!("HEAD {DATA_KEY}"), + format!("HEAD {DATA_KEY}"), + format!("GET {DATA_KEY}"), + format!("GET {DATA_KEY}"), + ] + ); + assert!(origin.take_requests().is_empty()); + + // Listing, copies, deletes, and files that are not on the allowlist always use origin. + let statuses = file_io + .list_status("oss://bkt/db.db/t/bucket-0/") + .await + .unwrap(); + assert_eq!(statuses.len(), 1); + write(&file_io, LATEST, b"1").await; + assert_eq!(read(&file_io, LATEST).await, "1"); + write(&file_io, SNAPSHOT, b"{}").await; + assert_eq!(read(&file_io, SNAPSHOT).await, "{}"); + assert_eq!(file_io.get_status(SNAPSHOT).await.unwrap().size, 2); + let unknown = "oss://bkt/db.db/t/bucket-0/part-00000.parquet"; + write(&file_io, unknown, b"x").await; + assert_eq!(read(&file_io, unknown).await, "x"); + file_io + .copy_file( + DATA, + "oss://bkt/db.db/t/bucket-0/data-123e4567-e89b-12d3-a456-426614174000-2.parquet", + ) + .await + .unwrap(); + file_io.delete_file(DATA).await.unwrap(); + assert!(cache.take_requests().is_empty()); + assert_eq!( + origin.take_requests(), + [ + "LIST bkt/db.db/t/bucket-0/".to_string(), + "PUT bkt/db.db/t/snapshot/LATEST".to_string(), + "GET bkt/db.db/t/snapshot/LATEST".to_string(), + "PUT bkt/db.db/t/snapshot/snapshot-1".to_string(), + "GET bkt/db.db/t/snapshot/snapshot-1".to_string(), + "HEAD bkt/db.db/t/snapshot/snapshot-1".to_string(), + "PUT bkt/db.db/t/bucket-0/part-00000.parquet".to_string(), + "GET bkt/db.db/t/bucket-0/part-00000.parquet".to_string(), + format!("GET {DATA_KEY}"), + "PUT bkt/db.db/t/bucket-0/data-123e4567-e89b-12d3-a456-426614174000-2.parquet" + .to_string(), + format!("DELETE {DATA_KEY}"), + ] + ); +} + +#[tokio::test] +async fn test_policy_selects_operation_classes() { + let (origin, cache, file_io) = routed("read").await; + + write(&file_io, DATA, b"data").await; + assert_eq!(file_io.get_status(DATA).await.unwrap().size, 4); + assert_eq!( + origin.take_requests(), + [format!("PUT {DATA_KEY}"), format!("HEAD {DATA_KEY}")] + ); + + assert_eq!(read(&file_io, DATA).await, "data"); + assert_eq!(cache.take_requests(), [format!("GET {DATA_KEY}")]); +} + +#[tokio::test] +async fn test_target_not_found_is_final() { + let (origin, cache, file_io) = routed("meta,read").await; + write(&file_io, DATA, b"data").await; + origin.take_requests(); + cache.hide(DATA_KEY); + + assert!(is_not_found(file_io.get_status(DATA).await.unwrap_err())); + let input = file_io.new_input(DATA).unwrap(); + assert!(is_not_found(input.read().await.unwrap_err())); + let reader = input.reader().await.unwrap(); + assert!(is_not_found(reader.read(0..4).await.unwrap_err())); + assert_eq!( + cache.take_requests(), + [ + format!("HEAD {DATA_KEY}"), + format!("GET {DATA_KEY}"), + format!("GET {DATA_KEY}") + ] + ); + assert!(origin.take_requests().is_empty()); +} + +#[tokio::test] +async fn test_target_errors_are_not_retried_on_origin() { + let (origin, cache, file_io) = routed("meta,read").await; + write(&file_io, DATA, b"data").await; + origin.take_requests(); + cache.set_unavailable(true); + + assert!(file_io.get_status(DATA).await.is_err()); + assert!(file_io.new_input(DATA).unwrap().read().await.is_err()); + // The target gets the initial attempt plus fs.oss.retry.count retries; origin none. + let served = cache.take_requests(); + let count = |method: &str| { + served + .iter() + .filter(|r| **r == format!("{method} {DATA_KEY}")) + .count() + }; + assert_eq!((count("HEAD"), count("GET")), (4, 4)); + assert!(origin.take_requests().is_empty()); +} + +#[tokio::test] +async fn test_writes_never_use_an_unavailable_target() { + let (origin, cache, file_io) = routed("meta,read").await; + cache.set_unavailable(true); + + write(&file_io, DATA, b"data").await; + let mut writer = file_io + .new_output(MANIFEST) + .unwrap() + .writer() + .await + .unwrap(); + writer.write(Bytes::from_static(b"manifest")).await.unwrap(); + writer.close().await.unwrap(); + assert!(file_io + .new_output(MANIFEST) + .unwrap() + .exists() + .await + .unwrap()); + assert!(cache.take_requests().is_empty()); + assert_eq!(origin.object(MANIFEST_KEY).unwrap(), "manifest"); +} + +#[tokio::test] +async fn test_targets_serve_the_file_types_of_their_routes() { + let (origin, accel, cluster, file_io) = multi_target().await; + write(&file_io, DATA, b"data").await; + write(&file_io, MANIFEST, b"manifest").await; + write(&file_io, INDEX, b"index").await; + assert_eq!(origin.take_requests().len(), 3); + + assert_eq!(read(&file_io, MANIFEST).await, "manifest"); + assert_eq!(file_io.get_status(MANIFEST).await.unwrap().size, 8); + assert_eq!(read(&file_io, DATA).await, "data"); + assert_eq!(file_io.get_status(DATA).await.unwrap().size, 4); + // No route names bucket indexes; existence checks follow the routes like file status. + assert_eq!(read(&file_io, INDEX).await, "index"); + assert!(file_io.exists(MANIFEST).await.unwrap()); + assert_eq!( + accel.take_requests(), + [ + format!("GET {MANIFEST_KEY}"), + format!("HEAD {MANIFEST_KEY}"), + format!("HEAD {MANIFEST_KEY}") + ] + ); + assert_eq!( + cluster.take_requests(), + [format!("GET {DATA_KEY}"), format!("HEAD {DATA_KEY}")] + ); + assert_eq!( + origin.take_requests(), + ["GET bkt/db.db/t/index/index-1".to_string()] + ); +} + +#[tokio::test] +async fn test_target_without_endpoint_is_skipped() { + let origin = TestOss::start().await; + let cluster = TestOss::start_sharing(&origin).await; + let file_io = oss_file_io(&[ + ("fs.oss.endpoint", origin.endpoint()), + ("io-cache.enabled", "true"), + ("io-cache.targets", "accel,cluster"), + ("io-cache.target.accel.path-style-access", "true"), + ("io-cache.target.cluster.endpoint", cluster.endpoint()), + ("io-cache.target.cluster.path-style-access", "true"), + ("io-cache.policy", "read"), + ]); + write(&file_io, DATA, b"data").await; + + // Without routes, the first target with an endpoint takes every type. + assert_eq!(read(&file_io, DATA).await, "data"); + assert_eq!(cluster.take_requests(), [format!("GET {DATA_KEY}")]); +} + +#[tokio::test] +async fn test_local_cache_reads_through_the_target() { + let (origin, cache, mut file_io) = routed("meta,read").await; + file_io.cache = Some(Arc::new(local_cache())); + write(&file_io, DATA, b"0123456789").await; + origin.take_requests(); + + assert_eq!(read(&file_io, DATA).await, "0123456789"); + let reader = file_io.new_input(DATA).unwrap().reader().await.unwrap(); + assert_eq!(reader.read(0..4).await.unwrap(), "0123"); + // The size and the bytes come from the target once; later reads are local. + assert_eq!( + cache.take_requests(), + [format!("HEAD {DATA_KEY}"), format!("GET {DATA_KEY}")] + ); + assert!(origin.take_requests().is_empty()); +} + +fn local_cache() -> crate::io::cache::LocalCache { + use crate::io::cache::LocalCacheConfig; + use crate::{CatalogOptions, Options}; + + let mut options = Options::new(); + options.set(CatalogOptions::LOCAL_CACHE_ENABLED, "true"); + options.set(CatalogOptions::LOCAL_CACHE_WHITELIST, "*"); + crate::io::cache::LocalCache::new(LocalCacheConfig::from_options(&options).unwrap().unwrap()) + .unwrap() +} + +#[tokio::test] +async fn test_origin_only_view_skips_the_target() { + let (origin, cache, file_io) = routed("meta,read").await; + + let file_io = file_io.origin_only(); + write(&file_io, DATA, b"data").await; + assert_eq!(file_io.get_status(DATA).await.unwrap().size, 4); + assert_eq!(read(&file_io, DATA).await, "data"); + assert!(cache.take_requests().is_empty()); + assert_eq!(origin.take_requests().len(), 3); +} + +#[tokio::test] +async fn test_dlf_oss_endpoint_disables_routing() { + let target = TestOss::start().await; + let cache = TestOss::start_sharing(&target).await; + let file_io = oss_file_io(&[ + ("fs.oss.endpoint", cache.endpoint()), + ("dlf.oss-endpoint", target.endpoint()), + ("io-cache.enabled", "true"), + ("io-cache.endpoint", cache.endpoint()), + ("io-cache.origin.endpoint", UNREACHABLE), + ("io-cache.policy", "meta,read"), + ]); + + write(&file_io, DATA, b"data").await; + assert_eq!(read(&file_io, DATA).await, "data"); + assert!(cache.take_requests().is_empty()); + assert_eq!(target.take_requests().len(), 2); +} + +#[tokio::test] +async fn test_routing_is_off_unless_enabled() { + let server = TestOss::start().await; + let file_io = oss_file_io(&[ + ("fs.oss.endpoint", server.endpoint()), + ("io-cache.endpoint", UNREACHABLE), + ("io-cache.origin.endpoint", UNREACHABLE), + ("io-cache.policy", "meta,read"), + ]); + + let (op, _) = file_io.create_routed_static(DATA).unwrap(); + assert!(op.target().is_none()); + write(&file_io, DATA, b"data").await; + assert_eq!(read(&file_io, DATA).await, "data"); + assert_eq!(server.take_requests().len(), 2); +} + +#[tokio::test] +async fn test_cache_key_is_identical_on_target_and_origin() { + let (_origin, _cache, file_io) = routed("read").await; + + let (op, relative_path) = file_io.create_routed_static(DATA).unwrap(); + let target = op.target().unwrap(); + let namespace = file_io.cache_namespace_for_path(DATA).unwrap(); + assert_eq!( + cache_object_path(&namespace, target, &relative_path), + cache_object_path(&namespace, op.origin_operator(), &relative_path) + ); +} diff --git a/crates/paimon/src/io/mod.rs b/crates/paimon/src/io/mod.rs index 2718ab6b1..81572f367 100644 --- a/crates/paimon/src/io/mod.rs +++ b/crates/paimon/src/io/mod.rs @@ -16,7 +16,11 @@ // under the License. pub(crate) mod cache; +mod cache_routing; +pub(crate) use cache_routing::{RoutedOperator, IO_CACHE_ENABLED}; mod file_io; +#[cfg(all(test, feature = "storage-oss"))] +pub(crate) mod oss_test_server; pub(crate) mod uri_reader; pub use file_io::*; diff --git a/crates/paimon/src/io/oss_test_server.rs b/crates/paimon/src/io/oss_test_server.rs new file mode 100644 index 000000000..e224772a2 --- /dev/null +++ b/crates/paimon/src/io/oss_test_server.rs @@ -0,0 +1,268 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! A path-style OSS stand-in for tests that records every request it serves. + +use std::collections::{BTreeMap, BTreeSet}; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::{Arc, Mutex}; + +use axum::body::{Body, Bytes}; +use axum::extract::State; +use axum::http::{header, HeaderMap, Method, Response, StatusCode, Uri}; +use axum::Router; +use percent_encoding::percent_decode_str; + +type Objects = Arc>>; + +#[derive(Default)] +struct ServerState { + objects: Objects, + requests: Mutex>, + unavailable: AtomicBool, + hidden: Mutex>, +} + +pub(crate) struct TestOss { + endpoint: String, + state: Arc, + server: tokio::task::JoinHandle<()>, +} + +impl Drop for TestOss { + fn drop(&mut self) { + self.server.abort(); + } +} + +impl TestOss { + pub(crate) async fn start() -> Self { + Self::serve(Objects::default()).await + } + + /// A server over the same objects, like a cache in front of `origin`. + pub(crate) async fn start_sharing(origin: &TestOss) -> Self { + Self::serve(origin.state.objects.clone()).await + } + + async fn serve(objects: Objects) -> Self { + let state = Arc::new(ServerState { + objects, + ..ServerState::default() + }); + let app = Router::new().fallback(handle).with_state(state.clone()); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let endpoint = format!("http://{}", listener.local_addr().unwrap()); + let server = tokio::spawn(async move { axum::serve(listener, app).await.unwrap() }); + Self { + endpoint, + state, + server, + } + } + + pub(crate) fn endpoint(&self) -> &str { + &self.endpoint + } + + /// Answers every request with 503 while set. + pub(crate) fn set_unavailable(&self, unavailable: bool) { + self.state.unavailable.store(unavailable, Ordering::SeqCst); + } + + /// Answers HEAD and GET of `bucket_and_key` with 404, like a cache that remembers it missing. + pub(crate) fn hide(&self, bucket_and_key: &str) { + self.state + .hidden + .lock() + .unwrap() + .insert(bucket_and_key.to_string()); + } + + /// Requests served so far, as `METHOD bucket/key`, or `LIST bucket/prefix`. + pub(crate) fn take_requests(&self) -> Vec { + std::mem::take(&mut *self.state.requests.lock().unwrap()) + } + + pub(crate) fn object(&self, bucket_and_key: &str) -> Option { + self.state + .objects + .lock() + .unwrap() + .get(bucket_and_key) + .cloned() + } +} + +async fn handle( + State(state): State>, + method: Method, + uri: Uri, + headers: HeaderMap, + body: Bytes, +) -> Response { + let path = percent_decode_str(uri.path().trim_start_matches('/')) + .decode_utf8_lossy() + .into_owned(); + let query = uri.query().unwrap_or_default(); + let is_list = method == Method::GET && query.contains("list-type"); + let label = if is_list { + let prefix = query_value(query, "prefix").unwrap_or_default(); + format!("LIST {}/{prefix}", path.trim_end_matches('/')) + } else { + format!("{method} {path}") + }; + state.requests.lock().unwrap().push(label); + + if state.unavailable.load(Ordering::SeqCst) { + return error(StatusCode::SERVICE_UNAVAILABLE, "ServiceUnavailable"); + } + if is_list { + return list(&state.objects, path.trim_end_matches('/'), query); + } + let mut objects = state.objects.lock().unwrap(); + if method == Method::POST && query.starts_with("delete") { + let mut deleted = String::new(); + let body = String::from_utf8_lossy(&body); + for key in body + .split("") + .skip(1) + .filter_map(|s| s.split_once("")) + { + objects.remove(&format!("{}/{}", path.trim_end_matches('/'), key.0)); + deleted.push_str(&format!("{}", key.0)); + } + return response(StatusCode::OK) + .body(Body::from(format!( + "{deleted}" + ))) + .unwrap(); + } + match method { + Method::PUT => { + objects.insert(path, body); + response(StatusCode::OK).body(Body::empty()).unwrap() + } + Method::DELETE => { + objects.remove(&path); + response(StatusCode::NO_CONTENT) + .body(Body::empty()) + .unwrap() + } + Method::HEAD | Method::GET => { + let hidden = state.hidden.lock().unwrap().contains(&path); + let Some(data) = objects.get(&path).cloned().filter(|_| !hidden) else { + return error(StatusCode::NOT_FOUND, "NoSuchKey"); + }; + if method == Method::HEAD { + return response(StatusCode::OK) + .header(header::CONTENT_LENGTH, data.len()) + .body(Body::empty()) + .unwrap(); + } + match headers.get(header::RANGE).and_then(|v| v.to_str().ok()) { + Some(range) => ranged(data, range), + None => response(StatusCode::OK).body(Body::from(data)).unwrap(), + } + } + _ => error(StatusCode::METHOD_NOT_ALLOWED, "MethodNotAllowed"), + } +} + +fn response(status: StatusCode) -> axum::http::response::Builder { + Response::builder() + .status(status) + .header(header::ETAG, "\"etag\"") + .header(header::LAST_MODIFIED, "Wed, 30 Sep 2026 00:00:00 GMT") +} + +fn error(status: StatusCode, code: &str) -> Response { + Response::builder() + .status(status) + .body(Body::from(format!( + "{code}{code}" + ))) + .unwrap() +} + +fn ranged(data: Bytes, range: &str) -> Response { + let (start, end) = range + .trim_start_matches("bytes=") + .split_once('-') + .unwrap_or_default(); + let start: usize = start.parse().unwrap_or(0); + let end: usize = end + .parse::() + .map_or(data.len(), |end| (end + 1).min(data.len())); + if start >= end { + return error(StatusCode::RANGE_NOT_SATISFIABLE, "InvalidRange"); + } + response(StatusCode::PARTIAL_CONTENT) + .header( + header::CONTENT_RANGE, + format!("bytes {start}-{}/{}", end - 1, data.len()), + ) + .body(Body::from(data.slice(start..end))) + .unwrap() +} + +fn query_value(query: &str, key: &str) -> Option { + query.split('&').find_map(|pair| { + let (k, v) = pair.split_once('=')?; + (k == key).then(|| percent_decode_str(v).decode_utf8_lossy().into_owned()) + }) +} + +/// ListObjectsV2 over one bucket; a single page is enough for tests. +fn list(objects: &Objects, bucket: &str, query: &str) -> Response { + let prefix = query_value(query, "prefix").unwrap_or_default(); + let delimiter = query_value(query, "delimiter").unwrap_or_default(); + let bucket_prefix = format!("{bucket}/"); + let mut contents = String::new(); + let mut common_prefixes = BTreeSet::new(); + for (path, data) in objects.lock().unwrap().iter() { + let Some(key) = path.strip_prefix(&bucket_prefix) else { + continue; + }; + let Some(rest) = key.strip_prefix(&prefix) else { + continue; + }; + match rest + .find(delimiter.as_str()) + .filter(|_| !delimiter.is_empty()) + { + Some(index) => { + common_prefixes.insert(format!("{prefix}{}", &rest[..=index])); + } + None => contents.push_str(&format!( + "{key}2026-09-30T00:00:00.000Z\ + \"etag\"{}", + data.len() + )), + } + } + let common_prefixes: String = common_prefixes + .into_iter() + .map(|prefix| format!("{prefix}")) + .collect(); + response(StatusCode::OK) + .body(Body::from(format!( + "{prefix}false\ + {contents}{common_prefixes}" + ))) + .unwrap() +} diff --git a/crates/paimon/src/io/storage.rs b/crates/paimon/src/io/storage.rs index afa9b25a5..d94e7f916 100644 --- a/crates/paimon/src/io/storage.rs +++ b/crates/paimon/src/io/storage.rs @@ -15,8 +15,26 @@ // specific language governing permissions and limitations // under the License. +#[cfg(any( + feature = "storage-azdls", + feature = "storage-cos", + feature = "storage-gcs", + feature = "storage-oss", + feature = "storage-obs", + feature = "storage-s3" +))] +use std::borrow::Borrow; use std::borrow::Cow; use std::collections::HashMap; +#[cfg(any( + feature = "storage-azdls", + feature = "storage-cos", + feature = "storage-gcs", + feature = "storage-oss", + feature = "storage-obs", + feature = "storage-s3" +))] +use std::hash::Hash; #[cfg(any( feature = "storage-azdls", feature = "storage-cos", @@ -42,7 +60,7 @@ use super::AzdlsStorageConfig; #[cfg(feature = "storage-jindo")] use super::JindoStorageConfig; #[cfg(feature = "storage-oss")] -use super::OssStorageConfig; +use super::{OssRoute, OssStorageConfig}; use opendal::Operator; #[cfg(feature = "storage-cos")] use opendal_service_cos::CosConfig; @@ -65,6 +83,7 @@ use url::Url; use crate::error; +use super::cache_routing::RoutedOperator; use super::FileIOBuilder; /// The storage carries all supported storage services in paimon @@ -80,7 +99,7 @@ pub enum Storage { #[cfg(feature = "storage-oss")] Oss { config: Box, - operators: Mutex>, + operators: Mutex>, }, #[cfg(feature = "storage-jindo")] Jindo { @@ -257,7 +276,8 @@ impl Storage { Storage::Oss { config, operators } => { let (bucket, relative_path) = Self::bucket_and_relative_path(path, "OSS", &["oss"])?; - let op = Self::cached_oss_operator(config, operators, path, &bucket)?; + let op = + Self::cached_oss_operator(config, operators, path, bucket, OssRoute::Origin)?; Ok((op, Cow::Borrowed(relative_path))) } #[cfg(feature = "storage-jindo")] @@ -354,6 +374,31 @@ impl Storage { } } + /// Like [`Self::create`], adding the io-cache target when a request on `path` may use one. + pub(crate) fn create_routed<'a>( + &self, + path: &'a str, + ) -> crate::Result<(RoutedOperator, Cow<'a, str>)> { + #[cfg(feature = "storage-oss")] + if let Storage::Oss { config, operators } = self { + if let Some(target) = config.target_route(path) { + let (bucket, relative_path) = + Self::bucket_and_relative_path(path, "OSS", &["oss"])?; + let operator = |route| { + Self::cached_oss_operator(config, operators, path, bucket.clone(), route) + }; + let routed = RoutedOperator::with_target( + operator(OssRoute::Origin)?, + operator(target.route)?, + target.classes, + ); + return Ok((routed, Cow::Borrowed(relative_path))); + } + } + let (op, relative_path) = self.create(path)?; + Ok((RoutedOperator::origin(op), relative_path)) + } + #[cfg(feature = "storage-memory")] fn memory_relative_path(path: &str) -> crate::Result<&str> { if let Some(stripped) = path.strip_prefix("memory:/") { @@ -446,10 +491,10 @@ impl Storage { feature = "storage-obs", feature = "storage-s3" ))] - fn lock_operator_cache<'a>( - operators: &'a Mutex>, + fn lock_operator_cache<'a, K>( + operators: &'a Mutex>, storage_name: &str, - ) -> crate::Result>> { + ) -> crate::Result>> { operators.lock().map_err(|_| error::Error::UnexpectedError { message: format!("Failed to lock {storage_name} operator cache"), source: None, @@ -464,31 +509,36 @@ impl Storage { feature = "storage-obs", feature = "storage-s3" ))] - fn cached_operator( - operators: &Mutex>, + fn cached_operator( + operators: &Mutex>, storage_name: &str, - cache_key: &str, + cache_key: &Q, build: impl FnOnce() -> crate::Result, - ) -> crate::Result { + ) -> crate::Result + where + K: Borrow + Eq + Hash, + Q: ToOwned + Eq + Hash + ?Sized, + { let mut operators = Self::lock_operator_cache(operators, storage_name)?; if let Some(op) = operators.get(cache_key) { return Ok(op.clone()); } let op = build()?; - operators.insert(cache_key.to_string(), op.clone()); + operators.insert(cache_key.to_owned(), op.clone()); Ok(op) } #[cfg(feature = "storage-oss")] fn cached_oss_operator( config: &OssStorageConfig, - operators: &Mutex>, + operators: &Mutex>, path: &str, - bucket: &str, + bucket: String, + route: OssRoute, ) -> crate::Result { - Self::cached_operator(operators, "OSS", bucket, || { - super::oss_config_build(config, path) + Self::cached_operator(operators, "OSS", &(bucket, route), || { + super::oss_config_build(config, path, route) }) } diff --git a/crates/paimon/src/io/storage_oss.rs b/crates/paimon/src/io/storage_oss.rs index 17b99bea7..427ba3040 100644 --- a/crates/paimon/src/io/storage_oss.rs +++ b/crates/paimon/src/io/storage_oss.rs @@ -23,14 +23,10 @@ use opendal_layer_retry::RetryLayer; use opendal_service_oss::OssConfig; use url::Url; +use super::cache_routing::{IoCacheRouting, OpClasses, OSS_ENDPOINT}; use crate::error::Error; use crate::Result; -/// Configuration key for OSS endpoint. -/// -/// Compatible with paimon-java's `fs.oss.endpoint`. -pub(crate) const OSS_ENDPOINT: &str = "fs.oss.endpoint"; - /// Configuration key for OSS access key ID. /// /// Compatible with paimon-java's `fs.oss.accessKeyId`. @@ -53,6 +49,9 @@ pub(crate) const OSS_RETRY_COUNT: &str = "fs.oss.retry.count"; /// Initial exponential retry interval in milliseconds. pub(crate) const OSS_RETRY_INTERVAL_MILLIS: &str = "fs.oss.retry.interval.millisecond"; +/// Use path-style (`endpoint/bucket/key`) instead of virtual-host (`bucket.endpoint/key`) URLs. +pub(crate) const OSS_PATH_STYLE_ACCESS: &str = "fs.oss.path-style-access"; + const DEFAULT_OSS_RETRY_COUNT: usize = 10; const DEFAULT_OSS_RETRY_INTERVAL_MILLIS: u64 = 500; @@ -62,6 +61,58 @@ pub struct OssStorageConfig { retry_count: usize, retry_interval: Duration, user_agent: String, + cache: Option, +} + +#[derive(Debug)] +struct OssCacheConfig { + routing: IoCacheRouting, +} + +/// Endpoint an OSS operator talks to. +#[derive(Clone, Copy, Debug, Eq, Hash, PartialEq)] +pub enum OssRoute { + /// Origin, or the only endpoint when io-cache routing is off. + Origin, + /// The io-cache target at this index, for reads and file status. + Target(usize), +} + +/// The io-cache target that some request on a path may use. +pub(crate) struct OssTargetRoute { + pub(crate) route: OssRoute, + pub(crate) classes: OpClasses, +} + +impl OssStorageConfig { + /// The io-cache target of `path`, when some request on it may leave origin. + pub(crate) fn target_route(&self, path: &str) -> Option { + let cache = self.cache.as_ref()?; + let (index, classes) = cache.routing.cache_target(path)?; + Some(OssTargetRoute { + route: OssRoute::Target(index), + classes, + }) + } + + /// Service config of the operators on `route`. + fn service_for(&self, route: OssRoute) -> Result { + let OssRoute::Target(index) = route else { + return Ok(self.service.clone()); + }; + let target = self + .cache + .as_ref() + .and_then(|cache| cache.routing.targets().get(index)) + .filter(|target| target.endpoint.is_some()) + .ok_or_else(|| Error::ConfigInvalid { + message: format!("io-cache target {index} has no endpoint"), + })?; + let mut service = self.service.clone(); + service.endpoint = target.endpoint.clone(); + service.addressing_style = target.path_style_access.then(|| "path".to_string()); + Ok(service) + } } /// Parse paimon catalog options into an [`OssStorageConfig`]. @@ -74,13 +125,22 @@ pub(crate) fn oss_config_parse(mut props: HashMap) -> Result) -> Result Result { +pub(crate) fn oss_config_build( + cfg: &OssStorageConfig, + path: &str, + route: OssRoute, +) -> Result { let url = Url::parse(path).map_err(|_| Error::ConfigInvalid { message: format!("Invalid OSS url: {path}"), })?; @@ -138,7 +211,7 @@ pub(crate) fn oss_config_build(cfg: &OssStorageConfig, path: &str) -> Result, entries: &[(&str, &str)]) { + for (key, value) in entries { + props.insert(key.to_string(), value.to_string()); + } + } + + #[test] + fn test_oss_config_parse_io_cache_endpoints() { + let mut props = required_props(); + set( + &mut props, + &[ + ("io-cache.endpoint", "http://cache"), + ("io-cache.policy", "read"), + ], + ); + // Without io-cache.enabled nothing is routed. + assert!(oss_config_parse(props.clone()).unwrap().cache.is_none()); + + set(&mut props, &[("io-cache.enabled", "true")]); + let cfg = oss_config_parse(props.clone()).unwrap(); + assert_eq!( + cfg.service.endpoint.as_deref(), + Some("https://oss-cn-hangzhou.aliyuncs.com") + ); + let cache = cfg.cache.as_ref().unwrap(); + assert_eq!( + cache.routing.targets()[0].endpoint.as_deref(), + Some("http://cache") + ); + + props.insert( + "io-cache.origin.endpoint".to_string(), + "http://origin".to_string(), + ); + let cfg = oss_config_parse(props.clone()).unwrap(); + assert_eq!(cfg.service.endpoint.as_deref(), Some("http://origin")); + assert!(cfg.cache.is_some()); + + props.insert( + "dlf.oss-endpoint".to_string(), + "http://override".to_string(), + ); + let cfg = oss_config_parse(props).unwrap(); + assert_eq!(cfg.service.endpoint.as_deref(), Some("http://override")); + assert!(cfg.cache.is_none()); + } + + #[test] + fn test_oss_target_service_settings() { + let mut props = required_props(); + set( + &mut props, + &[ + (OSS_PATH_STYLE_ACCESS, "true"), + ("io-cache.enabled", "true"), + ("io-cache.policy", "meta,read"), + ("io-cache.targets", "accel,cluster,off"), + ("io-cache.target.accel.endpoint", "https://accel"), + ("io-cache.target.accel.region", "cn-hangzhou"), + ("io-cache.target.cluster.endpoint", "http://10.0.0.1:8080"), + ("io-cache.target.cluster.path-style-access", "TRUE"), + ("io-cache.target.off.endpoint", ""), + ], + ); + let cfg = oss_config_parse(props).unwrap(); + assert_eq!(cfg.cache.as_ref().unwrap().routing.targets().len(), 3); + + let origin = cfg.service_for(OssRoute::Origin).unwrap(); + assert_eq!(origin.addressing_style.as_deref(), Some("path")); + let accel = cfg.service_for(OssRoute::Target(0)).unwrap(); + assert_eq!(accel.endpoint.as_deref(), Some("https://accel")); + assert_eq!(accel.addressing_style, None); + assert_eq!(accel.access_key_id.as_deref(), Some("test-ak")); + let cluster = cfg.service_for(OssRoute::Target(1)).unwrap(); + assert_eq!(cluster.endpoint.as_deref(), Some("http://10.0.0.1:8080")); + assert_eq!(cluster.addressing_style.as_deref(), Some("path")); + assert!(cfg.service_for(OssRoute::Target(2)).is_err()); + assert!(cfg.service_for(OssRoute::Target(3)).is_err()); + } + #[test] fn test_oss_retry_defaults_and_validation() { let cfg = oss_config_parse(required_props()).unwrap(); @@ -248,21 +424,26 @@ mod tests { let mut cfg = OssConfig::default(); cfg.endpoint = Some("https://oss-cn-hangzhou.aliyuncs.com".to_string()); - let op = oss_config_build(&storage_config(cfg), "oss://my-bucket/some/path").unwrap(); + let op = oss_config_build( + &storage_config(cfg), + "oss://my-bucket/some/path", + OssRoute::Origin, + ) + .unwrap(); assert_eq!(op.info().name(), "my-bucket"); } #[test] fn test_oss_config_build_invalid_url() { let cfg = storage_config(OssConfig::default()); - let result = oss_config_build(&cfg, "not-a-valid-url"); + let result = oss_config_build(&cfg, "not-a-valid-url", OssRoute::Origin); assert!(result.is_err()); } #[test] fn test_oss_config_build_missing_bucket() { let cfg = storage_config(OssConfig::default()); - let result = oss_config_build(&cfg, "oss:///path/without/bucket"); + let result = oss_config_build(&cfg, "oss:///path/without/bucket", OssRoute::Origin); assert!(result.is_err()); } @@ -281,7 +462,8 @@ mod tests { cfg.addressing_style = Some("path".to_string()); cfg.skip_signature = true; - let op = oss_config_build(&storage_config(cfg), "oss://bucket/path").unwrap(); + let op = + oss_config_build(&storage_config(cfg), "oss://bucket/path", OssRoute::Origin).unwrap(); assert_eq!(op.read("object").await.unwrap().to_bytes(), "ok"); assert_eq!(attempts.load(Ordering::SeqCst), 2); } @@ -304,7 +486,7 @@ mod tests { cfg.service.addressing_style = Some("path".to_string()); cfg.service.skip_signature = true; - let op = oss_config_build(&cfg, "oss://bucket/path").unwrap(); + let op = oss_config_build(&cfg, "oss://bucket/path", OssRoute::Origin).unwrap(); assert!(op.read("object").await.is_err()); assert_eq!(attempts.load(Ordering::SeqCst), 2); } @@ -337,7 +519,7 @@ mod tests { cfg.service.addressing_style = Some("path".to_string()); cfg.service.skip_signature = true; - let op = oss_config_build(&cfg, "oss://bucket/path").unwrap(); + let op = oss_config_build(&cfg, "oss://bucket/path", OssRoute::Origin).unwrap(); assert!(!op.exists("object").await.unwrap()); assert_eq!( *user_agents.lock().unwrap(), diff --git a/crates/paimon/src/table/snapshot_manager.rs b/crates/paimon/src/table/snapshot_manager.rs index 1ca5344db..6fe61f8c0 100644 --- a/crates/paimon/src/table/snapshot_manager.rs +++ b/crates/paimon/src/table/snapshot_manager.rs @@ -538,6 +538,43 @@ mod tests { .build() } + #[cfg(feature = "storage-oss")] + #[tokio::test] + async fn test_snapshot_commit_and_discovery_never_use_the_io_cache() { + use crate::io::oss_test_server::TestOss; + + let origin = TestOss::start().await; + let cache = TestOss::start_sharing(&origin).await; + let table = "oss://bkt/db.db/t"; + let file_io = FileIOBuilder::new("oss") + .with_props([ + ("fs.oss.endpoint", cache.endpoint()), + ("fs.oss.accessKeyId", "ak"), + ("fs.oss.accessKeySecret", "sk"), + ("fs.oss.path-style-access", "true"), + ("fs.oss.retry.count", "1"), + ("io-cache.enabled", "true"), + ("io-cache.endpoint", cache.endpoint()), + ("io-cache.target.default.path-style-access", "true"), + ("io-cache.origin.endpoint", origin.endpoint()), + ("io-cache.policy", "meta,read"), + ]) + .build() + .unwrap(); + let sm = SnapshotManager::new(file_io.clone(), table.to_string()); + assert!(sm.commit_snapshot(&test_snapshot(1)).await.unwrap()); + assert!(sm.commit_snapshot(&test_snapshot(2)).await.unwrap()); + // A cache that probed snapshot-2 before it was written would still report it missing. + cache.hide("bkt/db.db/t/snapshot/snapshot-2"); + + assert_eq!(sm.get_latest_snapshot_id().await.unwrap(), Some(2)); + assert_eq!(sm.get_snapshot(2).await.unwrap().id(), 2); + assert!(cache.take_requests().is_empty()); + assert!(origin + .take_requests() + .contains(&"GET bkt/db.db/t/snapshot/snapshot-2".to_string())); + } + struct RestFixture { table: crate::table::Table, response: std::sync::Arc>, diff --git a/crates/paimon/testdata/io_cache/routable-types.json b/crates/paimon/testdata/io_cache/routable-types.json new file mode 100644 index 000000000..bb780c885 --- /dev/null +++ b/crates/paimon/testdata/io_cache/routable-types.json @@ -0,0 +1,69 @@ +{ + "table_root": "oss://bkt/db1.db/t1", + "cases": [ + {"path": "oss://bkt/db1.db/t1/snapshot/snapshot-12", "options": {}, "type": null, "reason": "snapshot: sequential metadata may be read before it exists"}, + {"path": "oss://bkt/db1.db/t1/snapshot/LATEST", "options": {}, "type": null, "reason": "hint file, rewritten in place"}, + {"path": "oss://bkt/db1.db/t1/snapshot/EARLIEST", "options": {}, "type": null, "reason": "hint file, rewritten in place"}, + {"path": "oss://bkt/db1.db/t1/branch/branch-dev/snapshot/LATEST", "options": {}, "type": null, "reason": "hint file on a branch"}, + {"path": "oss://bkt/db1.db/t1/schema/schema-3", "options": {}, "type": null, "reason": "schema: named by sequence id"}, + {"path": "oss://bkt/db1.db/t1/manifest/manifest-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0", "options": {}, "type": "meta", "reason": "manifest"}, + {"path": "oss://bkt/db1.db/t1/manifest/manifest-list-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-1", "options": {}, "type": "meta", "reason": "manifest list"}, + {"path": "oss://bkt/db1.db/t1/manifest/index-manifest-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0", "options": {}, "type": "meta", "reason": "index manifest; 'manifest' wins over 'index-'"}, + {"path": "oss://bkt/db1.db/t1/manifest/manifest-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.avro.sidecar", "options": {}, "type": "meta", "reason": "manifest sidecar"}, + {"path": "oss://bkt/db1.db/t1/statistics/stat-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0", "options": {}, "type": "meta", "reason": "statistics"}, + {"path": "oss://bkt/db1.db/t1/tag/tag-2026-09-30", "options": {}, "type": null, "reason": "tags can be replaced"}, + {"path": "oss://bkt/db1.db/t1/tag/tag-success-file/t1_SUCCESS", "options": {}, "type": null, "reason": "tag success file"}, + {"path": "oss://bkt/db1.db/t1/consumer/consumer-job1", "options": {}, "type": null, "reason": "consumer progress, rewritten"}, + {"path": "oss://bkt/db1.db/t1/service/service-primary-key-lookup", "options": {}, "type": null, "reason": "service file, rewritten"}, + {"path": "oss://bkt/db1.db/t1/dt=2026-09-30/_SUCCESS", "options": {}, "type": null, "reason": "partition mark-done file"}, + {"path": "oss://bkt/db1.db/t1/changelog/changelog-5", "options": {}, "type": null, "reason": "decoupled changelog metadata, named by sequence id"}, + {"path": "oss://bkt/db1.db/t1/changelog/LATEST", "options": {}, "type": null, "reason": "changelog hint file"}, + {"path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "options": {}, "type": "data", "reason": "data file"}, + {"path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/changelog-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "options": {}, "type": "data", "reason": "changelog data file in a bucket"}, + {"path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.blob", "options": {}, "type": "data", "reason": "blob file"}, + {"path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet.index", "options": {}, "type": "file-index", "reason": "data-file index sidecar"}, + {"path": "oss://bkt/db1.db/t1/index/index-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0", "options": {}, "type": "bucket-index", "reason": "hash index or deletion vectors"}, + {"path": "oss://bkt/db1.db/t1/index/btree-global-index-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b.index", "options": {}, "type": "global-index", "reason": "global btree index"}, + {"path": "oss://bkt/db1.db/t1/index/lumina-global-index-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b.index", "options": {}, "type": "global-index", "reason": "global vector index"}, + {"path": "oss://bkt/db1.db/t1/snapshot/.snapshot-13.8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b.tmp", "options": {}, "type": null, "reason": "temp file of an atomic write"}, + {"path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/.data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet.8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b.tmp", "options": {}, "type": null, "reason": "temp data file"}, + {"path": "oss://bkt/db1.db/t1/metadata/version-hint.text", "options": {}, "type": null, "reason": "Iceberg-compatible metadata hint"}, + {"path": "oss://bkt/db1.db/t1/metadata/v3.metadata.json", "options": {}, "type": null, "reason": "Iceberg-compatible metadata, not a Paimon name"}, + {"path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/000000_0", "options": {}, "type": null, "reason": "migrated Hive file: unknown name"}, + {"path": "oss://bkt/db1.db/t1/dt=2026-09-30/part-00000-1a2b.snappy.parquet", "options": {}, "type": null, "reason": "format-table style name: unknown"}, + {"path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/custom-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.orc", "options": {}, "type": null, "reason": "custom data-file.prefix without the option"}, + {"path": "oss://bkt/db1.db/t1/README", "options": {}, "type": null, "reason": "unknown name"}, + {"path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/manifest-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.orc", "options": {"data-file.prefix": "manifest-"}, "type": "data", "reason": "a data file named like a manifest by data-file.prefix is data"}, + {"path": "oss://bkt/db1.db/t1/manifest/manifest-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0", "options": {"data-file.prefix": "manifest-"}, "type": "meta", "reason": "manifests stay meta with data-file.prefix manifest-"}, + {"path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/index-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.orc", "options": {"data-file.prefix": "index-"}, "type": "data", "reason": "a data file named like a bucket index by data-file.prefix is data"}, + {"path": "oss://bkt/db1.db/t1/index/index-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0", "options": {"data-file.prefix": "index-"}, "type": "bucket-index", "reason": "bucket indexes stay bucket-index with data-file.prefix index-"}, + {"path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/stat-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.orc", "options": {"data-file.prefix": "stat-"}, "type": "data", "reason": "a data file named like statistics by data-file.prefix is data"}, + {"path": "oss://bkt/db1.db/t1/statistics/stat-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0", "options": {"data-file.prefix": "stat-"}, "type": "meta", "reason": "statistics stay meta with data-file.prefix stat-"}, + {"path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/manifest-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.orc.index", "options": {"data-file.prefix": "manifest-"}, "type": "file-index", "reason": "the file index of such a data file stays file-index"}, + {"path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/ unknown.parquet", "options": {"data-file.prefix": " "}, "type": null, "reason": "a whitespace-only prefix does not make unknown files cacheable"}, + {"path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/custom-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.orc", "options": {"data-file.prefix": "custom-"}, "type": "data", "reason": "custom data-file.prefix from the merged options"}, + {"path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/cl-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.orc", "options": {"changelog-file.prefix": "cl-"}, "type": "data", "reason": "custom changelog-file.prefix from the merged options"}, + {"path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.orc", "options": {"data-file.prefix": "custom-"}, "type": "data", "reason": "default prefixes stay routable next to a custom one"}, + {"path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/7f3a/manifest-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.orc", "options": {"data-file.prefix": "manifest-"}, "type": "data", "reason": "an external data path may put an entropy directory between the bucket and the file"}, + {"path": "oss://bkt/db1.db/t1/bucket-postpone/index-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.orc", "options": {"data-file.prefix": "index-"}, "type": "data", "reason": "data file of a postpone bucket"}, + {"path": "oss://bkt/db1.db/t1/postpone/stat-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.orc", "options": {"data-file.prefix": "stat-"}, "type": "data", "reason": "data file of a legacy postpone bucket"}, + {"path": "oss://bkt/db1.db/t1/dt=2026-09-30/8b1f7c2e/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "options": {}, "type": "data", "reason": "the name decides, not the directory"}, + {"path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/snapshot-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.orc", "options": {"data-file.prefix": "snapshot-"}, "type": "data", "reason": "snapshots are named by sequence id, so a uuid name with this prefix is data"}, + {"path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/schema-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.orc", "options": {"data-file.prefix": "schema-"}, "type": "data", "reason": "schemas are named by sequence id, so a uuid name with this prefix is data"}, + {"path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/snapshot-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.orc.index", "options": {"data-file.prefix": "snapshot-"}, "type": "file-index", "reason": "file index of a data file named like a snapshot"}, + {"path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/global-index-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.orc.index", "options": {"data-file.prefix": "global-index-"}, "type": "file-index", "reason": "file index of a data file named like a global index"}, + {"path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/tag-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.orc", "options": {"data-file.prefix": "tag-"}, "type": null, "reason": "names rewritten in place never use a cache, whatever the prefix"}, + {"path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/consumer-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.orc", "options": {"data-file.prefix": "consumer-"}, "type": null, "reason": "names rewritten in place never use a cache, whatever the prefix"}, + {"path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/index-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0", "options": {"data-file.prefix": "index-"}, "type": "bucket-index", "reason": "no extension: a bucket index, not a data file"}, + {"path": "oss://bkt/db1.db/t1/manifest/manifest-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.avro.sidecar", "options": {"data-file.prefix": "manifest-"}, "type": "meta", "reason": "a manifest sidecar is metadata whatever the data prefix"}, + {"path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet.tmp-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b", "options": {}, "type": null, "reason": "temp name with a .tmp- suffix"}, + {"path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet.tmp.8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b", "options": {}, "type": null, "reason": "temp name with a .tmp. infix"}, + {"path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-1.parquet", "options": {}, "type": null, "reason": "not a name Paimon writers produce"}, + {"path": "oss://bkt/bucket-0/db/t/changelog/changelog-5", "options": {}, "type": null, "reason": "decoupled changelog metadata below a bucket-* warehouse directory"}, + {"path": "oss://bkt/warehouse/bucket-0/db/t1/tag/tag-file", "options": {"data-file.prefix": "tag-"}, "type": null, "reason": "tag metadata with a matching data prefix"}, + {"path": "oss://bkt/warehouse/bucket-0/db/t1/consumer/consumer-file", "options": {"data-file.prefix": "consumer-"}, "type": null, "reason": "consumer metadata with a matching data prefix"}, + {"path": "oss://bkt/warehouse/bucket-0/db/t1/service/service-file", "options": {"data-file.prefix": "service-"}, "type": null, "reason": "service metadata with a matching data prefix"}, + {"path": "oss://bkt/warehouse/bucket-0/db/t1/metadata/version-hint.text", "options": {"data-file.prefix": "version-"}, "type": null, "reason": "Iceberg metadata hint with a matching data prefix"}, + {"path": "oss://bkt/warehouse/bucket-0/db/t1/branch/branch-feature", "options": {"data-file.prefix": "branch-"}, "type": null, "reason": "branch directory with a matching data prefix"} + ] +} diff --git a/crates/paimon/testdata/io_cache/routing.json b/crates/paimon/testdata/io_cache/routing.json new file mode 100644 index 000000000..0614fff87 --- /dev/null +++ b/crates/paimon/testdata/io_cache/routing.json @@ -0,0 +1,90 @@ +{ + "table_root": "oss://bkt/db1.db/t1", + "cases": [ + {"name": "multi: a data file named like a manifest goes to the data target", "options": {"fs.oss.endpoint": "https://accelerator.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.targets": "accel,cluster", "io-cache.target.accel.endpoint": "https://accelerator.example.com", "io-cache.target.accel.region": "cn-hangzhou", "io-cache.target.cluster.endpoint": "http://cluster.example.com", "io-cache.target.cluster.path-style-access": "true", "io-cache.policy": "meta,read", "io-cache.whitelist": "*", "io-cache.routes": "meta=accel;data,bucket-index,global-index,file-index=cluster", "data-file.prefix": "manifest-"}, "op": "read", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/manifest-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.orc", "expect": "http://cluster.example.com"}, + {"name": "multi: manifests keep the meta target with data-file.prefix manifest-", "options": {"fs.oss.endpoint": "https://accelerator.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.targets": "accel,cluster", "io-cache.target.accel.endpoint": "https://accelerator.example.com", "io-cache.target.accel.region": "cn-hangzhou", "io-cache.target.cluster.endpoint": "http://cluster.example.com", "io-cache.target.cluster.path-style-access": "true", "io-cache.policy": "meta,read", "io-cache.whitelist": "*", "io-cache.routes": "meta=accel;data,bucket-index,global-index,file-index=cluster", "data-file.prefix": "manifest-"}, "op": "read", "path": "oss://bkt/db1.db/t1/manifest/manifest-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0", "expect": "https://accelerator.example.com"}, + {"name": "multi: a data file named like a bucket index goes to the data target", "options": {"fs.oss.endpoint": "https://accelerator.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.targets": "accel,cluster", "io-cache.target.accel.endpoint": "https://accelerator.example.com", "io-cache.target.accel.region": "cn-hangzhou", "io-cache.target.cluster.endpoint": "http://cluster.example.com", "io-cache.target.cluster.path-style-access": "true", "io-cache.policy": "meta,read", "io-cache.whitelist": "*", "io-cache.routes": "data=cluster;bucket-index=accel", "data-file.prefix": "index-"}, "op": "read", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/index-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.orc", "expect": "http://cluster.example.com"}, + {"name": "multi: bucket indexes keep their target with data-file.prefix index-", "options": {"fs.oss.endpoint": "https://accelerator.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.targets": "accel,cluster", "io-cache.target.accel.endpoint": "https://accelerator.example.com", "io-cache.target.accel.region": "cn-hangzhou", "io-cache.target.cluster.endpoint": "http://cluster.example.com", "io-cache.target.cluster.path-style-access": "true", "io-cache.policy": "meta,read", "io-cache.whitelist": "*", "io-cache.routes": "data=cluster;bucket-index=accel", "data-file.prefix": "index-"}, "op": "read", "path": "oss://bkt/db1.db/t1/index/index-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0", "expect": "https://accelerator.example.com"}, + {"name": "multi: a data file named like a snapshot goes to the data target", "options": {"fs.oss.endpoint": "https://accelerator.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.targets": "accel,cluster", "io-cache.target.accel.endpoint": "https://accelerator.example.com", "io-cache.target.accel.region": "cn-hangzhou", "io-cache.target.cluster.endpoint": "http://cluster.example.com", "io-cache.target.cluster.path-style-access": "true", "io-cache.policy": "meta,read", "io-cache.whitelist": "*", "io-cache.routes": "meta=accel;data,bucket-index,global-index,file-index=cluster", "data-file.prefix": "snapshot-"}, "op": "read", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/snapshot-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.orc", "expect": "http://cluster.example.com"}, + {"name": "multi: a data file below an entropy directory goes to the data target", "options": {"fs.oss.endpoint": "https://accelerator.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.targets": "accel,cluster", "io-cache.target.accel.endpoint": "https://accelerator.example.com", "io-cache.target.accel.region": "cn-hangzhou", "io-cache.target.cluster.endpoint": "http://cluster.example.com", "io-cache.target.cluster.path-style-access": "true", "io-cache.policy": "meta,read", "io-cache.whitelist": "*", "io-cache.routes": "meta=accel;data,bucket-index,global-index,file-index=cluster", "data-file.prefix": "manifest-"}, "op": "read", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/7f3a/manifest-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.orc", "expect": "http://cluster.example.com"}, + {"name": "multi: a manifest sidecar keeps the meta target with data-file.prefix manifest-", "options": {"fs.oss.endpoint": "https://accelerator.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.targets": "accel,cluster", "io-cache.target.accel.endpoint": "https://accelerator.example.com", "io-cache.target.accel.region": "cn-hangzhou", "io-cache.target.cluster.endpoint": "http://cluster.example.com", "io-cache.target.cluster.path-style-access": "true", "io-cache.policy": "meta,read", "io-cache.whitelist": "*", "io-cache.routes": "meta=accel;data,bucket-index,global-index,file-index=cluster", "data-file.prefix": "manifest-"}, "op": "read", "path": "oss://bkt/db1.db/t1/manifest/manifest-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.avro.sidecar", "expect": "https://accelerator.example.com"}, + {"name": "multi: a data file named like statistics goes to the data target", "options": {"fs.oss.endpoint": "https://accelerator.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.targets": "accel,cluster", "io-cache.target.accel.endpoint": "https://accelerator.example.com", "io-cache.target.accel.region": "cn-hangzhou", "io-cache.target.cluster.endpoint": "http://cluster.example.com", "io-cache.target.cluster.path-style-access": "true", "io-cache.policy": "meta,read", "io-cache.whitelist": "*", "io-cache.routes": "meta=accel;data,bucket-index,global-index,file-index=cluster", "data-file.prefix": "stat-"}, "op": "read", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/stat-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.orc", "expect": "http://cluster.example.com"}, + {"name": "single: a dls path is not routed", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": "http://cache.example.com", "io-cache.policy": "meta,read", "io-cache.whitelist": "meta,data"}, "op": "read", "path": "dls://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "expect": "https://oss-cn-hangzhou-internal.aliyuncs.com"}, + {"name": "policy: trimmed case-insensitive tokens", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": "http://cache.example.com", "io-cache.policy": " READ , Meta ", "io-cache.whitelist": "meta,data"}, "op": "meta", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "expect": "http://cache.example.com"}, + {"name": "policy: none overrides read", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": "http://cache.example.com", "io-cache.policy": "read,NONE", "io-cache.whitelist": "meta,data"}, "op": "exists", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "expect": "http://cache.example.com"}, + {"name": "policy: substrings are not operations", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": "http://cache.example.com", "io-cache.policy": "thread,metadata", "io-cache.whitelist": "meta,data"}, "op": "exists", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "expect": "http://cache.example.com"}, + {"name": "policy: unknown tokens do not disable read", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": "http://cache.example.com", "io-cache.policy": "read,nonetheless", "io-cache.whitelist": "meta,data"}, "op": "exists", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "expect": "https://oss-cn-hangzhou-internal.aliyuncs.com"}, + {"name": "single: whitespace endpoint is missing", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": " ", "io-cache.policy": "meta,read", "io-cache.whitelist": "meta,data"}, "op": "read", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "expect": "https://oss-cn-hangzhou-internal.aliyuncs.com"}, + {"name": "multi: whitespace endpoint is skipped", "options": {"fs.oss.endpoint": "https://accelerator.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.targets": "accel,cluster", "io-cache.target.accel.endpoint": " ", "io-cache.target.accel.region": "cn-hangzhou", "io-cache.target.cluster.endpoint": "http://cluster.example.com", "io-cache.target.cluster.path-style-access": "true", "io-cache.policy": "meta,read", "io-cache.whitelist": "*"}, "op": "read", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "expect": "http://cluster.example.com"}, + {"name": "multi: target endpoint is trimmed", "options": {"fs.oss.endpoint": "https://accelerator.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.targets": "accel,cluster", "io-cache.target.accel.endpoint": "https://accelerator.example.com", "io-cache.target.accel.region": "cn-hangzhou", "io-cache.target.cluster.endpoint": " http://cluster.example.com ", "io-cache.target.cluster.path-style-access": "true", "io-cache.policy": "meta,read", "io-cache.whitelist": "*", "io-cache.routes": "meta=accel;data,bucket-index,global-index,file-index=cluster"}, "op": "read", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "expect": "http://cluster.example.com"}, + {"name": "single: whitespace origin uses legacy endpoint", "options": {"fs.oss.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": " ", "io-cache.endpoint": "http://cache.example.com", "io-cache.policy": "meta,read", "io-cache.whitelist": "meta,data"}, "op": "list", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "expect": "https://oss-cn-hangzhou-internal.aliyuncs.com"}, + {"name": "single: data read goes to the cache", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": "http://cache.example.com", "io-cache.policy": "meta,read", "io-cache.whitelist": "meta,data"}, "op": "read", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "expect": "http://cache.example.com"}, + {"name": "single: data status goes to the cache", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": "http://cache.example.com", "io-cache.policy": "meta,read", "io-cache.whitelist": "meta,data"}, "op": "meta", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "expect": "http://cache.example.com"}, + {"name": "single: manifest read goes to the cache", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": "http://cache.example.com", "io-cache.policy": "meta,read", "io-cache.whitelist": "meta,data"}, "op": "read", "path": "oss://bkt/db1.db/t1/manifest/manifest-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0", "expect": "http://cache.example.com"}, + {"name": "single: manifest list read goes to the cache", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": "http://cache.example.com", "io-cache.policy": "meta,read", "io-cache.whitelist": "meta,data"}, "op": "read", "path": "oss://bkt/db1.db/t1/manifest/manifest-list-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-1", "expect": "http://cache.example.com"}, + {"name": "single: snapshot read stays at origin", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": "http://cache.example.com", "io-cache.policy": "meta,read", "io-cache.whitelist": "meta,data"}, "op": "read", "path": "oss://bkt/db1.db/t1/snapshot/snapshot-12", "expect": "https://oss-cn-hangzhou-internal.aliyuncs.com"}, + {"name": "single: snapshot status stays at origin", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": "http://cache.example.com", "io-cache.policy": "meta,read", "io-cache.whitelist": "meta,data"}, "op": "meta", "path": "oss://bkt/db1.db/t1/snapshot/snapshot-12", "expect": "https://oss-cn-hangzhou-internal.aliyuncs.com"}, + {"name": "single: hint file stays at origin", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": "http://cache.example.com", "io-cache.policy": "meta,read", "io-cache.whitelist": "meta,data"}, "op": "read", "path": "oss://bkt/db1.db/t1/snapshot/LATEST", "expect": "https://oss-cn-hangzhou-internal.aliyuncs.com"}, + {"name": "single: index type outside the whitelist stays at origin", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": "http://cache.example.com", "io-cache.policy": "meta,read", "io-cache.whitelist": "meta,data"}, "op": "read", "path": "oss://bkt/db1.db/t1/index/index-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0", "expect": "https://oss-cn-hangzhou-internal.aliyuncs.com"}, + {"name": "single: unknown name stays at origin", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": "http://cache.example.com", "io-cache.policy": "meta,read", "io-cache.whitelist": "meta,data"}, "op": "read", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/000000_0", "expect": "https://oss-cn-hangzhou-internal.aliyuncs.com"}, + {"name": "single: temp file stays at origin", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": "http://cache.example.com", "io-cache.policy": "meta,read", "io-cache.whitelist": "meta,data"}, "op": "read", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/.data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet.8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b.tmp", "expect": "https://oss-cn-hangzhou-internal.aliyuncs.com"}, + {"name": "single: a path in another bucket is routed like the table's", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": "http://cache.example.com", "io-cache.policy": "meta,read", "io-cache.whitelist": "meta,data"}, "op": "read", "path": "oss://other-bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "expect": "http://cache.example.com"}, + {"name": "single: disabled by default", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": "http://cache.example.com", "io-cache.policy": "meta,read", "io-cache.whitelist": "meta,data"}, "op": "read", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "expect": "http://cache.example.com"}, + {"name": "single: disabled keeps fs.oss.endpoint for everything", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.enabled": "false", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": "http://cache.example.com", "io-cache.policy": "meta,read", "io-cache.whitelist": "meta,data"}, "op": "exists", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "expect": "http://cache.example.com"}, + {"name": "single: no policy means no routing", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": "http://cache.example.com", "io-cache.whitelist": "meta,data"}, "op": "exists", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "expect": "http://cache.example.com"}, + {"name": "single: policy none means no routing", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": "http://cache.example.com", "io-cache.policy": "none", "io-cache.whitelist": "meta,data"}, "op": "exists", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "expect": "http://cache.example.com"}, + {"name": "single: write-only policy means no routing", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": "http://cache.example.com", "io-cache.policy": "write", "io-cache.whitelist": "meta,data"}, "op": "exists", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "expect": "http://cache.example.com"}, + {"name": "single: read-only policy keeps status at origin", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": "http://cache.example.com", "io-cache.policy": "read", "io-cache.whitelist": "meta,data"}, "op": "meta", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "expect": "https://oss-cn-hangzhou-internal.aliyuncs.com"}, + {"name": "single: read-only policy still routes reads", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": "http://cache.example.com", "io-cache.policy": "read", "io-cache.whitelist": "meta,data"}, "op": "read", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "expect": "http://cache.example.com"}, + {"name": "single: origin defaults to fs.oss.endpoint", "options": {"fs.oss.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.enabled": "true", "io-cache.endpoint": "http://cache.example.com", "io-cache.policy": "meta,read", "io-cache.whitelist": "meta,data"}, "op": "list", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "expect": "https://oss-cn-hangzhou-internal.aliyuncs.com"}, + {"name": "single: client endpoint override wins", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": "http://cache.example.com", "io-cache.policy": "meta,read", "io-cache.whitelist": "meta,data", "dlf.oss-endpoint": "https://oss-cn-hangzhou.aliyuncs.com"}, "op": "read", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "expect": "https://oss-cn-hangzhou.aliyuncs.com"}, + {"name": "single: absent whitelist allows every routable type", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": "http://cache.example.com", "io-cache.policy": "meta,read"}, "op": "read", "path": "oss://bkt/db1.db/t1/index/index-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0", "expect": "http://cache.example.com"}, + {"name": "single: star whitelist allows every routable type", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": "http://cache.example.com", "io-cache.policy": "meta,read", "io-cache.whitelist": "*"}, "op": "read", "path": "oss://bkt/db1.db/t1/index/btree-global-index-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b.index", "expect": "http://cache.example.com"}, + {"name": "single: file index sidecar with star whitelist", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": "http://cache.example.com", "io-cache.policy": "meta,read", "io-cache.whitelist": "*"}, "op": "read", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet.index", "expect": "http://cache.example.com"}, + {"name": "single: custom data prefix without the option stays at origin", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": "http://cache.example.com", "io-cache.policy": "meta,read", "io-cache.whitelist": "meta,data"}, "op": "read", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/custom-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.orc", "expect": "https://oss-cn-hangzhou-internal.aliyuncs.com"}, + {"name": "single: custom data prefix from options is routed", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": "http://cache.example.com", "io-cache.policy": "meta,read", "io-cache.whitelist": "meta,data", "data-file.prefix": "custom-"}, "op": "read", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/custom-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.orc", "expect": "http://cache.example.com"}, + {"name": "single: empty cache endpoint sends routed reads to origin", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": "", "io-cache.policy": "meta,read", "io-cache.whitelist": "meta,data", "io-cache.routes": "*=default"}, "op": "read", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "expect": "https://oss-cn-hangzhou-internal.aliyuncs.com"}, + {"name": "multi: manifest read goes to the accelerator", "options": {"fs.oss.endpoint": "https://accelerator.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.targets": "accel,cluster", "io-cache.target.accel.endpoint": "https://accelerator.example.com", "io-cache.target.accel.region": "cn-hangzhou", "io-cache.target.cluster.endpoint": "http://cluster.example.com", "io-cache.target.cluster.path-style-access": "true", "io-cache.policy": "meta,read", "io-cache.whitelist": "*", "io-cache.routes": "meta=accel;data,bucket-index,global-index,file-index=cluster"}, "op": "read", "path": "oss://bkt/db1.db/t1/manifest/manifest-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0", "expect": "https://accelerator.example.com"}, + {"name": "multi: manifest status goes to the accelerator", "options": {"fs.oss.endpoint": "https://accelerator.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.targets": "accel,cluster", "io-cache.target.accel.endpoint": "https://accelerator.example.com", "io-cache.target.accel.region": "cn-hangzhou", "io-cache.target.cluster.endpoint": "http://cluster.example.com", "io-cache.target.cluster.path-style-access": "true", "io-cache.policy": "meta,read", "io-cache.whitelist": "*", "io-cache.routes": "meta=accel;data,bucket-index,global-index,file-index=cluster"}, "op": "meta", "path": "oss://bkt/db1.db/t1/manifest/manifest-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0", "expect": "https://accelerator.example.com"}, + {"name": "multi: data read goes to the cluster", "options": {"fs.oss.endpoint": "https://accelerator.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.targets": "accel,cluster", "io-cache.target.accel.endpoint": "https://accelerator.example.com", "io-cache.target.accel.region": "cn-hangzhou", "io-cache.target.cluster.endpoint": "http://cluster.example.com", "io-cache.target.cluster.path-style-access": "true", "io-cache.policy": "meta,read", "io-cache.whitelist": "*", "io-cache.routes": "meta=accel;data,bucket-index,global-index,file-index=cluster"}, "op": "read", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "expect": "http://cluster.example.com"}, + {"name": "multi: deletion vectors go to the cluster", "options": {"fs.oss.endpoint": "https://accelerator.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.targets": "accel,cluster", "io-cache.target.accel.endpoint": "https://accelerator.example.com", "io-cache.target.accel.region": "cn-hangzhou", "io-cache.target.cluster.endpoint": "http://cluster.example.com", "io-cache.target.cluster.path-style-access": "true", "io-cache.policy": "meta,read", "io-cache.whitelist": "*", "io-cache.routes": "meta=accel;data,bucket-index,global-index,file-index=cluster"}, "op": "read", "path": "oss://bkt/db1.db/t1/index/index-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0", "expect": "http://cluster.example.com"}, + {"name": "multi: global index goes to the cluster", "options": {"fs.oss.endpoint": "https://accelerator.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.targets": "accel,cluster", "io-cache.target.accel.endpoint": "https://accelerator.example.com", "io-cache.target.accel.region": "cn-hangzhou", "io-cache.target.cluster.endpoint": "http://cluster.example.com", "io-cache.target.cluster.path-style-access": "true", "io-cache.policy": "meta,read", "io-cache.whitelist": "*", "io-cache.routes": "meta=accel;data,bucket-index,global-index,file-index=cluster"}, "op": "read", "path": "oss://bkt/db1.db/t1/index/btree-global-index-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b.index", "expect": "http://cluster.example.com"}, + {"name": "multi: manifest existence goes to the accelerator", "options": {"fs.oss.endpoint": "https://accelerator.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.targets": "accel,cluster", "io-cache.target.accel.endpoint": "https://accelerator.example.com", "io-cache.target.accel.region": "cn-hangzhou", "io-cache.target.cluster.endpoint": "http://cluster.example.com", "io-cache.target.cluster.path-style-access": "true", "io-cache.policy": "meta,read", "io-cache.whitelist": "*", "io-cache.routes": "meta=accel;data,bucket-index,global-index,file-index=cluster"}, "op": "exists", "path": "oss://bkt/db1.db/t1/manifest/manifest-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0", "expect": "https://accelerator.example.com"}, + {"name": "multi: data existence goes to the cluster", "options": {"fs.oss.endpoint": "https://accelerator.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.targets": "accel,cluster", "io-cache.target.accel.endpoint": "https://accelerator.example.com", "io-cache.target.accel.region": "cn-hangzhou", "io-cache.target.cluster.endpoint": "http://cluster.example.com", "io-cache.target.cluster.path-style-access": "true", "io-cache.policy": "meta,read", "io-cache.whitelist": "*", "io-cache.routes": "meta=accel;data,bucket-index,global-index,file-index=cluster"}, "op": "exists", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "expect": "http://cluster.example.com"}, + {"name": "single: data existence goes to the cache", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": "http://cache.example.com", "io-cache.policy": "meta,read", "io-cache.whitelist": "meta,data"}, "op": "exists", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "expect": "http://cache.example.com"}, + {"name": "single: snapshot existence stays at origin", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": "http://cache.example.com", "io-cache.policy": "meta,read", "io-cache.whitelist": "meta,data"}, "op": "exists", "path": "oss://bkt/db1.db/t1/snapshot/snapshot-12", "expect": "https://oss-cn-hangzhou-internal.aliyuncs.com"}, + {"name": "single: hint file existence stays at origin", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": "http://cache.example.com", "io-cache.policy": "meta,read", "io-cache.whitelist": "meta,data"}, "op": "exists", "path": "oss://bkt/db1.db/t1/snapshot/LATEST", "expect": "https://oss-cn-hangzhou-internal.aliyuncs.com"}, + {"name": "single: read-only policy keeps existence at origin", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": "http://cache.example.com", "io-cache.policy": "read", "io-cache.whitelist": "meta,data"}, "op": "exists", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "expect": "https://oss-cn-hangzhou-internal.aliyuncs.com"}, + {"name": "multi: snapshot stays at origin", "options": {"fs.oss.endpoint": "https://accelerator.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.targets": "accel,cluster", "io-cache.target.accel.endpoint": "https://accelerator.example.com", "io-cache.target.accel.region": "cn-hangzhou", "io-cache.target.cluster.endpoint": "http://cluster.example.com", "io-cache.target.cluster.path-style-access": "true", "io-cache.policy": "meta,read", "io-cache.whitelist": "*", "io-cache.routes": "meta=accel;data,bucket-index,global-index,file-index=cluster"}, "op": "read", "path": "oss://bkt/db1.db/t1/snapshot/snapshot-12", "expect": "https://oss-cn-hangzhou-internal.aliyuncs.com"}, + {"name": "multi: without routes the first usable target takes all", "options": {"fs.oss.endpoint": "https://accelerator.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.targets": "accel,cluster", "io-cache.target.accel.endpoint": "https://accelerator.example.com", "io-cache.target.accel.region": "cn-hangzhou", "io-cache.target.cluster.endpoint": "http://cluster.example.com", "io-cache.target.cluster.path-style-access": "true", "io-cache.policy": "meta,read", "io-cache.whitelist": "*"}, "op": "read", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "expect": "https://accelerator.example.com"}, + {"name": "multi: route to an undeclared target stays at origin", "options": {"fs.oss.endpoint": "https://accelerator.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.targets": "accel,cluster", "io-cache.target.accel.endpoint": "https://accelerator.example.com", "io-cache.target.accel.region": "cn-hangzhou", "io-cache.target.cluster.endpoint": "http://cluster.example.com", "io-cache.target.cluster.path-style-access": "true", "io-cache.policy": "meta,read", "io-cache.whitelist": "*", "io-cache.routes": "data=x"}, "op": "read", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "expect": "https://oss-cn-hangzhou-internal.aliyuncs.com"}, + {"name": "multi: type without a matching route stays at origin", "options": {"fs.oss.endpoint": "https://accelerator.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.targets": "accel,cluster", "io-cache.target.accel.endpoint": "https://accelerator.example.com", "io-cache.target.accel.region": "cn-hangzhou", "io-cache.target.cluster.endpoint": "http://cluster.example.com", "io-cache.target.cluster.path-style-access": "true", "io-cache.policy": "meta,read", "io-cache.whitelist": "*", "io-cache.routes": "meta=accel;data=cluster"}, "op": "read", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet.index", "expect": "https://oss-cn-hangzhou-internal.aliyuncs.com"}, + {"name": "multi: targets win over the single-target shorthand", "options": {"fs.oss.endpoint": "https://accelerator.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.targets": "accel,cluster", "io-cache.target.accel.endpoint": "https://accelerator.example.com", "io-cache.target.accel.region": "cn-hangzhou", "io-cache.target.cluster.endpoint": "http://cluster.example.com", "io-cache.target.cluster.path-style-access": "true", "io-cache.policy": "meta,read", "io-cache.whitelist": "*", "io-cache.endpoint": "http://cache.example.com"}, "op": "read", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "expect": "https://accelerator.example.com"}, + {"name": "multi: target without endpoint is skipped when routes are absent", "options": {"fs.oss.endpoint": "https://accelerator.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.targets": "accel,cluster", "io-cache.target.accel.region": "cn-hangzhou", "io-cache.target.cluster.endpoint": "http://cluster.example.com", "io-cache.target.cluster.path-style-access": "true", "io-cache.policy": "meta,read", "io-cache.whitelist": "*"}, "op": "read", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "expect": "http://cluster.example.com"}, + {"name": "multi: route to a target without endpoint stays at origin", "options": {"fs.oss.endpoint": "https://accelerator.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.targets": "accel,cluster", "io-cache.target.accel.endpoint": "https://accelerator.example.com", "io-cache.target.accel.region": "cn-hangzhou", "io-cache.target.cluster.path-style-access": "true", "io-cache.policy": "meta,read", "io-cache.whitelist": "*", "io-cache.routes": "meta=accel;data,bucket-index,global-index,file-index=cluster"}, "op": "read", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "expect": "https://oss-cn-hangzhou-internal.aliyuncs.com"}, + {"name": "multi: invalid target names are ignored", "options": {"fs.oss.endpoint": "https://accelerator.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.targets": "Bad_Name,cluster", "io-cache.target.accel.endpoint": "https://accelerator.example.com", "io-cache.target.accel.region": "cn-hangzhou", "io-cache.target.cluster.endpoint": "http://cluster.example.com", "io-cache.target.cluster.path-style-access": "true", "io-cache.policy": "meta,read", "io-cache.whitelist": "*"}, "op": "read", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "expect": "http://cluster.example.com"}, + {"name": "multi: star route sends every type to one target", "options": {"fs.oss.endpoint": "https://accelerator.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.targets": "accel,cluster", "io-cache.target.accel.endpoint": "https://accelerator.example.com", "io-cache.target.accel.region": "cn-hangzhou", "io-cache.target.cluster.endpoint": "http://cluster.example.com", "io-cache.target.cluster.path-style-access": "true", "io-cache.policy": "meta,read", "io-cache.whitelist": "*", "io-cache.routes": "*=cluster"}, "op": "read", "path": "oss://bkt/db1.db/t1/manifest/manifest-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0", "expect": "http://cluster.example.com"}, + {"name": "multi: first matching route wins", "options": {"fs.oss.endpoint": "https://accelerator.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.targets": "accel,cluster", "io-cache.target.accel.endpoint": "https://accelerator.example.com", "io-cache.target.accel.region": "cn-hangzhou", "io-cache.target.cluster.endpoint": "http://cluster.example.com", "io-cache.target.cluster.path-style-access": "true", "io-cache.policy": "meta,read", "io-cache.whitelist": "*", "io-cache.routes": "data=accel;data,meta=cluster"}, "op": "read", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "expect": "https://accelerator.example.com"}, + {"name": "multi: malformed routes are skipped", "options": {"fs.oss.endpoint": "https://accelerator.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.targets": "accel,cluster", "io-cache.target.accel.endpoint": "https://accelerator.example.com", "io-cache.target.accel.region": "cn-hangzhou", "io-cache.target.cluster.endpoint": "http://cluster.example.com", "io-cache.target.cluster.path-style-access": "true", "io-cache.policy": "meta,read", "io-cache.whitelist": "*", "io-cache.routes": "=cluster;meta=accel;junk"}, "op": "read", "path": "oss://bkt/db1.db/t1/manifest/manifest-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0", "expect": "https://accelerator.example.com"}, + {"name": "multi: malformed routes do not match other types", "options": {"fs.oss.endpoint": "https://accelerator.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.targets": "accel,cluster", "io-cache.target.accel.endpoint": "https://accelerator.example.com", "io-cache.target.accel.region": "cn-hangzhou", "io-cache.target.cluster.endpoint": "http://cluster.example.com", "io-cache.target.cluster.path-style-access": "true", "io-cache.policy": "meta,read", "io-cache.whitelist": "*", "io-cache.routes": "=cluster;meta=accel;junk"}, "op": "read", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "expect": "https://oss-cn-hangzhou-internal.aliyuncs.com"}, + {"name": "multi: whitelist filters before routes", "options": {"fs.oss.endpoint": "https://accelerator.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.targets": "accel,cluster", "io-cache.target.accel.endpoint": "https://accelerator.example.com", "io-cache.target.accel.region": "cn-hangzhou", "io-cache.target.cluster.endpoint": "http://cluster.example.com", "io-cache.target.cluster.path-style-access": "true", "io-cache.policy": "meta,read", "io-cache.whitelist": "meta", "io-cache.routes": "meta=accel;data,bucket-index,global-index,file-index=cluster"}, "op": "read", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "expect": "https://oss-cn-hangzhou-internal.aliyuncs.com"}, + {"name": "single: write stays at origin", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": "http://cache.example.com", "io-cache.policy": "meta,read", "io-cache.whitelist": "meta,data"}, "op": "write", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "expect": "https://oss-cn-hangzhou-internal.aliyuncs.com"}, + {"name": "multi: write stays at origin", "options": {"fs.oss.endpoint": "https://accelerator.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.targets": "accel,cluster", "io-cache.target.accel.endpoint": "https://accelerator.example.com", "io-cache.target.accel.region": "cn-hangzhou", "io-cache.target.cluster.endpoint": "http://cluster.example.com", "io-cache.target.cluster.path-style-access": "true", "io-cache.policy": "meta,read", "io-cache.whitelist": "*", "io-cache.routes": "meta=accel;data,bucket-index,global-index,file-index=cluster"}, "op": "write", "path": "oss://bkt/db1.db/t1/manifest/manifest-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0", "expect": "https://oss-cn-hangzhou-internal.aliyuncs.com"}, + {"name": "single: list stays at origin", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": "http://cache.example.com", "io-cache.policy": "meta,read", "io-cache.whitelist": "meta,data"}, "op": "list", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "expect": "https://oss-cn-hangzhou-internal.aliyuncs.com"}, + {"name": "multi: list stays at origin", "options": {"fs.oss.endpoint": "https://accelerator.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.targets": "accel,cluster", "io-cache.target.accel.endpoint": "https://accelerator.example.com", "io-cache.target.accel.region": "cn-hangzhou", "io-cache.target.cluster.endpoint": "http://cluster.example.com", "io-cache.target.cluster.path-style-access": "true", "io-cache.policy": "meta,read", "io-cache.whitelist": "*", "io-cache.routes": "meta=accel;data,bucket-index,global-index,file-index=cluster"}, "op": "list", "path": "oss://bkt/db1.db/t1/manifest/manifest-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0", "expect": "https://oss-cn-hangzhou-internal.aliyuncs.com"}, + {"name": "single: delete stays at origin", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": "http://cache.example.com", "io-cache.policy": "meta,read", "io-cache.whitelist": "meta,data"}, "op": "delete", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "expect": "https://oss-cn-hangzhou-internal.aliyuncs.com"}, + {"name": "multi: delete stays at origin", "options": {"fs.oss.endpoint": "https://accelerator.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.targets": "accel,cluster", "io-cache.target.accel.endpoint": "https://accelerator.example.com", "io-cache.target.accel.region": "cn-hangzhou", "io-cache.target.cluster.endpoint": "http://cluster.example.com", "io-cache.target.cluster.path-style-access": "true", "io-cache.policy": "meta,read", "io-cache.whitelist": "*", "io-cache.routes": "meta=accel;data,bucket-index,global-index,file-index=cluster"}, "op": "delete", "path": "oss://bkt/db1.db/t1/manifest/manifest-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0", "expect": "https://oss-cn-hangzhou-internal.aliyuncs.com"}, + {"name": "single: rename stays at origin", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": "http://cache.example.com", "io-cache.policy": "meta,read", "io-cache.whitelist": "meta,data"}, "op": "rename", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "expect": "https://oss-cn-hangzhou-internal.aliyuncs.com"}, + {"name": "multi: rename stays at origin", "options": {"fs.oss.endpoint": "https://accelerator.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.targets": "accel,cluster", "io-cache.target.accel.endpoint": "https://accelerator.example.com", "io-cache.target.accel.region": "cn-hangzhou", "io-cache.target.cluster.endpoint": "http://cluster.example.com", "io-cache.target.cluster.path-style-access": "true", "io-cache.policy": "meta,read", "io-cache.whitelist": "*", "io-cache.routes": "meta=accel;data,bucket-index,global-index,file-index=cluster"}, "op": "rename", "path": "oss://bkt/db1.db/t1/manifest/manifest-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0", "expect": "https://oss-cn-hangzhou-internal.aliyuncs.com"}, + {"name": "single: mkdirs stays at origin", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": "http://cache.example.com", "io-cache.policy": "meta,read", "io-cache.whitelist": "meta,data"}, "op": "mkdirs", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "expect": "https://oss-cn-hangzhou-internal.aliyuncs.com"}, + {"name": "multi: mkdirs stays at origin", "options": {"fs.oss.endpoint": "https://accelerator.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.targets": "accel,cluster", "io-cache.target.accel.endpoint": "https://accelerator.example.com", "io-cache.target.accel.region": "cn-hangzhou", "io-cache.target.cluster.endpoint": "http://cluster.example.com", "io-cache.target.cluster.path-style-access": "true", "io-cache.policy": "meta,read", "io-cache.whitelist": "*", "io-cache.routes": "meta=accel;data,bucket-index,global-index,file-index=cluster"}, "op": "mkdirs", "path": "oss://bkt/db1.db/t1/manifest/manifest-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0", "expect": "https://oss-cn-hangzhou-internal.aliyuncs.com"}, + {"name": "single: copy stays at origin", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": "http://cache.example.com", "io-cache.policy": "meta,read", "io-cache.whitelist": "meta,data"}, "op": "copy", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "expect": "https://oss-cn-hangzhou-internal.aliyuncs.com"}, + {"name": "multi: copy stays at origin", "options": {"fs.oss.endpoint": "https://accelerator.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.targets": "accel,cluster", "io-cache.target.accel.endpoint": "https://accelerator.example.com", "io-cache.target.accel.region": "cn-hangzhou", "io-cache.target.cluster.endpoint": "http://cluster.example.com", "io-cache.target.cluster.path-style-access": "true", "io-cache.policy": "meta,read", "io-cache.whitelist": "*", "io-cache.routes": "meta=accel;data,bucket-index,global-index,file-index=cluster"}, "op": "copy", "path": "oss://bkt/db1.db/t1/manifest/manifest-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0", "expect": "https://oss-cn-hangzhou-internal.aliyuncs.com"}, + {"name": "single: atomic-write stays at origin", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": "http://cache.example.com", "io-cache.policy": "meta,read", "io-cache.whitelist": "meta,data"}, "op": "atomic-write", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "expect": "https://oss-cn-hangzhou-internal.aliyuncs.com"}, + {"name": "multi: atomic-write stays at origin", "options": {"fs.oss.endpoint": "https://accelerator.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.targets": "accel,cluster", "io-cache.target.accel.endpoint": "https://accelerator.example.com", "io-cache.target.accel.region": "cn-hangzhou", "io-cache.target.cluster.endpoint": "http://cluster.example.com", "io-cache.target.cluster.path-style-access": "true", "io-cache.policy": "meta,read", "io-cache.whitelist": "*", "io-cache.routes": "meta=accel;data,bucket-index,global-index,file-index=cluster"}, "op": "atomic-write", "path": "oss://bkt/db1.db/t1/manifest/manifest-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0", "expect": "https://oss-cn-hangzhou-internal.aliyuncs.com"}, + {"name": "single: two-phase-write stays at origin", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": "http://cache.example.com", "io-cache.policy": "meta,read", "io-cache.whitelist": "meta,data"}, "op": "two-phase-write", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "expect": "https://oss-cn-hangzhou-internal.aliyuncs.com"}, + {"name": "multi: two-phase-write stays at origin", "options": {"fs.oss.endpoint": "https://accelerator.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.targets": "accel,cluster", "io-cache.target.accel.endpoint": "https://accelerator.example.com", "io-cache.target.accel.region": "cn-hangzhou", "io-cache.target.cluster.endpoint": "http://cluster.example.com", "io-cache.target.cluster.path-style-access": "true", "io-cache.policy": "meta,read", "io-cache.whitelist": "*", "io-cache.routes": "meta=accel;data,bucket-index,global-index,file-index=cluster"}, "op": "two-phase-write", "path": "oss://bkt/db1.db/t1/manifest/manifest-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0", "expect": "https://oss-cn-hangzhou-internal.aliyuncs.com"}, + {"name": "single: presign stays at origin", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": "http://cache.example.com", "io-cache.policy": "meta,read", "io-cache.whitelist": "meta,data"}, "op": "presign", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "expect": "https://oss-cn-hangzhou-internal.aliyuncs.com"}, + {"name": "multi: presign stays at origin", "options": {"fs.oss.endpoint": "https://accelerator.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.targets": "accel,cluster", "io-cache.target.accel.endpoint": "https://accelerator.example.com", "io-cache.target.accel.region": "cn-hangzhou", "io-cache.target.cluster.endpoint": "http://cluster.example.com", "io-cache.target.cluster.path-style-access": "true", "io-cache.policy": "meta,read", "io-cache.whitelist": "*", "io-cache.routes": "meta=accel;data,bucket-index,global-index,file-index=cluster"}, "op": "presign", "path": "oss://bkt/db1.db/t1/manifest/manifest-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0", "expect": "https://oss-cn-hangzhou-internal.aliyuncs.com"} + ] +} diff --git a/docs/src/getting-started.md b/docs/src/getting-started.md index 2d05ec114..9e04f4081 100644 --- a/docs/src/getting-started.md +++ b/docs/src/getting-started.md @@ -229,6 +229,8 @@ options.set("fs.oss.endpoint", "oss-cn-hangzhou.aliyuncs.com"); // Optional: configure retries for temporary OSS failures. options.set("fs.oss.retry.count", "10"); options.set("fs.oss.retry.interval.millisecond", "500"); +// Optional: path-style URLs (`endpoint/bucket/key`) instead of `bucket.endpoint/key`. +options.set("fs.oss.path-style-access", "true"); let catalog = CatalogFactory::create(options).await?; // Tencent Cloud COS @@ -368,6 +370,40 @@ Decoded `.blob` indexes are cached per catalog. Set `cache.blob-index.max-size` to control the budget (default `64 MiB`; `0` disables caching). BLOB payloads are not cached. +### OSS Cache Routing + +A REST catalog can vend OSS cache targets with a table token. The default +OpenDAL OSS backend uses the following routing options. These rules do not +apply when `fs.oss.impl=jindo`. + +| Option | Default | Description | +|--------|---------|-------------| +| `io-cache.enabled` | `false` | Master switch. A value set in the catalog options wins over the token, so clients can turn routing off. | +| `io-cache.origin.endpoint` | `fs.oss.endpoint` | OSS endpoint for requests that do not use a target. | +| `io-cache.policy` | none | Operations that may use a target: `read`, `meta` (file status), or `none`. | +| `io-cache.whitelist` | all types | File types that may use a target: `meta`, `global-index`, `bucket-index`, `data`, `file-index`, or `*`. | +| `io-cache.targets` | none | Comma-separated target names matching `[a-z][a-z0-9-]*`. | +| `io-cache.target..endpoint` | none | Target endpoint, such as `http://host:port`; `https` when the scheme is omitted. | +| `io-cache.target..path-style-access` | `false` | Path-style URLs (`endpoint/bucket/key`) for the target. | +| `io-cache.endpoint` | none | A single target named `default`, ignored when `io-cache.targets` is set. | +| `io-cache.routes` | first target with an endpoint | Ordered rules `types=target;...`, such as `meta=accel;data,file-index=cluster`. The first rule listing the file type picks the target; without a matching rule the request uses origin. | + +`io-cache.target..region` is accepted but ignored, because OpenDAL signs +OSS requests without a region. + +Only reads, file status and existence checks of files that Paimon writes once +under a unique name can use a target: manifests, statistics, indexes, and data files. A data file is +recognized by the name Paimon writers give it: the `data-` or `changelog-` +prefix, or `data-file.prefix` or `changelog-file.prefix` when these are in the +options, followed by `{uuid}-{count}.{extension}`. Everything else uses +origin: writes and a writer's own existence checks, listing, deletes, renames and copies, files +rewritten in place (such as `LATEST`, tags and temporary files), sequentially +numbered metadata (`snapshot-N`, `schema-N`, `changelog/changelog-N`), unknown +file names. A target is used like any +other endpoint: requests to it use the same retry settings, and its errors are +returned rather than retried on origin. Setting `dlf.oss-endpoint` on the +client turns routing off. + ### Manage Databases ```rust From 621c22deffb9765bc0097376fdced1837f9f28f3 Mon Sep 17 00:00:00 2001 From: Sun Dapeng Date: Tue, 6 Oct 2026 20:04:35 +0800 Subject: [PATCH 02/21] [core] Route writes of cacheable files when io-cache.policy has write Some cache endpoints, such as an OSS accelerator, also take writes and send them to OSS over their own write path, whose bandwidth adds to the direct one. With write in io-cache.policy, OutputFile writers of cacheable files (data files, manifests, indexes) use the file type's target. OutputFile::exists, the existence check a writer makes, stays on origin, as do deletes, renames, copies, listings and writes of snapshots, hints and temp files. A policy with only write routes writes only. --- crates/paimon/src/io/cache_routing.rs | 16 +++++++++++----- crates/paimon/src/io/file_io.rs | 4 ++-- .../src/io/file_io/cache_routing_tests.rs | 19 +++++++++++++++++++ crates/paimon/testdata/io_cache/routing.json | 17 ++++++++++++++++- docs/src/getting-started.md | 9 +++++---- 5 files changed, 53 insertions(+), 12 deletions(-) diff --git a/crates/paimon/src/io/cache_routing.rs b/crates/paimon/src/io/cache_routing.rs index 4302d10ac..af6e5e2d1 100644 --- a/crates/paimon/src/io/cache_routing.rs +++ b/crates/paimon/src/io/cache_routing.rs @@ -54,13 +54,15 @@ static DATA_FILE_SUFFIX: LazyLock = LazyLock::new(|| { .expect("valid data file pattern") }); -/// Operation class of a FileIO request; only `Meta` and `Read` can leave origin. +/// Operation class of a FileIO request; only `Meta`, `Read` and `Write` can leave origin. #[derive(Clone, Copy, Debug, Eq, Hash, PartialEq)] pub(crate) enum OpClass { /// File status and existence checks, including the size lookup before a cached read. Meta, Read, - /// Writes and their existence checks, listing, deletes, renames, mkdirs, copies. + /// Writes of new files, including streaming and multipart writes. + Write, + /// A writer's existence checks, listing, deletes, renames, mkdirs, copies. Origin, } @@ -69,6 +71,7 @@ impl OpClass { match self { Self::Meta => 1, Self::Read => 2, + Self::Write => 4, Self::Origin => 0, } } @@ -198,7 +201,7 @@ impl IoCacheRouting { pub(crate) fn cache_target(&self, path: &str) -> Option<(usize, OpClasses)> { let mut found = None; let mut classes = OpClasses::default(); - for op in [OpClass::Meta, OpClass::Read] { + for op in [OpClass::Meta, OpClass::Read, OpClass::Write] { if let Target::Cache(index) = self.route(op, path) { found = Some(index); classes.insert(op); @@ -279,6 +282,7 @@ fn parse_policy(value: Option<&str>) -> OpClasses { "none" => return OpClasses::default(), "meta" => policy.insert(OpClass::Meta), "read" => policy.insert(OpClass::Read), + "write" => policy.insert(OpClass::Write), _ => {} } } @@ -449,8 +453,10 @@ mod tests { match name { "read" => OpClass::Read, "meta" | "exists" => OpClass::Meta, - "write" | "list" | "delete" | "rename" | "mkdirs" | "copy" | "atomic-write" - | "two-phase-write" | "presign" => OpClass::Origin, + "write" | "two-phase-write" => OpClass::Write, + "list" | "delete" | "rename" | "mkdirs" | "copy" | "atomic-write" | "presign" => { + OpClass::Origin + } other => panic!("unknown operation {other}"), } } diff --git a/crates/paimon/src/io/file_io.rs b/crates/paimon/src/io/file_io.rs index a317fb8d0..a20996bac 100644 --- a/crates/paimon/src/io/file_io.rs +++ b/crates/paimon/src/io/file_io.rs @@ -1435,7 +1435,7 @@ impl OutputFile { pub async fn writer(&self) -> crate::Result> { let (op, relative_path, cache_path) = self.source.resolve(&self.path).await?; let writer: Box = Box::new( - op.origin_operator() + op.operator(OpClass::Write) .writer_with(&relative_path) .chunk(8 * 1024 * 1024) .await?, @@ -1454,7 +1454,7 @@ impl OutputFile { pub(crate) async fn async_writer(&self) -> crate::Result> { let (op, relative_path, cache_path) = self.source.resolve(&self.path).await?; let writer: Box = Box::new( - op.origin_operator() + op.operator(OpClass::Write) .writer_with(&relative_path) .chunk(8 * 1024 * 1024) .concurrent(1) diff --git a/crates/paimon/src/io/file_io/cache_routing_tests.rs b/crates/paimon/src/io/file_io/cache_routing_tests.rs index 291fbf6ba..b99ac85d4 100644 --- a/crates/paimon/src/io/file_io/cache_routing_tests.rs +++ b/crates/paimon/src/io/file_io/cache_routing_tests.rs @@ -232,6 +232,25 @@ async fn test_target_errors_are_not_retried_on_origin() { assert!(origin.take_requests().is_empty()); } +#[tokio::test] +async fn test_writes_use_the_target_with_write_policy() { + let (origin, cache, file_io) = routed("meta,read,write").await; + + write(&file_io, DATA, b"data").await; + write(&file_io, SNAPSHOT, b"{}").await; + // A writer checks existence on origin; snapshots are never written through a cache. + assert!(file_io.new_output(DATA).unwrap().exists().await.unwrap()); + assert_eq!(cache.take_requests(), [format!("PUT {DATA_KEY}")]); + assert_eq!( + origin.take_requests(), + [ + "PUT bkt/db.db/t/snapshot/snapshot-1".to_string(), + format!("HEAD {DATA_KEY}") + ] + ); + assert_eq!(origin.object(DATA_KEY).unwrap(), "data"); +} + #[tokio::test] async fn test_writes_never_use_an_unavailable_target() { let (origin, cache, file_io) = routed("meta,read").await; diff --git a/crates/paimon/testdata/io_cache/routing.json b/crates/paimon/testdata/io_cache/routing.json index 0614fff87..afd4b88eb 100644 --- a/crates/paimon/testdata/io_cache/routing.json +++ b/crates/paimon/testdata/io_cache/routing.json @@ -33,7 +33,22 @@ {"name": "single: disabled keeps fs.oss.endpoint for everything", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.enabled": "false", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": "http://cache.example.com", "io-cache.policy": "meta,read", "io-cache.whitelist": "meta,data"}, "op": "exists", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "expect": "http://cache.example.com"}, {"name": "single: no policy means no routing", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": "http://cache.example.com", "io-cache.whitelist": "meta,data"}, "op": "exists", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "expect": "http://cache.example.com"}, {"name": "single: policy none means no routing", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": "http://cache.example.com", "io-cache.policy": "none", "io-cache.whitelist": "meta,data"}, "op": "exists", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "expect": "http://cache.example.com"}, - {"name": "single: write-only policy means no routing", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": "http://cache.example.com", "io-cache.policy": "write", "io-cache.whitelist": "meta,data"}, "op": "exists", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "expect": "http://cache.example.com"}, + {"name": "single: write-only policy keeps status at origin", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": "http://cache.example.com", "io-cache.policy": "write", "io-cache.whitelist": "meta,data"}, "op": "exists", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "expect": "https://oss-cn-hangzhou-internal.aliyuncs.com"}, + {"name": "single: write-only policy routes writes", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": "http://cache.example.com", "io-cache.policy": "write", "io-cache.whitelist": "meta,data"}, "op": "write", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "expect": "http://cache.example.com"}, + {"name": "write: data file write goes to the cache", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": "http://cache.example.com", "io-cache.policy": "meta,read,write", "io-cache.whitelist": "meta,data"}, "op": "write", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "expect": "http://cache.example.com"}, + {"name": "write: two-phase data file write goes to the cache", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": "http://cache.example.com", "io-cache.policy": "meta,read,write", "io-cache.whitelist": "meta,data"}, "op": "two-phase-write", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "expect": "http://cache.example.com"}, + {"name": "write: manifest write goes to the cache", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": "http://cache.example.com", "io-cache.policy": "meta,read,write", "io-cache.whitelist": "meta,data"}, "op": "write", "path": "oss://bkt/db1.db/t1/manifest/manifest-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0", "expect": "http://cache.example.com"}, + {"name": "write: snapshot write stays at origin", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": "http://cache.example.com", "io-cache.policy": "meta,read,write", "io-cache.whitelist": "meta,data"}, "op": "write", "path": "oss://bkt/db1.db/t1/snapshot/snapshot-12", "expect": "https://oss-cn-hangzhou-internal.aliyuncs.com"}, + {"name": "write: hint file write stays at origin", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": "http://cache.example.com", "io-cache.policy": "meta,read,write", "io-cache.whitelist": "meta,data"}, "op": "write", "path": "oss://bkt/db1.db/t1/snapshot/LATEST", "expect": "https://oss-cn-hangzhou-internal.aliyuncs.com"}, + {"name": "write: temp file write stays at origin", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": "http://cache.example.com", "io-cache.policy": "meta,read,write", "io-cache.whitelist": "meta,data"}, "op": "write", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/.data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet.8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b.tmp", "expect": "https://oss-cn-hangzhou-internal.aliyuncs.com"}, + {"name": "write: atomic write stays at origin", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": "http://cache.example.com", "io-cache.policy": "meta,read,write", "io-cache.whitelist": "meta,data"}, "op": "atomic-write", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "expect": "https://oss-cn-hangzhou-internal.aliyuncs.com"}, + {"name": "write: delete stays at origin", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": "http://cache.example.com", "io-cache.policy": "meta,read,write", "io-cache.whitelist": "meta,data"}, "op": "delete", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "expect": "https://oss-cn-hangzhou-internal.aliyuncs.com"}, + {"name": "write: rename stays at origin", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": "http://cache.example.com", "io-cache.policy": "meta,read,write", "io-cache.whitelist": "meta,data"}, "op": "rename", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "expect": "https://oss-cn-hangzhou-internal.aliyuncs.com"}, + {"name": "write: copy stays at origin", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": "http://cache.example.com", "io-cache.policy": "meta,read,write", "io-cache.whitelist": "meta,data"}, "op": "copy", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "expect": "https://oss-cn-hangzhou-internal.aliyuncs.com"}, + {"name": "write: list stays at origin", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": "http://cache.example.com", "io-cache.policy": "meta,read,write", "io-cache.whitelist": "meta,data"}, "op": "list", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "expect": "https://oss-cn-hangzhou-internal.aliyuncs.com"}, + {"name": "write: whitelist filters writes", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": "http://cache.example.com", "io-cache.policy": "meta,read,write", "io-cache.whitelist": "meta"}, "op": "write", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "expect": "https://oss-cn-hangzhou-internal.aliyuncs.com"}, + {"name": "write: data write follows the data route", "options": {"fs.oss.endpoint": "https://accelerator.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.targets": "accel,cluster", "io-cache.target.accel.endpoint": "https://accelerator.example.com", "io-cache.target.accel.region": "cn-hangzhou", "io-cache.target.cluster.endpoint": "http://cluster.example.com", "io-cache.target.cluster.path-style-access": "true", "io-cache.policy": "meta,read,write", "io-cache.whitelist": "*", "io-cache.routes": "meta=accel;data,bucket-index,global-index,file-index=cluster"}, "op": "write", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "expect": "http://cluster.example.com"}, + {"name": "write: manifest write follows the meta route", "options": {"fs.oss.endpoint": "https://accelerator.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.targets": "accel,cluster", "io-cache.target.accel.endpoint": "https://accelerator.example.com", "io-cache.target.accel.region": "cn-hangzhou", "io-cache.target.cluster.endpoint": "http://cluster.example.com", "io-cache.target.cluster.path-style-access": "true", "io-cache.policy": "meta,read,write", "io-cache.whitelist": "*", "io-cache.routes": "meta=accel;data,bucket-index,global-index,file-index=cluster"}, "op": "write", "path": "oss://bkt/db1.db/t1/manifest/manifest-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0", "expect": "https://accelerator.example.com"}, {"name": "single: read-only policy keeps status at origin", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": "http://cache.example.com", "io-cache.policy": "read", "io-cache.whitelist": "meta,data"}, "op": "meta", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "expect": "https://oss-cn-hangzhou-internal.aliyuncs.com"}, {"name": "single: read-only policy still routes reads", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": "http://cache.example.com", "io-cache.policy": "read", "io-cache.whitelist": "meta,data"}, "op": "read", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "expect": "http://cache.example.com"}, {"name": "single: origin defaults to fs.oss.endpoint", "options": {"fs.oss.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.enabled": "true", "io-cache.endpoint": "http://cache.example.com", "io-cache.policy": "meta,read", "io-cache.whitelist": "meta,data"}, "op": "list", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "expect": "https://oss-cn-hangzhou-internal.aliyuncs.com"}, diff --git a/docs/src/getting-started.md b/docs/src/getting-started.md index 9e04f4081..cbd322fbe 100644 --- a/docs/src/getting-started.md +++ b/docs/src/getting-started.md @@ -380,7 +380,7 @@ apply when `fs.oss.impl=jindo`. |--------|---------|-------------| | `io-cache.enabled` | `false` | Master switch. A value set in the catalog options wins over the token, so clients can turn routing off. | | `io-cache.origin.endpoint` | `fs.oss.endpoint` | OSS endpoint for requests that do not use a target. | -| `io-cache.policy` | none | Operations that may use a target: `read`, `meta` (file status), or `none`. | +| `io-cache.policy` | none | Operations that may use a target: `read`, `meta` (file status and existence checks), `write`, or `none`. | | `io-cache.whitelist` | all types | File types that may use a target: `meta`, `global-index`, `bucket-index`, `data`, `file-index`, or `*`. | | `io-cache.targets` | none | Comma-separated target names matching `[a-z][a-z0-9-]*`. | | `io-cache.target..endpoint` | none | Target endpoint, such as `http://host:port`; `https` when the scheme is omitted. | @@ -391,12 +391,13 @@ apply when `fs.oss.impl=jindo`. `io-cache.target..region` is accepted but ignored, because OpenDAL signs OSS requests without a region. -Only reads, file status and existence checks of files that Paimon writes once -under a unique name can use a target: manifests, statistics, indexes, and data files. A data file is +Only reads, file status and existence checks, and with `write` in the policy also +writes, of files that Paimon writes once under a unique name can use a target: +manifests, statistics, indexes, and data files. A data file is recognized by the name Paimon writers give it: the `data-` or `changelog-` prefix, or `data-file.prefix` or `changelog-file.prefix` when these are in the options, followed by `{uuid}-{count}.{extension}`. Everything else uses -origin: writes and a writer's own existence checks, listing, deletes, renames and copies, files +origin: a writer's own existence checks, listing, deletes, renames and copies, files rewritten in place (such as `LATEST`, tags and temporary files), sequentially numbered metadata (`snapshot-N`, `schema-N`, `changelog/changelog-N`), unknown file names. A target is used like any From 16bc555cb10113536480ab3a84b79c4cecdcf7dc Mon Sep 17 00:00:00 2001 From: Sun Dapeng Date: Tue, 6 Oct 2026 23:57:48 +0800 Subject: [PATCH 03/21] [core] Write the io-cache routing test cases inline Replace the JSON test vectors with assertions grouped by topic in the cache_routing tests, and remove testdata/io_cache. --- crates/paimon/src/io/cache_routing.rs | 518 +++++++++++++++--- .../testdata/io_cache/routable-types.json | 69 --- crates/paimon/testdata/io_cache/routing.json | 105 ---- 3 files changed, 452 insertions(+), 240 deletions(-) delete mode 100644 crates/paimon/testdata/io_cache/routable-types.json delete mode 100644 crates/paimon/testdata/io_cache/routing.json diff --git a/crates/paimon/src/io/cache_routing.rs b/crates/paimon/src/io/cache_routing.rs index af6e5e2d1..848d2135f 100644 --- a/crates/paimon/src/io/cache_routing.rs +++ b/crates/paimon/src/io/cache_routing.rs @@ -419,92 +419,397 @@ impl RoutedOperator { mod tests { use super::*; - use serde::Deserialize; + use std::collections::BTreeMap; + + const TABLE_ROOT: &str = "oss://bkt/db1.db/t1"; + const UUID: &str = "8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b"; + + // paths below are relative to TABLE_ROOT, and {uuid} stands for UUID + const DATA_PATH: &str = "dt=1/bucket-0/data-{uuid}-0.parquet"; + const MANIFEST_PATH: &str = "manifest/manifest-{uuid}-0"; + const INDEX_PATH: &str = "index/index-{uuid}-0"; + const GLOBAL_INDEX_PATH: &str = "index/btree-global-index-{uuid}.index"; + const TEMP_PATH: &str = "dt=1/bucket-0/.data-{uuid}-0.parquet.{uuid}.tmp"; + const SNAPSHOT_PATH: &str = "snapshot/snapshot-12"; + const LATEST_PATH: &str = "snapshot/LATEST"; + + const OSS: &str = "https://oss-cn-hangzhou-internal.aliyuncs.com"; + const CACHE: &str = "http://cache.example.com"; + const ACCEL: &str = "https://accelerator.example.com"; + const CLUSTER: &str = "http://cluster.example.com"; + const WRITE: &str = "io-cache.policy=meta,read,write"; - #[derive(Deserialize)] - struct Vectors { - cases: Vec, + fn routing(options: &[(&str, &str)]) -> IoCacheRouting { + IoCacheRouting::from_props( + &options + .iter() + .map(|(k, v)| (k.to_string(), v.to_string())) + .collect(), + ) } - #[derive(Deserialize)] - struct RoutingCase { - name: String, - options: HashMap, - op: String, - path: String, - expect: String, + fn route(routing: &IoCacheRouting, op: OpClass, path: &str) -> Option { + routing.endpoint(routing.route(op, path)).map(String::from) } - #[derive(Deserialize)] - struct RoutableTypeCase { - path: String, - options: HashMap, - #[serde(rename = "type")] - file_type: Option, + #[test] + fn test_one_cache_target() { + let options = single(&[]); + assert_endpoint(&options, OpClass::Read, DATA_PATH, CACHE); + assert_endpoint(&options, OpClass::Meta, DATA_PATH, CACHE); + assert_endpoint(&options, OpClass::Read, MANIFEST_PATH, CACHE); + assert_endpoint( + &options, + OpClass::Read, + "manifest/manifest-list-{uuid}-1", + CACHE, + ); + let other_bucket = format!("oss://other-bkt/db1.db/t1/{DATA_PATH}"); + assert_endpoint(&options, OpClass::Read, &other_bucket, CACHE); + assert_endpoint(&options, OpClass::Read, SNAPSHOT_PATH, OSS); + assert_endpoint(&options, OpClass::Meta, SNAPSHOT_PATH, OSS); + assert_endpoint(&options, OpClass::Read, LATEST_PATH, OSS); + assert_endpoint(&options, OpClass::Meta, LATEST_PATH, OSS); + assert_endpoint(&options, OpClass::Read, TEMP_PATH, OSS); + assert_endpoint(&options, OpClass::Read, "dt=1/bucket-0/000000_0", OSS); + let dls = format!("dls://bkt/db1.db/t1/{DATA_PATH}"); + assert_endpoint(&options, OpClass::Read, &dls, OSS); + // the whitelist has no index + assert_endpoint(&options, OpClass::Read, INDEX_PATH, OSS); + assert_endpoint(&options, OpClass::Write, DATA_PATH, OSS); + assert_endpoint(&options, OpClass::Origin, DATA_PATH, OSS); } - fn load(name: &str) -> Vectors { - let path = format!("{}/testdata/io_cache/{name}", env!("CARGO_MANIFEST_DIR")); - serde_json::from_str(&std::fs::read_to_string(&path).unwrap()).unwrap() + #[test] + fn test_policy_and_whitelist() { + assert_endpoint( + &single(&["-io-cache.enabled"]), + OpClass::Read, + DATA_PATH, + CACHE, + ); + assert_endpoint( + &single(&["io-cache.enabled=false"]), + OpClass::Meta, + DATA_PATH, + CACHE, + ); + assert_endpoint( + &single(&["-io-cache.policy"]), + OpClass::Meta, + DATA_PATH, + CACHE, + ); + assert_endpoint( + &single(&["io-cache.policy=none"]), + OpClass::Meta, + DATA_PATH, + CACHE, + ); + // policy tokens are trimmed, case-insensitive and whole words + let tokens = single(&["io-cache.policy= READ , Meta "]); + assert_endpoint(&tokens, OpClass::Meta, DATA_PATH, CACHE); + assert_endpoint( + &single(&["io-cache.policy=read,NONE"]), + OpClass::Meta, + DATA_PATH, + CACHE, + ); + let words = single(&["io-cache.policy=thread,metadata"]); + assert_endpoint(&words, OpClass::Meta, DATA_PATH, CACHE); + let unknown = single(&["io-cache.policy=read,nonetheless"]); + assert_endpoint(&unknown, OpClass::Meta, DATA_PATH, OSS); + + let read_only = single(&["io-cache.policy=read"]); + assert_endpoint(&read_only, OpClass::Read, DATA_PATH, CACHE); + assert_endpoint(&read_only, OpClass::Meta, DATA_PATH, OSS); + let write_only = single(&["io-cache.policy=write"]); + assert_endpoint(&write_only, OpClass::Write, DATA_PATH, CACHE); + assert_endpoint(&write_only, OpClass::Meta, DATA_PATH, OSS); + + let any_type = single(&["-io-cache.whitelist"]); + assert_endpoint(&any_type, OpClass::Read, INDEX_PATH, CACHE); + let star = single(&["io-cache.whitelist=*"]); + assert_endpoint(&star, OpClass::Read, GLOBAL_INDEX_PATH, CACHE); + let file_index = format!("{DATA_PATH}.index"); + assert_endpoint(&star, OpClass::Read, &file_index, CACHE); } - /// Maps the operation names of the vectors to classes. - fn op_class(name: &str) -> OpClass { - match name { - "read" => OpClass::Read, - "meta" | "exists" => OpClass::Meta, - "write" | "two-phase-write" => OpClass::Write, - "list" | "delete" | "rename" | "mkdirs" | "copy" | "atomic-write" | "presign" => { - OpClass::Origin - } - other => panic!("unknown operation {other}"), - } + #[test] + fn test_endpoints() { + let blank = single(&["io-cache.endpoint= "]); + assert_endpoint(&blank, OpClass::Read, DATA_PATH, OSS); + let empty = single(&["io-cache.endpoint=", "io-cache.routes=*=default"]); + assert_endpoint(&empty, OpClass::Read, DATA_PATH, OSS); + // the origin defaults to fs.oss.endpoint + let oss = format!("fs.oss.endpoint={OSS}"); + let without_origin = single(&[oss.as_str(), "-io-cache.origin.endpoint"]); + assert_endpoint(&without_origin, OpClass::Origin, DATA_PATH, OSS); + let blank_origin = single(&[oss.as_str(), "io-cache.origin.endpoint= "]); + assert_endpoint(&blank_origin, OpClass::Origin, DATA_PATH, OSS); + // a client-side endpoint turns routing off + let overridden = single(&["dlf.oss-endpoint=https://oss-cn-hangzhou.aliyuncs.com"]); + let override_endpoint = "https://oss-cn-hangzhou.aliyuncs.com"; + assert_endpoint(&overridden, OpClass::Read, DATA_PATH, override_endpoint); } - fn routing(options: &[(&str, &str)]) -> IoCacheRouting { - IoCacheRouting::from_props( - &options - .iter() - .map(|(k, v)| (k.to_string(), v.to_string())) - .collect(), - ) + #[test] + fn test_write_policy() { + let options = single(&[WRITE]); + assert_endpoint(&options, OpClass::Write, DATA_PATH, CACHE); + assert_endpoint(&options, OpClass::Write, MANIFEST_PATH, CACHE); + assert_endpoint(&options, OpClass::Write, SNAPSHOT_PATH, OSS); + assert_endpoint(&options, OpClass::Write, LATEST_PATH, OSS); + assert_endpoint(&options, OpClass::Write, TEMP_PATH, OSS); + // atomic writes, copies and the existence check before a write stay at origin + assert_endpoint(&options, OpClass::Origin, DATA_PATH, OSS); + let meta_only = single(&[WRITE, "io-cache.whitelist=meta"]); + assert_endpoint(&meta_only, OpClass::Write, DATA_PATH, OSS); + + assert_endpoint(&multi(&[WRITE]), OpClass::Write, DATA_PATH, CLUSTER); + assert_endpoint(&multi(&[WRITE]), OpClass::Write, MANIFEST_PATH, ACCEL); } - fn route(routing: &IoCacheRouting, op: OpClass, path: &str) -> Option { - routing.endpoint(routing.route(op, path)).map(String::from) + #[test] + fn test_two_cache_targets() { + let options = multi(&[]); + assert_endpoint(&options, OpClass::Read, MANIFEST_PATH, ACCEL); + assert_endpoint(&options, OpClass::Meta, MANIFEST_PATH, ACCEL); + assert_endpoint(&options, OpClass::Read, DATA_PATH, CLUSTER); + assert_endpoint(&options, OpClass::Meta, DATA_PATH, CLUSTER); + assert_endpoint(&options, OpClass::Read, INDEX_PATH, CLUSTER); + assert_endpoint(&options, OpClass::Read, GLOBAL_INDEX_PATH, CLUSTER); + assert_endpoint(&options, OpClass::Read, SNAPSHOT_PATH, OSS); + assert_endpoint(&options, OpClass::Write, MANIFEST_PATH, OSS); + assert_endpoint(&options, OpClass::Origin, MANIFEST_PATH, OSS); + let meta_only = multi(&["io-cache.whitelist=meta"]); + assert_endpoint(&meta_only, OpClass::Read, DATA_PATH, OSS); } #[test] - fn test_routing_vectors() { - let vectors: Vectors = load("routing.json"); - assert!(!vectors.cases.is_empty()); - for case in vectors.cases { - let routing = IoCacheRouting::from_props(&case.options); - let target = routing.route(op_class(&case.op), &case.path); - assert_eq!( - routing.endpoint(target), - Some(case.expect.as_str()), - "{}", - case.name - ); + fn test_targets_and_routes() { + // without routes the first target with an endpoint takes every type + let no_routes = "-io-cache.routes"; + assert_endpoint(&multi(&[no_routes]), OpClass::Read, DATA_PATH, ACCEL); + let shorthand = format!("io-cache.endpoint={CACHE}"); + let with_shorthand = multi(&[no_routes, shorthand.as_str()]); + assert_endpoint(&with_shorthand, OpClass::Read, DATA_PATH, ACCEL); + let without_accel = multi(&[no_routes, "-io-cache.target.accel.endpoint"]); + assert_endpoint(&without_accel, OpClass::Read, DATA_PATH, CLUSTER); + let blank_accel = multi(&[no_routes, "io-cache.target.accel.endpoint= "]); + assert_endpoint(&blank_accel, OpClass::Read, DATA_PATH, CLUSTER); + let bad_name = multi(&[no_routes, "io-cache.targets=Bad_Name,cluster"]); + assert_endpoint(&bad_name, OpClass::Read, DATA_PATH, CLUSTER); + let padded = format!("io-cache.target.cluster.endpoint= {CLUSTER} "); + assert_endpoint( + &multi(&[padded.as_str()]), + OpClass::Read, + DATA_PATH, + CLUSTER, + ); + + // a type without a usable route goes to the origin + let undeclared = multi(&["io-cache.routes=data=x"]); + assert_endpoint(&undeclared, OpClass::Read, DATA_PATH, OSS); + let without_cluster = multi(&["-io-cache.target.cluster.endpoint"]); + assert_endpoint(&without_cluster, OpClass::Read, DATA_PATH, OSS); + let meta_and_data = multi(&["io-cache.routes=meta=accel;data=cluster"]); + let file_index = format!("{DATA_PATH}.index"); + assert_endpoint(&meta_and_data, OpClass::Read, &file_index, OSS); + + let star = multi(&["io-cache.routes=*=cluster"]); + assert_endpoint(&star, OpClass::Read, MANIFEST_PATH, CLUSTER); + let first_wins = multi(&["io-cache.routes=data=accel;data,meta=cluster"]); + assert_endpoint(&first_wins, OpClass::Read, DATA_PATH, ACCEL); + let malformed = multi(&["io-cache.routes==cluster;meta=accel;junk"]); + assert_endpoint(&malformed, OpClass::Read, MANIFEST_PATH, ACCEL); + assert_endpoint(&malformed, OpClass::Read, DATA_PATH, OSS); + } + + #[test] + fn test_data_files_named_like_metadata() { + let manifest = multi(&["data-file.prefix=manifest-"]); + let named_like_manifest = "dt=1/bucket-0/manifest-{uuid}-0.orc"; + assert_endpoint(&manifest, OpClass::Read, named_like_manifest, CLUSTER); + let entropy = "dt=1/bucket-0/7f3a/manifest-{uuid}-0.orc"; + assert_endpoint(&manifest, OpClass::Read, entropy, CLUSTER); + assert_endpoint(&manifest, OpClass::Read, MANIFEST_PATH, ACCEL); + let sidecar = format!("{MANIFEST_PATH}.avro.sidecar"); + assert_endpoint(&manifest, OpClass::Read, &sidecar, ACCEL); + let snapshot = multi(&["data-file.prefix=snapshot-"]); + let named_like_snapshot = "dt=1/bucket-0/snapshot-{uuid}-0.orc"; + assert_endpoint(&snapshot, OpClass::Read, named_like_snapshot, CLUSTER); + let stat = multi(&["data-file.prefix=stat-"]); + assert_endpoint( + &stat, + OpClass::Read, + "dt=1/bucket-0/stat-{uuid}-0.orc", + CLUSTER, + ); + let index = multi(&[ + "io-cache.routes=data=cluster;bucket-index=accel", + "data-file.prefix=index-", + ]); + assert_endpoint( + &index, + OpClass::Read, + "dt=1/bucket-0/index-{uuid}-0.orc", + CLUSTER, + ); + assert_endpoint(&index, OpClass::Read, INDEX_PATH, ACCEL); + + let custom = "dt=1/bucket-0/custom-{uuid}-0.orc"; + assert_endpoint(&single(&[]), OpClass::Read, custom, OSS); + let custom_prefix = single(&["data-file.prefix=custom-"]); + assert_endpoint(&custom_prefix, OpClass::Read, custom, CACHE); + } + + #[test] + fn test_cacheable_type() { + // named by sequence id or rewritten in place + for path in [ + "snapshot/snapshot-12", + "snapshot/LATEST", + "snapshot/EARLIEST", + "branch/branch-dev/snapshot/LATEST", + "schema/schema-3", + "changelog/changelog-5", + "changelog/LATEST", + "tag/tag-2026-09-30", + "tag/tag-success-file/t1_SUCCESS", + "consumer/consumer-job1", + "service/service-primary-key-lookup", + "dt=1/_SUCCESS", + "oss://bkt/bucket-0/db/t/changelog/changelog-5", + ] { + assert_type(path, None, &[]); } + // temporary or not written by Paimon + for path in [ + "snapshot/.snapshot-13.{uuid}.tmp", + "dt=1/bucket-0/.data-{uuid}-0.parquet.{uuid}.tmp", + "dt=1/bucket-0/data-{uuid}-0.parquet.tmp-{uuid}", + "dt=1/bucket-0/data-{uuid}-0.parquet.tmp.{uuid}", + "metadata/version-hint.text", + "metadata/v3.metadata.json", + "dt=1/bucket-0/000000_0", + "dt=1/part-00000-1a2b.snappy.parquet", + "dt=1/bucket-0/data-1.parquet", + "dt=1/bucket-0/custom-{uuid}-0.orc", + "README", + ] { + assert_type(path, None, &[]); + } + + assert_type("manifest/manifest-{uuid}-0", Some(FileType::Meta), &[]); + assert_type("manifest/manifest-list-{uuid}-1", Some(FileType::Meta), &[]); + assert_type( + "manifest/index-manifest-{uuid}-0", + Some(FileType::Meta), + &[], + ); + let sidecar = "manifest/manifest-{uuid}-0.avro.sidecar"; + assert_type(sidecar, Some(FileType::Meta), &[]); + assert_type("statistics/stat-{uuid}-0", Some(FileType::Meta), &[]); + assert_type( + "dt=1/bucket-0/data-{uuid}-0.parquet", + Some(FileType::Data), + &[], + ); + assert_type( + "dt=1/bucket-0/changelog-{uuid}-0.parquet", + Some(FileType::Data), + &[], + ); + assert_type( + "dt=1/bucket-0/data-{uuid}-0.blob", + Some(FileType::Data), + &[], + ); + assert_type( + "dt=1/8b1f7c2e/data-{uuid}-0.parquet", + Some(FileType::Data), + &[], + ); + let file_index = "dt=1/bucket-0/data-{uuid}-0.parquet.index"; + assert_type(file_index, Some(FileType::FileIndex), &[]); + assert_type("index/index-{uuid}-0", Some(FileType::BucketIndex), &[]); + let btree = "index/btree-global-index-{uuid}.index"; + assert_type(btree, Some(FileType::GlobalIndex), &[]); + let lumina = "index/lumina-global-index-{uuid}.index"; + assert_type(lumina, Some(FileType::GlobalIndex), &[]); } #[test] - fn test_routable_type_vectors() { - let vectors: Vectors = load("routable-types.json"); - assert!(!vectors.cases.is_empty()); - for case in vectors.cases { - let expected = case - .file_type - .as_deref() - .and_then(|name| FileType::parse_whitelist(name).into_iter().next()); - assert_eq!( - routable_type(&case.path, &data_prefixes(&case.options)), - expected, - "{}", - case.path - ); + fn test_cacheable_type_with_file_prefixes() { + let data = Some(FileType::Data); + let custom = &["data-file.prefix=custom-"]; + assert_type("dt=1/bucket-0/custom-{uuid}-0.orc", data, custom); + assert_type("dt=1/bucket-0/data-{uuid}-0.orc", data, custom); + let changelog = &["changelog-file.prefix=cl-"]; + assert_type("dt=1/bucket-0/cl-{uuid}-0.orc", data, changelog); + assert_type( + "dt=1/bucket-0/ unknown.parquet", + None, + &["data-file.prefix= "], + ); + + // the name decides, so data files may be named like metadata + let manifest = &["data-file.prefix=manifest-"]; + assert_type("dt=1/bucket-0/manifest-{uuid}-0.orc", data, manifest); + assert_type("dt=1/bucket-0/7f3a/manifest-{uuid}-0.orc", data, manifest); + let file_index = Some(FileType::FileIndex); + assert_type( + "dt=1/bucket-0/manifest-{uuid}-0.orc.index", + file_index, + manifest, + ); + let meta = Some(FileType::Meta); + assert_type("manifest/manifest-{uuid}-0", meta, manifest); + assert_type("manifest/manifest-{uuid}-0.avro.sidecar", meta, manifest); + let index = &["data-file.prefix=index-"]; + let bucket_index = Some(FileType::BucketIndex); + assert_type("dt=1/bucket-0/index-{uuid}-0.orc", data, index); + assert_type("bucket-postpone/index-{uuid}-0.orc", data, index); + assert_type("index/index-{uuid}-0", bucket_index, index); + assert_type("dt=1/bucket-0/index-{uuid}-0", bucket_index, index); + let stat = &["data-file.prefix=stat-"]; + assert_type("dt=1/bucket-0/stat-{uuid}-0.orc", data, stat); + assert_type("postpone/stat-{uuid}-0.orc", data, stat); + assert_type("statistics/stat-{uuid}-0", meta, stat); + let snapshot = &["data-file.prefix=snapshot-"]; + assert_type("dt=1/bucket-0/snapshot-{uuid}-0.orc", data, snapshot); + assert_type( + "dt=1/bucket-0/snapshot-{uuid}-0.orc.index", + file_index, + snapshot, + ); + let schema = &["data-file.prefix=schema-"]; + assert_type("dt=1/bucket-0/schema-{uuid}-0.orc", data, schema); + let global_index = &["data-file.prefix=global-index-"]; + assert_type( + "dt=1/bucket-0/global-index-{uuid}-0.orc.index", + file_index, + global_index, + ); + + // names rewritten in place never use a cache, whatever the prefix + assert_type( + "dt=1/bucket-0/tag-{uuid}-0.orc", + None, + &["data-file.prefix=tag-"], + ); + let consumer = &["data-file.prefix=consumer-"]; + assert_type("dt=1/bucket-0/consumer-{uuid}-0.orc", None, consumer); + for (name, prefix) in [ + ("tag/tag-file", "data-file.prefix=tag-"), + ("consumer/consumer-file", "data-file.prefix=consumer-"), + ("service/service-file", "data-file.prefix=service-"), + ("metadata/version-hint.text", "data-file.prefix=version-"), + ("branch/branch-feature", "data-file.prefix=branch-"), + ] { + let path = format!("oss://bkt/warehouse/bucket-0/db/t1/{name}"); + assert_type(&path, None, &[prefix]); } } @@ -631,4 +936,85 @@ mod tests { assert_eq!(route(&overridden, OpClass::Read, data).unwrap(), "override"); assert_eq!(overridden.origin(), Some("override")); } + + /// One cache target, which older clients also get as fs.oss.endpoint. + fn single(changes: &[&str]) -> HashMap { + let base = [ + (OSS_ENDPOINT, CACHE), + (IO_CACHE_ENABLED, "true"), + (IO_CACHE_ENDPOINT, CACHE), + (IO_CACHE_ORIGIN_ENDPOINT, OSS), + (IO_CACHE_POLICY, "meta,read"), + (IO_CACHE_WHITELIST, "meta,data"), + ]; + with(&base, changes) + } + + /// Metadata on an accelerator, data and indexes on a cache cluster. + fn multi(changes: &[&str]) -> HashMap { + let base = [ + (OSS_ENDPOINT, ACCEL), + (IO_CACHE_ENABLED, "true"), + (IO_CACHE_ORIGIN_ENDPOINT, OSS), + (IO_CACHE_TARGETS, "accel,cluster"), + ("io-cache.target.accel.endpoint", ACCEL), + ("io-cache.target.accel.region", "cn-hangzhou"), + ("io-cache.target.cluster.endpoint", CLUSTER), + ("io-cache.target.cluster.path-style-access", "true"), + (IO_CACHE_POLICY, "meta,read"), + (IO_CACHE_WHITELIST, "*"), + ( + IO_CACHE_ROUTES, + "meta=accel;data,bucket-index,global-index,file-index=cluster", + ), + ]; + with(&base, changes) + } + + // "key=value" sets an option and "-key" removes it + fn with(base: &[(&str, &str)], changes: &[&str]) -> HashMap { + let mut options: HashMap = base + .iter() + .map(|(k, v)| (k.to_string(), v.to_string())) + .collect(); + for change in changes { + if let Some(key) = change.strip_prefix('-') { + options.remove(key); + } else { + let (key, value) = change.split_once('=').unwrap(); + options.insert(key.to_string(), value.to_string()); + } + } + options + } + + fn resolve(path: &str) -> String { + let path = path.replace("{uuid}", UUID); + if path.contains("://") { + path + } else { + format!("{TABLE_ROOT}/{path}") + } + } + + #[track_caller] + fn assert_endpoint(options: &HashMap, op: OpClass, path: &str, expect: &str) { + let routing = IoCacheRouting::from_props(options); + let sorted: BTreeMap<_, _> = options.iter().collect(); + assert_eq!( + routing.endpoint(routing.route(op, &resolve(path))), + Some(expect), + "{op:?} {path} with {sorted:?}" + ); + } + + #[track_caller] + fn assert_type(path: &str, expect: Option, changes: &[&str]) { + let prefixes = data_prefixes(&with(&[], changes)); + assert_eq!( + routable_type(&resolve(path), &prefixes), + expect, + "{path} with {changes:?}" + ); + } } diff --git a/crates/paimon/testdata/io_cache/routable-types.json b/crates/paimon/testdata/io_cache/routable-types.json deleted file mode 100644 index bb780c885..000000000 --- a/crates/paimon/testdata/io_cache/routable-types.json +++ /dev/null @@ -1,69 +0,0 @@ -{ - "table_root": "oss://bkt/db1.db/t1", - "cases": [ - {"path": "oss://bkt/db1.db/t1/snapshot/snapshot-12", "options": {}, "type": null, "reason": "snapshot: sequential metadata may be read before it exists"}, - {"path": "oss://bkt/db1.db/t1/snapshot/LATEST", "options": {}, "type": null, "reason": "hint file, rewritten in place"}, - {"path": "oss://bkt/db1.db/t1/snapshot/EARLIEST", "options": {}, "type": null, "reason": "hint file, rewritten in place"}, - {"path": "oss://bkt/db1.db/t1/branch/branch-dev/snapshot/LATEST", "options": {}, "type": null, "reason": "hint file on a branch"}, - {"path": "oss://bkt/db1.db/t1/schema/schema-3", "options": {}, "type": null, "reason": "schema: named by sequence id"}, - {"path": "oss://bkt/db1.db/t1/manifest/manifest-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0", "options": {}, "type": "meta", "reason": "manifest"}, - {"path": "oss://bkt/db1.db/t1/manifest/manifest-list-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-1", "options": {}, "type": "meta", "reason": "manifest list"}, - {"path": "oss://bkt/db1.db/t1/manifest/index-manifest-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0", "options": {}, "type": "meta", "reason": "index manifest; 'manifest' wins over 'index-'"}, - {"path": "oss://bkt/db1.db/t1/manifest/manifest-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.avro.sidecar", "options": {}, "type": "meta", "reason": "manifest sidecar"}, - {"path": "oss://bkt/db1.db/t1/statistics/stat-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0", "options": {}, "type": "meta", "reason": "statistics"}, - {"path": "oss://bkt/db1.db/t1/tag/tag-2026-09-30", "options": {}, "type": null, "reason": "tags can be replaced"}, - {"path": "oss://bkt/db1.db/t1/tag/tag-success-file/t1_SUCCESS", "options": {}, "type": null, "reason": "tag success file"}, - {"path": "oss://bkt/db1.db/t1/consumer/consumer-job1", "options": {}, "type": null, "reason": "consumer progress, rewritten"}, - {"path": "oss://bkt/db1.db/t1/service/service-primary-key-lookup", "options": {}, "type": null, "reason": "service file, rewritten"}, - {"path": "oss://bkt/db1.db/t1/dt=2026-09-30/_SUCCESS", "options": {}, "type": null, "reason": "partition mark-done file"}, - {"path": "oss://bkt/db1.db/t1/changelog/changelog-5", "options": {}, "type": null, "reason": "decoupled changelog metadata, named by sequence id"}, - {"path": "oss://bkt/db1.db/t1/changelog/LATEST", "options": {}, "type": null, "reason": "changelog hint file"}, - {"path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "options": {}, "type": "data", "reason": "data file"}, - {"path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/changelog-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "options": {}, "type": "data", "reason": "changelog data file in a bucket"}, - {"path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.blob", "options": {}, "type": "data", "reason": "blob file"}, - {"path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet.index", "options": {}, "type": "file-index", "reason": "data-file index sidecar"}, - {"path": "oss://bkt/db1.db/t1/index/index-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0", "options": {}, "type": "bucket-index", "reason": "hash index or deletion vectors"}, - {"path": "oss://bkt/db1.db/t1/index/btree-global-index-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b.index", "options": {}, "type": "global-index", "reason": "global btree index"}, - {"path": "oss://bkt/db1.db/t1/index/lumina-global-index-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b.index", "options": {}, "type": "global-index", "reason": "global vector index"}, - {"path": "oss://bkt/db1.db/t1/snapshot/.snapshot-13.8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b.tmp", "options": {}, "type": null, "reason": "temp file of an atomic write"}, - {"path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/.data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet.8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b.tmp", "options": {}, "type": null, "reason": "temp data file"}, - {"path": "oss://bkt/db1.db/t1/metadata/version-hint.text", "options": {}, "type": null, "reason": "Iceberg-compatible metadata hint"}, - {"path": "oss://bkt/db1.db/t1/metadata/v3.metadata.json", "options": {}, "type": null, "reason": "Iceberg-compatible metadata, not a Paimon name"}, - {"path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/000000_0", "options": {}, "type": null, "reason": "migrated Hive file: unknown name"}, - {"path": "oss://bkt/db1.db/t1/dt=2026-09-30/part-00000-1a2b.snappy.parquet", "options": {}, "type": null, "reason": "format-table style name: unknown"}, - {"path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/custom-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.orc", "options": {}, "type": null, "reason": "custom data-file.prefix without the option"}, - {"path": "oss://bkt/db1.db/t1/README", "options": {}, "type": null, "reason": "unknown name"}, - {"path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/manifest-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.orc", "options": {"data-file.prefix": "manifest-"}, "type": "data", "reason": "a data file named like a manifest by data-file.prefix is data"}, - {"path": "oss://bkt/db1.db/t1/manifest/manifest-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0", "options": {"data-file.prefix": "manifest-"}, "type": "meta", "reason": "manifests stay meta with data-file.prefix manifest-"}, - {"path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/index-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.orc", "options": {"data-file.prefix": "index-"}, "type": "data", "reason": "a data file named like a bucket index by data-file.prefix is data"}, - {"path": "oss://bkt/db1.db/t1/index/index-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0", "options": {"data-file.prefix": "index-"}, "type": "bucket-index", "reason": "bucket indexes stay bucket-index with data-file.prefix index-"}, - {"path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/stat-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.orc", "options": {"data-file.prefix": "stat-"}, "type": "data", "reason": "a data file named like statistics by data-file.prefix is data"}, - {"path": "oss://bkt/db1.db/t1/statistics/stat-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0", "options": {"data-file.prefix": "stat-"}, "type": "meta", "reason": "statistics stay meta with data-file.prefix stat-"}, - {"path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/manifest-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.orc.index", "options": {"data-file.prefix": "manifest-"}, "type": "file-index", "reason": "the file index of such a data file stays file-index"}, - {"path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/ unknown.parquet", "options": {"data-file.prefix": " "}, "type": null, "reason": "a whitespace-only prefix does not make unknown files cacheable"}, - {"path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/custom-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.orc", "options": {"data-file.prefix": "custom-"}, "type": "data", "reason": "custom data-file.prefix from the merged options"}, - {"path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/cl-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.orc", "options": {"changelog-file.prefix": "cl-"}, "type": "data", "reason": "custom changelog-file.prefix from the merged options"}, - {"path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.orc", "options": {"data-file.prefix": "custom-"}, "type": "data", "reason": "default prefixes stay routable next to a custom one"}, - {"path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/7f3a/manifest-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.orc", "options": {"data-file.prefix": "manifest-"}, "type": "data", "reason": "an external data path may put an entropy directory between the bucket and the file"}, - {"path": "oss://bkt/db1.db/t1/bucket-postpone/index-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.orc", "options": {"data-file.prefix": "index-"}, "type": "data", "reason": "data file of a postpone bucket"}, - {"path": "oss://bkt/db1.db/t1/postpone/stat-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.orc", "options": {"data-file.prefix": "stat-"}, "type": "data", "reason": "data file of a legacy postpone bucket"}, - {"path": "oss://bkt/db1.db/t1/dt=2026-09-30/8b1f7c2e/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "options": {}, "type": "data", "reason": "the name decides, not the directory"}, - {"path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/snapshot-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.orc", "options": {"data-file.prefix": "snapshot-"}, "type": "data", "reason": "snapshots are named by sequence id, so a uuid name with this prefix is data"}, - {"path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/schema-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.orc", "options": {"data-file.prefix": "schema-"}, "type": "data", "reason": "schemas are named by sequence id, so a uuid name with this prefix is data"}, - {"path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/snapshot-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.orc.index", "options": {"data-file.prefix": "snapshot-"}, "type": "file-index", "reason": "file index of a data file named like a snapshot"}, - {"path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/global-index-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.orc.index", "options": {"data-file.prefix": "global-index-"}, "type": "file-index", "reason": "file index of a data file named like a global index"}, - {"path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/tag-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.orc", "options": {"data-file.prefix": "tag-"}, "type": null, "reason": "names rewritten in place never use a cache, whatever the prefix"}, - {"path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/consumer-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.orc", "options": {"data-file.prefix": "consumer-"}, "type": null, "reason": "names rewritten in place never use a cache, whatever the prefix"}, - {"path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/index-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0", "options": {"data-file.prefix": "index-"}, "type": "bucket-index", "reason": "no extension: a bucket index, not a data file"}, - {"path": "oss://bkt/db1.db/t1/manifest/manifest-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.avro.sidecar", "options": {"data-file.prefix": "manifest-"}, "type": "meta", "reason": "a manifest sidecar is metadata whatever the data prefix"}, - {"path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet.tmp-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b", "options": {}, "type": null, "reason": "temp name with a .tmp- suffix"}, - {"path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet.tmp.8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b", "options": {}, "type": null, "reason": "temp name with a .tmp. infix"}, - {"path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-1.parquet", "options": {}, "type": null, "reason": "not a name Paimon writers produce"}, - {"path": "oss://bkt/bucket-0/db/t/changelog/changelog-5", "options": {}, "type": null, "reason": "decoupled changelog metadata below a bucket-* warehouse directory"}, - {"path": "oss://bkt/warehouse/bucket-0/db/t1/tag/tag-file", "options": {"data-file.prefix": "tag-"}, "type": null, "reason": "tag metadata with a matching data prefix"}, - {"path": "oss://bkt/warehouse/bucket-0/db/t1/consumer/consumer-file", "options": {"data-file.prefix": "consumer-"}, "type": null, "reason": "consumer metadata with a matching data prefix"}, - {"path": "oss://bkt/warehouse/bucket-0/db/t1/service/service-file", "options": {"data-file.prefix": "service-"}, "type": null, "reason": "service metadata with a matching data prefix"}, - {"path": "oss://bkt/warehouse/bucket-0/db/t1/metadata/version-hint.text", "options": {"data-file.prefix": "version-"}, "type": null, "reason": "Iceberg metadata hint with a matching data prefix"}, - {"path": "oss://bkt/warehouse/bucket-0/db/t1/branch/branch-feature", "options": {"data-file.prefix": "branch-"}, "type": null, "reason": "branch directory with a matching data prefix"} - ] -} diff --git a/crates/paimon/testdata/io_cache/routing.json b/crates/paimon/testdata/io_cache/routing.json deleted file mode 100644 index afd4b88eb..000000000 --- a/crates/paimon/testdata/io_cache/routing.json +++ /dev/null @@ -1,105 +0,0 @@ -{ - "table_root": "oss://bkt/db1.db/t1", - "cases": [ - {"name": "multi: a data file named like a manifest goes to the data target", "options": {"fs.oss.endpoint": "https://accelerator.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.targets": "accel,cluster", "io-cache.target.accel.endpoint": "https://accelerator.example.com", "io-cache.target.accel.region": "cn-hangzhou", "io-cache.target.cluster.endpoint": "http://cluster.example.com", "io-cache.target.cluster.path-style-access": "true", "io-cache.policy": "meta,read", "io-cache.whitelist": "*", "io-cache.routes": "meta=accel;data,bucket-index,global-index,file-index=cluster", "data-file.prefix": "manifest-"}, "op": "read", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/manifest-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.orc", "expect": "http://cluster.example.com"}, - {"name": "multi: manifests keep the meta target with data-file.prefix manifest-", "options": {"fs.oss.endpoint": "https://accelerator.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.targets": "accel,cluster", "io-cache.target.accel.endpoint": "https://accelerator.example.com", "io-cache.target.accel.region": "cn-hangzhou", "io-cache.target.cluster.endpoint": "http://cluster.example.com", "io-cache.target.cluster.path-style-access": "true", "io-cache.policy": "meta,read", "io-cache.whitelist": "*", "io-cache.routes": "meta=accel;data,bucket-index,global-index,file-index=cluster", "data-file.prefix": "manifest-"}, "op": "read", "path": "oss://bkt/db1.db/t1/manifest/manifest-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0", "expect": "https://accelerator.example.com"}, - {"name": "multi: a data file named like a bucket index goes to the data target", "options": {"fs.oss.endpoint": "https://accelerator.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.targets": "accel,cluster", "io-cache.target.accel.endpoint": "https://accelerator.example.com", "io-cache.target.accel.region": "cn-hangzhou", "io-cache.target.cluster.endpoint": "http://cluster.example.com", "io-cache.target.cluster.path-style-access": "true", "io-cache.policy": "meta,read", "io-cache.whitelist": "*", "io-cache.routes": "data=cluster;bucket-index=accel", "data-file.prefix": "index-"}, "op": "read", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/index-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.orc", "expect": "http://cluster.example.com"}, - {"name": "multi: bucket indexes keep their target with data-file.prefix index-", "options": {"fs.oss.endpoint": "https://accelerator.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.targets": "accel,cluster", "io-cache.target.accel.endpoint": "https://accelerator.example.com", "io-cache.target.accel.region": "cn-hangzhou", "io-cache.target.cluster.endpoint": "http://cluster.example.com", "io-cache.target.cluster.path-style-access": "true", "io-cache.policy": "meta,read", "io-cache.whitelist": "*", "io-cache.routes": "data=cluster;bucket-index=accel", "data-file.prefix": "index-"}, "op": "read", "path": "oss://bkt/db1.db/t1/index/index-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0", "expect": "https://accelerator.example.com"}, - {"name": "multi: a data file named like a snapshot goes to the data target", "options": {"fs.oss.endpoint": "https://accelerator.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.targets": "accel,cluster", "io-cache.target.accel.endpoint": "https://accelerator.example.com", "io-cache.target.accel.region": "cn-hangzhou", "io-cache.target.cluster.endpoint": "http://cluster.example.com", "io-cache.target.cluster.path-style-access": "true", "io-cache.policy": "meta,read", "io-cache.whitelist": "*", "io-cache.routes": "meta=accel;data,bucket-index,global-index,file-index=cluster", "data-file.prefix": "snapshot-"}, "op": "read", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/snapshot-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.orc", "expect": "http://cluster.example.com"}, - {"name": "multi: a data file below an entropy directory goes to the data target", "options": {"fs.oss.endpoint": "https://accelerator.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.targets": "accel,cluster", "io-cache.target.accel.endpoint": "https://accelerator.example.com", "io-cache.target.accel.region": "cn-hangzhou", "io-cache.target.cluster.endpoint": "http://cluster.example.com", "io-cache.target.cluster.path-style-access": "true", "io-cache.policy": "meta,read", "io-cache.whitelist": "*", "io-cache.routes": "meta=accel;data,bucket-index,global-index,file-index=cluster", "data-file.prefix": "manifest-"}, "op": "read", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/7f3a/manifest-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.orc", "expect": "http://cluster.example.com"}, - {"name": "multi: a manifest sidecar keeps the meta target with data-file.prefix manifest-", "options": {"fs.oss.endpoint": "https://accelerator.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.targets": "accel,cluster", "io-cache.target.accel.endpoint": "https://accelerator.example.com", "io-cache.target.accel.region": "cn-hangzhou", "io-cache.target.cluster.endpoint": "http://cluster.example.com", "io-cache.target.cluster.path-style-access": "true", "io-cache.policy": "meta,read", "io-cache.whitelist": "*", "io-cache.routes": "meta=accel;data,bucket-index,global-index,file-index=cluster", "data-file.prefix": "manifest-"}, "op": "read", "path": "oss://bkt/db1.db/t1/manifest/manifest-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.avro.sidecar", "expect": "https://accelerator.example.com"}, - {"name": "multi: a data file named like statistics goes to the data target", "options": {"fs.oss.endpoint": "https://accelerator.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.targets": "accel,cluster", "io-cache.target.accel.endpoint": "https://accelerator.example.com", "io-cache.target.accel.region": "cn-hangzhou", "io-cache.target.cluster.endpoint": "http://cluster.example.com", "io-cache.target.cluster.path-style-access": "true", "io-cache.policy": "meta,read", "io-cache.whitelist": "*", "io-cache.routes": "meta=accel;data,bucket-index,global-index,file-index=cluster", "data-file.prefix": "stat-"}, "op": "read", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/stat-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.orc", "expect": "http://cluster.example.com"}, - {"name": "single: a dls path is not routed", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": "http://cache.example.com", "io-cache.policy": "meta,read", "io-cache.whitelist": "meta,data"}, "op": "read", "path": "dls://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "expect": "https://oss-cn-hangzhou-internal.aliyuncs.com"}, - {"name": "policy: trimmed case-insensitive tokens", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": "http://cache.example.com", "io-cache.policy": " READ , Meta ", "io-cache.whitelist": "meta,data"}, "op": "meta", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "expect": "http://cache.example.com"}, - {"name": "policy: none overrides read", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": "http://cache.example.com", "io-cache.policy": "read,NONE", "io-cache.whitelist": "meta,data"}, "op": "exists", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "expect": "http://cache.example.com"}, - {"name": "policy: substrings are not operations", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": "http://cache.example.com", "io-cache.policy": "thread,metadata", "io-cache.whitelist": "meta,data"}, "op": "exists", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "expect": "http://cache.example.com"}, - {"name": "policy: unknown tokens do not disable read", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": "http://cache.example.com", "io-cache.policy": "read,nonetheless", "io-cache.whitelist": "meta,data"}, "op": "exists", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "expect": "https://oss-cn-hangzhou-internal.aliyuncs.com"}, - {"name": "single: whitespace endpoint is missing", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": " ", "io-cache.policy": "meta,read", "io-cache.whitelist": "meta,data"}, "op": "read", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "expect": "https://oss-cn-hangzhou-internal.aliyuncs.com"}, - {"name": "multi: whitespace endpoint is skipped", "options": {"fs.oss.endpoint": "https://accelerator.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.targets": "accel,cluster", "io-cache.target.accel.endpoint": " ", "io-cache.target.accel.region": "cn-hangzhou", "io-cache.target.cluster.endpoint": "http://cluster.example.com", "io-cache.target.cluster.path-style-access": "true", "io-cache.policy": "meta,read", "io-cache.whitelist": "*"}, "op": "read", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "expect": "http://cluster.example.com"}, - {"name": "multi: target endpoint is trimmed", "options": {"fs.oss.endpoint": "https://accelerator.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.targets": "accel,cluster", "io-cache.target.accel.endpoint": "https://accelerator.example.com", "io-cache.target.accel.region": "cn-hangzhou", "io-cache.target.cluster.endpoint": " http://cluster.example.com ", "io-cache.target.cluster.path-style-access": "true", "io-cache.policy": "meta,read", "io-cache.whitelist": "*", "io-cache.routes": "meta=accel;data,bucket-index,global-index,file-index=cluster"}, "op": "read", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "expect": "http://cluster.example.com"}, - {"name": "single: whitespace origin uses legacy endpoint", "options": {"fs.oss.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": " ", "io-cache.endpoint": "http://cache.example.com", "io-cache.policy": "meta,read", "io-cache.whitelist": "meta,data"}, "op": "list", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "expect": "https://oss-cn-hangzhou-internal.aliyuncs.com"}, - {"name": "single: data read goes to the cache", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": "http://cache.example.com", "io-cache.policy": "meta,read", "io-cache.whitelist": "meta,data"}, "op": "read", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "expect": "http://cache.example.com"}, - {"name": "single: data status goes to the cache", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": "http://cache.example.com", "io-cache.policy": "meta,read", "io-cache.whitelist": "meta,data"}, "op": "meta", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "expect": "http://cache.example.com"}, - {"name": "single: manifest read goes to the cache", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": "http://cache.example.com", "io-cache.policy": "meta,read", "io-cache.whitelist": "meta,data"}, "op": "read", "path": "oss://bkt/db1.db/t1/manifest/manifest-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0", "expect": "http://cache.example.com"}, - {"name": "single: manifest list read goes to the cache", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": "http://cache.example.com", "io-cache.policy": "meta,read", "io-cache.whitelist": "meta,data"}, "op": "read", "path": "oss://bkt/db1.db/t1/manifest/manifest-list-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-1", "expect": "http://cache.example.com"}, - {"name": "single: snapshot read stays at origin", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": "http://cache.example.com", "io-cache.policy": "meta,read", "io-cache.whitelist": "meta,data"}, "op": "read", "path": "oss://bkt/db1.db/t1/snapshot/snapshot-12", "expect": "https://oss-cn-hangzhou-internal.aliyuncs.com"}, - {"name": "single: snapshot status stays at origin", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": "http://cache.example.com", "io-cache.policy": "meta,read", "io-cache.whitelist": "meta,data"}, "op": "meta", "path": "oss://bkt/db1.db/t1/snapshot/snapshot-12", "expect": "https://oss-cn-hangzhou-internal.aliyuncs.com"}, - {"name": "single: hint file stays at origin", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": "http://cache.example.com", "io-cache.policy": "meta,read", "io-cache.whitelist": "meta,data"}, "op": "read", "path": "oss://bkt/db1.db/t1/snapshot/LATEST", "expect": "https://oss-cn-hangzhou-internal.aliyuncs.com"}, - {"name": "single: index type outside the whitelist stays at origin", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": "http://cache.example.com", "io-cache.policy": "meta,read", "io-cache.whitelist": "meta,data"}, "op": "read", "path": "oss://bkt/db1.db/t1/index/index-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0", "expect": "https://oss-cn-hangzhou-internal.aliyuncs.com"}, - {"name": "single: unknown name stays at origin", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": "http://cache.example.com", "io-cache.policy": "meta,read", "io-cache.whitelist": "meta,data"}, "op": "read", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/000000_0", "expect": "https://oss-cn-hangzhou-internal.aliyuncs.com"}, - {"name": "single: temp file stays at origin", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": "http://cache.example.com", "io-cache.policy": "meta,read", "io-cache.whitelist": "meta,data"}, "op": "read", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/.data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet.8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b.tmp", "expect": "https://oss-cn-hangzhou-internal.aliyuncs.com"}, - {"name": "single: a path in another bucket is routed like the table's", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": "http://cache.example.com", "io-cache.policy": "meta,read", "io-cache.whitelist": "meta,data"}, "op": "read", "path": "oss://other-bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "expect": "http://cache.example.com"}, - {"name": "single: disabled by default", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": "http://cache.example.com", "io-cache.policy": "meta,read", "io-cache.whitelist": "meta,data"}, "op": "read", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "expect": "http://cache.example.com"}, - {"name": "single: disabled keeps fs.oss.endpoint for everything", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.enabled": "false", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": "http://cache.example.com", "io-cache.policy": "meta,read", "io-cache.whitelist": "meta,data"}, "op": "exists", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "expect": "http://cache.example.com"}, - {"name": "single: no policy means no routing", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": "http://cache.example.com", "io-cache.whitelist": "meta,data"}, "op": "exists", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "expect": "http://cache.example.com"}, - {"name": "single: policy none means no routing", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": "http://cache.example.com", "io-cache.policy": "none", "io-cache.whitelist": "meta,data"}, "op": "exists", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "expect": "http://cache.example.com"}, - {"name": "single: write-only policy keeps status at origin", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": "http://cache.example.com", "io-cache.policy": "write", "io-cache.whitelist": "meta,data"}, "op": "exists", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "expect": "https://oss-cn-hangzhou-internal.aliyuncs.com"}, - {"name": "single: write-only policy routes writes", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": "http://cache.example.com", "io-cache.policy": "write", "io-cache.whitelist": "meta,data"}, "op": "write", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "expect": "http://cache.example.com"}, - {"name": "write: data file write goes to the cache", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": "http://cache.example.com", "io-cache.policy": "meta,read,write", "io-cache.whitelist": "meta,data"}, "op": "write", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "expect": "http://cache.example.com"}, - {"name": "write: two-phase data file write goes to the cache", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": "http://cache.example.com", "io-cache.policy": "meta,read,write", "io-cache.whitelist": "meta,data"}, "op": "two-phase-write", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "expect": "http://cache.example.com"}, - {"name": "write: manifest write goes to the cache", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": "http://cache.example.com", "io-cache.policy": "meta,read,write", "io-cache.whitelist": "meta,data"}, "op": "write", "path": "oss://bkt/db1.db/t1/manifest/manifest-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0", "expect": "http://cache.example.com"}, - {"name": "write: snapshot write stays at origin", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": "http://cache.example.com", "io-cache.policy": "meta,read,write", "io-cache.whitelist": "meta,data"}, "op": "write", "path": "oss://bkt/db1.db/t1/snapshot/snapshot-12", "expect": "https://oss-cn-hangzhou-internal.aliyuncs.com"}, - {"name": "write: hint file write stays at origin", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": "http://cache.example.com", "io-cache.policy": "meta,read,write", "io-cache.whitelist": "meta,data"}, "op": "write", "path": "oss://bkt/db1.db/t1/snapshot/LATEST", "expect": "https://oss-cn-hangzhou-internal.aliyuncs.com"}, - {"name": "write: temp file write stays at origin", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": "http://cache.example.com", "io-cache.policy": "meta,read,write", "io-cache.whitelist": "meta,data"}, "op": "write", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/.data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet.8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b.tmp", "expect": "https://oss-cn-hangzhou-internal.aliyuncs.com"}, - {"name": "write: atomic write stays at origin", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": "http://cache.example.com", "io-cache.policy": "meta,read,write", "io-cache.whitelist": "meta,data"}, "op": "atomic-write", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "expect": "https://oss-cn-hangzhou-internal.aliyuncs.com"}, - {"name": "write: delete stays at origin", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": "http://cache.example.com", "io-cache.policy": "meta,read,write", "io-cache.whitelist": "meta,data"}, "op": "delete", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "expect": "https://oss-cn-hangzhou-internal.aliyuncs.com"}, - {"name": "write: rename stays at origin", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": "http://cache.example.com", "io-cache.policy": "meta,read,write", "io-cache.whitelist": "meta,data"}, "op": "rename", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "expect": "https://oss-cn-hangzhou-internal.aliyuncs.com"}, - {"name": "write: copy stays at origin", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": "http://cache.example.com", "io-cache.policy": "meta,read,write", "io-cache.whitelist": "meta,data"}, "op": "copy", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "expect": "https://oss-cn-hangzhou-internal.aliyuncs.com"}, - {"name": "write: list stays at origin", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": "http://cache.example.com", "io-cache.policy": "meta,read,write", "io-cache.whitelist": "meta,data"}, "op": "list", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "expect": "https://oss-cn-hangzhou-internal.aliyuncs.com"}, - {"name": "write: whitelist filters writes", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": "http://cache.example.com", "io-cache.policy": "meta,read,write", "io-cache.whitelist": "meta"}, "op": "write", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "expect": "https://oss-cn-hangzhou-internal.aliyuncs.com"}, - {"name": "write: data write follows the data route", "options": {"fs.oss.endpoint": "https://accelerator.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.targets": "accel,cluster", "io-cache.target.accel.endpoint": "https://accelerator.example.com", "io-cache.target.accel.region": "cn-hangzhou", "io-cache.target.cluster.endpoint": "http://cluster.example.com", "io-cache.target.cluster.path-style-access": "true", "io-cache.policy": "meta,read,write", "io-cache.whitelist": "*", "io-cache.routes": "meta=accel;data,bucket-index,global-index,file-index=cluster"}, "op": "write", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "expect": "http://cluster.example.com"}, - {"name": "write: manifest write follows the meta route", "options": {"fs.oss.endpoint": "https://accelerator.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.targets": "accel,cluster", "io-cache.target.accel.endpoint": "https://accelerator.example.com", "io-cache.target.accel.region": "cn-hangzhou", "io-cache.target.cluster.endpoint": "http://cluster.example.com", "io-cache.target.cluster.path-style-access": "true", "io-cache.policy": "meta,read,write", "io-cache.whitelist": "*", "io-cache.routes": "meta=accel;data,bucket-index,global-index,file-index=cluster"}, "op": "write", "path": "oss://bkt/db1.db/t1/manifest/manifest-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0", "expect": "https://accelerator.example.com"}, - {"name": "single: read-only policy keeps status at origin", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": "http://cache.example.com", "io-cache.policy": "read", "io-cache.whitelist": "meta,data"}, "op": "meta", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "expect": "https://oss-cn-hangzhou-internal.aliyuncs.com"}, - {"name": "single: read-only policy still routes reads", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": "http://cache.example.com", "io-cache.policy": "read", "io-cache.whitelist": "meta,data"}, "op": "read", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "expect": "http://cache.example.com"}, - {"name": "single: origin defaults to fs.oss.endpoint", "options": {"fs.oss.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.enabled": "true", "io-cache.endpoint": "http://cache.example.com", "io-cache.policy": "meta,read", "io-cache.whitelist": "meta,data"}, "op": "list", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "expect": "https://oss-cn-hangzhou-internal.aliyuncs.com"}, - {"name": "single: client endpoint override wins", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": "http://cache.example.com", "io-cache.policy": "meta,read", "io-cache.whitelist": "meta,data", "dlf.oss-endpoint": "https://oss-cn-hangzhou.aliyuncs.com"}, "op": "read", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "expect": "https://oss-cn-hangzhou.aliyuncs.com"}, - {"name": "single: absent whitelist allows every routable type", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": "http://cache.example.com", "io-cache.policy": "meta,read"}, "op": "read", "path": "oss://bkt/db1.db/t1/index/index-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0", "expect": "http://cache.example.com"}, - {"name": "single: star whitelist allows every routable type", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": "http://cache.example.com", "io-cache.policy": "meta,read", "io-cache.whitelist": "*"}, "op": "read", "path": "oss://bkt/db1.db/t1/index/btree-global-index-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b.index", "expect": "http://cache.example.com"}, - {"name": "single: file index sidecar with star whitelist", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": "http://cache.example.com", "io-cache.policy": "meta,read", "io-cache.whitelist": "*"}, "op": "read", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet.index", "expect": "http://cache.example.com"}, - {"name": "single: custom data prefix without the option stays at origin", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": "http://cache.example.com", "io-cache.policy": "meta,read", "io-cache.whitelist": "meta,data"}, "op": "read", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/custom-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.orc", "expect": "https://oss-cn-hangzhou-internal.aliyuncs.com"}, - {"name": "single: custom data prefix from options is routed", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": "http://cache.example.com", "io-cache.policy": "meta,read", "io-cache.whitelist": "meta,data", "data-file.prefix": "custom-"}, "op": "read", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/custom-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.orc", "expect": "http://cache.example.com"}, - {"name": "single: empty cache endpoint sends routed reads to origin", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": "", "io-cache.policy": "meta,read", "io-cache.whitelist": "meta,data", "io-cache.routes": "*=default"}, "op": "read", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "expect": "https://oss-cn-hangzhou-internal.aliyuncs.com"}, - {"name": "multi: manifest read goes to the accelerator", "options": {"fs.oss.endpoint": "https://accelerator.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.targets": "accel,cluster", "io-cache.target.accel.endpoint": "https://accelerator.example.com", "io-cache.target.accel.region": "cn-hangzhou", "io-cache.target.cluster.endpoint": "http://cluster.example.com", "io-cache.target.cluster.path-style-access": "true", "io-cache.policy": "meta,read", "io-cache.whitelist": "*", "io-cache.routes": "meta=accel;data,bucket-index,global-index,file-index=cluster"}, "op": "read", "path": "oss://bkt/db1.db/t1/manifest/manifest-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0", "expect": "https://accelerator.example.com"}, - {"name": "multi: manifest status goes to the accelerator", "options": {"fs.oss.endpoint": "https://accelerator.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.targets": "accel,cluster", "io-cache.target.accel.endpoint": "https://accelerator.example.com", "io-cache.target.accel.region": "cn-hangzhou", "io-cache.target.cluster.endpoint": "http://cluster.example.com", "io-cache.target.cluster.path-style-access": "true", "io-cache.policy": "meta,read", "io-cache.whitelist": "*", "io-cache.routes": "meta=accel;data,bucket-index,global-index,file-index=cluster"}, "op": "meta", "path": "oss://bkt/db1.db/t1/manifest/manifest-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0", "expect": "https://accelerator.example.com"}, - {"name": "multi: data read goes to the cluster", "options": {"fs.oss.endpoint": "https://accelerator.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.targets": "accel,cluster", "io-cache.target.accel.endpoint": "https://accelerator.example.com", "io-cache.target.accel.region": "cn-hangzhou", "io-cache.target.cluster.endpoint": "http://cluster.example.com", "io-cache.target.cluster.path-style-access": "true", "io-cache.policy": "meta,read", "io-cache.whitelist": "*", "io-cache.routes": "meta=accel;data,bucket-index,global-index,file-index=cluster"}, "op": "read", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "expect": "http://cluster.example.com"}, - {"name": "multi: deletion vectors go to the cluster", "options": {"fs.oss.endpoint": "https://accelerator.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.targets": "accel,cluster", "io-cache.target.accel.endpoint": "https://accelerator.example.com", "io-cache.target.accel.region": "cn-hangzhou", "io-cache.target.cluster.endpoint": "http://cluster.example.com", "io-cache.target.cluster.path-style-access": "true", "io-cache.policy": "meta,read", "io-cache.whitelist": "*", "io-cache.routes": "meta=accel;data,bucket-index,global-index,file-index=cluster"}, "op": "read", "path": "oss://bkt/db1.db/t1/index/index-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0", "expect": "http://cluster.example.com"}, - {"name": "multi: global index goes to the cluster", "options": {"fs.oss.endpoint": "https://accelerator.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.targets": "accel,cluster", "io-cache.target.accel.endpoint": "https://accelerator.example.com", "io-cache.target.accel.region": "cn-hangzhou", "io-cache.target.cluster.endpoint": "http://cluster.example.com", "io-cache.target.cluster.path-style-access": "true", "io-cache.policy": "meta,read", "io-cache.whitelist": "*", "io-cache.routes": "meta=accel;data,bucket-index,global-index,file-index=cluster"}, "op": "read", "path": "oss://bkt/db1.db/t1/index/btree-global-index-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b.index", "expect": "http://cluster.example.com"}, - {"name": "multi: manifest existence goes to the accelerator", "options": {"fs.oss.endpoint": "https://accelerator.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.targets": "accel,cluster", "io-cache.target.accel.endpoint": "https://accelerator.example.com", "io-cache.target.accel.region": "cn-hangzhou", "io-cache.target.cluster.endpoint": "http://cluster.example.com", "io-cache.target.cluster.path-style-access": "true", "io-cache.policy": "meta,read", "io-cache.whitelist": "*", "io-cache.routes": "meta=accel;data,bucket-index,global-index,file-index=cluster"}, "op": "exists", "path": "oss://bkt/db1.db/t1/manifest/manifest-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0", "expect": "https://accelerator.example.com"}, - {"name": "multi: data existence goes to the cluster", "options": {"fs.oss.endpoint": "https://accelerator.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.targets": "accel,cluster", "io-cache.target.accel.endpoint": "https://accelerator.example.com", "io-cache.target.accel.region": "cn-hangzhou", "io-cache.target.cluster.endpoint": "http://cluster.example.com", "io-cache.target.cluster.path-style-access": "true", "io-cache.policy": "meta,read", "io-cache.whitelist": "*", "io-cache.routes": "meta=accel;data,bucket-index,global-index,file-index=cluster"}, "op": "exists", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "expect": "http://cluster.example.com"}, - {"name": "single: data existence goes to the cache", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": "http://cache.example.com", "io-cache.policy": "meta,read", "io-cache.whitelist": "meta,data"}, "op": "exists", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "expect": "http://cache.example.com"}, - {"name": "single: snapshot existence stays at origin", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": "http://cache.example.com", "io-cache.policy": "meta,read", "io-cache.whitelist": "meta,data"}, "op": "exists", "path": "oss://bkt/db1.db/t1/snapshot/snapshot-12", "expect": "https://oss-cn-hangzhou-internal.aliyuncs.com"}, - {"name": "single: hint file existence stays at origin", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": "http://cache.example.com", "io-cache.policy": "meta,read", "io-cache.whitelist": "meta,data"}, "op": "exists", "path": "oss://bkt/db1.db/t1/snapshot/LATEST", "expect": "https://oss-cn-hangzhou-internal.aliyuncs.com"}, - {"name": "single: read-only policy keeps existence at origin", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": "http://cache.example.com", "io-cache.policy": "read", "io-cache.whitelist": "meta,data"}, "op": "exists", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "expect": "https://oss-cn-hangzhou-internal.aliyuncs.com"}, - {"name": "multi: snapshot stays at origin", "options": {"fs.oss.endpoint": "https://accelerator.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.targets": "accel,cluster", "io-cache.target.accel.endpoint": "https://accelerator.example.com", "io-cache.target.accel.region": "cn-hangzhou", "io-cache.target.cluster.endpoint": "http://cluster.example.com", "io-cache.target.cluster.path-style-access": "true", "io-cache.policy": "meta,read", "io-cache.whitelist": "*", "io-cache.routes": "meta=accel;data,bucket-index,global-index,file-index=cluster"}, "op": "read", "path": "oss://bkt/db1.db/t1/snapshot/snapshot-12", "expect": "https://oss-cn-hangzhou-internal.aliyuncs.com"}, - {"name": "multi: without routes the first usable target takes all", "options": {"fs.oss.endpoint": "https://accelerator.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.targets": "accel,cluster", "io-cache.target.accel.endpoint": "https://accelerator.example.com", "io-cache.target.accel.region": "cn-hangzhou", "io-cache.target.cluster.endpoint": "http://cluster.example.com", "io-cache.target.cluster.path-style-access": "true", "io-cache.policy": "meta,read", "io-cache.whitelist": "*"}, "op": "read", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "expect": "https://accelerator.example.com"}, - {"name": "multi: route to an undeclared target stays at origin", "options": {"fs.oss.endpoint": "https://accelerator.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.targets": "accel,cluster", "io-cache.target.accel.endpoint": "https://accelerator.example.com", "io-cache.target.accel.region": "cn-hangzhou", "io-cache.target.cluster.endpoint": "http://cluster.example.com", "io-cache.target.cluster.path-style-access": "true", "io-cache.policy": "meta,read", "io-cache.whitelist": "*", "io-cache.routes": "data=x"}, "op": "read", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "expect": "https://oss-cn-hangzhou-internal.aliyuncs.com"}, - {"name": "multi: type without a matching route stays at origin", "options": {"fs.oss.endpoint": "https://accelerator.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.targets": "accel,cluster", "io-cache.target.accel.endpoint": "https://accelerator.example.com", "io-cache.target.accel.region": "cn-hangzhou", "io-cache.target.cluster.endpoint": "http://cluster.example.com", "io-cache.target.cluster.path-style-access": "true", "io-cache.policy": "meta,read", "io-cache.whitelist": "*", "io-cache.routes": "meta=accel;data=cluster"}, "op": "read", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet.index", "expect": "https://oss-cn-hangzhou-internal.aliyuncs.com"}, - {"name": "multi: targets win over the single-target shorthand", "options": {"fs.oss.endpoint": "https://accelerator.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.targets": "accel,cluster", "io-cache.target.accel.endpoint": "https://accelerator.example.com", "io-cache.target.accel.region": "cn-hangzhou", "io-cache.target.cluster.endpoint": "http://cluster.example.com", "io-cache.target.cluster.path-style-access": "true", "io-cache.policy": "meta,read", "io-cache.whitelist": "*", "io-cache.endpoint": "http://cache.example.com"}, "op": "read", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "expect": "https://accelerator.example.com"}, - {"name": "multi: target without endpoint is skipped when routes are absent", "options": {"fs.oss.endpoint": "https://accelerator.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.targets": "accel,cluster", "io-cache.target.accel.region": "cn-hangzhou", "io-cache.target.cluster.endpoint": "http://cluster.example.com", "io-cache.target.cluster.path-style-access": "true", "io-cache.policy": "meta,read", "io-cache.whitelist": "*"}, "op": "read", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "expect": "http://cluster.example.com"}, - {"name": "multi: route to a target without endpoint stays at origin", "options": {"fs.oss.endpoint": "https://accelerator.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.targets": "accel,cluster", "io-cache.target.accel.endpoint": "https://accelerator.example.com", "io-cache.target.accel.region": "cn-hangzhou", "io-cache.target.cluster.path-style-access": "true", "io-cache.policy": "meta,read", "io-cache.whitelist": "*", "io-cache.routes": "meta=accel;data,bucket-index,global-index,file-index=cluster"}, "op": "read", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "expect": "https://oss-cn-hangzhou-internal.aliyuncs.com"}, - {"name": "multi: invalid target names are ignored", "options": {"fs.oss.endpoint": "https://accelerator.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.targets": "Bad_Name,cluster", "io-cache.target.accel.endpoint": "https://accelerator.example.com", "io-cache.target.accel.region": "cn-hangzhou", "io-cache.target.cluster.endpoint": "http://cluster.example.com", "io-cache.target.cluster.path-style-access": "true", "io-cache.policy": "meta,read", "io-cache.whitelist": "*"}, "op": "read", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "expect": "http://cluster.example.com"}, - {"name": "multi: star route sends every type to one target", "options": {"fs.oss.endpoint": "https://accelerator.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.targets": "accel,cluster", "io-cache.target.accel.endpoint": "https://accelerator.example.com", "io-cache.target.accel.region": "cn-hangzhou", "io-cache.target.cluster.endpoint": "http://cluster.example.com", "io-cache.target.cluster.path-style-access": "true", "io-cache.policy": "meta,read", "io-cache.whitelist": "*", "io-cache.routes": "*=cluster"}, "op": "read", "path": "oss://bkt/db1.db/t1/manifest/manifest-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0", "expect": "http://cluster.example.com"}, - {"name": "multi: first matching route wins", "options": {"fs.oss.endpoint": "https://accelerator.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.targets": "accel,cluster", "io-cache.target.accel.endpoint": "https://accelerator.example.com", "io-cache.target.accel.region": "cn-hangzhou", "io-cache.target.cluster.endpoint": "http://cluster.example.com", "io-cache.target.cluster.path-style-access": "true", "io-cache.policy": "meta,read", "io-cache.whitelist": "*", "io-cache.routes": "data=accel;data,meta=cluster"}, "op": "read", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "expect": "https://accelerator.example.com"}, - {"name": "multi: malformed routes are skipped", "options": {"fs.oss.endpoint": "https://accelerator.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.targets": "accel,cluster", "io-cache.target.accel.endpoint": "https://accelerator.example.com", "io-cache.target.accel.region": "cn-hangzhou", "io-cache.target.cluster.endpoint": "http://cluster.example.com", "io-cache.target.cluster.path-style-access": "true", "io-cache.policy": "meta,read", "io-cache.whitelist": "*", "io-cache.routes": "=cluster;meta=accel;junk"}, "op": "read", "path": "oss://bkt/db1.db/t1/manifest/manifest-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0", "expect": "https://accelerator.example.com"}, - {"name": "multi: malformed routes do not match other types", "options": {"fs.oss.endpoint": "https://accelerator.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.targets": "accel,cluster", "io-cache.target.accel.endpoint": "https://accelerator.example.com", "io-cache.target.accel.region": "cn-hangzhou", "io-cache.target.cluster.endpoint": "http://cluster.example.com", "io-cache.target.cluster.path-style-access": "true", "io-cache.policy": "meta,read", "io-cache.whitelist": "*", "io-cache.routes": "=cluster;meta=accel;junk"}, "op": "read", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "expect": "https://oss-cn-hangzhou-internal.aliyuncs.com"}, - {"name": "multi: whitelist filters before routes", "options": {"fs.oss.endpoint": "https://accelerator.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.targets": "accel,cluster", "io-cache.target.accel.endpoint": "https://accelerator.example.com", "io-cache.target.accel.region": "cn-hangzhou", "io-cache.target.cluster.endpoint": "http://cluster.example.com", "io-cache.target.cluster.path-style-access": "true", "io-cache.policy": "meta,read", "io-cache.whitelist": "meta", "io-cache.routes": "meta=accel;data,bucket-index,global-index,file-index=cluster"}, "op": "read", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "expect": "https://oss-cn-hangzhou-internal.aliyuncs.com"}, - {"name": "single: write stays at origin", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": "http://cache.example.com", "io-cache.policy": "meta,read", "io-cache.whitelist": "meta,data"}, "op": "write", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "expect": "https://oss-cn-hangzhou-internal.aliyuncs.com"}, - {"name": "multi: write stays at origin", "options": {"fs.oss.endpoint": "https://accelerator.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.targets": "accel,cluster", "io-cache.target.accel.endpoint": "https://accelerator.example.com", "io-cache.target.accel.region": "cn-hangzhou", "io-cache.target.cluster.endpoint": "http://cluster.example.com", "io-cache.target.cluster.path-style-access": "true", "io-cache.policy": "meta,read", "io-cache.whitelist": "*", "io-cache.routes": "meta=accel;data,bucket-index,global-index,file-index=cluster"}, "op": "write", "path": "oss://bkt/db1.db/t1/manifest/manifest-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0", "expect": "https://oss-cn-hangzhou-internal.aliyuncs.com"}, - {"name": "single: list stays at origin", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": "http://cache.example.com", "io-cache.policy": "meta,read", "io-cache.whitelist": "meta,data"}, "op": "list", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "expect": "https://oss-cn-hangzhou-internal.aliyuncs.com"}, - {"name": "multi: list stays at origin", "options": {"fs.oss.endpoint": "https://accelerator.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.targets": "accel,cluster", "io-cache.target.accel.endpoint": "https://accelerator.example.com", "io-cache.target.accel.region": "cn-hangzhou", "io-cache.target.cluster.endpoint": "http://cluster.example.com", "io-cache.target.cluster.path-style-access": "true", "io-cache.policy": "meta,read", "io-cache.whitelist": "*", "io-cache.routes": "meta=accel;data,bucket-index,global-index,file-index=cluster"}, "op": "list", "path": "oss://bkt/db1.db/t1/manifest/manifest-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0", "expect": "https://oss-cn-hangzhou-internal.aliyuncs.com"}, - {"name": "single: delete stays at origin", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": "http://cache.example.com", "io-cache.policy": "meta,read", "io-cache.whitelist": "meta,data"}, "op": "delete", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "expect": "https://oss-cn-hangzhou-internal.aliyuncs.com"}, - {"name": "multi: delete stays at origin", "options": {"fs.oss.endpoint": "https://accelerator.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.targets": "accel,cluster", "io-cache.target.accel.endpoint": "https://accelerator.example.com", "io-cache.target.accel.region": "cn-hangzhou", "io-cache.target.cluster.endpoint": "http://cluster.example.com", "io-cache.target.cluster.path-style-access": "true", "io-cache.policy": "meta,read", "io-cache.whitelist": "*", "io-cache.routes": "meta=accel;data,bucket-index,global-index,file-index=cluster"}, "op": "delete", "path": "oss://bkt/db1.db/t1/manifest/manifest-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0", "expect": "https://oss-cn-hangzhou-internal.aliyuncs.com"}, - {"name": "single: rename stays at origin", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": "http://cache.example.com", "io-cache.policy": "meta,read", "io-cache.whitelist": "meta,data"}, "op": "rename", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "expect": "https://oss-cn-hangzhou-internal.aliyuncs.com"}, - {"name": "multi: rename stays at origin", "options": {"fs.oss.endpoint": "https://accelerator.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.targets": "accel,cluster", "io-cache.target.accel.endpoint": "https://accelerator.example.com", "io-cache.target.accel.region": "cn-hangzhou", "io-cache.target.cluster.endpoint": "http://cluster.example.com", "io-cache.target.cluster.path-style-access": "true", "io-cache.policy": "meta,read", "io-cache.whitelist": "*", "io-cache.routes": "meta=accel;data,bucket-index,global-index,file-index=cluster"}, "op": "rename", "path": "oss://bkt/db1.db/t1/manifest/manifest-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0", "expect": "https://oss-cn-hangzhou-internal.aliyuncs.com"}, - {"name": "single: mkdirs stays at origin", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": "http://cache.example.com", "io-cache.policy": "meta,read", "io-cache.whitelist": "meta,data"}, "op": "mkdirs", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "expect": "https://oss-cn-hangzhou-internal.aliyuncs.com"}, - {"name": "multi: mkdirs stays at origin", "options": {"fs.oss.endpoint": "https://accelerator.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.targets": "accel,cluster", "io-cache.target.accel.endpoint": "https://accelerator.example.com", "io-cache.target.accel.region": "cn-hangzhou", "io-cache.target.cluster.endpoint": "http://cluster.example.com", "io-cache.target.cluster.path-style-access": "true", "io-cache.policy": "meta,read", "io-cache.whitelist": "*", "io-cache.routes": "meta=accel;data,bucket-index,global-index,file-index=cluster"}, "op": "mkdirs", "path": "oss://bkt/db1.db/t1/manifest/manifest-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0", "expect": "https://oss-cn-hangzhou-internal.aliyuncs.com"}, - {"name": "single: copy stays at origin", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": "http://cache.example.com", "io-cache.policy": "meta,read", "io-cache.whitelist": "meta,data"}, "op": "copy", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "expect": "https://oss-cn-hangzhou-internal.aliyuncs.com"}, - {"name": "multi: copy stays at origin", "options": {"fs.oss.endpoint": "https://accelerator.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.targets": "accel,cluster", "io-cache.target.accel.endpoint": "https://accelerator.example.com", "io-cache.target.accel.region": "cn-hangzhou", "io-cache.target.cluster.endpoint": "http://cluster.example.com", "io-cache.target.cluster.path-style-access": "true", "io-cache.policy": "meta,read", "io-cache.whitelist": "*", "io-cache.routes": "meta=accel;data,bucket-index,global-index,file-index=cluster"}, "op": "copy", "path": "oss://bkt/db1.db/t1/manifest/manifest-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0", "expect": "https://oss-cn-hangzhou-internal.aliyuncs.com"}, - {"name": "single: atomic-write stays at origin", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": "http://cache.example.com", "io-cache.policy": "meta,read", "io-cache.whitelist": "meta,data"}, "op": "atomic-write", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "expect": "https://oss-cn-hangzhou-internal.aliyuncs.com"}, - {"name": "multi: atomic-write stays at origin", "options": {"fs.oss.endpoint": "https://accelerator.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.targets": "accel,cluster", "io-cache.target.accel.endpoint": "https://accelerator.example.com", "io-cache.target.accel.region": "cn-hangzhou", "io-cache.target.cluster.endpoint": "http://cluster.example.com", "io-cache.target.cluster.path-style-access": "true", "io-cache.policy": "meta,read", "io-cache.whitelist": "*", "io-cache.routes": "meta=accel;data,bucket-index,global-index,file-index=cluster"}, "op": "atomic-write", "path": "oss://bkt/db1.db/t1/manifest/manifest-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0", "expect": "https://oss-cn-hangzhou-internal.aliyuncs.com"}, - {"name": "single: two-phase-write stays at origin", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": "http://cache.example.com", "io-cache.policy": "meta,read", "io-cache.whitelist": "meta,data"}, "op": "two-phase-write", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "expect": "https://oss-cn-hangzhou-internal.aliyuncs.com"}, - {"name": "multi: two-phase-write stays at origin", "options": {"fs.oss.endpoint": "https://accelerator.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.targets": "accel,cluster", "io-cache.target.accel.endpoint": "https://accelerator.example.com", "io-cache.target.accel.region": "cn-hangzhou", "io-cache.target.cluster.endpoint": "http://cluster.example.com", "io-cache.target.cluster.path-style-access": "true", "io-cache.policy": "meta,read", "io-cache.whitelist": "*", "io-cache.routes": "meta=accel;data,bucket-index,global-index,file-index=cluster"}, "op": "two-phase-write", "path": "oss://bkt/db1.db/t1/manifest/manifest-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0", "expect": "https://oss-cn-hangzhou-internal.aliyuncs.com"}, - {"name": "single: presign stays at origin", "options": {"fs.oss.endpoint": "http://cache.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.endpoint": "http://cache.example.com", "io-cache.policy": "meta,read", "io-cache.whitelist": "meta,data"}, "op": "presign", "path": "oss://bkt/db1.db/t1/dt=2026-09-30/bucket-0/data-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0.parquet", "expect": "https://oss-cn-hangzhou-internal.aliyuncs.com"}, - {"name": "multi: presign stays at origin", "options": {"fs.oss.endpoint": "https://accelerator.example.com", "io-cache.enabled": "true", "io-cache.origin.endpoint": "https://oss-cn-hangzhou-internal.aliyuncs.com", "io-cache.targets": "accel,cluster", "io-cache.target.accel.endpoint": "https://accelerator.example.com", "io-cache.target.accel.region": "cn-hangzhou", "io-cache.target.cluster.endpoint": "http://cluster.example.com", "io-cache.target.cluster.path-style-access": "true", "io-cache.policy": "meta,read", "io-cache.whitelist": "*", "io-cache.routes": "meta=accel;data,bucket-index,global-index,file-index=cluster"}, "op": "presign", "path": "oss://bkt/db1.db/t1/manifest/manifest-8b1f7c2e-3a4d-4e5f-9a0b-1c2d3e4f5a6b-0", "expect": "https://oss-cn-hangzhou-internal.aliyuncs.com"} - ] -} From ed39f90fb895b02d2d12a3c607d613c51d4fa868 Mon Sep 17 00:00:00 2001 From: Sun Dapeng Date: Wed, 7 Oct 2026 15:17:26 +0800 Subject: [PATCH 04/21] [core] Route exists to a target only with exists in io-cache.policy exists answers whether a file is still there: commit recovery checks the files it replays. A cache that keeps the metadata of deleted objects answers yes after the file was deleted on origin, so a replay could publish a snapshot that references a deleted file. meta now covers file status only; FileIO::exists and InputFile::exists use a target only when io-cache.policy lists exists, as in the Java and Python clients. --- .../src/catalog/rest/rest_token_file_io.rs | 11 +-- crates/paimon/src/io/cache_routing.rs | 24 ++++++- crates/paimon/src/io/file_io.rs | 4 +- .../src/io/file_io/cache_routing_tests.rs | 72 ++++++++++++++++--- 4 files changed, 92 insertions(+), 19 deletions(-) diff --git a/crates/paimon/src/catalog/rest/rest_token_file_io.rs b/crates/paimon/src/catalog/rest/rest_token_file_io.rs index 869f4d0fb..43952f6b1 100644 --- a/crates/paimon/src/catalog/rest/rest_token_file_io.rs +++ b/crates/paimon/src/catalog/rest/rest_token_file_io.rs @@ -476,14 +476,15 @@ mod tests { ); assert_eq!( cache.take_requests(), - [ - "HEAD bkt/db.db/t/bucket-0/data-123e4567-e89b-12d3-a456-426614174000-1.parquet", - "GET bkt/db.db/t/bucket-0/data-123e4567-e89b-12d3-a456-426614174000-1.parquet" - ] + ["GET bkt/db.db/t/bucket-0/data-123e4567-e89b-12d3-a456-426614174000-1.parquet"] ); + // The token's policy has no exists, so the existence check stays on origin. assert_eq!( origin.take_requests(), - ["PUT bkt/db.db/t/bucket-0/data-123e4567-e89b-12d3-a456-426614174000-1.parquet"] + [ + "PUT bkt/db.db/t/bucket-0/data-123e4567-e89b-12d3-a456-426614174000-1.parquet", + "HEAD bkt/db.db/t/bucket-0/data-123e4567-e89b-12d3-a456-426614174000-1.parquet" + ] ); server.abort(); diff --git a/crates/paimon/src/io/cache_routing.rs b/crates/paimon/src/io/cache_routing.rs index 848d2135f..47cac87c1 100644 --- a/crates/paimon/src/io/cache_routing.rs +++ b/crates/paimon/src/io/cache_routing.rs @@ -57,8 +57,10 @@ static DATA_FILE_SUFFIX: LazyLock = LazyLock::new(|| { /// Operation class of a FileIO request; only `Meta`, `Read` and `Write` can leave origin. #[derive(Clone, Copy, Debug, Eq, Hash, PartialEq)] pub(crate) enum OpClass { - /// File status and existence checks, including the size lookup before a cached read. + /// File status lookups, including the size lookup before a cached read. Meta, + /// Existence checks, which ask whether a file is still there. + Exists, Read, /// Writes of new files, including streaming and multipart writes. Write, @@ -72,6 +74,7 @@ impl OpClass { Self::Meta => 1, Self::Read => 2, Self::Write => 4, + Self::Exists => 8, Self::Origin => 0, } } @@ -201,7 +204,12 @@ impl IoCacheRouting { pub(crate) fn cache_target(&self, path: &str) -> Option<(usize, OpClasses)> { let mut found = None; let mut classes = OpClasses::default(); - for op in [OpClass::Meta, OpClass::Read, OpClass::Write] { + for op in [ + OpClass::Meta, + OpClass::Exists, + OpClass::Read, + OpClass::Write, + ] { if let Target::Cache(index) = self.route(op, path) { found = Some(index); classes.insert(op); @@ -281,6 +289,7 @@ fn parse_policy(value: Option<&str>) -> OpClasses { match token.trim().to_ascii_lowercase().as_str() { "none" => return OpClasses::default(), "meta" => policy.insert(OpClass::Meta), + "exists" => policy.insert(OpClass::Exists), "read" => policy.insert(OpClass::Read), "write" => policy.insert(OpClass::Write), _ => {} @@ -438,6 +447,7 @@ mod tests { const ACCEL: &str = "https://accelerator.example.com"; const CLUSTER: &str = "http://cluster.example.com"; const WRITE: &str = "io-cache.policy=meta,read,write"; + const EXISTS: &str = "io-cache.policy=meta,read,exists"; fn routing(options: &[(&str, &str)]) -> IoCacheRouting { IoCacheRouting::from_props( @@ -457,6 +467,10 @@ mod tests { let options = single(&[]); assert_endpoint(&options, OpClass::Read, DATA_PATH, CACHE); assert_endpoint(&options, OpClass::Meta, DATA_PATH, CACHE); + // exists asks whether a file is still there, so it needs its own policy token + assert_endpoint(&options, OpClass::Exists, DATA_PATH, OSS); + assert_endpoint(&single(&[EXISTS]), OpClass::Exists, DATA_PATH, CACHE); + assert_endpoint(&single(&[EXISTS]), OpClass::Exists, SNAPSHOT_PATH, OSS); assert_endpoint(&options, OpClass::Read, MANIFEST_PATH, CACHE); assert_endpoint( &options, @@ -526,6 +540,9 @@ mod tests { let write_only = single(&["io-cache.policy=write"]); assert_endpoint(&write_only, OpClass::Write, DATA_PATH, CACHE); assert_endpoint(&write_only, OpClass::Meta, DATA_PATH, OSS); + let exists_only = single(&["io-cache.policy=exists"]); + assert_endpoint(&exists_only, OpClass::Exists, DATA_PATH, CACHE); + assert_endpoint(&exists_only, OpClass::Meta, DATA_PATH, OSS); let any_type = single(&["-io-cache.whitelist"]); assert_endpoint(&any_type, OpClass::Read, INDEX_PATH, CACHE); @@ -575,6 +592,9 @@ mod tests { let options = multi(&[]); assert_endpoint(&options, OpClass::Read, MANIFEST_PATH, ACCEL); assert_endpoint(&options, OpClass::Meta, MANIFEST_PATH, ACCEL); + assert_endpoint(&options, OpClass::Exists, MANIFEST_PATH, OSS); + assert_endpoint(&multi(&[EXISTS]), OpClass::Exists, MANIFEST_PATH, ACCEL); + assert_endpoint(&multi(&[EXISTS]), OpClass::Exists, DATA_PATH, CLUSTER); assert_endpoint(&options, OpClass::Read, DATA_PATH, CLUSTER); assert_endpoint(&options, OpClass::Meta, DATA_PATH, CLUSTER); assert_endpoint(&options, OpClass::Read, INDEX_PATH, CLUSTER); diff --git a/crates/paimon/src/io/file_io.rs b/crates/paimon/src/io/file_io.rs index a20996bac..c98fa38c0 100644 --- a/crates/paimon/src/io/file_io.rs +++ b/crates/paimon/src/io/file_io.rs @@ -660,7 +660,7 @@ impl FileIO { pub async fn exists(&self, path: &str) -> Result { let (op, relative_path) = self.create_routed(path).await?; - op.operator(OpClass::Meta) + op.operator(OpClass::Exists) .exists(relative_path.as_ref()) .await .map_err(|error| { @@ -1314,7 +1314,7 @@ impl InputFile { pub async fn exists(&self) -> crate::Result { let (op, relative_path, _) = self.source.resolve(&self.path).await?; - Ok(op.operator(OpClass::Meta).exists(&relative_path).await?) + Ok(op.operator(OpClass::Exists).exists(&relative_path).await?) } pub async fn metadata(&self) -> crate::Result { diff --git a/crates/paimon/src/io/file_io/cache_routing_tests.rs b/crates/paimon/src/io/file_io/cache_routing_tests.rs index b99ac85d4..70f0bc492 100644 --- a/crates/paimon/src/io/file_io/cache_routing_tests.rs +++ b/crates/paimon/src/io/file_io/cache_routing_tests.rs @@ -101,19 +101,21 @@ async fn test_only_reads_and_status_use_the_target() { let (origin, cache, file_io) = routed("meta,read").await; write(&file_io, DATA, b"0123456789").await; - // A writer checks existence on origin; readers check it like file status. + // A writer checks existence on origin, and so does exists without the exists token. assert!(file_io.new_output(DATA).unwrap().exists().await.unwrap()); - assert_eq!( - origin.take_requests(), - [format!("PUT {DATA_KEY}"), format!("HEAD {DATA_KEY}")] - ); assert!(file_io.exists(DATA).await.unwrap()); let input = file_io.new_input(DATA).unwrap(); assert!(input.exists().await.unwrap()); assert_eq!( - cache.take_requests(), - [format!("HEAD {DATA_KEY}"), format!("HEAD {DATA_KEY}")] + origin.take_requests(), + [ + format!("PUT {DATA_KEY}"), + format!("HEAD {DATA_KEY}"), + format!("HEAD {DATA_KEY}"), + format!("HEAD {DATA_KEY}") + ] ); + assert!(cache.take_requests().is_empty()); assert_eq!(file_io.get_status(DATA).await.unwrap().size, 10); assert_eq!(input.metadata().await.unwrap().size, 10); @@ -287,14 +289,13 @@ async fn test_targets_serve_the_file_types_of_their_routes() { assert_eq!(file_io.get_status(MANIFEST).await.unwrap().size, 8); assert_eq!(read(&file_io, DATA).await, "data"); assert_eq!(file_io.get_status(DATA).await.unwrap().size, 4); - // No route names bucket indexes; existence checks follow the routes like file status. + // No route names bucket indexes; without the exists token existence checks use origin. assert_eq!(read(&file_io, INDEX).await, "index"); assert!(file_io.exists(MANIFEST).await.unwrap()); assert_eq!( accel.take_requests(), [ format!("GET {MANIFEST_KEY}"), - format!("HEAD {MANIFEST_KEY}"), format!("HEAD {MANIFEST_KEY}") ] ); @@ -304,10 +305,61 @@ async fn test_targets_serve_the_file_types_of_their_routes() { ); assert_eq!( origin.take_requests(), - ["GET bkt/db.db/t/index/index-1".to_string()] + [ + "GET bkt/db.db/t/index/index-1".to_string(), + format!("HEAD {MANIFEST_KEY}") + ] ); } +#[tokio::test] +async fn test_exists_ignores_a_cache_that_kept_a_deleted_file() { + let origin = TestOss::start().await; + // Its own storage stands in for a cache that kept a file after origin deleted it. + let cache = TestOss::start().await; + write( + &oss_file_io(&[("fs.oss.endpoint", cache.endpoint())]), + DATA, + b"stale", + ) + .await; + let routed = |policy| { + oss_file_io(&[ + ("fs.oss.endpoint", cache.endpoint()), + ("io-cache.enabled", "true"), + ("io-cache.endpoint", cache.endpoint()), + ("io-cache.target.default.path-style-access", "true"), + ("io-cache.origin.endpoint", origin.endpoint()), + ("io-cache.policy", policy), + ]) + }; + + let file_io = routed("meta,read"); + assert!(!file_io.exists(DATA).await.unwrap()); + assert!(!file_io.new_input(DATA).unwrap().exists().await.unwrap()); + // With exists in the policy the target answers, so it is vended only for consistent targets. + assert!(routed("meta,read,exists").exists(DATA).await.unwrap()); +} + +#[tokio::test] +async fn test_exists_uses_the_target_with_the_exists_token() { + let (origin, cache, file_io) = routed("meta,read,exists").await; + + write(&file_io, DATA, b"data").await; + assert!(file_io.new_output(DATA).unwrap().exists().await.unwrap()); + assert_eq!( + origin.take_requests(), + [format!("PUT {DATA_KEY}"), format!("HEAD {DATA_KEY}")] + ); + assert!(file_io.exists(DATA).await.unwrap()); + assert!(file_io.new_input(DATA).unwrap().exists().await.unwrap()); + assert_eq!( + cache.take_requests(), + [format!("HEAD {DATA_KEY}"), format!("HEAD {DATA_KEY}")] + ); + assert!(origin.take_requests().is_empty()); +} + #[tokio::test] async fn test_target_without_endpoint_is_skipped() { let origin = TestOss::start().await; From 33f4fb95fb42272cfe3ce02b645e0eb33a3b7dc7 Mon Sep 17 00:00:00 2001 From: Sun Dapeng Date: Wed, 7 Oct 2026 19:45:52 +0800 Subject: [PATCH 05/21] [core] Route metadata and index files only when named as Paimon writes them The broad FileType classifier also matched names such as manifest.parquet, stat-2024.parquet or foo.index, which a Format Table may hold as ordinary files and replace in place, so a routed scan could return the old file from a cache. A metadata or index file now uses a target only under the exact name Paimon writes: manifest-, manifest-list-, index-manifest- and stat-{uuid}-{n} (a manifest also with .avro.sidecar), index-{uuid}-{n}, and {kind}-global-index-{uuid}.index. Data files and their file indexes keep the existing name rule; every other name goes to the origin. test_format_table_file_named_like_a_manifest_uses_origin replaces such a file on the origin while a target still holds the old one. --- crates/paimon/src/io/cache_routing.rs | 58 ++++++++++++++++--- .../src/io/file_io/cache_routing_tests.rs | 47 +++++++++++++-- 2 files changed, 93 insertions(+), 12 deletions(-) diff --git a/crates/paimon/src/io/cache_routing.rs b/crates/paimon/src/io/cache_routing.rs index 47cac87c1..75355b3b9 100644 --- a/crates/paimon/src/io/cache_routing.rs +++ b/crates/paimon/src/io/cache_routing.rs @@ -49,12 +49,27 @@ const DEFAULT_DATA_PREFIXES: [&str; 2] = ["data-", "changelog-"]; const MANIFEST_SIDECAR_SUFFIX: &str = ".avro.sidecar"; // Data files are named {prefix}{uuid}-{count}.{extension}; this matches what follows the prefix. +const UUID: &str = "[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}"; static DATA_FILE_SUFFIX: LazyLock = LazyLock::new(|| { - Regex::new(r"^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}-[0-9]+\..+$") - .expect("valid data file pattern") + Regex::new(&format!(r"^{UUID}-[0-9]+\..+$")).expect("valid data file pattern") +}); +// Other types are routed only under the names Paimon writes; Format Table files may be replaced +// in place. +static META_FILE: LazyLock = LazyLock::new(|| { + Regex::new(&format!( + r"^(?:(?:manifest|manifest-list|index-manifest|stat)-{UUID}-[0-9]+|manifest-{UUID}-[0-9]+\.avro\.sidecar)$" + )) + .expect("valid metadata file pattern") +}); +static BUCKET_INDEX_FILE: LazyLock = LazyLock::new(|| { + Regex::new(&format!(r"^index-{UUID}-[0-9]+$")).expect("valid bucket index file pattern") +}); +static GLOBAL_INDEX_FILE: LazyLock = LazyLock::new(|| { + Regex::new(&format!(r"^[a-z0-9_-]+-global-index-{UUID}\.index$")) + .expect("valid global index file pattern") }); -/// Operation class of a FileIO request; only `Meta`, `Read` and `Write` can leave origin. +/// Operation class of a FileIO request; only `Meta`, `Exists`, `Read` and `Write` can leave origin. #[derive(Clone, Copy, Debug, Eq, Hash, PartialEq)] pub(crate) enum OpClass { /// File status lookups, including the size lookup before a cached read. @@ -252,10 +267,15 @@ pub(crate) fn routable_type(path: &str, data_prefixes: &[String]) -> Option None, - file_type => Some(file_type), - } + let file_type = FileType::classify(path); + let paimon_name = match file_type { + FileType::Meta => META_FILE.is_match(name), + FileType::BucketIndex => BUCKET_INDEX_FILE.is_match(name), + FileType::GlobalIndex => GLOBAL_INDEX_FILE.is_match(name), + // Data files and their file indexes are recognized by `is_data_file_name`. + _ => false, + }; + paimon_name.then_some(file_type) } // Manifests, indexes and statistics share the uuid-count shape but have no extension. @@ -490,6 +510,10 @@ mod tests { assert_endpoint(&options, OpClass::Read, &dls, OSS); // the whitelist has no index assert_endpoint(&options, OpClass::Read, INDEX_PATH, OSS); + // a Format Table file named like a manifest may be replaced in place + let external = "review_external/manifest.parquet"; + assert_endpoint(&options, OpClass::Read, external, OSS); + assert_endpoint(&options, OpClass::Meta, external, OSS); assert_endpoint(&options, OpClass::Write, DATA_PATH, OSS); assert_endpoint(&options, OpClass::Origin, DATA_PATH, OSS); } @@ -720,6 +744,23 @@ mod tests { ] { assert_type(path, None, &[]); } + // Format Table files named like metadata or indexes, which may be replaced in place + for path in [ + "dt=1/manifest.parquet", + "dt=1/stat-2024.parquet", + "dt=1/index-a.csv", + "dt=1/foo.index", + "review_external/manifest.parquet", + "manifest/manifest-old", + "manifest/manifest-old.avro.sidecar", + "manifest/manifest-list-{uuid}-1.avro.sidecar", + "statistics/stat-old", + "index/index-old", + "index/my-global-index.index", + "index/btree-global-index-{uuid}-0.index", + ] { + assert_type(path, None, &[]); + } assert_type("manifest/manifest-{uuid}-0", Some(FileType::Meta), &[]); assert_type("manifest/manifest-list-{uuid}-1", Some(FileType::Meta), &[]); @@ -884,7 +925,8 @@ mod tests { ]); let both = OpClasses(OpClass::Meta.bit() | OpClass::Read.bit()); assert_eq!( - routing.cache_target("oss://b/t/manifest/manifest-1"), + routing + .cache_target("oss://b/t/manifest/manifest-123e4567-e89b-12d3-a456-426614174000-1"), Some((0, both)) ); assert_eq!( diff --git a/crates/paimon/src/io/file_io/cache_routing_tests.rs b/crates/paimon/src/io/file_io/cache_routing_tests.rs index 70f0bc492..4c7534d09 100644 --- a/crates/paimon/src/io/file_io/cache_routing_tests.rs +++ b/crates/paimon/src/io/file_io/cache_routing_tests.rs @@ -20,9 +20,9 @@ use crate::io::oss_test_server::TestOss; const DATA: &str = "oss://bkt/db.db/t/bucket-0/data-123e4567-e89b-12d3-a456-426614174000-1.parquet"; const DATA_KEY: &str = "bkt/db.db/t/bucket-0/data-123e4567-e89b-12d3-a456-426614174000-1.parquet"; -const MANIFEST: &str = "oss://bkt/db.db/t/manifest/manifest-1"; -const MANIFEST_KEY: &str = "bkt/db.db/t/manifest/manifest-1"; -const INDEX: &str = "oss://bkt/db.db/t/index/index-1"; +const MANIFEST: &str = "oss://bkt/db.db/t/manifest/manifest-123e4567-e89b-12d3-a456-426614174000-1"; +const MANIFEST_KEY: &str = "bkt/db.db/t/manifest/manifest-123e4567-e89b-12d3-a456-426614174000-1"; +const INDEX: &str = "oss://bkt/db.db/t/index/index-123e4567-e89b-12d3-a456-426614174000-1"; const LATEST: &str = "oss://bkt/db.db/t/snapshot/LATEST"; const SNAPSHOT: &str = "oss://bkt/db.db/t/snapshot/snapshot-1"; const UNREACHABLE: &str = "http://127.0.0.1:1"; @@ -306,12 +306,51 @@ async fn test_targets_serve_the_file_types_of_their_routes() { assert_eq!( origin.take_requests(), [ - "GET bkt/db.db/t/index/index-1".to_string(), + "GET bkt/db.db/t/index/index-123e4567-e89b-12d3-a456-426614174000-1".to_string(), format!("HEAD {MANIFEST_KEY}") ] ); } +#[tokio::test] +async fn test_format_table_file_named_like_a_manifest_uses_origin() { + let origin = TestOss::start().await; + // Its own storage stands in for a cache that kept the file before it was replaced. + let cache = TestOss::start().await; + let path = "oss://bkt/db.db/review_external/manifest.parquet"; + let key = "bkt/db.db/review_external/manifest.parquet"; + write( + &oss_file_io(&[("fs.oss.endpoint", cache.endpoint())]), + path, + b"id=1", + ) + .await; + write( + &oss_file_io(&[("fs.oss.endpoint", origin.endpoint())]), + path, + b"id=2", + ) + .await; + cache.take_requests(); + origin.take_requests(); + let file_io = oss_file_io(&[ + ("fs.oss.endpoint", cache.endpoint()), + ("io-cache.enabled", "true"), + ("io-cache.endpoint", cache.endpoint()), + ("io-cache.target.default.path-style-access", "true"), + ("io-cache.origin.endpoint", origin.endpoint()), + ("io-cache.policy", "meta,read"), + ]); + + assert_eq!(read(&file_io, path).await, "id=2"); + assert_eq!(file_io.get_status(path).await.unwrap().size, 4); + assert!(cache.take_requests().is_empty()); + assert_eq!( + origin.take_requests(), + [format!("GET {key}"), format!("HEAD {key}")] + ); +} + #[tokio::test] async fn test_exists_ignores_a_cache_that_kept_a_deleted_file() { let origin = TestOss::start().await; From 129fad90f32aa48fb9c511c199bab31afe44a992 Mon Sep 17 00:00:00 2001 From: Sun Dapeng Date: Thu, 8 Oct 2026 16:19:23 +0800 Subject: [PATCH 06/21] [core] Keep Format Table requests on the origin with io-cache routing A Format Table may hold files that other writers copy in or replace under the same name, so even a Paimon-shaped name such as data-{uuid}-1.parquet does not make its content immutable, and a routed scan could return the old copy a cache still holds. Table::new and Table::from_resolved_schema now give a Format Table the origin-only view of its FileIO, so its reads, file status lookups and writes all use the origin whatever io-cache.policy says; other tables keep the per-request routing. format_table_reads_replaced_files_from_origin_with_io_cache reproduces the review probe: the target keeps id=1, the origin holds the replacement id=2, the scan returns [2], an append stays on the origin, and the target sees no request. --- crates/paimon/src/io/file_io.rs | 2 +- .../src/table/format_table_write_tests.rs | 55 +++++++++++++++++++ crates/paimon/src/table/mod.rs | 11 ++++ docs/src/getting-started.md | 5 +- 4 files changed, 70 insertions(+), 3 deletions(-) diff --git a/crates/paimon/src/io/file_io.rs b/crates/paimon/src/io/file_io.rs index 0f1702f23..9871cea6a 100644 --- a/crates/paimon/src/io/file_io.rs +++ b/crates/paimon/src/io/file_io.rs @@ -317,7 +317,7 @@ impl FileIO { } } - /// A view of this FileIO that sends every request to origin, as copies require. + /// A view of this FileIO that sends every request to origin, for copies and Format Tables. pub(crate) fn origin_only(&self) -> Self { let mut file_io = self.clone(); file_io.origin_only = true; diff --git a/crates/paimon/src/table/format_table_write_tests.rs b/crates/paimon/src/table/format_table_write_tests.rs index 94baa8e88..bb788e589 100644 --- a/crates/paimon/src/table/format_table_write_tests.rs +++ b/crates/paimon/src/table/format_table_write_tests.rs @@ -1840,3 +1840,58 @@ async fn avro_format_table_append_is_readable_through_native_scan() { assert_eq!(visible_files(&table, "dt=a").await.len(), 1); assert_eq!(visible_files(&table, "dt=b").await.len(), 1); } + +#[cfg(feature = "storage-oss")] +#[tokio::test] +async fn format_table_reads_replaced_files_from_origin_with_io_cache() { + use crate::io::oss_test_server::TestOss; + + let oss_io = |props: &[(&str, &str)]| { + FileIOBuilder::new("oss") + .with_props([ + ("fs.oss.accessKeyId", "ak"), + ("fs.oss.accessKeySecret", "sk"), + ("fs.oss.path-style-access", "true"), + ]) + .with_props(props.iter().copied()) + .build() + .unwrap() + }; + let parquet = |ids: &[i32]| { + let batch = unpartitioned_batch(ids); + let mut bytes = Vec::new(); + let mut writer = + parquet::arrow::ArrowWriter::try_new(&mut bytes, batch.schema(), None).unwrap(); + writer.write(&batch).unwrap(); + writer.close().unwrap(); + Bytes::from(bytes) + }; + let origin = TestOss::start().await; + // Its own storage stands in for a cache that kept the file before origin replaced it. + let cache = TestOss::start().await; + let location = "oss://bkt/db.db/review_external"; + let file = format!("{location}/data-123e4567-e89b-12d3-a456-426614174000-1.parquet"); + for (server, ids) in [(&cache, [1]), (&origin, [2])] { + let io = oss_io(&[("fs.oss.endpoint", server.endpoint())]); + io.new_output(&file) + .unwrap() + .write(parquet(&ids)) + .await + .unwrap(); + } + cache.take_requests(); + + let io = oss_io(&[ + ("fs.oss.endpoint", cache.endpoint()), + ("io-cache.enabled", "true"), + ("io-cache.endpoint", cache.endpoint()), + ("io-cache.target.default.path-style-access", "true"), + ("io-cache.origin.endpoint", origin.endpoint()), + ("io-cache.policy", "meta,exists,read,write"), + ]); + let table = table(io, location, false, &[]); + assert_eq!(ids(&table).await, [2]); + append(&table, &unpartitioned_batch(&[3])).await; + assert_eq!(ids(&table).await, [2, 3]); + assert!(cache.take_requests().is_empty()); +} diff --git a/crates/paimon/src/table/mod.rs b/crates/paimon/src/table/mod.rs index ed7e6a063..c706f14a3 100644 --- a/crates/paimon/src/table/mod.rs +++ b/crates/paimon/src/table/mod.rs @@ -264,6 +264,15 @@ pub struct Table { applied_dynamic_option_keys: HashSet, } +// Format Table files may be replaced in place under the same name, so they never use io-cache. +fn table_file_io(file_io: FileIO, schema: &TableSchema) -> FileIO { + if CoreOptions::new(schema.options()).is_format_table() { + file_io.origin_only() + } else { + file_io + } +} + impl Table { /// Create a new table. pub fn new( @@ -273,6 +282,7 @@ impl Table { schema: TableSchema, rest_env: Option, ) -> Self { + let file_io = table_file_io(file_io, &schema); let schema_manager = SchemaManager::new(file_io.clone(), location.clone()); let branch = DEFAULT_MAIN_BRANCH.to_string(); Self { @@ -310,6 +320,7 @@ impl Table { identifier.validate()?; validate_branch_name(&branch)?; schema.validate_resolved_structure()?; + let file_io = table_file_io(file_io, &schema); let schema_manager = SchemaManager::new(file_io.clone(), location.clone()); let schema_manager = if branch == DEFAULT_MAIN_BRANCH { schema_manager diff --git a/docs/src/getting-started.md b/docs/src/getting-started.md index 0a9049575..73a878f5c 100644 --- a/docs/src/getting-started.md +++ b/docs/src/getting-started.md @@ -427,8 +427,9 @@ rewritten in place (such as `LATEST`, tags and temporary files), sequentially numbered metadata (`snapshot-N`, `schema-N`, `changelog/changelog-N`), unknown file names. A target is used like any other endpoint: requests to it use the same retry settings, and its errors are -returned rather than retried on origin. Setting `dlf.oss-endpoint` on the -client turns routing off. +returned rather than retried on origin. Format Tables never use a target, +because their files may be replaced under the same name. Setting +`dlf.oss-endpoint` on the client turns routing off. ### Manage Databases From b63dae4ece632a931148343abded7cbb07ebf50a Mon Sep 17 00:00:00 2001 From: Sun Dapeng Date: Thu, 8 Oct 2026 16:19:23 +0800 Subject: [PATCH 07/21] [core] Route the changelog manifests TableCommit writes TableCommit names input-changelog manifests manifest-{uuid}-changelog-{count}, optionally with .avro.sidecar, but the metadata name rule only accepted manifest-{uuid}-{count}, so these immutable files always went to the origin. The rule now accepts the -changelog infix for manifests (not for manifest lists). test_input_changelog_manifests_use_the_io_cache_target commits a primary-key table with changelog-producer=input and manifest sidecars under meta,read,write, reads the changelog incrementally, and checks that the changelog manifest and its sidecar are written and read through the target and never reach the origin. --- crates/paimon/src/io/cache_routing.rs | 10 ++- crates/paimon/src/table/table_write.rs | 89 ++++++++++++++++++++++++++ 2 files changed, 97 insertions(+), 2 deletions(-) diff --git a/crates/paimon/src/io/cache_routing.rs b/crates/paimon/src/io/cache_routing.rs index 75355b3b9..2ebf4eee2 100644 --- a/crates/paimon/src/io/cache_routing.rs +++ b/crates/paimon/src/io/cache_routing.rs @@ -54,10 +54,10 @@ static DATA_FILE_SUFFIX: LazyLock = LazyLock::new(|| { Regex::new(&format!(r"^{UUID}-[0-9]+\..+$")).expect("valid data file pattern") }); // Other types are routed only under the names Paimon writes; Format Table files may be replaced -// in place. +// in place. TableCommit names changelog manifests manifest-{uuid}-changelog-{count}. static META_FILE: LazyLock = LazyLock::new(|| { Regex::new(&format!( - r"^(?:(?:manifest|manifest-list|index-manifest|stat)-{UUID}-[0-9]+|manifest-{UUID}-[0-9]+\.avro\.sidecar)$" + r"^(?:(?:manifest-list|index-manifest|stat)-{UUID}-[0-9]+|manifest-{UUID}(?:-changelog)?-[0-9]+(?:\.avro\.sidecar)?)$" )) .expect("valid metadata file pattern") }); @@ -754,6 +754,8 @@ mod tests { "manifest/manifest-old", "manifest/manifest-old.avro.sidecar", "manifest/manifest-list-{uuid}-1.avro.sidecar", + "manifest/manifest-list-{uuid}-changelog-1", + "manifest/manifest-{uuid}-changelog", "statistics/stat-old", "index/index-old", "index/my-global-index.index", @@ -771,6 +773,10 @@ mod tests { ); let sidecar = "manifest/manifest-{uuid}-0.avro.sidecar"; assert_type(sidecar, Some(FileType::Meta), &[]); + let changelog = "manifest/manifest-{uuid}-changelog-0"; + assert_type(changelog, Some(FileType::Meta), &[]); + let changelog_sidecar = "manifest/manifest-{uuid}-changelog-0.avro.sidecar"; + assert_type(changelog_sidecar, Some(FileType::Meta), &[]); assert_type("statistics/stat-{uuid}-0", Some(FileType::Meta), &[]); assert_type( "dt=1/bucket-0/data-{uuid}-0.parquet", diff --git a/crates/paimon/src/table/table_write.rs b/crates/paimon/src/table/table_write.rs index 9d7d2f9e5..d2a8dbd86 100644 --- a/crates/paimon/src/table/table_write.rs +++ b/crates/paimon/src/table/table_write.rs @@ -4151,6 +4151,95 @@ pub(in crate::table) mod tests { assert_eq!(collect_i32(&changelog_batches, 3), vec![10, 20]); } + #[cfg(feature = "storage-oss")] + #[tokio::test] + async fn test_input_changelog_manifests_use_the_io_cache_target() { + use crate::io::oss_test_server::TestOss; + use crate::table::IncrementalScanMode; + use futures::TryStreamExt; + + let origin = TestOss::start().await; + let cache = TestOss::start_sharing(&origin).await; + let file_io = FileIOBuilder::new("oss") + .with_props([ + ("fs.oss.endpoint", cache.endpoint()), + ("fs.oss.accessKeyId", "ak"), + ("fs.oss.accessKeySecret", "sk"), + ("fs.oss.path-style-access", "true"), + ("io-cache.enabled", "true"), + ("io-cache.endpoint", cache.endpoint()), + ("io-cache.target.default.path-style-access", "true"), + ("io-cache.origin.endpoint", origin.endpoint()), + ("io-cache.policy", "meta,read,write"), + ]) + .build() + .unwrap(); + let table = Table::new( + file_io, + Identifier::new("db", "input_changelog"), + "oss://bkt/db.db/input_changelog".to_string(), + pk_changelog_schema(&[ + ("changelog-producer", "input"), + ("manifest.sidecar.enabled", "true"), + ]), + None, + ); + let mut table_write = TableWrite::new(&table, "test-user".to_string()).unwrap(); + table_write + .write_arrow_batch(&make_batch(vec![1, 1], vec![10, 20])) + .await + .unwrap(); + let messages = table_write.prepare_commit().await.unwrap(); + TableCommit::new(table.clone(), "test-user".to_string()) + .commit(messages) + .await + .unwrap(); + + let read_builder = table.new_read_builder(); + let plan = read_builder + .new_incremental_scan(IncrementalScanMode::Changelog, 0, 1) + .plan() + .await + .unwrap(); + let batches: Vec = read_builder + .new_read() + .unwrap() + .to_incremental_arrow(&plan) + .unwrap() + .try_collect() + .await + .unwrap(); + assert_eq!(batches.iter().map(RecordBatch::num_rows).sum::(), 2); + + // TableCommit names them manifest-{uuid}-changelog-{count}, plus .avro.sidecar. + let changelog_manifests = |requests: Vec| { + requests + .into_iter() + .filter(|r| r.contains("/manifest/manifest-") && r.contains("-changelog-")) + .map(|r| { + let (method, key) = r.split_once(' ').unwrap(); + let suffix = key.rsplit_once("-changelog-").unwrap().1; + format!("{method} -changelog-{suffix}") + }) + .collect::>() + }; + let cache_requests = changelog_manifests(cache.take_requests()); + for request in [ + "PUT -changelog-0", + "PUT -changelog-0.avro.sidecar", + "GET -changelog-0", + ] { + assert!( + cache_requests.iter().any(|r| r == request), + "{request} not in {cache_requests:?}" + ); + } + assert_eq!( + changelog_manifests(origin.take_requests()), + Vec::::new() + ); + } + #[tokio::test] async fn test_input_changelog_metadata_counts_retract_rows() { let file_io = test_file_io(); From 6ed10b41bddba481f221dfa626c66c49b60bd52d Mon Sep 17 00:00:00 2001 From: Sun Dapeng Date: Thu, 8 Oct 2026 16:19:23 +0800 Subject: [PATCH 08/21] [core] Document exists as its own io-cache.policy value meta now covers file status only and exists is a separate token, which the OSS Cache Routing section still described as one. --- docs/src/getting-started.md | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/docs/src/getting-started.md b/docs/src/getting-started.md index 73a878f5c..b5e6fb2ca 100644 --- a/docs/src/getting-started.md +++ b/docs/src/getting-started.md @@ -405,7 +405,7 @@ apply when `fs.oss.impl=jindo`. |--------|---------|-------------| | `io-cache.enabled` | `false` | Master switch. A value set in the catalog options wins over the token, so clients can turn routing off. | | `io-cache.origin.endpoint` | `fs.oss.endpoint` | OSS endpoint for requests that do not use a target. | -| `io-cache.policy` | none | Operations that may use a target: `read`, `meta` (file status and existence checks), `write`, or `none`. | +| `io-cache.policy` | none | Operations that may use a target: `read`, `meta` (file status), `exists` (existence checks), `write`, or `none`. | | `io-cache.whitelist` | all types | File types that may use a target: `meta`, `global-index`, `bucket-index`, `data`, `file-index`, or `*`. | | `io-cache.targets` | none | Comma-separated target names matching `[a-z][a-z0-9-]*`. | | `io-cache.target..endpoint` | none | Target endpoint, such as `http://host:port`; `https` when the scheme is omitted. | @@ -416,8 +416,9 @@ apply when `fs.oss.impl=jindo`. `io-cache.target..region` is accepted but ignored, because OpenDAL signs OSS requests without a region. -Only reads, file status and existence checks, and with `write` in the policy also -writes, of files that Paimon writes once under a unique name can use a target: +Only reads, file status lookups, existence checks (with `exists` in the policy) +and writes (with `write` in the policy) of files that Paimon writes once under a +unique name can use a target: manifests, statistics, indexes, and data files. A data file is recognized by the name Paimon writers give it: the `data-` or `changelog-` prefix, or `data-file.prefix` or `changelog-file.prefix` when these are in the From b85e26be5207ed7baa2048c1551c27550fc00ca5 Mon Sep 17 00:00:00 2001 From: Sun Dapeng Date: Thu, 8 Oct 2026 19:03:33 +0800 Subject: [PATCH 09/21] [core] Let the catalog decide whether Format Tables use io-cache targets This reverts 129fad9. Format Table files may be replaced in place by scheduled jobs, but whether that can return old content depends on the target: one that forwards every request to OSS, such as an OSS accelerator, never does. The REST catalog that vends the targets knows which kind it vends and already resolves the io-cache options per table, database and catalog, so it decides whether a Format Table uses a target, as it does for exists. FileIO routes Format Table files by name like those of any other table again; the docs say who decides. --- crates/paimon/src/io/file_io.rs | 2 +- .../src/table/format_table_write_tests.rs | 55 ------------------- crates/paimon/src/table/mod.rs | 11 ---- docs/src/getting-started.md | 7 ++- 4 files changed, 5 insertions(+), 70 deletions(-) diff --git a/crates/paimon/src/io/file_io.rs b/crates/paimon/src/io/file_io.rs index 9871cea6a..0f1702f23 100644 --- a/crates/paimon/src/io/file_io.rs +++ b/crates/paimon/src/io/file_io.rs @@ -317,7 +317,7 @@ impl FileIO { } } - /// A view of this FileIO that sends every request to origin, for copies and Format Tables. + /// A view of this FileIO that sends every request to origin, as copies require. pub(crate) fn origin_only(&self) -> Self { let mut file_io = self.clone(); file_io.origin_only = true; diff --git a/crates/paimon/src/table/format_table_write_tests.rs b/crates/paimon/src/table/format_table_write_tests.rs index bb788e589..94baa8e88 100644 --- a/crates/paimon/src/table/format_table_write_tests.rs +++ b/crates/paimon/src/table/format_table_write_tests.rs @@ -1840,58 +1840,3 @@ async fn avro_format_table_append_is_readable_through_native_scan() { assert_eq!(visible_files(&table, "dt=a").await.len(), 1); assert_eq!(visible_files(&table, "dt=b").await.len(), 1); } - -#[cfg(feature = "storage-oss")] -#[tokio::test] -async fn format_table_reads_replaced_files_from_origin_with_io_cache() { - use crate::io::oss_test_server::TestOss; - - let oss_io = |props: &[(&str, &str)]| { - FileIOBuilder::new("oss") - .with_props([ - ("fs.oss.accessKeyId", "ak"), - ("fs.oss.accessKeySecret", "sk"), - ("fs.oss.path-style-access", "true"), - ]) - .with_props(props.iter().copied()) - .build() - .unwrap() - }; - let parquet = |ids: &[i32]| { - let batch = unpartitioned_batch(ids); - let mut bytes = Vec::new(); - let mut writer = - parquet::arrow::ArrowWriter::try_new(&mut bytes, batch.schema(), None).unwrap(); - writer.write(&batch).unwrap(); - writer.close().unwrap(); - Bytes::from(bytes) - }; - let origin = TestOss::start().await; - // Its own storage stands in for a cache that kept the file before origin replaced it. - let cache = TestOss::start().await; - let location = "oss://bkt/db.db/review_external"; - let file = format!("{location}/data-123e4567-e89b-12d3-a456-426614174000-1.parquet"); - for (server, ids) in [(&cache, [1]), (&origin, [2])] { - let io = oss_io(&[("fs.oss.endpoint", server.endpoint())]); - io.new_output(&file) - .unwrap() - .write(parquet(&ids)) - .await - .unwrap(); - } - cache.take_requests(); - - let io = oss_io(&[ - ("fs.oss.endpoint", cache.endpoint()), - ("io-cache.enabled", "true"), - ("io-cache.endpoint", cache.endpoint()), - ("io-cache.target.default.path-style-access", "true"), - ("io-cache.origin.endpoint", origin.endpoint()), - ("io-cache.policy", "meta,exists,read,write"), - ]); - let table = table(io, location, false, &[]); - assert_eq!(ids(&table).await, [2]); - append(&table, &unpartitioned_batch(&[3])).await; - assert_eq!(ids(&table).await, [2, 3]); - assert!(cache.take_requests().is_empty()); -} diff --git a/crates/paimon/src/table/mod.rs b/crates/paimon/src/table/mod.rs index c706f14a3..ed7e6a063 100644 --- a/crates/paimon/src/table/mod.rs +++ b/crates/paimon/src/table/mod.rs @@ -264,15 +264,6 @@ pub struct Table { applied_dynamic_option_keys: HashSet, } -// Format Table files may be replaced in place under the same name, so they never use io-cache. -fn table_file_io(file_io: FileIO, schema: &TableSchema) -> FileIO { - if CoreOptions::new(schema.options()).is_format_table() { - file_io.origin_only() - } else { - file_io - } -} - impl Table { /// Create a new table. pub fn new( @@ -282,7 +273,6 @@ impl Table { schema: TableSchema, rest_env: Option, ) -> Self { - let file_io = table_file_io(file_io, &schema); let schema_manager = SchemaManager::new(file_io.clone(), location.clone()); let branch = DEFAULT_MAIN_BRANCH.to_string(); Self { @@ -320,7 +310,6 @@ impl Table { identifier.validate()?; validate_branch_name(&branch)?; schema.validate_resolved_structure()?; - let file_io = table_file_io(file_io, &schema); let schema_manager = SchemaManager::new(file_io.clone(), location.clone()); let schema_manager = if branch == DEFAULT_MAIN_BRANCH { schema_manager diff --git a/docs/src/getting-started.md b/docs/src/getting-started.md index b5e6fb2ca..b8b31914f 100644 --- a/docs/src/getting-started.md +++ b/docs/src/getting-started.md @@ -428,9 +428,10 @@ rewritten in place (such as `LATEST`, tags and temporary files), sequentially numbered metadata (`snapshot-N`, `schema-N`, `changelog/changelog-N`), unknown file names. A target is used like any other endpoint: requests to it use the same retry settings, and its errors are -returned rather than retried on origin. Format Tables never use a target, -because their files may be replaced under the same name. Setting -`dlf.oss-endpoint` on the client turns routing off. +returned rather than retried on origin. Format Tables follow the same rules; +their files may be replaced in place, so the catalog that vends the token +decides per table whether they use a target. Setting `dlf.oss-endpoint` on the +client turns routing off. ### Manage Databases From 194d85dce68e403e1fbf2ad07b01aeda62b60b3c Mon Sep 17 00:00:00 2001 From: Sun Dapeng Date: Tue, 13 Oct 2026 01:48:09 +0800 Subject: [PATCH 10/21] [core] Keep Format and Object Table requests on the origin with io-cache routing Other jobs may replace the files of a Format Table or an Object Table in place under the same name, so a Paimon-shaped name does not make them immutable. Leaving the choice to the catalog that vends the targets does not hold: FileSystemCatalog shares one FileIO across its tables, and REST external tables build theirs from the catalog options without a table token, so a catalog with a read cache had no way to exempt them. Table::new, Table::from_resolved_schema and copy_with_resolved_schema give a Format Table the origin-only view of its FileIO again, and ObjectTable keeps one for its objects, so their reads, file status lookups and writes use the origin whatever io-cache.policy says. A Format Table that should go through an OSS accelerator gets it by making the accelerator the origin with io-cache.origin.endpoint. format_table_reads_replaced_files_from_origin creates and loads a Format Table through FileSystemCatalog with catalog-level io-cache and meta,read. The target keeps row 7 and origin holds its replacement, row 9, under a Paimon-shaped name; the loaded table, a copy and a table built from the resolved schema all read 9, and the target sees no request. Without the change they read 7. object_table_reads_objects_from_origin does the same for an Object Table's FileIO. --- .../src/table/io_cache_routing_tests.rs | 217 ++++++++++++++++++ crates/paimon/src/table/mod.rs | 14 ++ crates/paimon/src/table/object_table.rs | 3 +- docs/src/getting-started.md | 10 +- 4 files changed, 239 insertions(+), 5 deletions(-) create mode 100644 crates/paimon/src/table/io_cache_routing_tests.rs diff --git a/crates/paimon/src/table/io_cache_routing_tests.rs b/crates/paimon/src/table/io_cache_routing_tests.rs new file mode 100644 index 000000000..e624ad12d --- /dev/null +++ b/crates/paimon/src/table/io_cache_routing_tests.rs @@ -0,0 +1,217 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! io-cache routing of the files that FileSystemCatalog tables and their writers produce. + +use std::collections::HashMap; +use std::sync::Arc; + +use arrow_array::{Int32Array, RecordBatch}; +use arrow_schema::{DataType as ArrowDataType, Field as ArrowField, Schema as ArrowSchema}; +use bytes::Bytes; +use futures::TryStreamExt; + +use super::Table; +use crate::catalog::{Catalog, FileSystemCatalog, Identifier, LoadedTable}; +use crate::common::{CatalogOptions, Options}; +use crate::io::oss_test_server::TestOss; +use crate::io::FileIOBuilder; +use crate::spec::{DataType, IntType, Schema}; + +const WAREHOUSE: &str = "bkt/warehouse"; +const FILE_NAME: &str = "data-123e4567-e89b-12d3-a456-426614174000-1.parquet"; + +/// Options that make `cache` the only io-cache target and send everything else to `origin`. +fn io_cache_options(origin: &TestOss, cache: &TestOss, options: &[(&str, &str)]) -> Options { + let mut io_cache_options = Options::new(); + io_cache_options.set(CatalogOptions::WAREHOUSE, format!("oss://{WAREHOUSE}")); + for (key, value) in [ + ("fs.oss.endpoint", origin.endpoint()), + ("fs.oss.accessKeyId", "ak"), + ("fs.oss.accessKeySecret", "sk"), + ("fs.oss.path-style-access", "true"), + ("io-cache.enabled", "true"), + ("io-cache.endpoint", cache.endpoint()), + ("io-cache.target.default.path-style-access", "true"), + ] + .iter() + .chain(options) + { + io_cache_options.set(*key, *value); + } + io_cache_options +} + +async fn catalog(origin: &TestOss, cache: &TestOss, options: &[(&str, &str)]) -> FileSystemCatalog { + let catalog = FileSystemCatalog::new(io_cache_options(origin, cache, options)).unwrap(); + catalog + .create_database("db", false, HashMap::new()) + .await + .unwrap(); + catalog +} + +async fn create_table(catalog: &FileSystemCatalog, name: &str, schema: Schema) -> Table { + let identifier = Identifier::new("db", name); + catalog + .create_table(&identifier, schema, false) + .await + .unwrap(); + catalog.get_table(&identifier).await.unwrap() +} + +/// Writes `content` to `path` on `server` only, so the other endpoint keeps what it had. +async fn put(server: &TestOss, path: &str, content: Bytes) { + let file_io = FileIOBuilder::new("oss") + .with_props([ + ("fs.oss.endpoint", server.endpoint()), + ("fs.oss.accessKeyId", "ak"), + ("fs.oss.accessKeySecret", "sk"), + ("fs.oss.path-style-access", "true"), + ]) + .build() + .unwrap(); + file_io + .new_output(path) + .unwrap() + .write(content) + .await + .unwrap(); +} + +fn parquet(ids: &[i32]) -> Bytes { + let schema = Arc::new(ArrowSchema::new(vec![ArrowField::new( + "id", + ArrowDataType::Int32, + true, + )])); + let batch = RecordBatch::try_new( + schema.clone(), + vec![Arc::new(Int32Array::from(ids.to_vec()))], + ) + .unwrap(); + let mut bytes = Vec::new(); + let mut writer = parquet::arrow::ArrowWriter::try_new(&mut bytes, schema, None).unwrap(); + writer.write(&batch).unwrap(); + writer.close().unwrap(); + Bytes::from(bytes) +} + +async fn read(table: &Table) -> Vec { + let builder = table.new_read_builder(); + let plan = builder.new_scan().plan().await.unwrap(); + builder + .new_read() + .unwrap() + .to_arrow(plan.splits()) + .unwrap() + .try_collect() + .await + .unwrap() +} + +async fn ids(table: &Table) -> Vec { + let mut ids: Vec = read(table) + .await + .iter() + .flat_map(|batch| { + let column = batch.column_by_name("id").unwrap(); + let column = column.as_any().downcast_ref::().unwrap(); + column.values().to_vec() + }) + .collect(); + ids.sort(); + ids +} + +#[tokio::test] +async fn format_table_reads_replaced_files_from_origin() { + let origin = TestOss::start().await; + // Its own storage stands in for a cache that kept the file before origin replaced it. + let cache = TestOss::start().await; + let policy = [("io-cache.policy", "meta,read")]; + let catalog = catalog(&origin, &cache, &policy).await; + let schema = Schema::builder() + .column("id", DataType::Int(IntType::new())) + .option("type", "format-table") + .option("file.format", "parquet") + .build() + .unwrap(); + let table = create_table(&catalog, "external", schema).await; + + // An upstream job replaced the file under the same, Paimon-shaped name. + let file = format!("{}/{FILE_NAME}", table.location()); + put(&cache, &file, parquet(&[7])).await; + put(&origin, &file, parquet(&[9])).await; + cache.take_requests(); + + assert_eq!(ids(&table).await, [9]); + let copy = table.copy_with_options(HashMap::from([( + "read.batch-size".to_string(), + "16".to_string(), + )])); + assert_eq!(ids(©).await, [9]); + // A caller that resolved the schema itself gets the same view of the catalog's FileIO. + let file_io = FileIOBuilder::new("oss") + .with_props(io_cache_options(&origin, &cache, &policy).to_map().iter()) + .build() + .unwrap(); + let resolved = Table::from_resolved_schema( + file_io, + table.identifier().clone(), + table.location().to_string(), + table.schema().clone(), + "main", + ) + .unwrap(); + assert_eq!(ids(&resolved).await, [9]); + assert_eq!(cache.take_requests(), Vec::::new()); +} + +#[tokio::test] +async fn object_table_reads_objects_from_origin() { + let origin = TestOss::start().await; + let cache = TestOss::start().await; + let catalog = catalog(&origin, &cache, &[("io-cache.policy", "meta,read")]).await; + let identifier = Identifier::new("db", "objects"); + let schema = Schema::builder() + .option("type", "object-table") + .build() + .unwrap(); + catalog + .create_table(&identifier, schema, false) + .await + .unwrap(); + let LoadedTable::Object(table) = catalog.load_table(&identifier).await.unwrap() else { + panic!("expected an object table"); + }; + + let object = format!("{}/{FILE_NAME}", table.location()); + put(&cache, &object, Bytes::from("old")).await; + put(&origin, &object, Bytes::from("new")).await; + cache.take_requests(); + + let input = table.file_io().new_input(&object).unwrap(); + assert_eq!(input.read().await.unwrap(), "new"); + assert_eq!(input.metadata().await.unwrap().size, 3); + let objects = table.list_objects().await.unwrap(); + assert!( + objects.iter().any(|entry| entry.name() == FILE_NAME), + "{objects:?}" + ); + assert_eq!(cache.take_requests(), Vec::::new()); +} diff --git a/crates/paimon/src/table/mod.rs b/crates/paimon/src/table/mod.rs index ff1db75b0..cc575eb65 100644 --- a/crates/paimon/src/table/mod.rs +++ b/crates/paimon/src/table/mod.rs @@ -85,6 +85,8 @@ mod hybrid_search_builder; mod incremental_scan; pub(crate) mod index_file_path; mod inline_blob; +#[cfg(all(test, feature = "storage-oss"))] +mod io_cache_routing_tests; mod kv_file_reader; mod kv_file_writer; mod lumina_index_build_builder; @@ -294,6 +296,15 @@ pub struct Table { applied_dynamic_option_keys: HashSet, } +// Format Table files may be replaced in place under the same name, so they never use io-cache. +fn table_file_io(file_io: &FileIO, schema: &TableSchema) -> FileIO { + if CoreOptions::new(schema.options()).is_format_table() { + file_io.origin_only() + } else { + file_io.clone() + } +} + impl Table { /// Create a new table. pub fn new( @@ -303,6 +314,7 @@ impl Table { schema: TableSchema, rest_env: Option, ) -> Self { + let file_io = table_file_io(&file_io, &schema); let schema_manager = SchemaManager::new(file_io.clone(), location.clone()); let branch = DEFAULT_MAIN_BRANCH.to_string(); Self { @@ -340,6 +352,7 @@ impl Table { identifier.validate()?; validate_branch_name(&branch)?; schema.validate_resolved_structure()?; + let file_io = table_file_io(&file_io, &schema); let schema_manager = SchemaManager::new(file_io.clone(), location.clone()); let schema_manager = if branch == DEFAULT_MAIN_BRANCH { schema_manager @@ -657,6 +670,7 @@ impl Table { schema_manager.with_branch(branch) }; Ok(Self { + file_io: table_file_io(&self.file_io, &schema), schema, schema_manager, branch: branch.to_string(), diff --git a/crates/paimon/src/table/object_table.rs b/crates/paimon/src/table/object_table.rs index e0a550cfe..22465b0b2 100644 --- a/crates/paimon/src/table/object_table.rs +++ b/crates/paimon/src/table/object_table.rs @@ -126,7 +126,8 @@ impl ObjectTable { })? .to_string(); Ok(Self { - file_io, + // Other writers manage these objects and may replace them, so they never use io-cache. + file_io: file_io.origin_only(), identifier, location, comment: schema.comment().map(str::to_string), diff --git a/docs/src/getting-started.md b/docs/src/getting-started.md index 8c3df168e..b71634d8d 100644 --- a/docs/src/getting-started.md +++ b/docs/src/getting-started.md @@ -444,10 +444,12 @@ rewritten in place (such as `LATEST`, tags and temporary files), sequentially numbered metadata (`snapshot-N`, `schema-N`, `changelog/changelog-N`), unknown file names. A target is used like any other endpoint: requests to it use the same retry settings, and its errors are -returned rather than retried on origin. Format Tables follow the same rules; -their files may be replaced in place, so the catalog that vends the token -decides per table whether they use a target. Setting `dlf.oss-endpoint` on the -client turns routing off. +returned rather than retried on origin. Format Tables and Object Tables never +use a target, because other jobs may replace their files in place under the +same name; to send their requests through an endpoint that answers as OSS +does, such as an OSS accelerator, make it the origin with +`io-cache.origin.endpoint`. Setting `dlf.oss-endpoint` on the client turns +routing off. ### Manage Databases From 6341ba4483cd5eb9444e944bd64e5866363375d2 Mon Sep 17 00:00:00 2001 From: Sun Dapeng Date: Tue, 13 Oct 2026 01:52:47 +0800 Subject: [PATCH 11/21] [core] Route data files named with the table's own file prefixes A table's data-file.prefix and changelog-file.prefix live in its schema, but the io-cache router only read them from the FileIO options, so a table with a custom prefix sent every request for its data files to origin unless the prefix was copied into the catalog or token options. Each table's view of its FileIO now carries the prefixes of its schema, and every request passes them to the router, through RESTTokenFileIO too. copy_with_options and copy_with_resolved_schema add the prefixes of the new schema, so files written under a dynamic prefix use the target while earlier files keep using it. data_files_named_with_the_schema_prefix_use_the_target writes, commits and scans a FileSystemCatalog table whose schema alone sets data-file.prefix to custom-, under meta,read,write and the data whitelist: the PUT and GET of its data files, and of a copy with a dynamic- prefix, go to the target and none to origin. test_token_file_io_routes_files_named_with_table_prefixes covers the REST token path. --- .../src/catalog/rest/rest_token_file_io.rs | 47 +++++++- crates/paimon/src/io/cache_routing.rs | 73 +++++++++---- crates/paimon/src/io/file_io.rs | 48 +++++++-- .../src/io/file_io/cache_routing_tests.rs | 4 +- crates/paimon/src/io/storage.rs | 3 +- crates/paimon/src/io/storage_oss.rs | 8 +- .../src/table/io_cache_routing_tests.rs | 101 +++++++++++++++++- crates/paimon/src/table/mod.rs | 16 +-- docs/src/getting-started.md | 8 +- 9 files changed, 259 insertions(+), 49 deletions(-) diff --git a/crates/paimon/src/catalog/rest/rest_token_file_io.rs b/crates/paimon/src/catalog/rest/rest_token_file_io.rs index 030cdad67..9cf94c1a3 100644 --- a/crates/paimon/src/catalog/rest/rest_token_file_io.rs +++ b/crates/paimon/src/catalog/rest/rest_token_file_io.rs @@ -206,10 +206,14 @@ impl FileIOProvider for RESTTokenFileIO { Ok((op, relative_path, Some(cache_namespace))) } - async fn create_routed(&self, path: &str) -> Result<(RoutedOperator, String, Option)> { + async fn create_routed( + &self, + path: &str, + file_prefixes: &[String], + ) -> Result<(RoutedOperator, String, Option)> { let file_io = self.current_file_io().await?; let namespace = file_io.cache_namespace_for_path(path)?; - let (op, relative_path) = file_io.create_routed_static(path)?; + let (op, relative_path) = file_io.create_routed_static(path, file_prefixes)?; Ok((op, relative_path, Some(namespace))) } } @@ -555,6 +559,45 @@ mod tests { server.abort(); } + #[tokio::test] + async fn test_token_file_io_routes_files_named_with_table_prefixes() { + let origin = TestOss::start().await; + let cache = TestOss::start_sharing(&origin).await; + let (file_io, server) = io_cache_file_io(&origin, &cache, &[]).await; + + let data = + "oss://bkt/db.db/t/bucket-0/custom-123e4567-e89b-12d3-a456-426614174000-1.parquet"; + file_io + .new_output(data) + .unwrap() + .write(Bytes::from_static(b"data")) + .await + .unwrap(); + // Without the table's prefix the name is not known as a data file. + file_io.new_input(data).unwrap().read().await.unwrap(); + assert!(cache.take_requests().is_empty()); + + let table_file_io = file_io.with_file_prefixes(["custom-"]); + assert_eq!( + table_file_io.new_input(data).unwrap().read().await.unwrap(), + "data" + ); + assert_eq!( + cache.take_requests(), + ["GET bkt/db.db/t/bucket-0/custom-123e4567-e89b-12d3-a456-426614174000-1.parquet"] + ); + // An origin-only view, as a Format Table gets, keeps the prefix but not the target. + table_file_io + .origin_only() + .new_input(data) + .unwrap() + .read() + .await + .unwrap(); + assert!(cache.take_requests().is_empty()); + server.abort(); + } + #[tokio::test] async fn test_dlf_oss_endpoint_turns_off_io_cache_routing() { let origin = TestOss::start().await; diff --git a/crates/paimon/src/io/cache_routing.rs b/crates/paimon/src/io/cache_routing.rs index 2ebf4eee2..9a2b021ad 100644 --- a/crates/paimon/src/io/cache_routing.rs +++ b/crates/paimon/src/io/cache_routing.rs @@ -185,8 +185,9 @@ impl IoCacheRouting { &self.targets } - /// Applies the routing rules in order, the first match winning. - pub(crate) fn route(&self, op: OpClass, path: &str) -> Target { + /// Applies the routing rules in order, the first match winning; `table_prefixes` are the data + /// file prefixes of the table that `path` belongs to. + pub(crate) fn route(&self, op: OpClass, path: &str, table_prefixes: &[String]) -> Target { if self.override_endpoint.is_some() { return Target::Override; } @@ -197,8 +198,8 @@ impl IoCacheRouting { if !self.policy.contains(op) || !path.starts_with("oss://") { return Target::Origin; } - let Some(file_type) = - routable_type(path, &self.data_prefixes).filter(|t| self.whitelist.contains(t)) + let Some(file_type) = routable_type(path, &self.data_prefixes, table_prefixes) + .filter(|t| self.whitelist.contains(t)) else { return Target::Origin; }; @@ -216,7 +217,11 @@ impl IoCacheRouting { } /// The target that [`Self::route`] picks for `path`, and the operation classes it serves. - pub(crate) fn cache_target(&self, path: &str) -> Option<(usize, OpClasses)> { + pub(crate) fn cache_target( + &self, + path: &str, + table_prefixes: &[String], + ) -> Option<(usize, OpClasses)> { let mut found = None; let mut classes = OpClasses::default(); for op in [ @@ -225,7 +230,7 @@ impl IoCacheRouting { OpClass::Read, OpClass::Write, ] { - if let Target::Cache(index) = self.route(op, path) { + if let Target::Cache(index) = self.route(op, path, table_prefixes) { found = Some(index); classes.insert(op); } @@ -235,7 +240,7 @@ impl IoCacheRouting { /// Endpoint of requests that never leave origin. pub(crate) fn origin(&self) -> Option<&str> { - self.endpoint(self.route(OpClass::Origin, "")) + self.endpoint(self.route(OpClass::Origin, "", &[])) } pub(crate) fn endpoint(&self, target: Target) -> Option<&str> { @@ -249,7 +254,11 @@ impl IoCacheRouting { } /// The type a cache may serve, or `None` when the file must be read from the origin. -pub(crate) fn routable_type(path: &str, data_prefixes: &[String]) -> Option { +pub(crate) fn routable_type( + path: &str, + data_prefixes: &[String], + table_prefixes: &[String], +) -> Option { let path = path.trim_end_matches('/'); let name = path.rsplit('/').next().unwrap_or(path); // Temp files are renamed away once written, whatever prefix they carry. @@ -257,7 +266,7 @@ pub(crate) fn routable_type(path: &str, data_prefixes: &[String]) -> Option Option bool { +fn is_data_file_name<'a>(name: &str, mut prefixes: impl Iterator) -> bool { !name.ends_with(MANIFEST_SIDECAR_SUFFIX) - && data_prefixes.iter().any(|prefix| { + && prefixes.any(|prefix| { name.strip_prefix(prefix.as_str()) .is_some_and(|rest| DATA_FILE_SUFFIX.is_match(rest)) }) @@ -376,7 +385,7 @@ fn parse_routes(value: &str) -> Vec<(HashSet, String)> { .collect() } -/// `data-` and `changelog-`, plus the table's own prefixes when they are in the options. +/// `data-` and `changelog-`, plus the prefixes in the options, such as those of a table token. fn data_prefixes(props: &HashMap) -> Vec { let mut prefixes: Vec = DEFAULT_DATA_PREFIXES.map(String::from).to_vec(); for key in [DATA_FILE_PREFIX, CHANGELOG_FILE_PREFIX] { @@ -479,7 +488,9 @@ mod tests { } fn route(routing: &IoCacheRouting, op: OpClass, path: &str) -> Option { - routing.endpoint(routing.route(op, path)).map(String::from) + routing + .endpoint(routing.route(op, path, &[])) + .map(String::from) } #[test] @@ -706,6 +717,17 @@ mod tests { assert_endpoint(&single(&[]), OpClass::Read, custom, OSS); let custom_prefix = single(&["data-file.prefix=custom-"]); assert_endpoint(&custom_prefix, OpClass::Read, custom, CACHE); + + // a table passes the prefixes from its schema with each request + let routing = IoCacheRouting::from_props(&single(&[])); + let table_prefixes = ["custom-".to_string()]; + let target = routing.route(OpClass::Read, &resolve(custom), &table_prefixes); + assert_eq!(routing.endpoint(target), Some(CACHE)); + let manifest = routing.route(OpClass::Read, &resolve(MANIFEST_PATH), &table_prefixes); + assert_eq!(routing.endpoint(manifest), Some(CACHE)); + let snapshot = ["snapshot-".to_string()]; + let target = routing.route(OpClass::Read, &resolve(SNAPSHOT_PATH), &snapshot); + assert_eq!(routing.endpoint(target), Some(OSS)); } #[test] @@ -887,12 +909,13 @@ mod tests { "oss://b/t/snapshot/snapshot-1.tmp-123e4567-e89b-12d3-a456-426614174000", "oss://b/t/bucket-0/data-123e4567-e89b-12d3-a456-426614174000-1.parquet.tmp-123e4567-e89b-12d3-a456-426614174000", ] { - assert_eq!(routable_type(path, &prefixes), None, "{path}"); + assert_eq!(routable_type(path, &prefixes, &[]), None, "{path}"); } assert_eq!( routable_type( "oss://b/t/bucket-0/data-123e4567-e89b-12d3-a456-426614174000-0.parquet/", - &prefixes + &prefixes, + &[] ), Some(FileType::Data) ); @@ -931,18 +954,24 @@ mod tests { ]); let both = OpClasses(OpClass::Meta.bit() | OpClass::Read.bit()); assert_eq!( - routing - .cache_target("oss://b/t/manifest/manifest-123e4567-e89b-12d3-a456-426614174000-1"), + routing.cache_target( + "oss://b/t/manifest/manifest-123e4567-e89b-12d3-a456-426614174000-1", + &[] + ), Some((0, both)) ); assert_eq!( routing.cache_target( - "oss://b/t/bucket-0/data-123e4567-e89b-12d3-a456-426614174000-1.parquet" + "oss://b/t/bucket-0/data-123e4567-e89b-12d3-a456-426614174000-1.parquet", + &[] ), Some((1, both)) ); - assert_eq!(routing.cache_target("oss://b/t/snapshot/snapshot-1"), None); - assert_eq!(routing.cache_target("oss://b/t/index/index-1"), None); + assert_eq!( + routing.cache_target("oss://b/t/snapshot/snapshot-1", &[]), + None + ); + assert_eq!(routing.cache_target("oss://b/t/index/index-1", &[]), None); assert!(!routing.targets()[0].path_style_access); assert!(routing.targets()[1].path_style_access); } @@ -1070,7 +1099,7 @@ mod tests { let routing = IoCacheRouting::from_props(options); let sorted: BTreeMap<_, _> = options.iter().collect(); assert_eq!( - routing.endpoint(routing.route(op, &resolve(path))), + routing.endpoint(routing.route(op, &resolve(path), &[])), Some(expect), "{op:?} {path} with {sorted:?}" ); @@ -1080,7 +1109,7 @@ mod tests { fn assert_type(path: &str, expect: Option, changes: &[&str]) { let prefixes = data_prefixes(&with(&[], changes)); assert_eq!( - routable_type(&resolve(path), &prefixes), + routable_type(&resolve(path), &prefixes, &[]), expect, "{path} with {changes:?}" ); diff --git a/crates/paimon/src/io/file_io.rs b/crates/paimon/src/io/file_io.rs index 0f1702f23..0a615145d 100644 --- a/crates/paimon/src/io/file_io.rs +++ b/crates/paimon/src/io/file_io.rs @@ -95,11 +95,13 @@ pub trait FileIOProvider: std::fmt::Debug + Send + Sync + 'static { Ok((op, relative_path, None)) } - /// Like [`Self::create`], adding the io-cache endpoint for requests that may use it. + /// Like [`Self::create`], adding the io-cache endpoint for requests that may use it; + /// `file_prefixes` are the data file prefixes of the table that `path` belongs to. #[doc(hidden)] async fn create_routed( &self, path: &str, + _file_prefixes: &[String], ) -> crate::Result<(RoutedOperator, String, Option)> { let (op, relative_path, namespace) = self.create_with_cache_namespace(path).await?; Ok((RoutedOperator::origin(op), relative_path, namespace)) @@ -120,6 +122,8 @@ pub struct FileIO { file_format_metadata_cache: Arc, blob_index_cache: Arc, origin_only: bool, + /// Data file prefixes of the table this view serves, for io-cache routing. + file_prefixes: Arc<[String]>, } pub(crate) const DEFAULT_FILE_FORMAT_METADATA_CACHE_MAX_BYTES: usize = 50 * 1024 * 1024; @@ -324,6 +328,22 @@ impl FileIO { file_io } + /// A view of this FileIO whose io-cache routing also knows data files named with `prefixes`. + pub(crate) fn with_file_prefixes<'a>( + &self, + prefixes: impl IntoIterator, + ) -> Self { + let mut file_prefixes = self.file_prefixes.to_vec(); + for prefix in prefixes { + if !prefix.trim().is_empty() && !file_prefixes.iter().any(|known| known == prefix) { + file_prefixes.push(prefix.to_string()); + } + } + let mut file_io = self.clone(); + file_io.file_prefixes = file_prefixes.into(); + file_io + } + fn storage(&self) -> crate::Result<&Storage> { let FileIOBackend::Storage(storage) = &self.backend else { return Err(Error::IoUnsupported { @@ -342,8 +362,9 @@ impl FileIO { pub(crate) fn create_routed_static( &self, path: &str, + file_prefixes: &[String], ) -> crate::Result<(RoutedOperator, String)> { - let (op, relative_path) = self.storage()?.create_routed(path)?; + let (op, relative_path) = self.storage()?.create_routed(path, file_prefixes)?; let op = if self.origin_only { op.without_cache() } else { @@ -386,11 +407,16 @@ impl FileIO { async fn create_routed(&self, path: &str) -> crate::Result<(RoutedOperator, String)> { match &self.backend { FileIOBackend::Provider(provider) => { - let (op, relative_path, _) = - resolve_provider_routed(provider.as_ref(), path, self.origin_only).await?; + let (op, relative_path, _) = resolve_provider_routed( + provider.as_ref(), + path, + self.origin_only, + &self.file_prefixes, + ) + .await?; Ok((op, relative_path)) } - FileIOBackend::Storage(_) => self.create_routed_static(path), + FileIOBackend::Storage(_) => self.create_routed_static(path, &self.file_prefixes), } } @@ -399,9 +425,10 @@ impl FileIO { FileIOBackend::Provider(provider) => Ok(FileSource::Provider { provider: provider.clone(), origin_only: self.origin_only, + file_prefixes: self.file_prefixes.clone(), }), FileIOBackend::Storage(storage) => { - let (op, relative_path) = self.create_routed_static(path)?; + let (op, relative_path) = self.create_routed_static(path, &self.file_prefixes)?; let namespace = self.cache_namespace_for_path(path)?; let cache_path = cache_object_path(&namespace, op.origin_operator(), &relative_path); @@ -852,12 +879,13 @@ async fn resolve_provider_routed( provider: &dyn FileIOProvider, path: &str, origin_only: bool, + file_prefixes: &[String], ) -> crate::Result<(RoutedOperator, String, Option)> { let (op, relative_path, namespace) = if origin_only { let (op, relative_path, namespace) = provider.create_with_cache_namespace(path).await?; (RoutedOperator::origin(op), relative_path, namespace) } else { - provider.create_routed(path).await? + provider.create_routed(path, file_prefixes).await? }; validate_provider_path(path, &relative_path)?; Ok((op, relative_path, namespace)) @@ -1075,6 +1103,7 @@ impl FileIOBuilder { file_format_metadata_cache, blob_index_cache, origin_only: false, + file_prefixes: Arc::default(), }) } } @@ -1397,6 +1426,7 @@ enum FileSource { Provider { provider: Arc, origin_only: bool, + file_prefixes: Arc<[String]>, }, } @@ -1406,9 +1436,11 @@ impl FileSource { Self::Provider { provider, origin_only, + file_prefixes, } => { let (op, relative_path, namespace) = - resolve_provider_routed(provider.as_ref(), path, *origin_only).await?; + resolve_provider_routed(provider.as_ref(), path, *origin_only, file_prefixes) + .await?; let cache_path = namespace.map(|namespace| { cache_object_path(&namespace, op.origin_operator(), &relative_path) }); diff --git a/crates/paimon/src/io/file_io/cache_routing_tests.rs b/crates/paimon/src/io/file_io/cache_routing_tests.rs index 4c7534d09..23bc73ddf 100644 --- a/crates/paimon/src/io/file_io/cache_routing_tests.rs +++ b/crates/paimon/src/io/file_io/cache_routing_tests.rs @@ -489,7 +489,7 @@ async fn test_routing_is_off_unless_enabled() { ("io-cache.policy", "meta,read"), ]); - let (op, _) = file_io.create_routed_static(DATA).unwrap(); + let (op, _) = file_io.create_routed_static(DATA, &[]).unwrap(); assert!(op.target().is_none()); write(&file_io, DATA, b"data").await; assert_eq!(read(&file_io, DATA).await, "data"); @@ -500,7 +500,7 @@ async fn test_routing_is_off_unless_enabled() { async fn test_cache_key_is_identical_on_target_and_origin() { let (_origin, _cache, file_io) = routed("read").await; - let (op, relative_path) = file_io.create_routed_static(DATA).unwrap(); + let (op, relative_path) = file_io.create_routed_static(DATA, &[]).unwrap(); let target = op.target().unwrap(); let namespace = file_io.cache_namespace_for_path(DATA).unwrap(); assert_eq!( diff --git a/crates/paimon/src/io/storage.rs b/crates/paimon/src/io/storage.rs index f2dc02775..948fc98bf 100644 --- a/crates/paimon/src/io/storage.rs +++ b/crates/paimon/src/io/storage.rs @@ -408,10 +408,11 @@ impl Storage { pub(crate) fn create_routed<'a>( &self, path: &'a str, + file_prefixes: &[String], ) -> crate::Result<(RoutedOperator, Cow<'a, str>)> { #[cfg(feature = "storage-oss")] if let Storage::Oss { config, operators } = self { - if let Some(target) = config.target_route(path) { + if let Some(target) = config.target_route(path, file_prefixes) { let (bucket, relative_path) = Self::bucket_and_relative_path(path, "OSS", &["oss"])?; let operator = |route| { diff --git a/crates/paimon/src/io/storage_oss.rs b/crates/paimon/src/io/storage_oss.rs index 427ba3040..2644ead24 100644 --- a/crates/paimon/src/io/storage_oss.rs +++ b/crates/paimon/src/io/storage_oss.rs @@ -86,9 +86,13 @@ pub(crate) struct OssTargetRoute { impl OssStorageConfig { /// The io-cache target of `path`, when some request on it may leave origin. - pub(crate) fn target_route(&self, path: &str) -> Option { + pub(crate) fn target_route( + &self, + path: &str, + file_prefixes: &[String], + ) -> Option { let cache = self.cache.as_ref()?; - let (index, classes) = cache.routing.cache_target(path)?; + let (index, classes) = cache.routing.cache_target(path, file_prefixes)?; Some(OssTargetRoute { route: OssRoute::Target(index), classes, diff --git a/crates/paimon/src/table/io_cache_routing_tests.rs b/crates/paimon/src/table/io_cache_routing_tests.rs index e624ad12d..25de4caad 100644 --- a/crates/paimon/src/table/io_cache_routing_tests.rs +++ b/crates/paimon/src/table/io_cache_routing_tests.rs @@ -25,12 +25,13 @@ use arrow_schema::{DataType as ArrowDataType, Field as ArrowField, Schema as Arr use bytes::Bytes; use futures::TryStreamExt; -use super::Table; +use super::table_write::tests::make_batch; +use super::{Table, TableCommit, TableWrite}; use crate::catalog::{Catalog, FileSystemCatalog, Identifier, LoadedTable}; use crate::common::{CatalogOptions, Options}; use crate::io::oss_test_server::TestOss; use crate::io::FileIOBuilder; -use crate::spec::{DataType, IntType, Schema}; +use crate::spec::{DataType, IntType, Schema, SchemaBuilder}; const WAREHOUSE: &str = "bkt/warehouse"; const FILE_NAME: &str = "data-123e4567-e89b-12d3-a456-426614174000-1.parquet"; @@ -215,3 +216,99 @@ async fn object_table_reads_objects_from_origin() { ); assert_eq!(cache.take_requests(), Vec::::new()); } + +/// A catalog whose data files may be written to and read from the target. +async fn data_target_catalog(origin: &TestOss, cache: &TestOss) -> FileSystemCatalog { + let options = [ + ("io-cache.policy", "meta,read,write"), + ("io-cache.whitelist", "data"), + ]; + catalog(origin, cache, &options).await +} + +fn int_schema() -> SchemaBuilder { + Schema::builder() + .column("id", DataType::Int(IntType::new())) + .column("value", DataType::Int(IntType::new())) +} + +/// Writes and commits `batch`, returning the names of the new data files. +async fn write_and_commit(table: &Table, batch: &RecordBatch) -> Vec { + let mut write = TableWrite::new(table, "io-cache-user".to_string()).unwrap(); + write.write_arrow_batch(batch).await.unwrap(); + let messages = write.prepare_commit().await.unwrap(); + let files = messages + .iter() + .flat_map(|message| message.new_files.iter().map(|file| file.file_name.clone())) + .collect(); + TableCommit::new(table.clone(), "io-cache-user".to_string()) + .commit(messages) + .await + .unwrap(); + files +} + +/// Methods of the requests on keys containing `fragment`, in order. +fn methods(requests: &[String], fragment: &str) -> Vec { + requests + .iter() + .filter(|request| request.contains(fragment)) + .map(|request| request.split_once(' ').unwrap().0.to_string()) + .collect() +} + +#[track_caller] +fn assert_reads_and_writes(requests: &[String], fragment: &str) { + let methods = methods(requests, fragment); + assert!( + methods.iter().any(|m| m == "PUT"), + "{fragment}: {methods:?}" + ); + assert!( + methods.iter().any(|m| m == "GET"), + "{fragment}: {methods:?}" + ); +} + +#[tokio::test] +async fn data_files_named_with_the_schema_prefix_use_the_target() { + let origin = TestOss::start().await; + let cache = TestOss::start_sharing(&origin).await; + let catalog = data_target_catalog(&origin, &cache).await; + // The prefix is only in the table schema, not in the catalog options. + let schema = int_schema() + .option("data-file.prefix", "custom-") + .build() + .unwrap(); + let table = create_table(&catalog, "custom_prefix", schema).await; + + let files = write_and_commit(&table, &make_batch(vec![1, 2], vec![10, 20])).await; + assert!( + files.iter().all(|file| file.starts_with("custom-")), + "{files:?}" + ); + assert_eq!(ids(&table).await, [1, 2]); + assert_reads_and_writes(&cache.take_requests(), "/custom-"); + assert_eq!( + methods(&origin.take_requests(), "/custom-"), + Vec::::new() + ); + + // A dynamic prefix adds to the table's own, so both sets of files keep using the target. + let dynamic = table.copy_with_options(HashMap::from([( + "data-file.prefix".to_string(), + "dynamic-".to_string(), + )])); + let files = write_and_commit(&dynamic, &make_batch(vec![3], vec![30])).await; + assert!( + files.iter().all(|file| file.starts_with("dynamic-")), + "{files:?}" + ); + assert_eq!(ids(&dynamic).await, [1, 2, 3]); + let requests = cache.take_requests(); + assert_reads_and_writes(&requests, "/dynamic-"); + assert!(methods(&requests, "/custom-").iter().any(|m| m == "GET")); + let requests = origin.take_requests(); + assert_eq!(methods(&requests, "/custom-"), Vec::::new()); + assert_eq!(methods(&requests, "/dynamic-"), Vec::::new()); +} diff --git a/crates/paimon/src/table/mod.rs b/crates/paimon/src/table/mod.rs index cc575eb65..162c3d6ae 100644 --- a/crates/paimon/src/table/mod.rs +++ b/crates/paimon/src/table/mod.rs @@ -296,13 +296,14 @@ pub struct Table { applied_dynamic_option_keys: HashSet, } -// Format Table files may be replaced in place under the same name, so they never use io-cache. +// Format Table files may be replaced in place under the same name, so they never use io-cache; +// other tables tell io-cache routing the prefixes their data files are named with. fn table_file_io(file_io: &FileIO, schema: &TableSchema) -> FileIO { - if CoreOptions::new(schema.options()).is_format_table() { - file_io.origin_only() - } else { - file_io.clone() + let options = CoreOptions::new(schema.options()); + if options.is_format_table() { + return file_io.origin_only(); } + file_io.with_file_prefixes([options.data_file_prefix(), options.changelog_file_prefix()]) } impl Table { @@ -632,11 +633,12 @@ impl Table { || k == "scan.mode" || k == "incremental-between-timestamp" }); + let schema = self.schema.copy_with_options(extra); Self { - file_io: self.file_io.clone(), + file_io: table_file_io(&self.file_io, &schema), identifier: self.identifier.clone(), location: self.location.clone(), - schema: self.schema.copy_with_options(extra), + schema, schema_manager: self.schema_manager.clone(), branch: self.branch.clone(), branch_reference: self.branch_reference, diff --git a/docs/src/getting-started.md b/docs/src/getting-started.md index b71634d8d..aae125fd5 100644 --- a/docs/src/getting-started.md +++ b/docs/src/getting-started.md @@ -436,9 +436,11 @@ Only reads, file status lookups, existence checks (with `exists` in the policy) and writes (with `write` in the policy) of files that Paimon writes once under a unique name can use a target: manifests, statistics, indexes, and data files. A data file is -recognized by the name Paimon writers give it: the `data-` or `changelog-` -prefix, or `data-file.prefix` or `changelog-file.prefix` when these are in the -options, followed by `{uuid}-{count}.{extension}`. Everything else uses +recognized by the name Paimon writers give it: a data or changelog file prefix +followed by `{uuid}-{count}.{extension}`. The prefixes are `data-` and +`changelog-`, the table's `data-file.prefix` and `changelog-file.prefix`, and +these two options when they are in the catalog options or the token. +Everything else uses origin: a writer's own existence checks, listing, deletes, renames and copies, files rewritten in place (such as `LATEST`, tags and temporary files), sequentially numbered metadata (`snapshot-N`, `schema-N`, `changelog/changelog-N`), unknown From 6c2f5c451a1f71c27344aa94af746092dae07184 Mon Sep 17 00:00:00 2001 From: Sun Dapeng Date: Tue, 13 Oct 2026 01:56:06 +0800 Subject: [PATCH 12/21] [core] Recognize postpone and managed BLOB file names in io-cache routing The postpone writer names its files {prefix}-u-{commit user}-s-{write id}-w-{uuid}-{count}.{extension} and the managed BLOB writer names its packs {prefix}{uuid}.managed.blob. Both are written once under a unique name, but neither matched {prefix}{uuid}-{count}.{extension}, so their reads and writes always used origin. Both are now data files, with the .index and .blobref files named after them. The other names the crate writes on current main (data, changelog, blob, bucket and deletion vector indexes, global indexes, manifests, statistics) already match. postpone_bucket_files_use_the_target writes a bucket=-2 primary-key table and managed_blob_packs_use_the_target writes and reads a primary-key BLOB table through FileSystemCatalog under meta,read,write: the postpone file and the packs are written and read on the target and never on origin. --- crates/paimon/src/io/cache_routing.rs | 36 +++++++- .../src/table/io_cache_routing_tests.rs | 85 ++++++++++++++++++- docs/src/getting-started.md | 9 +- 3 files changed, 120 insertions(+), 10 deletions(-) diff --git a/crates/paimon/src/io/cache_routing.rs b/crates/paimon/src/io/cache_routing.rs index 9a2b021ad..2dbf33af3 100644 --- a/crates/paimon/src/io/cache_routing.rs +++ b/crates/paimon/src/io/cache_routing.rs @@ -48,10 +48,14 @@ const CHANGELOG_FILE_PREFIX: &str = "changelog-file.prefix"; const DEFAULT_DATA_PREFIXES: [&str; 2] = ["data-", "changelog-"]; const MANIFEST_SIDECAR_SUFFIX: &str = ".avro.sidecar"; -// Data files are named {prefix}{uuid}-{count}.{extension}; this matches what follows the prefix. +// What follows a data file prefix: {uuid}-{count}.{extension}, also after the postpone writer's +// -u-{commit user}-s-{write id}-w-, or a managed BLOB pack {uuid}.managed.blob. const UUID: &str = "[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}"; static DATA_FILE_SUFFIX: LazyLock = LazyLock::new(|| { - Regex::new(&format!(r"^{UUID}-[0-9]+\..+$")).expect("valid data file pattern") + Regex::new(&format!( + r"^(?:(?:-u-.+-s-[0-9]+-w-)?{UUID}-[0-9]+\..+|{UUID}\.managed\.blob)$" + )) + .expect("valid data file pattern") }); // Other types are routed only under the names Paimon writes; Format Table files may be replaced // in place. TableCommit names changelog manifests manifest-{uuid}-changelog-{count}. @@ -721,8 +725,10 @@ mod tests { // a table passes the prefixes from its schema with each request let routing = IoCacheRouting::from_props(&single(&[])); let table_prefixes = ["custom-".to_string()]; - let target = routing.route(OpClass::Read, &resolve(custom), &table_prefixes); - assert_eq!(routing.endpoint(target), Some(CACHE)); + for path in [custom, "bucket-postpone/custom--u-user-s-1-w-{uuid}-0.orc"] { + let target = routing.route(OpClass::Read, &resolve(path), &table_prefixes); + assert_eq!(routing.endpoint(target), Some(CACHE), "{path}"); + } let manifest = routing.route(OpClass::Read, &resolve(MANIFEST_PATH), &table_prefixes); assert_eq!(routing.endpoint(manifest), Some(CACHE)); let snapshot = ["snapshot-".to_string()]; @@ -762,6 +768,12 @@ mod tests { "dt=1/part-00000-1a2b.snappy.parquet", "dt=1/bucket-0/data-1.parquet", "dt=1/bucket-0/custom-{uuid}-0.orc", + "bucket-postpone/data--u-user-s-x-w-{uuid}-0.parquet", + "bucket-postpone/data--u--s-1-w-{uuid}-0.parquet", + "bucket-postpone/custom--u-user-s-1-w-{uuid}-0.parquet", + "dt=1/bucket-0/data-{uuid}.blob", + "dt=1/bucket-0/data-{uuid}.managed.blob.{uuid}.tmp", + "dt=1/bucket-0/data-old.managed.blob", "README", ] { assert_type(path, None, &[]); @@ -822,6 +834,22 @@ mod tests { ); let file_index = "dt=1/bucket-0/data-{uuid}-0.parquet.index"; assert_type(file_index, Some(FileType::FileIndex), &[]); + let blob_references = "dt=1/bucket-0/data-{uuid}-0.parquet.blobref"; + assert_type(blob_references, Some(FileType::Data), &[]); + let pack = "dt=1/bucket-0/data-{uuid}.managed.blob"; + assert_type(pack, Some(FileType::Data), &[]); + // the postpone writer puts its commit user and write id before {uuid}-{count} + for postpone in [ + "bucket-postpone/data--u-io-cache-user-s-7-w-{uuid}-0.parquet", + "dt=1/bucket-postpone/data--u-{uuid}-s-2147483647-w-{uuid}-12.orc", + "bucket-postpone/changelog--u-user-s-0-w-{uuid}-0.parquet", + ] { + assert_type(postpone, Some(FileType::Data), &[]); + let index = format!("{postpone}.index"); + assert_type(&index, Some(FileType::FileIndex), &[]); + let blob_references = format!("{postpone}.blobref"); + assert_type(&blob_references, Some(FileType::Data), &[]); + } assert_type("index/index-{uuid}-0", Some(FileType::BucketIndex), &[]); let btree = "index/btree-global-index-{uuid}.index"; assert_type(btree, Some(FileType::GlobalIndex), &[]); diff --git a/crates/paimon/src/table/io_cache_routing_tests.rs b/crates/paimon/src/table/io_cache_routing_tests.rs index 25de4caad..7dfcc041c 100644 --- a/crates/paimon/src/table/io_cache_routing_tests.rs +++ b/crates/paimon/src/table/io_cache_routing_tests.rs @@ -20,7 +20,7 @@ use std::collections::HashMap; use std::sync::Arc; -use arrow_array::{Int32Array, RecordBatch}; +use arrow_array::{Array, Int32Array, LargeBinaryArray, RecordBatch}; use arrow_schema::{DataType as ArrowDataType, Field as ArrowField, Schema as ArrowSchema}; use bytes::Bytes; use futures::TryStreamExt; @@ -31,7 +31,7 @@ use crate::catalog::{Catalog, FileSystemCatalog, Identifier, LoadedTable}; use crate::common::{CatalogOptions, Options}; use crate::io::oss_test_server::TestOss; use crate::io::FileIOBuilder; -use crate::spec::{DataType, IntType, Schema, SchemaBuilder}; +use crate::spec::{BlobType, DataType, IntType, Schema, SchemaBuilder}; const WAREHOUSE: &str = "bkt/warehouse"; const FILE_NAME: &str = "data-123e4567-e89b-12d3-a456-426614174000-1.parquet"; @@ -312,3 +312,84 @@ async fn data_files_named_with_the_schema_prefix_use_the_target() { assert_eq!(methods(&requests, "/custom-"), Vec::::new()); assert_eq!(methods(&requests, "/dynamic-"), Vec::::new()); } + +#[tokio::test] +async fn postpone_bucket_files_use_the_target() { + let origin = TestOss::start().await; + let cache = TestOss::start_sharing(&origin).await; + let catalog = data_target_catalog(&origin, &cache).await; + let schema = int_schema() + .primary_key(["id"]) + .option("bucket", "-2") + .build() + .unwrap(); + let table = create_table(&catalog, "postpone", schema).await; + + let files = write_and_commit(&table, &make_batch(vec![1, 2], vec![10, 20])).await; + let [file] = files.as_slice() else { + panic!("expected one postpone file, got {files:?}"); + }; + assert!(file.starts_with("data--u-io-cache-user-s-"), "{file}"); + // Postpone files are read once compaction assigns their rows to buckets. + let path = format!("{}/bucket-postpone/{file}", table.location()); + table + .file_io() + .new_input(&path) + .unwrap() + .read() + .await + .unwrap(); + + assert_eq!(methods(&cache.take_requests(), file), ["PUT", "GET"]); + assert_eq!(methods(&origin.take_requests(), file), Vec::::new()); +} + +#[tokio::test] +async fn managed_blob_packs_use_the_target() { + let origin = TestOss::start().await; + let cache = TestOss::start_sharing(&origin).await; + let catalog = data_target_catalog(&origin, &cache).await; + let schema = Schema::builder() + .column("id", DataType::Int(IntType::new())) + .column("payload", DataType::Blob(BlobType::new())) + .primary_key(["id"]) + .option("bucket", "1") + .option("file.format", "parquet") + .build() + .unwrap(); + let table = create_table(&catalog, "managed_blob", schema).await; + + let batch = RecordBatch::try_new( + Arc::new(ArrowSchema::new(vec![ + ArrowField::new("id", ArrowDataType::Int32, false), + ArrowField::new("payload", ArrowDataType::LargeBinary, true), + ])), + vec![ + Arc::new(Int32Array::from(vec![1, 2])), + Arc::new(LargeBinaryArray::from(vec![ + Some(b"hello".as_slice()), + Some(b"world".as_slice()), + ])), + ], + ) + .unwrap(); + write_and_commit(&table, &batch).await; + let payloads: Vec> = read(&table) + .await + .iter() + .flat_map(|batch| { + let column = batch.column_by_name("payload").unwrap(); + let column = column.as_any().downcast_ref::().unwrap(); + (0..column.len()) + .map(|row| column.value(row).to_vec()) + .collect::>() + }) + .collect(); + assert_eq!(payloads, [b"hello".to_vec(), b"world".to_vec()]); + + assert_reads_and_writes(&cache.take_requests(), ".managed.blob"); + assert_eq!( + methods(&origin.take_requests(), ".managed.blob"), + Vec::::new() + ); +} diff --git a/docs/src/getting-started.md b/docs/src/getting-started.md index aae125fd5..f9562253b 100644 --- a/docs/src/getting-started.md +++ b/docs/src/getting-started.md @@ -437,10 +437,11 @@ and writes (with `write` in the policy) of files that Paimon writes once under a unique name can use a target: manifests, statistics, indexes, and data files. A data file is recognized by the name Paimon writers give it: a data or changelog file prefix -followed by `{uuid}-{count}.{extension}`. The prefixes are `data-` and -`changelog-`, the table's `data-file.prefix` and `changelog-file.prefix`, and -these two options when they are in the catalog options or the token. -Everything else uses +followed by `{uuid}-{count}.{extension}`, the same after the postpone writer's +`-u-{commit user}-s-{write id}-w-`, or `{uuid}.managed.blob` for a managed BLOB +pack. The prefixes are `data-` and `changelog-`, the table's +`data-file.prefix` and `changelog-file.prefix`, and these two options when they +are in the catalog options or the token. Everything else uses origin: a writer's own existence checks, listing, deletes, renames and copies, files rewritten in place (such as `LATEST`, tags and temporary files), sequentially numbered metadata (`snapshot-N`, `schema-N`, `changelog/changelog-N`), unknown From 869f7bec0dbfe1c69e91ee42bbb5f8debfa40124 Mon Sep 17 00:00:00 2001 From: Sun Dapeng Date: Sun, 25 Oct 2026 01:56:57 +0800 Subject: [PATCH 13/21] [core] Read BLOB descriptor sources from the origin under io-cache routing BlobFormatWriter copies the payload a BlobDescriptor points to into the .blob file of an append table or the .managed.blob pack of a primary-key table. The source is owned by whoever wrote it, not by the table, so its Paimon-shaped name says nothing about immutability: an import job may replace oss://bkt/import/data--1.png while an io-cache target still serves the old copy, and the table then persists the old payload for good. BlobFormatWriter, which both BLOB writer paths share, now reads descriptor sources through the origin-only view of its FileIO. The .blob files and managed packs the table writes are its own immutable files and keep their io-cache routing. TestOss can retain an old copy of an object, like a cache that has not seen the replacement. blob_descriptor_sources_are_read_from_origin writes a descriptor whose source is NEW on origin and OLD on the target into a primary-key and an append BLOB table: the source GET reaches origin only, the .blob PUT still goes to the target, and both the routed table and an origin-only client read NEW. Without the fix both tables persist OLD. --- crates/paimon/src/arrow/format/blob/writer.rs | 3 +- crates/paimon/src/io/oss_test_server.rs | 14 +- .../src/table/io_cache_routing_tests.rs | 130 ++++++++++++++---- 3 files changed, 118 insertions(+), 29 deletions(-) diff --git a/crates/paimon/src/arrow/format/blob/writer.rs b/crates/paimon/src/arrow/format/blob/writer.rs index c51e33e41..c4623ec9a 100644 --- a/crates/paimon/src/arrow/format/blob/writer.rs +++ b/crates/paimon/src/arrow/format/blob/writer.rs @@ -136,6 +136,7 @@ impl FormatWriterFactory for BlobWriterFactory { pub(crate) struct BlobFormatWriter { writer: Box, + /// Reads descriptor sources from origin: other writers own them and may replace them. file_io: Option, kind: BlobFieldKind, path: String, @@ -180,7 +181,7 @@ impl BlobFormatWriter { } Ok(Self { writer, - file_io, + file_io: file_io.map(|file_io| file_io.origin_only()), kind, path: output.location().to_string(), field_name: field.map_or_else(String::new, |field| field.name().to_string()), diff --git a/crates/paimon/src/io/oss_test_server.rs b/crates/paimon/src/io/oss_test_server.rs index e224772a2..db41c7ef7 100644 --- a/crates/paimon/src/io/oss_test_server.rs +++ b/crates/paimon/src/io/oss_test_server.rs @@ -35,6 +35,7 @@ struct ServerState { requests: Mutex>, unavailable: AtomicBool, hidden: Mutex>, + retained: Objects, } pub(crate) struct TestOss { @@ -93,6 +94,15 @@ impl TestOss { .insert(bucket_and_key.to_string()); } + /// Answers HEAD and GET of `bucket_and_key` with `content`, like a cache that kept an old copy. + pub(crate) fn retain(&self, bucket_and_key: &str, content: Bytes) { + self.state + .retained + .lock() + .unwrap() + .insert(bucket_and_key.to_string(), content); + } + /// Requests served so far, as `METHOD bucket/key`, or `LIST bucket/prefix`. pub(crate) fn take_requests(&self) -> Vec { std::mem::take(&mut *self.state.requests.lock().unwrap()) @@ -165,7 +175,9 @@ async fn handle( } Method::HEAD | Method::GET => { let hidden = state.hidden.lock().unwrap().contains(&path); - let Some(data) = objects.get(&path).cloned().filter(|_| !hidden) else { + let retained = state.retained.lock().unwrap().get(&path).cloned(); + let Some(data) = retained.or_else(|| objects.get(&path).cloned().filter(|_| !hidden)) + else { return error(StatusCode::NOT_FOUND, "NoSuchKey"); }; if method == Method::HEAD { diff --git a/crates/paimon/src/table/io_cache_routing_tests.rs b/crates/paimon/src/table/io_cache_routing_tests.rs index 7dfcc041c..9c17de67c 100644 --- a/crates/paimon/src/table/io_cache_routing_tests.rs +++ b/crates/paimon/src/table/io_cache_routing_tests.rs @@ -31,7 +31,7 @@ use crate::catalog::{Catalog, FileSystemCatalog, Identifier, LoadedTable}; use crate::common::{CatalogOptions, Options}; use crate::io::oss_test_server::TestOss; use crate::io::FileIOBuilder; -use crate::spec::{BlobType, DataType, IntType, Schema, SchemaBuilder}; +use crate::spec::{BlobDescriptor, BlobType, DataType, IntType, Schema, SchemaBuilder}; const WAREHOUSE: &str = "bkt/warehouse"; const FILE_NAME: &str = "data-123e4567-e89b-12d3-a456-426614174000-1.parquet"; @@ -75,15 +75,20 @@ async fn create_table(catalog: &FileSystemCatalog, name: &str, schema: Schema) - catalog.get_table(&identifier).await.unwrap() } +/// Options of a client without io-cache routing, which only talks to `server`. +fn plain_options(server: &TestOss) -> [(&str, &str); 4] { + [ + ("fs.oss.endpoint", server.endpoint()), + ("fs.oss.accessKeyId", "ak"), + ("fs.oss.accessKeySecret", "sk"), + ("fs.oss.path-style-access", "true"), + ] +} + /// Writes `content` to `path` on `server` only, so the other endpoint keeps what it had. async fn put(server: &TestOss, path: &str, content: Bytes) { let file_io = FileIOBuilder::new("oss") - .with_props([ - ("fs.oss.endpoint", server.endpoint()), - ("fs.oss.accessKeyId", "ak"), - ("fs.oss.accessKeySecret", "sk"), - ("fs.oss.path-style-access", "true"), - ]) + .with_props(plain_options(server)) .build() .unwrap(); file_io @@ -344,37 +349,36 @@ async fn postpone_bucket_files_use_the_target() { assert_eq!(methods(&origin.take_requests(), file), Vec::::new()); } -#[tokio::test] -async fn managed_blob_packs_use_the_target() { - let origin = TestOss::start().await; - let cache = TestOss::start_sharing(&origin).await; - let catalog = data_target_catalog(&origin, &cache).await; - let schema = Schema::builder() +fn blob_schema() -> SchemaBuilder { + Schema::builder() .column("id", DataType::Int(IntType::new())) .column("payload", DataType::Blob(BlobType::new())) +} + +/// A primary-key table, which copies its BLOBs into managed packs. +fn pk_blob_schema() -> SchemaBuilder { + blob_schema() .primary_key(["id"]) .option("bucket", "1") .option("file.format", "parquet") - .build() - .unwrap(); - let table = create_table(&catalog, "managed_blob", schema).await; +} - let batch = RecordBatch::try_new( +fn blob_batch(ids: &[i32], payloads: &[&[u8]]) -> RecordBatch { + RecordBatch::try_new( Arc::new(ArrowSchema::new(vec![ ArrowField::new("id", ArrowDataType::Int32, false), ArrowField::new("payload", ArrowDataType::LargeBinary, true), ])), vec![ - Arc::new(Int32Array::from(vec![1, 2])), - Arc::new(LargeBinaryArray::from(vec![ - Some(b"hello".as_slice()), - Some(b"world".as_slice()), - ])), + Arc::new(Int32Array::from(ids.to_vec())), + Arc::new(LargeBinaryArray::from(payloads.to_vec())), ], ) - .unwrap(); - write_and_commit(&table, &batch).await; - let payloads: Vec> = read(&table) + .unwrap() +} + +async fn payloads(table: &Table) -> Vec> { + read(table) .await .iter() .flat_map(|batch| { @@ -384,8 +388,21 @@ async fn managed_blob_packs_use_the_target() { .map(|row| column.value(row).to_vec()) .collect::>() }) - .collect(); - assert_eq!(payloads, [b"hello".to_vec(), b"world".to_vec()]); + .collect() +} + +#[tokio::test] +async fn managed_blob_packs_use_the_target() { + let origin = TestOss::start().await; + let cache = TestOss::start_sharing(&origin).await; + let catalog = data_target_catalog(&origin, &cache).await; + let table = create_table(&catalog, "managed_blob", pk_blob_schema().build().unwrap()).await; + + write_and_commit(&table, &blob_batch(&[1, 2], &[b"hello", b"world"])).await; + assert_eq!( + payloads(&table).await, + [b"hello".to_vec(), b"world".to_vec()] + ); assert_reads_and_writes(&cache.take_requests(), ".managed.blob"); assert_eq!( @@ -393,3 +410,62 @@ async fn managed_blob_packs_use_the_target() { Vec::::new() ); } + +#[tokio::test] +async fn blob_descriptor_sources_are_read_from_origin() { + let origin = TestOss::start().await; + let cache = TestOss::start_sharing(&origin).await; + let catalog = data_target_catalog(&origin, &cache).await; + let mut plain = Options::new(); + plain.set(CatalogOptions::WAREHOUSE, format!("oss://{WAREHOUSE}")); + for (key, value) in plain_options(&origin) { + plain.set(key, value); + } + let origin_catalog = FileSystemCatalog::new(plain).unwrap(); + // Primary-key tables copy descriptor sources into managed packs, append tables into .blob files. + let append_blob_schema = blob_schema() + .option("data-evolution.enabled", "true") + .option("row-tracking.enabled", "true"); + for (name, schema) in [ + ("pk_blob", pk_blob_schema()), + ("append_blob", append_blob_schema), + ] { + let table = create_table(&catalog, name, schema.build().unwrap()).await; + // An import job owns the source; the target kept the copy from before origin replaced it. + let source = + format!("oss://bkt/import/{name}/data-123e4567-e89b-12d3-a456-426614174000-1.png"); + let key = source.strip_prefix("oss://").unwrap(); + put(&origin, &source, Bytes::from("NEW")).await; + cache.retain(key, Bytes::from("OLD")); + origin.take_requests(); + cache.take_requests(); + + let descriptor = BlobDescriptor::new(source.clone(), 0, 3).serialize(); + write_and_commit(&table, &blob_batch(&[1], &[&descriptor])).await; + let (origin_requests, cache_requests) = (origin.take_requests(), cache.take_requests()); + assert_eq!(methods(&origin_requests, key), ["GET"], "{name}"); + assert_eq!( + methods(&cache_requests, key), + Vec::::new(), + "{name}" + ); + // The copy is a new immutable file of the table, so it is still written to the target. + let blob_puts = |requests: &[String]| { + requests + .iter() + .filter(|request| request.starts_with("PUT ") && request.ends_with(".blob")) + .count() + }; + assert_eq!(blob_puts(&cache_requests), 1, "{name}: {cache_requests:?}"); + assert_eq!( + blob_puts(&origin_requests), + 0, + "{name}: {origin_requests:?}" + ); + + assert_eq!(payloads(&table).await, [b"NEW".to_vec()], "{name}"); + let identifier = Identifier::new("db", name); + let origin_table = origin_catalog.get_table(&identifier).await.unwrap(); + assert_eq!(payloads(&origin_table).await, [b"NEW".to_vec()], "{name}"); + } +} From e871d80e3457565c4158c5034f90e0be23da056a Mon Sep 17 00:00:00 2001 From: Sun Dapeng Date: Sun, 25 Oct 2026 02:52:59 +0800 Subject: [PATCH 14/21] [core] Route data files named with the prefixes of every persisted schema A table's io-cache FileIO view only knew the data-file.prefix and changelog-file.prefix of the schema it was built from. After ALTER TABLE changes a prefix, the table loaded again, for example after a restart, sends the GETs of the data files committed under the old prefix to origin only, so those immutable files silently lose their io-cache traffic. Every table view now carries a shared, once-initialized loader of the prefixes of the table's persisted schemas. Table::new and from_resolved_schema attach it, copies share it, and copies that switch to another branch attach one for that branch's schemas. The loader runs at most once, on the first request whose file is named like a data file with none of the view's own prefixes, and only after the backend says io-cache routing is on; the token FileIO answers that from the token the request needs anyway. It lists and reads schema/ through an origin-only view, which has no loader and never uses a target, and shares the SchemaManager cache. A failure is logged once and those files stay on origin. Loading a table therefore does no extra I/O or token request, and a table without io-cache routing, or whose files all carry its current prefixes, never reads its schema history. Dynamic prefixes that were never persisted cannot be recovered; files named with them keep going to origin, which is safe. altered_data_file_prefix_keeps_routing_after_reloads mirrors the review probe with a FileSystemCatalog: data-file.prefix=older-, write and commit, ALTER to newer-, load_table and append, then a new catalog's get_table scans both rows, the schemas are listed from origin during that scan, and the GETs of the older- and newer- files reach the target, not origin. A catalog without io-cache routing scans the same rows without listing the schemas. rest_tables_keep_routing_for_prefixes_of_older_schemas builds a REST table whose token vends io-cache routing and whose response carries only the newer- schema: building it sends no OSS request and no token request beyond the one RESTTokenFileIO::build_file_io already makes, and reading the older- file reaches the target without another token request. --- .../src/catalog/rest/rest_token_file_io.rs | 4 + crates/paimon/src/io/cache_routing.rs | 44 ++++ crates/paimon/src/io/file_io.rs | 191 +++++++++++++--- crates/paimon/src/io/storage.rs | 9 + crates/paimon/src/io/storage_oss.rs | 5 + .../src/table/io_cache_routing_tests.rs | 208 +++++++++++++++++- crates/paimon/src/table/mod.rs | 32 ++- crates/paimon/src/table/schema_manager.rs | 9 + 8 files changed, 462 insertions(+), 40 deletions(-) diff --git a/crates/paimon/src/catalog/rest/rest_token_file_io.rs b/crates/paimon/src/catalog/rest/rest_token_file_io.rs index 9cf94c1a3..d09f382e2 100644 --- a/crates/paimon/src/catalog/rest/rest_token_file_io.rs +++ b/crates/paimon/src/catalog/rest/rest_token_file_io.rs @@ -216,6 +216,10 @@ impl FileIOProvider for RESTTokenFileIO { let (op, relative_path) = file_io.create_routed_static(path, file_prefixes)?; Ok((op, relative_path, Some(namespace))) } + + async fn routes_to_io_cache(&self) -> Result { + self.current_file_io().await?.routes_to_io_cache().await + } } #[cfg(test)] diff --git a/crates/paimon/src/io/cache_routing.rs b/crates/paimon/src/io/cache_routing.rs index 108a7feee..240d65dba 100644 --- a/crates/paimon/src/io/cache_routing.rs +++ b/crates/paimon/src/io/cache_routing.rs @@ -57,6 +57,11 @@ static DATA_FILE_SUFFIX: LazyLock = LazyLock::new(|| { )) .expect("valid data file pattern") }); +// The name of a data file or its file index, whatever its prefix. +static ANY_PREFIX_DATA_FILE: LazyLock = LazyLock::new(|| { + Regex::new(&format!(r"(?:{UUID}-[0-9]+\..+|{UUID}\.managed\.blob)$")) + .expect("valid data file pattern") +}); // Other types are routed only under the names Paimon writes; Format Table files may be replaced // in place. TableCommit names changelog manifests manifest-{uuid}-changelog-{count}. static META_FILE: LazyLock = LazyLock::new(|| { @@ -293,6 +298,16 @@ pub(crate) fn routable_type( paimon_name.then_some(file_type) } +/// Whether `path` is named like a data file but with none of `prefixes`, so other prefixes of +/// the table may still make it routable. +pub(crate) fn has_other_data_file_prefix(path: &str, prefixes: &[String]) -> bool { + let path = path.trim_end_matches('/'); + let name = path.rsplit('/').next().unwrap_or(path); + !name.ends_with(MANIFEST_SIDECAR_SUFFIX) + && ANY_PREFIX_DATA_FILE.is_match(name) + && !is_data_file_name(name, prefixes.iter()) +} + // Manifests, indexes and statistics share the uuid-count shape but have no extension. fn is_data_file_name<'a>(name: &str, mut prefixes: impl Iterator) -> bool { !name.ends_with(MANIFEST_SIDECAR_SUFFIX) @@ -932,6 +947,35 @@ mod tests { } } + #[test] + fn test_names_with_other_data_file_prefixes() { + let known = ["data-".to_string(), "newer-".to_string()]; + for path in [ + "bucket-0/older-{uuid}-0.parquet", + "bucket-0/older-{uuid}-0.parquet.index", + "bucket-postpone/older--u-user-s-0-w-{uuid}-0.parquet", + "bucket-0/older-{uuid}.managed.blob", + ] { + assert!(has_other_data_file_prefix(&resolve(path), &known), "{path}"); + } + for path in [ + "bucket-0/newer-{uuid}-0.parquet", + "bucket-0/data-{uuid}.managed.blob", + "manifest/manifest-{uuid}-0", + "manifest/manifest-{uuid}-0.avro.sidecar", + "manifest/manifest-list-{uuid}-1", + "index/btree-global-index-{uuid}.index", + "snapshot/snapshot-1", + "snapshot/LATEST", + "schema/schema-0", + ] { + assert!( + !has_other_data_file_prefix(&resolve(path), &known), + "{path}" + ); + } + } + #[test] fn test_rust_temp_names_are_not_routable() { let prefixes = data_prefixes(&HashMap::new()); diff --git a/crates/paimon/src/io/file_io.rs b/crates/paimon/src/io/file_io.rs index 43a0e9c18..99bdb4b6d 100644 --- a/crates/paimon/src/io/file_io.rs +++ b/crates/paimon/src/io/file_io.rs @@ -19,6 +19,7 @@ use crate::common::options::{parse_memory_size, CatalogOptions}; use crate::error::*; use lru::LruCache; use std::any::Any; +use std::borrow::Cow; use std::collections::HashMap; use std::future::Future; use std::num::NonZeroUsize; @@ -30,8 +31,9 @@ use std::time::SystemTime; use bytes::Bytes; use chrono::{DateTime, Utc}; +use futures::future::BoxFuture; use futures::stream::BoxStream; -use futures::{StreamExt, TryStreamExt}; +use futures::{FutureExt, StreamExt, TryStreamExt}; use opendal::raw::{new_std_io_error, normalize_path, normalize_root}; use opendal::Operator; use sha2::{Digest, Sha256}; @@ -40,7 +42,7 @@ use tokio_util::compat::FuturesAsyncWriteCompatExt; use url::Url; use super::cache::{CachedFileReader, LocalCache}; -use super::cache_routing::{OpClass, RoutedOperator}; +use super::cache_routing::{has_other_data_file_prefix, OpClass, RoutedOperator}; use super::Storage; #[cfg(all(test, feature = "storage-oss"))] @@ -111,6 +113,12 @@ pub trait FileIOProvider: std::fmt::Debug + Send + Sync + 'static { let (op, relative_path, namespace) = self.create_with_cache_namespace(path).await?; Ok((RoutedOperator::origin(op), relative_path, namespace)) } + + /// Whether some request may use an io-cache target. + #[doc(hidden)] + async fn routes_to_io_cache(&self) -> crate::Result { + Ok(false) + } } #[derive(Clone, Debug)] @@ -129,6 +137,63 @@ pub struct FileIO { origin_only: bool, /// Data file prefixes of the table this view serves, for io-cache routing. file_prefixes: Arc<[String]>, + /// Prefixes of the table's persisted schemas, loaded once a request may use io-cache. + schema_file_prefixes: Option>, +} + +type LoadFilePrefixes = Box BoxFuture<'static, Result>> + Send + Sync>; + +/// Data file prefixes of every persisted schema of a table, loaded at most once. +pub(crate) struct SchemaFilePrefixes { + load: LoadFilePrefixes, + loaded: tokio::sync::OnceCell>, +} + +impl SchemaFilePrefixes { + pub(crate) fn new(load: F) -> Self + where + F: Fn() -> Fut + Send + Sync + 'static, + Fut: Future>> + Send + 'static, + { + Self { + load: Box::new(move || load().boxed()), + loaded: tokio::sync::OnceCell::new(), + } + } + + async fn load(&self) -> &[String] { + self.loaded + .get_or_init(|| async { + // Files with a prefix missed here are read from origin, which only loses io-cache. + (self.load)().await.unwrap_or_else(|error| { + log::warn!("Failed to load schema file prefixes for io-cache routing: {error}"); + Vec::new() + }) + }) + .await + } +} + +impl std::fmt::Debug for SchemaFilePrefixes { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("SchemaFilePrefixes") + .field("loaded", &self.loaded.get()) + .finish_non_exhaustive() + } +} + +/// `own`, followed by the `persisted` prefixes it lacks. +fn with_persisted_prefixes<'a>(own: &'a [String], persisted: &[String]) -> Cow<'a, [String]> { + if persisted.iter().all(|prefix| own.contains(prefix)) { + return Cow::Borrowed(own); + } + let mut prefixes = own.to_vec(); + for prefix in persisted { + if !prefixes.contains(prefix) { + prefixes.push(prefix.clone()); + } + } + Cow::Owned(prefixes) } pub(crate) const DEFAULT_FILE_FORMAT_METADATA_CACHE_MAX_BYTES: usize = 50 * 1024 * 1024; @@ -434,9 +499,51 @@ impl FileIO { pub(crate) fn origin_only(&self) -> Self { let mut file_io = self.clone(); file_io.origin_only = true; + file_io.schema_file_prefixes = None; + file_io + } + + /// A view of this FileIO whose io-cache routing also knows the prefixes `schemas` loads, + /// on the first request that may use io-cache. + pub(crate) fn with_schema_file_prefixes(&self, schemas: SchemaFilePrefixes) -> Self { + let mut file_io = self.clone(); + if !self.origin_only { + file_io.schema_file_prefixes = Some(Arc::new(schemas)); + } file_io } + /// Whether routing `path` needs the persisted prefixes, which are not loaded yet. + fn awaits_schema_file_prefixes(&self, path: &str) -> bool { + self.schema_file_prefixes + .as_ref() + .is_some_and(|schemas| schemas.loaded.get().is_none()) + && has_other_data_file_prefix(path, &self.file_prefixes) + } + + /// Prefixes for io-cache routing, with the persisted ones loaded so far. + fn loaded_file_prefixes(&self) -> Cow<'_, [String]> { + match self + .schema_file_prefixes + .as_ref() + .and_then(|s| s.loaded.get()) + { + Some(persisted) => with_persisted_prefixes(&self.file_prefixes, persisted), + None => Cow::Borrowed(&self.file_prefixes), + } + } + + /// Prefixes for io-cache routing of `path`, loading the persisted ones first when only they + /// may make it routable. + async fn routing_file_prefixes(&self, path: &str) -> crate::Result> { + if self.awaits_schema_file_prefixes(path) && self.routes_to_io_cache().await? { + if let Some(schemas) = &self.schema_file_prefixes { + schemas.load().await; + } + } + Ok(self.loaded_file_prefixes()) + } + /// A view of this FileIO whose io-cache routing also knows data files named with `prefixes`. pub(crate) fn with_file_prefixes<'a>( &self, @@ -453,6 +560,17 @@ impl FileIO { file_io } + /// Whether some request through this view may use an io-cache target. + pub(crate) async fn routes_to_io_cache(&self) -> crate::Result { + if self.origin_only { + return Ok(false); + } + match &self.backend { + FileIOBackend::Provider(provider) => provider.routes_to_io_cache().await, + FileIOBackend::Storage(storage) => Ok(storage.routes_to_io_cache()), + } + } + fn storage(&self) -> crate::Result<&Storage> { let FileIOBackend::Storage(storage) = &self.backend else { return Err(Error::IoUnsupported { @@ -521,30 +639,61 @@ impl FileIO { } async fn create_routed(&self, path: &str) -> crate::Result<(RoutedOperator, String)> { + let file_prefixes = self.routing_file_prefixes(path).await?; match &self.backend { FileIOBackend::Provider(provider) => { let (op, relative_path, _) = resolve_provider_routed( provider.as_ref(), path, self.origin_only, - &self.file_prefixes, + &file_prefixes, ) .await?; Ok((op, relative_path)) } - FileIOBackend::Storage(_) => self.create_routed_static(path, &self.file_prefixes), + FileIOBackend::Storage(_) => self.create_routed_static(path, &file_prefixes), + } + } + + /// The operators, relative path and local cache path of a request on `path`. + async fn resolve_routed( + &self, + path: &str, + ) -> crate::Result<(RoutedOperator, String, Option)> { + let file_prefixes = self.routing_file_prefixes(path).await?; + match &self.backend { + FileIOBackend::Provider(provider) => { + let (op, relative_path, namespace) = resolve_provider_routed( + provider.as_ref(), + path, + self.origin_only, + &file_prefixes, + ) + .await?; + let cache_path = namespace.map(|namespace| { + cache_object_path(&namespace, op.origin_operator(), &relative_path) + }); + Ok((op, relative_path, cache_path)) + } + FileIOBackend::Storage(_) => { + let (op, relative_path) = self.create_routed_static(path, &file_prefixes)?; + let namespace = self.cache_namespace_for_path(path)?; + let cache_path = + cache_object_path(&namespace, op.origin_operator(), &relative_path); + Ok((op, relative_path, Some(cache_path))) + } } } fn file_source(&self, path: &str) -> crate::Result { match &self.backend { - FileIOBackend::Provider(provider) => Ok(FileSource::Provider { - provider: provider.clone(), - origin_only: self.origin_only, - file_prefixes: self.file_prefixes.clone(), - }), + FileIOBackend::Provider(_) => Ok(FileSource::Routed(self.clone())), FileIOBackend::Storage(storage) => { - let (op, relative_path) = self.create_routed_static(path, &self.file_prefixes)?; + if self.awaits_schema_file_prefixes(path) && storage.routes_to_io_cache() { + return Ok(FileSource::Routed(self.clone())); + } + let (op, relative_path) = + self.create_routed_static(path, &self.loaded_file_prefixes())?; let namespace = self.cache_namespace_for_path(path)?; let cache_path = cache_object_path(&namespace, op.origin_operator(), &relative_path); @@ -1276,6 +1425,7 @@ impl FileIOBuilder { blob_index_cache, origin_only: false, file_prefixes: Arc::default(), + schema_file_prefixes: None, }) } } @@ -1608,29 +1758,14 @@ enum FileSource { cache_path: String, local_fs: bool, }, - Provider { - provider: Arc, - origin_only: bool, - file_prefixes: Arc<[String]>, - }, + /// Resolved per request: a provider, or prefixes that still have to be loaded. + Routed(FileIO), } impl FileSource { async fn resolve(&self, path: &str) -> crate::Result<(RoutedOperator, String, Option)> { match self { - Self::Provider { - provider, - origin_only, - file_prefixes, - } => { - let (op, relative_path, namespace) = - resolve_provider_routed(provider.as_ref(), path, *origin_only, file_prefixes) - .await?; - let cache_path = namespace.map(|namespace| { - cache_object_path(&namespace, op.origin_operator(), &relative_path) - }); - Ok((op, relative_path, cache_path)) - } + Self::Routed(file_io) => file_io.resolve_routed(path).await, Self::Static { op, relative_path, diff --git a/crates/paimon/src/io/storage.rs b/crates/paimon/src/io/storage.rs index 8152f6be6..27688a4b5 100644 --- a/crates/paimon/src/io/storage.rs +++ b/crates/paimon/src/io/storage.rs @@ -430,6 +430,15 @@ impl Storage { } } + /// Whether some request may use an io-cache target. + pub(crate) fn routes_to_io_cache(&self) -> bool { + #[cfg(feature = "storage-oss")] + if let Storage::Oss { config, .. } = self { + return config.routes_to_io_cache(); + } + false + } + /// Like [`Self::create`], adding the io-cache target when a request on `path` may use one. pub(crate) fn create_routed<'a>( &self, diff --git a/crates/paimon/src/io/storage_oss.rs b/crates/paimon/src/io/storage_oss.rs index 2644ead24..59b517550 100644 --- a/crates/paimon/src/io/storage_oss.rs +++ b/crates/paimon/src/io/storage_oss.rs @@ -85,6 +85,11 @@ pub(crate) struct OssTargetRoute { } impl OssStorageConfig { + /// Whether some request may use an io-cache target. + pub(crate) fn routes_to_io_cache(&self) -> bool { + self.cache.is_some() + } + /// The io-cache target of `path`, when some request on it may leave origin. pub(crate) fn target_route( &self, diff --git a/crates/paimon/src/table/io_cache_routing_tests.rs b/crates/paimon/src/table/io_cache_routing_tests.rs index 9c17de67c..7fdc1e076 100644 --- a/crates/paimon/src/table/io_cache_routing_tests.rs +++ b/crates/paimon/src/table/io_cache_routing_tests.rs @@ -18,20 +18,28 @@ //! io-cache routing of the files that FileSystemCatalog tables and their writers produce. use std::collections::HashMap; +use std::sync::atomic::{AtomicUsize, Ordering}; use std::sync::Arc; use arrow_array::{Array, Int32Array, LargeBinaryArray, RecordBatch}; use arrow_schema::{DataType as ArrowDataType, Field as ArrowField, Schema as ArrowSchema}; +use axum::extract::State; +use axum::routing::get; +use axum::{Json, Router}; use bytes::Bytes; use futures::TryStreamExt; use super::table_write::tests::make_batch; -use super::{Table, TableCommit, TableWrite}; -use crate::catalog::{Catalog, FileSystemCatalog, Identifier, LoadedTable}; +use super::{RESTEnv, Table, TableCommit, TableWrite}; +use crate::api::rest_api::RESTApi; +use crate::api::{GetTableResponse, GetTableTokenResponse}; +use crate::catalog::{Catalog, FileSystemCatalog, FusePathResolver, Identifier, LoadedTable}; use crate::common::{CatalogOptions, Options}; use crate::io::oss_test_server::TestOss; -use crate::io::FileIOBuilder; -use crate::spec::{BlobDescriptor, BlobType, DataType, IntType, Schema, SchemaBuilder}; +use crate::io::{FileIOBuilder, FileIOCacheContext}; +use crate::spec::{ + BlobDescriptor, BlobType, DataType, IntType, Schema, SchemaBuilder, SchemaChange, +}; const WAREHOUSE: &str = "bkt/warehouse"; const FILE_NAME: &str = "data-123e4567-e89b-12d3-a456-426614174000-1.parquet"; @@ -222,13 +230,15 @@ async fn object_table_reads_objects_from_origin() { assert_eq!(cache.take_requests(), Vec::::new()); } +/// Routing that lets data files be written to and read from the target. +const DATA_TARGET: [(&str, &str); 2] = [ + ("io-cache.policy", "meta,read,write"), + ("io-cache.whitelist", "data"), +]; + /// A catalog whose data files may be written to and read from the target. async fn data_target_catalog(origin: &TestOss, cache: &TestOss) -> FileSystemCatalog { - let options = [ - ("io-cache.policy", "meta,read,write"), - ("io-cache.whitelist", "data"), - ]; - catalog(origin, cache, &options).await + catalog(origin, cache, &DATA_TARGET).await } fn int_schema() -> SchemaBuilder { @@ -469,3 +479,183 @@ async fn blob_descriptor_sources_are_read_from_origin() { assert_eq!(payloads(&origin_table).await, [b"NEW".to_vec()], "{name}"); } } + +/// Creates `name` with `data-file.prefix=older-`, commits a row, then persists `newer-`. +async fn table_with_altered_prefix(catalog: &FileSystemCatalog, name: &str) -> (Table, String) { + let schema = int_schema() + .option("data-file.prefix", "older-") + .build() + .unwrap(); + let table = create_table(catalog, name, schema).await; + let files = write_and_commit(&table, &make_batch(vec![1], vec![10])).await; + let [file] = files.as_slice() else { + panic!("expected one data file, got {files:?}"); + }; + assert!(file.starts_with("older-"), "{file}"); + let change = SchemaChange::set_option("data-file.prefix".to_string(), "newer-".to_string()); + catalog + .alter_table(table.identifier(), vec![change], false) + .await + .unwrap(); + (table, file.clone()) +} + +/// The request that lists the schemas of `table`. +fn schema_listing(table: &Table) -> String { + format!( + "LIST {}/schema/", + table.location().strip_prefix("oss://").unwrap() + ) +} + +#[tokio::test] +async fn altered_data_file_prefix_keeps_routing_after_reloads() { + let origin = TestOss::start().await; + let cache = TestOss::start_sharing(&origin).await; + let catalog = data_target_catalog(&origin, &cache).await; + let (table, older) = table_with_altered_prefix(&catalog, "altered_prefix").await; + let identifier = table.identifier().clone(); + + // Fresh loads only see the latest schema in their options, as after a restart. + let LoadedTable::Paimon(table) = catalog.load_table(&identifier).await.unwrap() else { + panic!("expected a Paimon table"); + }; + let newer = write_and_commit(&table, &make_batch(vec![2], vec![20])).await; + assert!( + newer.iter().all(|file| file.starts_with("newer-")), + "{newer:?}" + ); + let restarted = + FileSystemCatalog::new(io_cache_options(&origin, &cache, &DATA_TARGET)).unwrap(); + let table = restarted.get_table(&identifier).await.unwrap(); + cache.take_requests(); + origin.take_requests(); + + assert_eq!(ids(&table).await, [1, 2]); + let (cache_requests, origin_requests) = (cache.take_requests(), origin.take_requests()); + // The first request that may use io-cache loads the schemas, from origin. + let listing = schema_listing(&table); + assert!(origin_requests.contains(&listing), "{origin_requests:?}"); + assert!(!cache_requests.contains(&listing), "{cache_requests:?}"); + for file in newer.iter().chain([&older]) { + let methods_on_target = methods(&cache_requests, file); + assert!( + methods_on_target.iter().any(|m| m == "GET"), + "{file}: {methods_on_target:?}" + ); + assert_eq!( + methods(&origin_requests, file), + Vec::::new(), + "{file}" + ); + } + + // Without io-cache routing a table never loads its schema history. + let mut plain = Options::new(); + plain.set(CatalogOptions::WAREHOUSE, format!("oss://{WAREHOUSE}")); + for (key, value) in plain_options(&origin) { + plain.set(key, value); + } + let table = FileSystemCatalog::new(plain) + .unwrap() + .get_table(&identifier) + .await + .unwrap(); + origin.take_requests(); + assert_eq!(ids(&table).await, [1, 2]); + let origin_requests = origin.take_requests(); + assert!(!origin_requests.contains(&listing), "{origin_requests:?}"); +} + +/// The token a REST catalog vends, and the number of token requests it served. +type TokenState = (Arc>, Arc); + +async fn io_cache_token( + State((token, requests)): State, +) -> Json { + requests.fetch_add(1, Ordering::SeqCst); + let now = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_millis() as i64; + Json(GetTableTokenResponse { + token: token.as_ref().clone(), + expires_at_millis: Some(now + 24 * 3_600_000), + }) +} + +#[tokio::test] +async fn rest_tables_keep_routing_for_prefixes_of_older_schemas() { + let origin = TestOss::start().await; + let cache = TestOss::start_sharing(&origin).await; + let catalog = data_target_catalog(&origin, &cache).await; + let (table, older) = table_with_altered_prefix(&catalog, "rest_prefix").await; + let older = format!("{}/bucket-0/{older}", table.location()); + + // The REST catalog vends io-cache routing with the table token and returns the latest schema. + let token: HashMap = io_cache_options(&origin, &cache, &DATA_TARGET) + .to_map() + .clone(); + let token_requests = Arc::new(AtomicUsize::new(0)); + let app = Router::new() + .route( + "/v1/databases/db/tables/rest_prefix/token", + get(io_cache_token), + ) + .with_state((Arc::new(token), token_requests.clone())); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let address = listener.local_addr().unwrap(); + let server = tokio::spawn(async move { axum::serve(listener, app).await.unwrap() }); + let mut options = Options::new(); + options.set(CatalogOptions::URI, format!("http://{address}")); + options.set(CatalogOptions::TOKEN_PROVIDER, "bear"); + options.set(CatalogOptions::TOKEN, "test-token"); + let api = Arc::new(RESTApi::new(options.clone(), false).await.unwrap()); + let mut response: GetTableResponse = serde_json::from_value(serde_json::json!({ + "id": "rest-prefix-uuid", + "database": "db", + "name": "rest_prefix", + "path": table.location(), + "isExternal": false, + "schemaId": 1, + })) + .unwrap(); + let latest = int_schema() + .option("data-file.prefix", "newer-") + .build() + .unwrap(); + response.schema = Some(latest); + let fuse_resolver = Arc::new(FusePathResolver::new(api.options())); + let file_io_cache = FileIOCacheContext::from_props(options.to_map()).unwrap(); + origin.take_requests(); + cache.take_requests(); + let rest_table = RESTEnv::build_table( + table.identifier(), + response, + api, + options, + true, + None, + file_io_cache, + fuse_resolver, + ) + .await + .unwrap(); + // Loading reads no schemas; its only token request is the one RESTTokenFileIO makes anyway. + assert_eq!(origin.take_requests(), Vec::::new()); + assert_eq!(cache.take_requests(), Vec::::new()); + assert_eq!(token_requests.load(Ordering::SeqCst), 1); + + let input = rest_table.file_io().new_input(&older).unwrap(); + input.read().await.unwrap(); + let file = older.rsplit('/').next().unwrap(); + assert_eq!(methods(&cache.take_requests(), file), ["GET"]); + let origin_requests = origin.take_requests(); + assert_eq!(methods(&origin_requests, file), Vec::::new()); + assert!( + origin_requests.contains(&schema_listing(&table)), + "{origin_requests:?}" + ); + assert_eq!(token_requests.load(Ordering::SeqCst), 1); + server.abort(); +} diff --git a/crates/paimon/src/table/mod.rs b/crates/paimon/src/table/mod.rs index cfad6191c..1045eeb6b 100644 --- a/crates/paimon/src/table/mod.rs +++ b/crates/paimon/src/table/mod.rs @@ -290,7 +290,7 @@ pub use vector_search_builder::VectorSearchBuilder; pub use write_builder::WriteBuilder; use crate::catalog::{validate_branch_name, Identifier, DEFAULT_MAIN_BRANCH}; -use crate::io::FileIO; +use crate::io::{FileIO, SchemaFilePrefixes}; use crate::spec::{ CoreOptions, DataField, Snapshot, TableSchema, SCAN_SNAPSHOT_ID_OPTION, SCAN_TAG_NAME_OPTION, SCAN_TIMESTAMP_MILLIS_OPTION, SCAN_TIMESTAMP_OPTION, SCAN_VERSION_OPTION, @@ -350,6 +350,23 @@ fn table_file_io(file_io: &FileIO, schema: &TableSchema) -> FileIO { file_io.with_file_prefixes([options.data_file_prefix(), options.changelog_file_prefix()]) } +// Data files committed before a prefix option changed keep using io-cache; the schemas are read +// from origin on the first request that may use io-cache. +fn with_schema_history(file_io: &FileIO, schema_manager: &SchemaManager) -> FileIO { + let schemas = schema_manager.with_file_io(file_io.origin_only()); + file_io.with_schema_file_prefixes(SchemaFilePrefixes::new(move || { + let schemas = schemas.clone(); + async move { + let schemas = schemas.list_all().await?; + let prefixes = schemas.iter().flat_map(|schema| { + let options = CoreOptions::new(schema.options()); + [options.data_file_prefix(), options.changelog_file_prefix()].map(String::from) + }); + Ok(prefixes.collect()) + } + })) +} + impl Table { /// Create a new table. pub fn new( @@ -361,6 +378,7 @@ impl Table { ) -> Self { let file_io = table_file_io(&file_io, &schema); let schema_manager = SchemaManager::new(file_io.clone(), location.clone()); + let file_io = with_schema_history(&file_io, &schema_manager); let branch = DEFAULT_MAIN_BRANCH.to_string(); Self { file_io, @@ -404,6 +422,7 @@ impl Table { } else { schema_manager.with_branch(&branch) }; + let file_io = with_schema_history(&file_io, &schema_manager); let branch_reference = branch != DEFAULT_MAIN_BRANCH; Ok(Self { @@ -830,8 +849,14 @@ impl Table { } else { schema_manager.with_branch(branch) }; + let file_io = table_file_io(&self.file_io, &schema); + let file_io = if branch == self.branch { + file_io + } else { + with_schema_history(&file_io, &schema_manager) + }; Ok(Self { - file_io: table_file_io(&self.file_io, &schema), + file_io, schema, schema_manager, branch: branch.to_string(), @@ -1142,6 +1167,7 @@ impl Table { } else { schema_manager.with_branch(&branch) }; + table.file_io = with_schema_history(&table.file_io, &table.schema_manager); let mut options = table.schema.options().clone(); options.insert("branch".to_string(), branch); table.schema = table.schema.copy_with_replaced_options(options); @@ -1162,7 +1188,7 @@ impl Table { let mut options = schema.options().clone(); options.insert("branch".to_string(), branch.clone()); Ok(Self { - file_io: self.file_io.clone(), + file_io: with_schema_history(&self.file_io, &schema_manager), identifier: self.identifier.clone(), location: self.location.clone(), schema: schema.copy_with_replaced_options(options), diff --git a/crates/paimon/src/table/schema_manager.rs b/crates/paimon/src/table/schema_manager.rs index f727df68d..1ae3c8815 100644 --- a/crates/paimon/src/table/schema_manager.rs +++ b/crates/paimon/src/table/schema_manager.rs @@ -72,6 +72,15 @@ impl SchemaManager { Self::new(self.file_io.clone(), branch_path) } + /// This manager reading through `file_io`, sharing the cache of loaded schemas. + pub(crate) fn with_file_io(&self, file_io: FileIO) -> Self { + Self { + file_io, + table_path: self.table_path.clone(), + cache: self.cache.clone(), + } + } + /// Path to a specific schema file (e.g. `{table_path}/schema/schema-0`). pub fn schema_path(&self, schema_id: i64) -> String { format!("{}/{}{}", self.schema_directory(), SCHEMA_PREFIX, schema_id) From 11c86138ceb71a71934662f4853eba8f053915dc Mon Sep 17 00:00:00 2001 From: Sun Dapeng Date: Sun, 25 Oct 2026 02:55:09 +0800 Subject: [PATCH 15/21] [core] Accept only the contract's file types in io-cache.whitelist io-cache.whitelist and io-cache.routes reuse FileType::parse_whitelist, which since main split data into dedicated and structured data also accepts local-cache's structured-data selector. That value is not part of the io-cache contract (meta, data, bucket-index, global-index, file-index or *), so a whitelist naming it would route on this client but not on others. Parse io-cache file types through a wrapper that treats structured-data like any other unknown name: it is logged and ignored. data still selects both dedicated and structured data files. test_structured_data_is_not_an_io_cache_file_type pins it: whitelist=data routes Parquet and .blob files, whitelist=structured-data routes neither, structured-data,meta only routes metadata, and a route of structured-data alone is skipped like other malformed rules. --- crates/paimon/src/io/cache_routing.rs | 36 +++++++++++++++++++++++++-- 1 file changed, 34 insertions(+), 2 deletions(-) diff --git a/crates/paimon/src/io/cache_routing.rs b/crates/paimon/src/io/cache_routing.rs index 240d65dba..b61d0dfce 100644 --- a/crates/paimon/src/io/cache_routing.rs +++ b/crates/paimon/src/io/cache_routing.rs @@ -170,7 +170,7 @@ impl IoCacheRouting { default_endpoint, enabled: flag(props, IO_CACHE_ENABLED, false), policy: parse_policy(props.get(IO_CACHE_POLICY).map(String::as_str)), - whitelist: FileType::parse_whitelist( + whitelist: parse_file_types( &props .get(IO_CACHE_WHITELIST) .map_or("*".to_string(), |value| value.to_ascii_lowercase()), @@ -393,6 +393,21 @@ fn is_valid_target_name(name: &str) -> bool { .all(|c| c.is_ascii_lowercase() || c.is_ascii_digit() || c == '-') } +/// The contract's file types; unknown names, including local-cache's `structured-data`, are ignored. +fn parse_file_types(value: &str) -> HashSet { + value + .split(',') + .filter(|name| { + let local_cache_only = name.trim() == "structured-data"; + if local_cache_only { + log::warn!("Unknown io-cache file type 'structured-data'; use data"); + } + !local_cache_only + }) + .flat_map(FileType::parse_whitelist) + .collect() +} + /// Rules `types=target` separated by `;`; malformed rules are skipped. fn parse_routes(value: &str) -> Vec<(HashSet, String)> { value @@ -400,7 +415,7 @@ fn parse_routes(value: &str) -> Vec<(HashSet, String)> { .filter_map(|rule| { let (types, target) = rule.trim().split_once('=')?; let target = target.trim().to_ascii_lowercase(); - let types = FileType::parse_whitelist(&types.to_ascii_lowercase()); + let types = parse_file_types(&types.to_ascii_lowercase()); (!target.is_empty() && !types.is_empty()).then_some((types, target)) }) .collect() @@ -702,6 +717,23 @@ mod tests { assert_endpoint(&malformed, OpClass::Read, DATA_PATH, OSS); } + #[test] + fn test_structured_data_is_not_an_io_cache_file_type() { + let blob = "dt=1/bucket-0/data-{uuid}-0.blob"; + let data = single(&["io-cache.whitelist=data"]); + assert_endpoint(&data, OpClass::Read, DATA_PATH, CACHE); + assert_endpoint(&data, OpClass::Read, blob, CACHE); + let structured = single(&["io-cache.whitelist=structured-data"]); + assert_endpoint(&structured, OpClass::Read, DATA_PATH, OSS); + assert_endpoint(&structured, OpClass::Read, blob, OSS); + let with_meta = single(&["io-cache.whitelist=structured-data,meta"]); + assert_endpoint(&with_meta, OpClass::Read, DATA_PATH, OSS); + assert_endpoint(&with_meta, OpClass::Read, MANIFEST_PATH, CACHE); + // a route of only unknown types is malformed and skipped + let route = multi(&["io-cache.routes=structured-data=cluster;*=accel"]); + assert_endpoint(&route, OpClass::Read, DATA_PATH, ACCEL); + } + #[test] fn test_data_files_named_like_metadata() { let manifest = multi(&["data-file.prefix=manifest-"]); From d686bfa86072ff6f9a9649bd2d9b40e3bfcbf5cc Mon Sep 17 00:00:00 2001 From: Sun Dapeng Date: Sun, 25 Oct 2026 02:59:21 +0800 Subject: [PATCH 16/21] [core] Read the sources of BLOB descriptor fields from the origin A column listed in blob-descriptor-field keeps the BlobDescriptor the writer supplied; the object it points to is not copied into the table and is owned by whoever wrote it. Reads still resolved those descriptors through the table's io-cache routing, so a source with a Paimon-shaped name such as oss://bkt/import/data--1.png that was replaced on origin could be served from a stale target copy. With data-evolution.row-sidecar.enabled, RowSidecarWriter also copies these payloads into the .row file while writing, which could persist that stale copy, like the BLOB writers fixed earlier. The data evolution reader now resolves descriptor fields, and RowSidecarWriter reads their sources, through the origin-only view of the table's FileIO. Managed packs, .blob files and .row files written by the table keep their io-cache routing. blob_descriptor_fields_read_their_sources_from_origin writes a descriptor whose source is NEW on origin and OLD on the target into a descriptor field, with and without a ROW sidecar: the scan returns NEW, every source GET reaches origin and none the target, and the table's data files are still written to and read from the target. Without the reader change the scan returns OLD; without the sidecar change the sidecar write reads the source from the target. --- .../paimon/src/table/data_evolution_reader.rs | 2 + .../src/table/io_cache_routing_tests.rs | 53 +++++++++++++++++++ crates/paimon/src/table/row_sidecar.rs | 3 +- 3 files changed, 57 insertions(+), 1 deletion(-) diff --git a/crates/paimon/src/table/data_evolution_reader.rs b/crates/paimon/src/table/data_evolution_reader.rs index 6b0bff0c7..0d7f0ea32 100644 --- a/crates/paimon/src/table/data_evolution_reader.rs +++ b/crates/paimon/src/table/data_evolution_reader.rs @@ -1397,6 +1397,8 @@ async fn resolve_descriptor_columns( file_io: &FileIO, limiter: &BlobReadLimiter, ) -> crate::Result { + // Descriptor fields reference objects the table does not own, which may be replaced. + let file_io = file_io.origin_only(); resolve_descriptor_columns_with( batch, blob_descriptor_fields, diff --git a/crates/paimon/src/table/io_cache_routing_tests.rs b/crates/paimon/src/table/io_cache_routing_tests.rs index 7fdc1e076..04b1cf7c6 100644 --- a/crates/paimon/src/table/io_cache_routing_tests.rs +++ b/crates/paimon/src/table/io_cache_routing_tests.rs @@ -659,3 +659,56 @@ async fn rest_tables_keep_routing_for_prefixes_of_older_schemas() { assert_eq!(token_requests.load(Ordering::SeqCst), 1); server.abort(); } + +#[tokio::test] +async fn blob_descriptor_fields_read_their_sources_from_origin() { + let origin = TestOss::start().await; + let cache = TestOss::start_sharing(&origin).await; + let catalog = data_target_catalog(&origin, &cache).await; + // The table keeps the descriptors; a ROW sidecar also copies the payloads when writing. + for (name, row_sidecar) in [ + ("descriptor_field", "false"), + ("descriptor_sidecar", "true"), + ] { + let schema = blob_schema() + .option("data-evolution.enabled", "true") + .option("row-tracking.enabled", "true") + .option("blob-descriptor-field", "payload") + .option("data-evolution.row-sidecar.enabled", row_sidecar) + .build() + .unwrap(); + let table = create_table(&catalog, name, schema).await; + let source = + format!("oss://bkt/import/{name}/data-123e4567-e89b-12d3-a456-426614174000-1.png"); + let key = source.strip_prefix("oss://").unwrap(); + put(&origin, &source, Bytes::from("NEW")).await; + cache.retain(key, Bytes::from("OLD")); + origin.take_requests(); + cache.take_requests(); + + let descriptor = BlobDescriptor::new(source.clone(), 0, 3).serialize(); + let files = write_and_commit(&table, &blob_batch(&[1], &[&descriptor])).await; + assert_eq!(payloads(&table).await, [b"NEW".to_vec()], "{name}"); + let (origin_requests, cache_requests) = (origin.take_requests(), cache.take_requests()); + let source_reads = methods(&origin_requests, key); + assert!( + source_reads.iter().all(|m| m == "GET"), + "{name}: {source_reads:?}" + ); + assert!(!source_reads.is_empty(), "{name}"); + assert_eq!( + methods(&cache_requests, key), + Vec::::new(), + "{name}" + ); + // The table's own data files keep using the target. + for file in &files { + assert_reads_and_writes(&cache_requests, file); + assert_eq!( + methods(&origin_requests, file), + Vec::::new(), + "{file}" + ); + } + } +} diff --git a/crates/paimon/src/table/row_sidecar.rs b/crates/paimon/src/table/row_sidecar.rs index a337b9c9d..b2c8cf171 100644 --- a/crates/paimon/src/table/row_sidecar.rs +++ b/crates/paimon/src/table/row_sidecar.rs @@ -33,6 +33,7 @@ use std::sync::Arc; pub(super) struct RowSidecarWriter { pub file_name: String, pub writer: Box, + /// Reads descriptor field sources from origin: other writers own them and may replace them. io: FileIO, descriptor_fields: HashSet, view_fields: HashSet, @@ -64,7 +65,7 @@ impl RowSidecarWriter { Ok(Self { file_name: format!("{file_name}.row"), writer: with_write_resources(writer, resources), - io: io.clone(), + io: io.origin_only(), descriptor_fields: options.blob_descriptor_fields(), view_fields: options.blob_view_fields(), }) From bbc4a6932ce2fe7532bd919c83d217a532d77d1b Mon Sep 17 00:00:00 2001 From: Sun Dapeng Date: Mon, 26 Oct 2026 01:48:52 +0800 Subject: [PATCH 17/21] [core] Route only the files under a table's own location through io-cache A primary-key table with blob-descriptor-field keeps the descriptors in its rows and resolves them through managed_blob_reader::resolve_batch with the table's FileIO, so an object an import job owns, named like a Paimon file, was read from the target, which may still hold a copy the origin replaced. The previous round fixed this at three call sites; every other reader of descriptors, such as BlobReader over the table's FileIO behind the C and Go bindings, still routed them by name. Data files the table writes under data-file.external-paths went to the target too, which serves the table's location and may not serve those paths at all. A table's view of the FileIO now carries the table's location, and a request through it uses io-cache only for a path under that location; anything else uses the origin. A FileIO that serves no table, such as the one BlobReader builds from storage options, cannot tell who owns a URI taken from row data, so UriInput and BlobReader read such URIs from the origin. The managed packs and .blob files a table writes lie under its location and keep their routing. The three call-site views of the previous round are removed. --- crates/paimon/src/arrow/format/blob/writer.rs | 3 +- crates/paimon/src/io/file_io.rs | 36 ++++- crates/paimon/src/io/uri_reader.rs | 2 +- crates/paimon/src/table/blob_resolver.rs | 3 +- .../paimon/src/table/data_evolution_reader.rs | 2 - .../src/table/io_cache_routing_tests.rs | 144 +++++++++++++++++- crates/paimon/src/table/mod.rs | 18 ++- crates/paimon/src/table/row_sidecar.rs | 3 +- 8 files changed, 191 insertions(+), 20 deletions(-) diff --git a/crates/paimon/src/arrow/format/blob/writer.rs b/crates/paimon/src/arrow/format/blob/writer.rs index c4623ec9a..c51e33e41 100644 --- a/crates/paimon/src/arrow/format/blob/writer.rs +++ b/crates/paimon/src/arrow/format/blob/writer.rs @@ -136,7 +136,6 @@ impl FormatWriterFactory for BlobWriterFactory { pub(crate) struct BlobFormatWriter { writer: Box, - /// Reads descriptor sources from origin: other writers own them and may replace them. file_io: Option, kind: BlobFieldKind, path: String, @@ -181,7 +180,7 @@ impl BlobFormatWriter { } Ok(Self { writer, - file_io: file_io.map(|file_io| file_io.origin_only()), + file_io, kind, path: output.location().to_string(), field_name: field.map_or_else(String::new, |field| field.name().to_string()), diff --git a/crates/paimon/src/io/file_io.rs b/crates/paimon/src/io/file_io.rs index 99bdb4b6d..9df45d143 100644 --- a/crates/paimon/src/io/file_io.rs +++ b/crates/paimon/src/io/file_io.rs @@ -139,6 +139,8 @@ pub struct FileIO { file_prefixes: Arc<[String]>, /// Prefixes of the table's persisted schemas, loaded once a request may use io-cache. schema_file_prefixes: Option>, + /// Location of the table this view serves; requests for anything outside it use origin. + table_location: Option>, } type LoadFilePrefixes = Box BoxFuture<'static, Result>> + Send + Sync>; @@ -560,6 +562,33 @@ impl FileIO { file_io } + /// A view of this FileIO serving the table at `location`. + pub(crate) fn with_table_location(&self, location: &str) -> Self { + let mut file_io = self.clone(); + file_io.table_location = Some(location.trim_end_matches('/').into()); + file_io + } + + /// The view to read URIs taken from row data through. A FileIO that serves no table cannot + /// tell who owns them, and other writers may replace what they own, so it uses origin. + pub(crate) fn for_row_data(&self) -> Self { + if self.table_location.is_some() { + self.clone() + } else { + self.origin_only() + } + } + + /// Whether requests on `path` must use origin: a table's view routes only its own files. + fn origin_only_for(&self, path: &str) -> bool { + self.origin_only + || self.table_location.as_deref().is_some_and(|location| { + !path + .strip_prefix(location) + .is_some_and(|rest| rest.starts_with('/')) + }) + } + /// Whether some request through this view may use an io-cache target. pub(crate) async fn routes_to_io_cache(&self) -> crate::Result { if self.origin_only { @@ -592,7 +621,7 @@ impl FileIO { file_prefixes: &[String], ) -> crate::Result<(RoutedOperator, String)> { let (op, relative_path) = self.storage()?.create_routed(path, file_prefixes)?; - let op = if self.origin_only { + let op = if self.origin_only_for(path) { op.without_cache() } else { op @@ -645,7 +674,7 @@ impl FileIO { let (op, relative_path, _) = resolve_provider_routed( provider.as_ref(), path, - self.origin_only, + self.origin_only_for(path), &file_prefixes, ) .await?; @@ -666,7 +695,7 @@ impl FileIO { let (op, relative_path, namespace) = resolve_provider_routed( provider.as_ref(), path, - self.origin_only, + self.origin_only_for(path), &file_prefixes, ) .await?; @@ -1426,6 +1455,7 @@ impl FileIOBuilder { origin_only: false, file_prefixes: Arc::default(), schema_file_prefixes: None, + table_location: None, }) } } diff --git a/crates/paimon/src/io/uri_reader.rs b/crates/paimon/src/io/uri_reader.rs index d6e25cfed..cbdd0daab 100644 --- a/crates/paimon/src/io/uri_reader.rs +++ b/crates/paimon/src/io/uri_reader.rs @@ -42,7 +42,7 @@ impl UriInput { if http { Ok(Self::Http(HttpReader { uri: uri.into() })) } else { - Ok(Self::File(file_io.new_input(uri)?)) + Ok(Self::File(file_io.for_row_data().new_input(uri)?)) } } diff --git a/crates/paimon/src/table/blob_resolver.rs b/crates/paimon/src/table/blob_resolver.rs index 863c65533..fcb852665 100644 --- a/crates/paimon/src/table/blob_resolver.rs +++ b/crates/paimon/src/table/blob_resolver.rs @@ -80,7 +80,8 @@ impl BlobReader { None => FileIO::from_path(descriptor.uri()) .and_then(|builder| builder.with_props(self.storage_options.iter()).build()) .map_err(|error| blob_error_with_context(error, &[0], Some(descriptor.uri())))?, - }; + } + .for_row_data(); Ok(BlobStream { file_io, diff --git a/crates/paimon/src/table/data_evolution_reader.rs b/crates/paimon/src/table/data_evolution_reader.rs index 0d7f0ea32..6b0bff0c7 100644 --- a/crates/paimon/src/table/data_evolution_reader.rs +++ b/crates/paimon/src/table/data_evolution_reader.rs @@ -1397,8 +1397,6 @@ async fn resolve_descriptor_columns( file_io: &FileIO, limiter: &BlobReadLimiter, ) -> crate::Result { - // Descriptor fields reference objects the table does not own, which may be replaced. - let file_io = file_io.origin_only(); resolve_descriptor_columns_with( batch, blob_descriptor_fields, diff --git a/crates/paimon/src/table/io_cache_routing_tests.rs b/crates/paimon/src/table/io_cache_routing_tests.rs index 04b1cf7c6..675e65ce0 100644 --- a/crates/paimon/src/table/io_cache_routing_tests.rs +++ b/crates/paimon/src/table/io_cache_routing_tests.rs @@ -30,7 +30,7 @@ use bytes::Bytes; use futures::TryStreamExt; use super::table_write::tests::make_batch; -use super::{RESTEnv, Table, TableCommit, TableWrite}; +use super::{BlobReader, RESTEnv, Table, TableCommit, TableWrite}; use crate::api::rest_api::RESTApi; use crate::api::{GetTableResponse, GetTableTokenResponse}; use crate::catalog::{Catalog, FileSystemCatalog, FusePathResolver, Identifier, LoadedTable}; @@ -712,3 +712,145 @@ async fn blob_descriptor_fields_read_their_sources_from_origin() { } } } + +#[tokio::test] +async fn primary_key_descriptor_fields_read_their_sources_from_origin() { + let origin = TestOss::start().await; + let cache = TestOss::start_sharing(&origin).await; + let catalog = data_target_catalog(&origin, &cache).await; + // `payload` keeps descriptors of an import job's objects; `picture` is copied into managed packs. + let schema = pk_blob_schema() + .column("picture", DataType::Blob(BlobType::new())) + .option("blob-descriptor-field", "payload") + .build() + .unwrap(); + let table = create_table(&catalog, "pk_descriptor_field", schema).await; + let source = "oss://bkt/import/pk/data-123e4567-e89b-12d3-a456-426614174000-1.png"; + let key = source.strip_prefix("oss://").unwrap(); + put(&origin, source, Bytes::from("NEW")).await; + cache.retain(key, Bytes::from("OLD")); + let descriptor = BlobDescriptor::new(source.to_string(), 0, 3).serialize(); + let batch = RecordBatch::try_new( + Arc::new(ArrowSchema::new(vec![ + ArrowField::new("id", ArrowDataType::Int32, false), + ArrowField::new("payload", ArrowDataType::LargeBinary, true), + ArrowField::new("picture", ArrowDataType::LargeBinary, true), + ])), + vec![ + Arc::new(Int32Array::from(vec![1])), + Arc::new(LargeBinaryArray::from(vec![descriptor.as_slice()])), + Arc::new(LargeBinaryArray::from(vec![b"pixels".as_slice()])), + ], + ) + .unwrap(); + write_and_commit(&table, &batch).await; + origin.take_requests(); + cache.take_requests(); + + let batches = read(&table).await; + let values = |name: &str| { + batches + .iter() + .flat_map(|batch| { + let column = batch.column_by_name(name).unwrap(); + let column = column.as_any().downcast_ref::().unwrap(); + (0..column.len()) + .map(|row| column.value(row).to_vec()) + .collect::>() + }) + .collect::>() + }; + assert_eq!(values("payload"), [b"NEW".to_vec()]); + assert_eq!(values("picture"), [b"pixels".to_vec()]); + let (origin_requests, cache_requests) = (origin.take_requests(), cache.take_requests()); + assert_eq!(methods(&cache_requests, key), Vec::::new()); + assert!(methods(&origin_requests, key).iter().any(|m| m == "GET")); + // The table's own managed packs keep using the target. + assert!(methods(&cache_requests, ".managed.blob") + .iter() + .any(|m| m == "GET")); + assert_eq!( + methods(&origin_requests, ".managed.blob"), + Vec::::new() + ); +} + +#[tokio::test] +async fn blob_readers_read_objects_outside_the_table_from_origin() { + let origin = TestOss::start().await; + let cache = TestOss::start_sharing(&origin).await; + let catalog = data_target_catalog(&origin, &cache).await; + let table = create_table(&catalog, "blob_reader", pk_blob_schema().build().unwrap()).await; + let owned = format!( + "{}/bucket-0/data-123e4567-e89b-12d3-a456-426614174000.managed.blob", + table.location() + ); + put(&origin, &owned, Bytes::from("OWN")).await; + let external = "oss://bkt/import/reader/data-123e4567-e89b-12d3-a456-426614174000-1.png"; + let key = external.strip_prefix("oss://").unwrap(); + put(&origin, external, Bytes::from("NEW")).await; + cache.retain(key, Bytes::from("OLD")); + let descriptors = [owned.as_str(), external] + .map(|uri| BlobDescriptor::new(uri.to_string(), 0, 3).serialize()); + let expected = [b"OWN".to_vec(), b"NEW".to_vec()]; + let owned_key = owned.strip_prefix("oss://").unwrap(); + origin.take_requests(); + cache.take_requests(); + + // The C binding's paimon_table_new_blob_reader reads through the table's FileIO this way. + let reader = BlobReader::from_file_io(table.file_io().clone()); + assert_eq!(reader.read_blobs(&descriptors).await.unwrap(), expected); + let mut stream = reader.open_blob(&descriptors[1]).unwrap(); + assert_eq!(stream.read(16).await.unwrap(), b"NEW"); + let (origin_requests, cache_requests) = (origin.take_requests(), cache.take_requests()); + assert_eq!(methods(&cache_requests, key), Vec::::new()); + assert!(methods(&origin_requests, key).iter().any(|m| m == "GET")); + assert!(methods(&cache_requests, owned_key) + .iter() + .any(|m| m == "GET")); + assert_eq!(methods(&origin_requests, owned_key), Vec::::new()); + + // A reader built from storage options serves no table, so it never uses the target. + let options = io_cache_options(&origin, &cache, &DATA_TARGET) + .to_map() + .clone(); + let reader = BlobReader::new(options); + assert_eq!(reader.read_blobs(&descriptors).await.unwrap(), expected); + let mut stream = reader.open_blob(&descriptors[1]).unwrap(); + assert_eq!(stream.read(16).await.unwrap(), b"NEW"); + assert_eq!(cache.take_requests(), Vec::::new()); +} + +#[tokio::test] +async fn data_files_under_external_paths_use_the_origin() { + let origin = TestOss::start().await; + let cache = TestOss::start_sharing(&origin).await; + let catalog = data_target_catalog(&origin, &cache).await; + // The target serves the table's location; files the table writes elsewhere stay on origin. + let schema = int_schema() + .option("data-file.external-paths", "oss://bkt/external") + .option("data-file.external-paths.strategy", "round-robin") + .build() + .unwrap(); + let table = create_table(&catalog, "external_paths", schema).await; + let files = write_and_commit(&table, &make_batch(vec![1], vec![10])).await; + assert_eq!(ids(&table).await, [1]); + let (origin_requests, cache_requests) = (origin.take_requests(), cache.take_requests()); + for file in &files { + let external = "bkt/external/"; + let on_origin: Vec<_> = origin_requests + .iter() + .filter(|request| request.contains(file.as_str())) + .collect(); + assert!( + on_origin.iter().all(|request| request.contains(external)), + "{on_origin:?}" + ); + assert_reads_and_writes(&origin_requests, file); + assert_eq!( + methods(&cache_requests, file), + Vec::::new(), + "{file}" + ); + } +} diff --git a/crates/paimon/src/table/mod.rs b/crates/paimon/src/table/mod.rs index 1045eeb6b..2bf048ddc 100644 --- a/crates/paimon/src/table/mod.rs +++ b/crates/paimon/src/table/mod.rs @@ -341,13 +341,15 @@ pub struct Table { } // Format Table files may be replaced in place under the same name, so they never use io-cache; -// other tables tell io-cache routing the prefixes their data files are named with. -fn table_file_io(file_io: &FileIO, schema: &TableSchema) -> FileIO { +// other tables tell io-cache routing their location and the prefixes their data files are named with. +fn table_file_io(file_io: &FileIO, location: &str, schema: &TableSchema) -> FileIO { let options = CoreOptions::new(schema.options()); if options.is_format_table() { return file_io.origin_only(); } - file_io.with_file_prefixes([options.data_file_prefix(), options.changelog_file_prefix()]) + file_io + .with_table_location(location) + .with_file_prefixes([options.data_file_prefix(), options.changelog_file_prefix()]) } // Data files committed before a prefix option changed keep using io-cache; the schemas are read @@ -376,7 +378,7 @@ impl Table { schema: TableSchema, rest_env: Option, ) -> Self { - let file_io = table_file_io(&file_io, &schema); + let file_io = table_file_io(&file_io, &location, &schema); let schema_manager = SchemaManager::new(file_io.clone(), location.clone()); let file_io = with_schema_history(&file_io, &schema_manager); let branch = DEFAULT_MAIN_BRANCH.to_string(); @@ -415,7 +417,7 @@ impl Table { identifier.validate()?; validate_branch_name(&branch)?; schema.validate_resolved_structure()?; - let file_io = table_file_io(&file_io, &schema); + let file_io = table_file_io(&file_io, &location, &schema); let schema_manager = SchemaManager::new(file_io.clone(), location.clone()); let schema_manager = if branch == DEFAULT_MAIN_BRANCH { schema_manager @@ -776,7 +778,7 @@ impl Table { }); let schema = self.schema.copy_with_options(extra); Self { - file_io: table_file_io(&self.file_io, &schema), + file_io: table_file_io(&self.file_io, &self.location, &schema), identifier: self.identifier.clone(), location: self.location.clone(), schema, @@ -832,7 +834,7 @@ impl Table { .iter() .any(|key| schema.options().get(*key) != self.schema.options().get(*key)); return Ok(Self { - file_io: table_file_io(&self.file_io, &schema), + file_io: table_file_io(&self.file_io, &self.location, &schema), schema, travel_snapshot: if changed { None @@ -849,7 +851,7 @@ impl Table { } else { schema_manager.with_branch(branch) }; - let file_io = table_file_io(&self.file_io, &schema); + let file_io = table_file_io(&self.file_io, &self.location, &schema); let file_io = if branch == self.branch { file_io } else { diff --git a/crates/paimon/src/table/row_sidecar.rs b/crates/paimon/src/table/row_sidecar.rs index b2c8cf171..a337b9c9d 100644 --- a/crates/paimon/src/table/row_sidecar.rs +++ b/crates/paimon/src/table/row_sidecar.rs @@ -33,7 +33,6 @@ use std::sync::Arc; pub(super) struct RowSidecarWriter { pub file_name: String, pub writer: Box, - /// Reads descriptor field sources from origin: other writers own them and may replace them. io: FileIO, descriptor_fields: HashSet, view_fields: HashSet, @@ -65,7 +64,7 @@ impl RowSidecarWriter { Ok(Self { file_name: format!("{file_name}.row"), writer: with_write_resources(writer, resources), - io: io.origin_only(), + io: io.clone(), descriptor_fields: options.blob_descriptor_fields(), view_fields: options.blob_view_fields(), }) From 04517aa7038f70930d4a94c2fb11e19cbc62a04e Mon Sep 17 00:00:00 2001 From: Sun Dapeng Date: Mon, 26 Oct 2026 01:48:52 +0800 Subject: [PATCH 18/21] [core] Give every table and branch copy its io-cache view in one place Copying a REST table to a branch kept the source table's FileIO, so that its provider and FUSE mount carry over, and attached the branch's schema history. A branch whose schema exists only in the catalog has no schema directory, so the prefixes of its current schema were lost and its data files were read from the origin. table_view now builds the view for every constructor and every copy that moves to another branch: origin only for a Format Table, otherwise the table's location, the prefixes of the schema it uses and the lazily loaded prefixes of the branch's persisted schemas. The REST branch copy passes the schema the catalog returned. Copies within a branch keep the loaded history and add the prefixes of their options. --- .../src/table/io_cache_routing_tests.rs | 256 +++++++++++++++++- crates/paimon/src/table/mod.rs | 45 ++- 2 files changed, 286 insertions(+), 15 deletions(-) diff --git a/crates/paimon/src/table/io_cache_routing_tests.rs b/crates/paimon/src/table/io_cache_routing_tests.rs index 675e65ce0..6fd80b073 100644 --- a/crates/paimon/src/table/io_cache_routing_tests.rs +++ b/crates/paimon/src/table/io_cache_routing_tests.rs @@ -23,14 +23,14 @@ use std::sync::Arc; use arrow_array::{Array, Int32Array, LargeBinaryArray, RecordBatch}; use arrow_schema::{DataType as ArrowDataType, Field as ArrowField, Schema as ArrowSchema}; -use axum::extract::State; +use axum::extract::{Path, State}; use axum::routing::get; use axum::{Json, Router}; use bytes::Bytes; use futures::TryStreamExt; use super::table_write::tests::make_batch; -use super::{BlobReader, RESTEnv, Table, TableCommit, TableWrite}; +use super::{BlobReader, BranchManager, RESTEnv, Table, TableCommit, TableWrite}; use crate::api::rest_api::RESTApi; use crate::api::{GetTableResponse, GetTableTokenResponse}; use crate::catalog::{Catalog, FileSystemCatalog, FusePathResolver, Identifier, LoadedTable}; @@ -821,6 +821,258 @@ async fn blob_readers_read_objects_outside_the_table_from_origin() { assert_eq!(cache.take_requests(), Vec::::new()); } +#[tokio::test] +async fn every_table_copy_routes_files_named_with_the_schema_prefix() { + let origin = TestOss::start().await; + let cache = TestOss::start_sharing(&origin).await; + let catalog = data_target_catalog(&origin, &cache).await; + let schema = int_schema() + .option("data-file.prefix", "custom-") + .build() + .unwrap(); + let table = create_table(&catalog, "copies", schema).await; + let files = write_and_commit(&table, &make_batch(vec![1], vec![10])).await; + let snapshot = table + .snapshot_manager() + .get_latest_snapshot() + .await + .unwrap() + .unwrap(); + table.tag_manager().create("base", &snapshot).await.unwrap(); + BranchManager::new(table.file_io().clone(), table.location().into()) + .create_branch_from_tag("dev", "base") + .await + .unwrap(); + let option = |key: &str, value: &str| HashMap::from([(key.to_string(), value.to_string())]); + let file_io = FileIOBuilder::new("oss") + .with_props( + io_cache_options(&origin, &cache, &DATA_TARGET) + .to_map() + .iter(), + ) + .build() + .unwrap(); + let copies = [ + ( + "from_resolved_schema", + Table::from_resolved_schema( + file_io, + table.identifier().clone(), + table.location().to_string(), + table.schema().clone(), + "main", + ) + .unwrap(), + ), + ( + "copy_with_options", + table.copy_with_options(option("read.batch-size", "16")), + ), + ( + "snapshot", + table + .copy_with_time_travel(option("scan.snapshot-id", "1")) + .await + .unwrap(), + ), + ( + "tag", + table + .copy_with_time_travel(option("scan.tag-name", "base")) + .await + .unwrap(), + ), + ( + "copy_with_resolved_snapshot", + table.copy_with_resolved_snapshot(&snapshot).await.unwrap(), + ), + ( + "copy_with_pinned_snapshot", + table.copy_with_pinned_snapshot(Some(&snapshot)), + ), + ( + "copy_with_resolved_schema", + table + .copy_with_resolved_schema(table.schema().clone(), "main") + .unwrap(), + ), + ( + "copy_with_resolved_schema on a branch", + table + .copy_with_resolved_schema(table.schema().clone(), "dev") + .unwrap(), + ), + ( + "copy_with_branch", + table.copy_with_branch("dev").await.unwrap(), + ), + ( + "branch option", + table + .copy_with_time_travel(option("branch", "dev")) + .await + .unwrap(), + ), + ]; + for (name, copy) in copies { + origin.take_requests(); + cache.take_requests(); + assert_eq!(ids(©).await, [1], "{name}"); + let (origin_requests, cache_requests) = (origin.take_requests(), cache.take_requests()); + for file in &files { + let on_target = methods(&cache_requests, file); + assert!( + on_target.iter().any(|m| m == "GET"), + "{name}: {on_target:?}" + ); + assert_eq!( + methods(&origin_requests, file), + Vec::::new(), + "{name}" + ); + } + } +} + +/// Tables a REST catalog serves by name, and the io-cache token it vends for each of them. +type RestTables = ( + Arc>, + Arc>, +); + +async fn rest_table( + State((tables, _)): State, + Path((_, name)): Path<(String, String)>, +) -> Json { + Json(tables[&name].clone()) +} + +async fn rest_token(State((_, token)): State) -> Json { + Json(GetTableTokenResponse { + token: token.as_ref().clone(), + expires_at_millis: Some(i64::MAX / 2), + }) +} + +fn rest_table_response(id: &str, name: &str, location: &str, prefix: &str) -> GetTableResponse { + let mut response: GetTableResponse = serde_json::from_value(serde_json::json!({ + "id": id, + "database": "db", + "name": name, + "path": location, + "isExternal": false, + "schemaId": 0, + })) + .unwrap(); + response.schema = Some( + int_schema() + .option("data-file.prefix", prefix) + .build() + .unwrap(), + ); + response +} + +#[tokio::test] +async fn rest_branch_copies_route_files_named_with_the_branch_prefix() { + let origin = TestOss::start().await; + let cache = TestOss::start_sharing(&origin).await; + // The catalog keeps the branch schema; storage has no schema directory for it. + let location = format!("oss://{WAREHOUSE}/db.db/rest_branch"); + let file = "branch-123e4567-e89b-12d3-a456-426614174000-1.parquet"; + let path = format!("{location}/bucket-0/{file}"); + put(&origin, &path, parquet(&[1])).await; + let tables = HashMap::from([ + ( + "rest_branch".to_string(), + rest_table_response("main-uuid", "rest_branch", &location, "main-"), + ), + ( + "rest_branch$branch_dev".to_string(), + rest_table_response( + "branch-uuid", + "rest_branch$branch_dev", + &location, + "branch-", + ), + ), + ]); + let token = io_cache_options(&origin, &cache, &DATA_TARGET) + .to_map() + .clone(); + let app = Router::new() + .route("/v1/databases/:db/tables/:name", get(rest_table)) + .route("/v1/databases/:db/tables/:name/token", get(rest_token)) + .with_state((Arc::new(tables.clone()), Arc::new(token))); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let address = listener.local_addr().unwrap(); + let server = tokio::spawn(async move { axum::serve(listener, app).await.unwrap() }); + let mut options = Options::new(); + options.set(CatalogOptions::URI, format!("http://{address}")); + options.set(CatalogOptions::TOKEN_PROVIDER, "bear"); + options.set(CatalogOptions::TOKEN, "test-token"); + let api = Arc::new(RESTApi::new(options.clone(), false).await.unwrap()); + let fuse_resolver = Arc::new(FusePathResolver::new(api.options())); + let file_io_cache = FileIOCacheContext::from_props(options.to_map()).unwrap(); + let load = |name: &str, identifier: Identifier| { + let (response, api, options) = (tables[name].clone(), api.clone(), options.clone()); + let (file_io_cache, fuse_resolver) = (file_io_cache.clone(), fuse_resolver.clone()); + async move { + RESTEnv::build_table( + &identifier, + response, + api, + options, + true, + None, + file_io_cache, + fuse_resolver, + ) + .await + } + }; + let main = load("rest_branch", Identifier::new("db", "rest_branch")) + .await + .unwrap(); + let branch_identifier = Identifier::new("db", "rest_branch") + .with_branch("dev") + .unwrap(); + let loaded_branch = load("rest_branch$branch_dev", branch_identifier) + .await + .unwrap(); + let copies = [ + ("loaded branch", loaded_branch), + ( + "copy_with_branch", + main.copy_with_branch("dev").await.unwrap(), + ), + ( + "branch option", + main.copy_with_time_travel(HashMap::from([("branch".to_string(), "dev".to_string())])) + .await + .unwrap(), + ), + ]; + for (name, branch) in copies { + origin.take_requests(); + cache.take_requests(); + branch + .file_io() + .new_input(&path) + .unwrap() + .read() + .await + .unwrap(); + assert_eq!(methods(&cache.take_requests(), file), ["GET"], "{name}"); + assert_eq!( + methods(&origin.take_requests(), file), + Vec::::new(), + "{name}" + ); + } + server.abort(); +} + #[tokio::test] async fn data_files_under_external_paths_use_the_origin() { let origin = TestOss::start().await; diff --git a/crates/paimon/src/table/mod.rs b/crates/paimon/src/table/mod.rs index 2bf048ddc..e6ac8977c 100644 --- a/crates/paimon/src/table/mod.rs +++ b/crates/paimon/src/table/mod.rs @@ -352,6 +352,17 @@ fn table_file_io(file_io: &FileIO, location: &str, schema: &TableSchema) -> File .with_file_prefixes([options.data_file_prefix(), options.changelog_file_prefix()]) } +// The FileIO of a table whose schemas `schema_manager` keeps; every constructor, and every copy +// that moves to another branch, takes its FileIO from here. +fn table_view( + file_io: &FileIO, + location: &str, + schema: &TableSchema, + schema_manager: &SchemaManager, +) -> FileIO { + with_schema_history(&table_file_io(file_io, location, schema), schema_manager) +} + // Data files committed before a prefix option changed keep using io-cache; the schemas are read // from origin on the first request that may use io-cache. fn with_schema_history(file_io: &FileIO, schema_manager: &SchemaManager) -> FileIO { @@ -378,9 +389,11 @@ impl Table { schema: TableSchema, rest_env: Option, ) -> Self { - let file_io = table_file_io(&file_io, &location, &schema); - let schema_manager = SchemaManager::new(file_io.clone(), location.clone()); - let file_io = with_schema_history(&file_io, &schema_manager); + let schema_manager = SchemaManager::new( + table_file_io(&file_io, &location, &schema), + location.clone(), + ); + let file_io = table_view(&file_io, &location, &schema, &schema_manager); let branch = DEFAULT_MAIN_BRANCH.to_string(); Self { file_io, @@ -417,14 +430,16 @@ impl Table { identifier.validate()?; validate_branch_name(&branch)?; schema.validate_resolved_structure()?; - let file_io = table_file_io(&file_io, &location, &schema); - let schema_manager = SchemaManager::new(file_io.clone(), location.clone()); + let schema_manager = SchemaManager::new( + table_file_io(&file_io, &location, &schema), + location.clone(), + ); let schema_manager = if branch == DEFAULT_MAIN_BRANCH { schema_manager } else { schema_manager.with_branch(&branch) }; - let file_io = with_schema_history(&file_io, &schema_manager); + let file_io = table_view(&file_io, &location, &schema, &schema_manager); let branch_reference = branch != DEFAULT_MAIN_BRANCH; Ok(Self { @@ -851,11 +866,10 @@ impl Table { } else { schema_manager.with_branch(branch) }; - let file_io = table_file_io(&self.file_io, &self.location, &schema); let file_io = if branch == self.branch { - file_io + table_file_io(&self.file_io, &self.location, &schema) } else { - with_schema_history(&file_io, &schema_manager) + table_view(&self.file_io, &self.location, &schema, &schema_manager) }; Ok(Self { file_io, @@ -1162,14 +1176,19 @@ impl Table { source: None, }); } - table.file_io = self.file_io.clone(); - let schema_manager = SchemaManager::new(table.file_io.clone(), table.location.clone()); + let schema_manager = SchemaManager::new(self.file_io.clone(), table.location.clone()); table.schema_manager = if branch == DEFAULT_MAIN_BRANCH { schema_manager } else { schema_manager.with_branch(&branch) }; - table.file_io = with_schema_history(&table.file_io, &table.schema_manager); + // The branch schema may exist only in the catalog, so its prefixes come from it too. + table.file_io = table_view( + &self.file_io, + &table.location, + &table.schema, + &table.schema_manager, + ); let mut options = table.schema.options().clone(); options.insert("branch".to_string(), branch); table.schema = table.schema.copy_with_replaced_options(options); @@ -1190,7 +1209,7 @@ impl Table { let mut options = schema.options().clone(); options.insert("branch".to_string(), branch.clone()); Ok(Self { - file_io: with_schema_history(&self.file_io, &schema_manager), + file_io: table_view(&self.file_io, &self.location, &schema, &schema_manager), identifier: self.identifier.clone(), location: self.location.clone(), schema: schema.copy_with_replaced_options(options), From dd16a42f5e48603b1360854f63a65a42cc600a25 Mon Sep 17 00:00:00 2001 From: Sun Dapeng Date: Tue, 27 Oct 2026 00:25:03 +0800 Subject: [PATCH 19/21] [core] Send the status request of a ROW sidecar through the io-cache meta rule A selective read takes the size of the ROW sidecar it reads before reading it. That status request went through the origin operator, while the read of the same sidecar followed the read rule, so with io-cache.policy meta,read the HEAD of every sidecar stayed on origin. immutable_row_sidecar_size now resolves the routed operators like any other read and stats through OpClass::Meta, as get_status does. The local size cache keeps its origin-based key. The new routing test reads two rows of a 100-row file and checks that both the HEAD and the GET of the sidecar reach the target; it fails without the change. --- crates/paimon/src/io/file_io.rs | 4 +- .../src/table/io_cache_routing_tests.rs | 51 ++++++++++++++++++- 2 files changed, 53 insertions(+), 2 deletions(-) diff --git a/crates/paimon/src/io/file_io.rs b/crates/paimon/src/io/file_io.rs index 9df45d143..563dc28a8 100644 --- a/crates/paimon/src/io/file_io.rs +++ b/crates/paimon/src/io/file_io.rs @@ -868,7 +868,8 @@ impl FileIO { /// Return the size of an immutable ROW sidecar without repeating remote /// status requests for subsequent reads through the same cache context. pub(crate) async fn immutable_row_sidecar_size(&self, path: &str) -> Result { - let (op, relative_path, cache_key) = self.create_with_cache_path(path).await?; + // The status request follows the meta rule like `get_status`; the cache key stays origin's. + let (op, relative_path, cache_key) = self.resolve_routed(path).await?; if let Some(size) = cache_key .as_deref() .and_then(|key| self.file_format_metadata_cache.row_sidecar_size(key)) @@ -876,6 +877,7 @@ impl FileIO { return Ok(size); } let size = op + .operator(OpClass::Meta) .stat(relative_path.as_ref()) .await .map_err(|error| { diff --git a/crates/paimon/src/table/io_cache_routing_tests.rs b/crates/paimon/src/table/io_cache_routing_tests.rs index 6fd80b073..3c0773de1 100644 --- a/crates/paimon/src/table/io_cache_routing_tests.rs +++ b/crates/paimon/src/table/io_cache_routing_tests.rs @@ -30,7 +30,7 @@ use bytes::Bytes; use futures::TryStreamExt; use super::table_write::tests::make_batch; -use super::{BlobReader, BranchManager, RESTEnv, Table, TableCommit, TableWrite}; +use super::{BlobReader, BranchManager, RESTEnv, RowRange, Table, TableCommit, TableWrite}; use crate::api::rest_api::RESTApi; use crate::api::{GetTableResponse, GetTableTokenResponse}; use crate::catalog::{Catalog, FileSystemCatalog, FusePathResolver, Identifier, LoadedTable}; @@ -660,6 +660,55 @@ async fn rest_tables_keep_routing_for_prefixes_of_older_schemas() { server.abort(); } +#[tokio::test] +async fn row_sidecars_of_selective_reads_use_the_target() { + let origin = TestOss::start().await; + let cache = TestOss::start_sharing(&origin).await; + let catalog = catalog( + &origin, + &cache, + &[ + ("io-cache.policy", "meta,read"), + ("io-cache.whitelist", "data"), + ], + ) + .await; + let schema = int_schema() + .option("row-tracking.enabled", "true") + .option("data-evolution.enabled", "true") + .option("data-evolution.row-sidecar.enabled", "true") + .build() + .unwrap(); + let table = create_table(&catalog, "row_sidecar", schema).await; + let ids: Vec = (0..100).collect(); + write_and_commit(&table, &make_batch(ids.clone(), ids)).await; + origin.take_requests(); + cache.take_requests(); + + // Two of 100 rows: the read takes the sidecar's size, then its rows. + let mut builder = table.new_read_builder(); + builder.with_row_ranges(vec![RowRange::new(5, 6)]); + let plan = builder.new_scan().plan().await.unwrap(); + let batches: Vec = builder + .new_read() + .unwrap() + .to_arrow(plan.splits()) + .unwrap() + .try_collect() + .await + .unwrap(); + assert_eq!(batches.iter().map(RecordBatch::num_rows).sum::(), 2); + let (origin_requests, cache_requests) = (origin.take_requests(), cache.take_requests()); + assert_eq!(methods(&origin_requests, ".row"), Vec::::new()); + let on_target = methods(&cache_requests, ".row"); + assert_eq!( + on_target.first().map(String::as_str), + Some("HEAD"), + "{on_target:?}" + ); + assert!(on_target.iter().any(|m| m == "GET"), "{on_target:?}"); +} + #[tokio::test] async fn blob_descriptor_fields_read_their_sources_from_origin() { let origin = TestOss::start().await; From 62c44c780f980df8de0f72866947d60bef5b027b Mon Sep 17 00:00:00 2001 From: Sun Dapeng Date: Tue, 27 Oct 2026 00:37:51 +0800 Subject: [PATCH 20/21] [core] Cover the earlier prefixes a branch inherits in io-cache routing tests A branch created from a tag copies the schemas of the files it inherits, so its own schema history already holds the earlier data-file.prefix of main. The new routing test alters the prefix, creates a branch from a tag taken after the change and reads both files through copy_with_branch and the branch option; the file named with the earlier prefix reaches the target. It fails when the view carries no schema history. A branch whose schemas exist only in the catalog cannot read such a file at all: the reader loads the file's schema from the branch's own schema directory before it opens the file. So no fallback to main's schemas is needed. --- .../src/table/io_cache_routing_tests.rs | 55 +++++++++++++++++++ 1 file changed, 55 insertions(+) diff --git a/crates/paimon/src/table/io_cache_routing_tests.rs b/crates/paimon/src/table/io_cache_routing_tests.rs index 3c0773de1..47a908646 100644 --- a/crates/paimon/src/table/io_cache_routing_tests.rs +++ b/crates/paimon/src/table/io_cache_routing_tests.rs @@ -500,6 +500,61 @@ async fn table_with_altered_prefix(catalog: &FileSystemCatalog, name: &str) -> ( (table, file.clone()) } +/// A branch reads the files it inherited under an earlier prefix through the target: creating it +/// copied the schemas they were written with. +#[tokio::test] +async fn branches_route_files_inherited_under_an_earlier_prefix() { + let origin = TestOss::start().await; + let cache = TestOss::start_sharing(&origin).await; + let catalog = data_target_catalog(&origin, &cache).await; + let (table, older) = table_with_altered_prefix(&catalog, "inherited_prefix").await; + let identifier = table.identifier().clone(); + let table = catalog.get_table(&identifier).await.unwrap(); + let newer = write_and_commit(&table, &make_batch(vec![2], vec![20])).await; + let snapshot = table + .snapshot_manager() + .get_latest_snapshot() + .await + .unwrap() + .unwrap(); + table.tag_manager().create("base", &snapshot).await.unwrap(); + BranchManager::new(table.file_io().clone(), table.location().into()) + .create_branch_from_tag("dev", "base") + .await + .unwrap(); + let branches = [ + ( + "copy_with_branch", + table.copy_with_branch("dev").await.unwrap(), + ), + ( + "branch option", + table + .copy_with_time_travel(HashMap::from([("branch".to_string(), "dev".to_string())])) + .await + .unwrap(), + ), + ]; + for (name, branch) in branches { + origin.take_requests(); + cache.take_requests(); + assert_eq!(ids(&branch).await, [1, 2], "{name}"); + let (cache_requests, origin_requests) = (cache.take_requests(), origin.take_requests()); + for file in newer.iter().chain([&older]) { + let on_target = methods(&cache_requests, file); + assert!( + on_target.iter().any(|m| m == "GET"), + "{name}: {file}: {on_target:?}" + ); + assert_eq!( + methods(&origin_requests, file), + Vec::::new(), + "{name}: {file}" + ); + } + } +} + /// The request that lists the schemas of `table`. fn schema_listing(table: &Table) -> String { format!( From cac6733e25cbf447d97439d85e11dd9a6594598c Mon Sep 17 00:00:00 2001 From: Sun Dapeng Date: Tue, 27 Oct 2026 04:49:03 +0800 Subject: [PATCH 21/21] [core] Normalize bare io-cache target endpoints --- crates/paimon/src/io/cache_routing.rs | 36 +++++++++++++++++++++++---- 1 file changed, 31 insertions(+), 5 deletions(-) diff --git a/crates/paimon/src/io/cache_routing.rs b/crates/paimon/src/io/cache_routing.rs index b61d0dfce..57f07f7c6 100644 --- a/crates/paimon/src/io/cache_routing.rs +++ b/crates/paimon/src/io/cache_routing.rs @@ -358,7 +358,7 @@ fn parse_targets(props: &HashMap) -> Vec { .get(endpoint_key) .map(|endpoint| endpoint.trim()) .filter(|endpoint| !endpoint.is_empty()) - .map(String::from), + .map(normalize_target_endpoint), path_style_access: flag(props, &path_style_key, false), name, } @@ -385,6 +385,15 @@ fn parse_targets(props: &HashMap) -> Vec { .collect() } +/// Match the Java and Python clients: a target host without a scheme uses HTTPS. +fn normalize_target_endpoint(endpoint: &str) -> String { + if endpoint.contains("://") { + endpoint.to_string() + } else { + format!("https://{endpoint}") + } +} + /// `[a-z][a-z0-9-]*`. fn is_valid_target_name(name: &str) -> bool { name.starts_with(|c: char| c.is_ascii_lowercase()) @@ -1037,8 +1046,14 @@ mod tests { (IO_CACHE_WHITELIST, "DATA"), ]); let data = "oss://b/t/bucket-0/data-123e4567-e89b-12d3-a456-426614174000-1.parquet"; - assert_eq!(route(&routing, OpClass::Read, data).unwrap(), "cache"); - assert_eq!(route(&routing, OpClass::Meta, data).unwrap(), "cache"); + assert_eq!( + route(&routing, OpClass::Read, data).unwrap(), + "https://cache" + ); + assert_eq!( + route(&routing, OpClass::Meta, data).unwrap(), + "https://cache" + ); assert_eq!(route(&routing, OpClass::Origin, data).unwrap(), "origin"); assert_eq!( route(&routing, OpClass::Read, "oss://b/t/manifest/manifest-1").unwrap(), @@ -1084,19 +1099,30 @@ mod tests { #[test] fn test_shorthand_target_settings() { - let routing = routing(&[ + let with_scheme = routing(&[ (OSS_ENDPOINT, "origin"), (IO_CACHE_ENABLED, "true"), (IO_CACHE_ENDPOINT, "http://cache"), ("io-cache.target.default.path-style-access", "true"), (IO_CACHE_POLICY, "read"), ]); - let [target] = routing.targets() else { + let [target] = with_scheme.targets() else { panic!("expected one target"); }; assert_eq!(target.name, DEFAULT_TARGET); assert_eq!(target.endpoint.as_deref(), Some("http://cache")); assert!(target.path_style_access); + + let routing = routing(&[ + (OSS_ENDPOINT, "origin"), + (IO_CACHE_ENABLED, "true"), + (IO_CACHE_ENDPOINT, "cache.example.com"), + (IO_CACHE_POLICY, "read"), + ]); + assert_eq!( + routing.targets()[0].endpoint.as_deref(), + Some("https://cache.example.com") + ); } #[test]