From b0f9e4887f2dcabb977bac43becf00ee10e715c6 Mon Sep 17 00:00:00 2001 From: Anant Vindal Date: Wed, 11 Nov 2026 12:55:25 +0530 Subject: [PATCH 01/13] feat: resolve field type conflicts across ingestors in distributed mode When several ingestors see the same new field with different types in the same window (e.g. `a: Int64` on one, `a: Utf8` on another), each wrote its own schema and parquet, and the strict schema merge on the querier failed, making the stream unqueryable. Ingestors now ask a single schema authority before converting arrow files whose fields they have not resolved yet: - Authority is the Prism node if one is registered, otherwise the querier (lowest live node id). New endpoint POST /internal/schema/resolve, intra-cluster only, gated by the new admin-only ResolveSchema action. - The authority keeps a write-once field registry per stream. The first type claimed wins; a conflicting type is renamed to `_` (chained up to 3 times). Reserved columns (p_timestamp, time and custom partitions, etc.) are never renamed and come back as unresolvable. - The registry is persisted to `.stream/.schema` (with a `parseable.registry_version` metadata key) before replying, so an answer never changes, and is rebuilt from it on restart. - Ingestors persist the answers in `.field_resolution.json` in the stream's staging directory, rename columns at conversion with a zero-copy schema swap, align their in-memory schema with the authority's types, and only advertise resolved fields in their staged schema file. - If the authority is unreachable, conversion of the affected files is deferred to the next cycle. After P_SCHEMA_RESOLVE_MAX_DEFERRED_CYCLES (default 3) failed cycles, ingestion for that stream returns 503 with Retry-After until resolution succeeds. - At startup, deferred files in processing directories go back to the staging root for the periodic cycle to retry. The query side stays strict: fetch_schema still rejects conflicting types, so no read-time casts are introduced. Tolerant merges are used only on the write side. New options: P_SCHEMA_RESOLVE_MODE (off | prefer | require, default require; ignored in standalone mode), P_SCHEMA_RESOLVE_MAX_DEFERRED_CYCLES, P_SCHEMA_RESOLVE_CONCURRENCY. Also: - the schema key hash now separates field name and type, to avoid collisions - get_all_schemas skips unreadable schema files instead of panicking --- src/cli.rs | 28 +- src/event/mod.rs | 34 +- src/handlers/http/ingest.rs | 8 + src/handlers/http/mod.rs | 2 + src/handlers/http/modal/query_server.rs | 19 +- src/handlers/http/modal/utils/ingest_utils.rs | 9 + src/lib.rs | 1 + .../metastores/object_store_metastore.rs | 12 +- src/option.rs | 25 + src/parseable/staging/reader.rs | 85 ++- src/parseable/staging/writer.rs | 10 + src/parseable/streams.rs | 289 ++++++++- src/rbac/role.rs | 3 + src/schema_registry/authority.rs | 423 +++++++++++++ src/schema_registry/client.rs | 273 ++++++++ src/schema_registry/ingestor.rs | 581 ++++++++++++++++++ src/schema_registry/mod.rs | 151 +++++ src/schema_registry/protocol.rs | 143 +++++ src/schema_registry/registry.rs | 314 ++++++++++ src/schema_registry/view.rs | 332 ++++++++++ src/storage/object_storage.rs | 27 +- src/utils/mod.rs | 1 + 22 files changed, 2729 insertions(+), 41 deletions(-) create mode 100644 src/schema_registry/authority.rs create mode 100644 src/schema_registry/client.rs create mode 100644 src/schema_registry/ingestor.rs create mode 100644 src/schema_registry/mod.rs create mode 100644 src/schema_registry/protocol.rs create mode 100644 src/schema_registry/registry.rs create mode 100644 src/schema_registry/view.rs diff --git a/src/cli.rs b/src/cli.rs index 440769263..e46d62b71 100644 --- a/src/cli.rs +++ b/src/cli.rs @@ -26,7 +26,7 @@ use crate::connectors::kafka::config::KafkaConfig; use crate::{ oidc::{self, OpenidConfig}, - option::{Compression, Mode, validation}, + option::{Compression, Mode, SchemaResolveMode, validation}, storage::{AzureBlobConfig, FSConfig, GcsConfig, S3Config}, }; @@ -558,6 +558,32 @@ pub struct Options { )] pub querier_endpoint: String, + #[arg( + long, + env = "P_SCHEMA_RESOLVE_MODE", + default_value = "require", + value_parser = validation::schema_resolve_mode, + help = "Distributed ingestors: reconcile new field types with the querier before writing parquet (off, prefer, require)" + )] + pub schema_resolve_mode: SchemaResolveMode, + + #[arg( + long, + env = "P_SCHEMA_RESOLVE_MAX_DEFERRED_CYCLES", + default_value = "3", + help = "Failed resolution cycles after which a stream rejects ingestion with 503 (0 = never)" + )] + pub schema_resolve_max_deferred_cycles: u32, + + #[arg( + long, + env = "P_SCHEMA_RESOLVE_CONCURRENCY", + default_value = "8", + value_parser = value_parser!(u16).range(1..), + help = "Maximum concurrent schema resolution requests from one ingestor" + )] + pub schema_resolve_concurrency: u16, + #[arg( long, env = "P_PRISM_ENDPOINT", diff --git a/src/event/mod.rs b/src/event/mod.rs index f6d0c435e..42fb0260f 100644 --- a/src/event/mod.rs +++ b/src/event/mod.rs @@ -87,6 +87,9 @@ impl Event { let stream = PARSEABLE.get_stream(&self.stream_name, &self.tenant_id)?; let _ingestion_guard = stream.ingestion_guard()?; + stream.check_schema_resolution().map_err(|reason| { + EventError::SchemaResolutionBlocked(self.stream_name.clone(), reason) + })?; if self.is_first_event { commit_schema(&self.stream_name, self.rb.schema(), &self.tenant_id)?; @@ -145,13 +148,20 @@ impl Event { } } +/// Key of the arrow file a batch is written to. It covers names *and* types: a batch whose +/// field changed type (after schema resolution) must never be appended to a file whose IPC +/// header declares the old type. Each name/type is framed so `["ab","c"]` != `["a","bc"]`. pub fn get_schema_key(fields: &[Arc]) -> String { + use std::hash::{Hash, Hasher}; // Fields must be sorted let mut hasher = xxhash_rust::xxh3::Xxh3::new(); for field in fields.iter().sorted_by_key(|v| v.name()) { hasher.update(field.name().as_bytes()); + hasher.update(&[0xFF]); + field.data_type().hash(&mut hasher); + hasher.update(&[0xFE]); } - let hash = hasher.digest(); + let hash = hasher.finish(); format!("{hash:x}") } @@ -165,10 +175,26 @@ pub fn commit_schema( // global lock while waiting for the per-stream metadata lock can deadlock // concurrent ingestion that needs the map to finish processing an event. let stream = PARSEABLE.get_stream(stream_name, tenant_id)?; + let view = stream.resolution.view(); let mut metadata = stream.metadata.write().expect(LOCK_EXPECT); let map = &mut metadata.schema; + // A batch built against the schema from before a resolved type change can still carry + // the old type. Its arrow file is renamed at conversion, so it must not fail here. + let incoming: Vec> = schema + .fields() + .iter() + .filter(|f| { + !(map + .get(f.name()) + .is_some_and(|cur| cur.data_type() != f.data_type()) + && view + .renames + .contains_key(&(f.name().clone(), f.data_type().clone()))) + }) + .cloned() + .collect(); let current_schema = Schema::new(map.values().cloned().collect::()); - let schema = Schema::try_merge(vec![current_schema, schema.as_ref().clone()])?; + let schema = Schema::try_merge(vec![current_schema, Schema::new(incoming)])?; map.clear(); map.extend(schema.fields.iter().map(|f| (f.name().clone(), f.clone()))); Ok(()) @@ -188,5 +214,9 @@ pub mod error { ObjectStorage(#[from] ObjectStorageError), #[error(transparent)] StreamNotFound(#[from] StreamNotFound), + #[error( + "Stream {0} is temporarily not accepting data: its schema could not be reconciled with the cluster ({1}). Retry later" + )] + SchemaResolutionBlocked(String, String), } } diff --git a/src/handlers/http/ingest.rs b/src/handlers/http/ingest.rs index d808ea575..80ec36eb7 100644 --- a/src/handlers/http/ingest.rs +++ b/src/handlers/http/ingest.rs @@ -582,6 +582,8 @@ impl actix_web::ResponseError for PostError { StreamNotFound(_) | Event(EventError::StreamNotFound(_)) => StatusCode::NOT_FOUND, + Event(EventError::SchemaResolutionBlocked(_, _)) => StatusCode::SERVICE_UNAVAILABLE, + Event(_) | CreateStream(_) | CustomError(_) @@ -605,6 +607,12 @@ impl actix_web::ResponseError for PostError { .insert_header(ContentType::json()) .json(metastore_error.to_detail()) } + PostError::Event(EventError::SchemaResolutionBlocked(_, _)) => { + actix_web::HttpResponse::build(self.status_code()) + .insert_header(ContentType::plaintext()) + .insert_header((actix_web::http::header::RETRY_AFTER, "60")) + .body(self.to_string()) + } _ => actix_web::HttpResponse::build(self.status_code()) .insert_header(ContentType::plaintext()) .body(self.to_string()), diff --git a/src/handlers/http/mod.rs b/src/handlers/http/mod.rs index 6bf903095..441d6aa34 100644 --- a/src/handlers/http/mod.rs +++ b/src/handlers/http/mod.rs @@ -144,6 +144,8 @@ pub async fn fetch_schema( .get_all_schemas(stream_name, tenant_id) .await?; + // Strict on purpose: a type conflict must surface as an error, never be papered over + // by picking one type, which would make the query engine cast data at read time. let new_schema = Schema::try_merge(res)?; Ok(new_schema) } diff --git a/src/handlers/http/modal/query_server.rs b/src/handlers/http/modal/query_server.rs index 22dbbfd03..40c4f508d 100644 --- a/src/handlers/http/modal/query_server.rs +++ b/src/handlers/http/modal/query_server.rs @@ -22,7 +22,7 @@ use crate::handlers::airplane; use crate::handlers::http::cluster; use crate::handlers::http::logstream; use crate::handlers::http::max_event_payload_size; -use crate::handlers::http::middleware::{DisAllowRootUser, RouteExt}; +use crate::handlers::http::middleware::{DisAllowRootUser, IntraClusterRequest, RouteExt}; use crate::handlers::http::modal::initialize_hot_tier_metadata_on_startup; use crate::handlers::http::rbac::patch_user; use crate::handlers::http::{base_path, prism_base_path}; @@ -32,7 +32,7 @@ use crate::hottier::HotTierManager; use crate::hottier::HotTierMessage; use crate::hottier::hottier_runtime; use crate::rbac::role::Action; -use crate::{analytics, migration, storage}; +use crate::{analytics, migration, schema_registry, storage}; use actix_web::web::{ServiceConfig, resource}; use actix_web::{Scope, web}; use actix_web_prometheus::PrometheusMetrics; @@ -74,6 +74,7 @@ impl ParseableServer for QueryServer { .service(Server::get_alerts_webscope()) .service(Server::get_targets_webscope()) .service(Self::get_cluster_web_scope()) + .service(Self::get_schema_authority_scope()) .service(Server::get_demo_data_webscope()) .service(Server::get_alert_target_policy()), ) @@ -255,6 +256,20 @@ impl QueryServer { ) } + /// Intra-cluster schema authority: ingestors resolve new field types here. + pub fn get_schema_authority_scope() -> Scope { + web::scope("/internal/schema").service( + web::resource("/resolve") + .route( + web::post() + .to(schema_registry::authority::resolve_handler) + .authorize(Action::ResolveSchema) + .wrap(IntraClusterRequest), + ) + .app_data(web::JsonConfig::default().limit(1 << 20)), + ) + } + // get the logstream web scope pub fn get_logstream_webscope() -> Scope { web::scope("/logstream") diff --git a/src/handlers/http/modal/utils/ingest_utils.rs b/src/handlers/http/modal/utils/ingest_utils.rs index e428977b2..10ab755b5 100644 --- a/src/handlers/http/modal/utils/ingest_utils.rs +++ b/src/handlers/http/modal/utils/ingest_utils.rs @@ -28,6 +28,7 @@ use std::collections::HashMap; use tokio::sync::oneshot; use tracing::{error, instrument, warn}; +use crate::event::error::EventError; use crate::{ LOCK_EXPECT, event::{ @@ -540,6 +541,14 @@ pub fn validate_stream_for_ingestion( return Err(PostError::StreamBeingDeleted(stream_name.to_string())); } + // Reject early while the stream's schema cannot be reconciled with the cluster. + stream.check_schema_resolution().map_err(|reason| { + PostError::Event(EventError::SchemaResolutionBlocked( + stream_name.to_string(), + reason, + )) + })?; + // Validate that the stream's log source is compatible stream .get_log_source() diff --git a/src/lib.rs b/src/lib.rs index 58abee54d..929d51965 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -51,6 +51,7 @@ pub mod prism; pub mod query; pub mod rbac; mod response; +pub mod schema_registry; pub mod sse; mod static_schema; mod stats; diff --git a/src/metastore/metastores/object_store_metastore.rs b/src/metastore/metastores/object_store_metastore.rs index c8d43a4c7..7af9ab20c 100644 --- a/src/metastore/metastores/object_store_metastore.rs +++ b/src/metastore/metastores/object_store_metastore.rs @@ -1189,10 +1189,14 @@ impl Metastore for ObjectStoreMetastore { ) .await? .iter() - // we should be able to unwrap as we know the data is valid schema - .map(|byte_obj| { - serde_json::from_slice(byte_obj) - .unwrap_or_else(|_| panic!("got an invalid schema for stream: {stream_name}")) + .filter_map(|byte_obj| match serde_json::from_slice(byte_obj) { + Ok(schema) => Some(schema), + Err(err) => { + tracing::warn!( + "skipping unreadable schema file of stream {stream_name}: {err}" + ); + None + } }) .collect()) } diff --git a/src/option.rs b/src/option.rs index 53eb39fdc..42b15c97b 100644 --- a/src/option.rs +++ b/src/option.rs @@ -30,6 +30,19 @@ pub enum Mode { All, } +/// How an ingestor reconciles new field types with the schema authority. +/// Only distributed ingestors (`P_MODE=ingest`) use it; standalone mode has one schema. +#[derive(Debug, Default, Clone, Copy, PartialEq, Eq)] +pub enum SchemaResolveMode { + /// Escape hatch: every ingestor keeps its locally inferred types (legacy behaviour). + Off, + /// Resolve with the authority; fall back to local types only if the authority is too old. + Prefer, + /// Resolve with the authority; never convert unresolved fields (default). + #[default] + Require, +} + #[derive(Debug, thiserror::Error)] #[error( "Starting Standalone Mode is not permitted when Distributed Mode is enabled. Please restart the server with Distributed Mode enabled." @@ -148,6 +161,18 @@ pub mod validation { } } + pub fn schema_resolve_mode(s: &str) -> Result { + use super::SchemaResolveMode; + match s { + "off" => Ok(SchemaResolveMode::Off), + "prefer" => Ok(SchemaResolveMode::Prefer), + "require" => Ok(SchemaResolveMode::Require), + _ => Err( + "Invalid schema resolve mode, expected one of: off, prefer, require".to_string(), + ), + } + } + pub fn compression(s: &str) -> Result { match s { "uncompressed" => Ok(Compression::Uncompressed), diff --git a/src/parseable/staging/reader.rs b/src/parseable/staging/reader.rs index 00cd399da..8d084bcce 100644 --- a/src/parseable/staging/reader.rs +++ b/src/parseable/staging/reader.rs @@ -37,6 +37,7 @@ use tracing::{error, info_span}; use crate::{ event::DEFAULT_TIMESTAMP_KEY, + schema_registry::view::{ResolvedView, rename_batch}, utils::arrow::{adapt_batch, reverse}, }; @@ -44,8 +45,12 @@ use crate::{ /// Merges reverse readers while retaining the source-file validation result. pub struct MergedReverseRecordReader { pub readers: Vec>>>, + /// Resolved-name schema per reader (None = no rename needed). + renamed: Vec>>, pub readable_files: Vec, pub invalid_files: Vec, + /// Valid files whose columns cannot be renamed to the resolved schema. Left untouched. + pub unrenameable_files: Vec, } #[derive(Debug)] @@ -55,12 +60,16 @@ pub struct MergedForwardRecordReader { file_plans: Vec, pub readable_files: Vec, pub invalid_files: Vec, + /// Valid files whose columns cannot be renamed to the resolved schema. Left untouched. + pub unrenameable_files: Vec, } #[derive(Debug)] pub(crate) struct ForwardFilePlan { path: PathBuf, schema: Arc, + /// Resolved-name schema (None = no rename needed). + renamed: Option>, messages: Vec<(MessageHeader, usize, usize)>, record_batches: usize, } @@ -73,6 +82,14 @@ pub(crate) struct ForwardReaderLane { } impl ForwardFilePlan { + pub(crate) fn renamed_schema(&self) -> Option> { + self.renamed.clone() + } + + fn effective_schema(&self) -> &Schema { + self.renamed.as_deref().unwrap_or(self.schema.as_ref()) + } + /// Opens independent virtual Arrow streams over striped record batches. /// Every stream begins with the original schema and receives every /// dictionary update, preserving Arrow IPC decoder state. @@ -140,11 +157,18 @@ impl ForwardFilePlan { impl MergedForwardRecordReader { /// Opens valid Arrow files and excludes any crash-truncated tail message. + #[cfg(test)] pub fn try_new(file_paths: &[PathBuf]) -> Self { + Self::try_new_with_view(file_paths, &ResolvedView::default()) + } + + /// Like [`Self::try_new`], renaming columns as decided by the schema authority. + pub fn try_new_with_view(file_paths: &[PathBuf], view: &ResolvedView) -> Self { let _span = info_span!("open_forward_arrow_files", file_count = file_paths.len()).entered(); let mut file_plans = Vec::with_capacity(file_paths.len()); let mut readable_files = Vec::with_capacity(file_paths.len()); let mut invalid_files = Vec::new(); + let mut unrenameable_files = Vec::new(); for path in file_paths { match File::open(path) { Err(err) => { @@ -179,9 +203,19 @@ impl MergedForwardRecordReader { .iter() .filter(|(header, _, _)| *header == MessageHeader::RecordBatch) .count(); + let schema = schema_reader.schema(); + let renamed = match view.rename_schema(&schema) { + Ok(renamed) => renamed, + Err(err) => { + error!("Cannot apply resolved schema to {path:?}: {err}"); + unrenameable_files.push(path.clone()); + continue; + } + }; file_plans.push(ForwardFilePlan { path: path.clone(), - schema: schema_reader.schema(), + schema, + renamed, messages, record_batches, }); @@ -194,6 +228,7 @@ impl MergedForwardRecordReader { file_plans, readable_files, invalid_files, + unrenameable_files, } } @@ -205,24 +240,30 @@ impl MergedForwardRecordReader { self.file_plans.into_iter() } - /// Returns the union of schemas exposed by all readable Arrow streams. - pub fn merged_schema(&self) -> Schema { + /// Returns the union of (renamed) schemas exposed by all readable Arrow streams. + pub fn merged_schema(&self) -> Result { Schema::try_merge( self.file_plans .iter() - .map(|plan| plan.schema.as_ref().clone()), + .map(|plan| plan.effective_schema().clone()), ) - .unwrap() } } impl MergedReverseRecordReader { /// Opens valid Arrow files and separates files that fail structural validation. pub fn try_new(file_paths: &[PathBuf]) -> Self { + Self::try_new_with_view(file_paths, &ResolvedView::default()) + } + + /// Like [`Self::try_new`], renaming columns as decided by the schema authority. + pub fn try_new_with_view(file_paths: &[PathBuf], view: &ResolvedView) -> Self { let _span = info_span!("open_arrow_files", file_count = file_paths.len()).entered(); let mut readers = Vec::with_capacity(file_paths.len()); + let mut renamed = Vec::with_capacity(file_paths.len()); let mut readable_files = Vec::with_capacity(file_paths.len()); let mut invalid_files = Vec::new(); + let mut unrenameable_files = Vec::new(); for path in file_paths { match File::open(path) { Err(err) => { @@ -238,7 +279,16 @@ impl MergedReverseRecordReader { continue; } }; + let renamed_schema = match view.rename_schema(&reader.schema()) { + Ok(renamed_schema) => renamed_schema, + Err(err) => { + error!("Cannot apply resolved schema to {path:?}: {err}"); + unrenameable_files.push(path.clone()); + continue; + } + }; readers.push(reader); + renamed.push(renamed_schema); readable_files.push(path.clone()); } } @@ -246,8 +296,10 @@ impl MergedReverseRecordReader { Self { readers, + renamed, readable_files, invalid_files, + unrenameable_files, } } @@ -257,7 +309,15 @@ impl MergedReverseRecordReader { schema: Arc, time_partition: Option, ) -> impl Iterator> { - let adapted_readers = self.readers; + // Rename before adapting: `adapt_batch` matches columns by name and would + // otherwise fill a renamed column with nulls. + let adapted_readers = + self.readers + .into_iter() + .zip(self.renamed) + .map(|(reader, renamed)| { + reader.map(move |batch| batch.and_then(|b| rename_batch(&renamed, b))) + }); kmerge_by( adapted_readers, move |a: &Result, b: &Result| { @@ -278,14 +338,19 @@ impl MergedReverseRecordReader { .map(move |batch| batch.map(|batch| adapt_batch(schema.clone(), &batch))) } - /// Returns the union of schemas exposed by all readable Arrow streams. - pub fn merged_schema(&self) -> Schema { + /// Returns the union of (renamed) schemas exposed by all readable Arrow streams. + pub fn merged_schema(&self) -> Result { Schema::try_merge( self.readers .iter() - .map(|reader| reader.schema().as_ref().clone()), + .zip(&self.renamed) + .map(|(reader, renamed)| { + renamed + .as_ref() + .map(|s| s.as_ref().clone()) + .unwrap_or_else(|| reader.schema().as_ref().clone()) + }), ) - .unwrap() } } diff --git a/src/parseable/staging/writer.rs b/src/parseable/staging/writer.rs index 919e78c23..5c4f51e3b 100644 --- a/src/parseable/staging/writer.rs +++ b/src/parseable/staging/writer.rs @@ -74,6 +74,16 @@ pub struct Writer { pub disk: HashMap, } +impl Writer { + /// Disk-only writer, for nodes that never serve queries from staging. + pub fn without_memory() -> Self { + Self { + mem: None, + disk: HashMap::new(), + } + } +} + impl Default for Writer { fn default() -> Self { let mem = if *ENABLE_MEMORY_STAGING { diff --git a/src/parseable/streams.rs b/src/parseable/streams.rs index 66d722196..61e41348f 100644 --- a/src/parseable/streams.rs +++ b/src/parseable/streams.rs @@ -81,8 +81,14 @@ use crate::{ hottier::StreamHotTier, metadata::{LogStreamMetadata, SchemaVersion}, metrics, - option::Mode, + option::{Mode, SchemaResolveMode}, parseable::{DEFAULT_TENANT, PARSEABLE}, + schema_registry::{ + client::resolve_blocking, + ingestor::{ResolutionOutcome, ResolutionState, ResolveContext, schema_overrides}, + merge_schemas_tolerant, + view::{ResolvedView, rename_batch}, + }, storage::{StreamType, object_storage::to_bytes, retention::Retention}, sync::FLUSH_AND_CONVERT_RUNTIME, utils::{ @@ -415,6 +421,8 @@ pub struct Stream { // Ingestion/local sync hold read access; staging cleanup holds write access. // Acquire before schema_writer or writer when an operation needs both. staging_activity: RwLock<()>, + /// Cluster-wide schema resolution state (distributed ingestors only). + pub resolution: ResolutionState, } #[derive(Default)] @@ -475,10 +483,13 @@ struct ActiveMetricForwardFile { lanes: Vec, next_batch: usize, total_batches: usize, + /// Schema with resolved column names, applied to every batch before adapting it. + renamed: Option>, } impl ActiveMetricForwardFile { fn try_new(plan: ForwardFilePlan, readers_per_file: usize) -> Result { + let renamed = plan.renamed_schema(); let readers = plan .open_lanes(readers_per_file) .map_err(|err| ArrowError::IoError(err.to_string(), err))?; @@ -501,6 +512,7 @@ impl ActiveMetricForwardFile { lanes, next_batch: 0, total_batches, + renamed, }) } @@ -624,7 +636,12 @@ impl Iterator for MetricForwardRecordIterator { match &mut queued.state { MetricForwardFileState::Ready(active) => { if let Some(batch) = active.next_batch() { - return Some(batch.map(|batch| adapt_batch(self.schema.clone(), &batch))); + let renamed = &active.renamed; + return Some( + batch + .and_then(|batch| rename_batch(renamed, batch)) + .map(|batch| adapt_batch(self.schema.clone(), &batch)), + ); } } MetricForwardFileState::Failed(_) => {} @@ -648,11 +665,22 @@ impl Iterator for MetricForwardRecordIterator { } impl ConversionRecordReader { - fn for_stream(file_paths: &[PathBuf], forward: bool) -> Self { + fn for_stream(file_paths: &[PathBuf], forward: bool, view: &ResolvedView) -> Self { if forward { - Self::Forward(MergedForwardRecordReader::try_new(file_paths)) + Self::Forward(MergedForwardRecordReader::try_new_with_view( + file_paths, view, + )) } else { - Self::Reverse(MergedReverseRecordReader::try_new(file_paths)) + Self::Reverse(MergedReverseRecordReader::try_new_with_view( + file_paths, view, + )) + } + } + + fn unrenameable_files(&self) -> &[PathBuf] { + match self { + Self::Forward(reader) => &reader.unrenameable_files, + Self::Reverse(reader) => &reader.unrenameable_files, } } @@ -677,7 +705,7 @@ impl ConversionRecordReader { } } - fn merged_schema(&self) -> Schema { + fn merged_schema(&self) -> Result { match self { Self::Forward(reader) => reader.merged_schema(), Self::Reverse(reader) => reader.merged_schema(), @@ -724,8 +752,11 @@ impl StartupSyncPlan { let time_partition = self.stream.get_time_partition(); let custom_partition = self.stream.get_custom_partition(); + let staging_files = self + .stream + .resolve_inprocess_groups(self.staging_files, &self.tenant_id); let outcome = self.stream.convert_arrow_file_groups_to_parquet( - self.staging_files, + staging_files, time_partition.as_ref(), custom_partition.as_ref(), &self.tenant_id, @@ -745,13 +776,35 @@ impl Stream { ) -> StreamRef { let stream_name = stream_name.into(); let data_path = options.local_stream_data_path(&stream_name, tenant_id); + let resolving = + options.mode == Mode::Ingest && options.schema_resolve_mode != SchemaResolveMode::Off; + let resolution = if resolving { + ResolutionState::load(&data_path) + } else { + ResolutionState::default() + }; + let mut metadata = metadata; + // Re-apply decisions made before a restart, so stale local types are not used + // for conflict detection of new events. + for field in schema_overrides(&resolution.view(), &metadata.schema) { + if metadata.schema.contains_key(field.name()) { + metadata.schema.insert(field.name().clone(), field); + } + } + // The in-memory staging buffer only serves queries, which ingestors never run, and + // it cannot merge batches across a resolved type change. + let writer = if resolving { + Writer::without_memory() + } else { + Writer::default() + }; Arc::new(Self { stream_name: stream_name.clone(), metadata: RwLock::new(metadata), data_path, options, - writer: Mutex::new(Writer::default()), + writer: Mutex::new(writer), push_queue: Mutex::new(PushQueue::default()), push_queue_empty: Condvar::default(), #[cfg(test)] @@ -759,6 +812,7 @@ impl Stream { schema_writer: Mutex::new(()), ingestor_id, staging_activity: RwLock::new(()), + resolution, }) } @@ -1070,17 +1124,45 @@ impl Stream { group_minute: u128, init_signal: bool, shutdown_signal: bool, + ) -> Vec<(PathBuf, Vec)> { + self.arrow_files_grouped_for_conversion( + exclude, + group_minute, + init_signal, + shutdown_signal, + &None, + ) + } + + fn arrow_files_grouped_for_conversion( + &self, + exclude: SystemTime, + group_minute: u128, + init_signal: bool, + shutdown_signal: bool, + tenant_id: &Option, ) -> Vec<(PathBuf, Vec)> { if init_signal { // Startup owns only this one-time snapshot of processing files. // Root Arrow files are left for regular sync so startup never // races a live writer or a periodic cycle for the same file. - self.group_inprocess_arrow_files(&Ulid::new().to_string()) + let groups = self.group_inprocess_arrow_files(&Ulid::new().to_string()); + self.resolve_inprocess_groups(groups, tenant_id) } else { let arrow_files = self.fetch_arrow_files_for_conversion(exclude, shutdown_signal); if arrow_files.is_empty() { return Vec::new(); } + // Resolve before moving: deferred files must stay in the root directory so + // the next periodic cycle picks them up again. + let arrow_files = if self.schema_resolution_enabled() { + self.resolve_arrow_files(arrow_files, tenant_id).ready + } else { + arrow_files + }; + if arrow_files.is_empty() { + return Vec::new(); + } // Never reuse a directory that may belong to the immutable startup // snapshot (possible when a process restarts within the same minute). @@ -1107,6 +1189,129 @@ impl Stream { } } + /// True when this stream reconciles new field types with the cluster's schema authority. + pub fn schema_resolution_enabled(&self) -> bool { + self.options.mode == Mode::Ingest + && self.options.schema_resolve_mode != SchemaResolveMode::Off + && !self.get_static_schema_flag() + && self.get_stream_type() != StreamType::Internal + } + + /// Rejects ingestion while the stream's schema cannot be reconciled with the cluster. + pub fn check_schema_resolution(&self) -> Result<(), String> { + if self.resolution.is_blocked() { + Err(self + .resolution + .last_error() + .unwrap_or_else(|| "schema resolution pending".to_owned())) + } else { + Ok(()) + } + } + + /// Asks the schema authority about fields in `files` it has not resolved yet and + /// aligns the in-memory and staged schemas with the answer. Blocking: call only from + /// conversion (blocking) threads. + fn resolve_arrow_files( + &self, + files: Vec, + tenant_id: &Option, + ) -> ResolutionOutcome { + let resolver = |req| resolve_blocking(req); + let node_id = self + .ingestor_id + .clone() + .unwrap_or_else(|| PARSEABLE.options.address.clone()); + let ctx = ResolveContext { + stream: &self.stream_name, + tenant_id, + node_id, + data_path: &self.data_path, + mode: self.options.schema_resolve_mode, + max_deferred_cycles: self.options.schema_resolve_max_deferred_cycles, + resolver: &resolver, + }; + self.resolution.ensure_resolved(&ctx, files, |view| { + let overrides = schema_overrides(view, &self.get_schema_raw()); + if !overrides.is_empty() { + self.overwrite_schema_fields(&overrides); + } + if let Err(err) = self.stage_schema_file(self.get_schema().as_ref().clone()) { + warn!( + "failed to stage resolved schema for stream {}: {err}", + self.stream_name + ); + } + }) + } + + /// Startup path: files are already inside processing directories, so deferred ones + /// are moved back to the root directory for the periodic cycle to retry. + fn resolve_inprocess_groups( + &self, + groups: Vec<(PathBuf, Vec)>, + tenant_id: &Option, + ) -> Vec<(PathBuf, Vec)> { + if groups.is_empty() || !self.schema_resolution_enabled() { + return groups; + } + let files: Vec = groups.iter().flat_map(|(_, f)| f.clone()).collect(); + let outcome = self.resolve_arrow_files(files, tenant_id); + if outcome.deferred.is_empty() { + return groups; + } + self.return_files_to_root(&outcome.deferred); + let ready: HashSet = outcome.ready.into_iter().collect(); + groups + .into_iter() + .map(|(parquet, files)| { + ( + parquet, + files.into_iter().filter(|f| ready.contains(f)).collect(), + ) + }) + .filter(|(_, files): &(PathBuf, Vec)| !files.is_empty()) + .collect() + } + + fn return_files_to_root(&self, files: &[PathBuf]) { + for file in files { + let Some(name) = file.file_name() else { + continue; + }; + let mut target = self.data_path.join(name); + if target.exists() + && let Some(stem) = file.file_stem().and_then(|s| s.to_str()) + { + target = self + .data_path + .join(format!("{stem}.{}.{ARROW_FILE_EXTENSION}", Ulid::new())); + } + if let Err(err) = fs::rename(file, &target) { + error!( + "Failed to return deferred arrow file {} to {}: {err}", + file.display(), + target.display() + ); + continue; + } + if let Some(dir) = file.parent() + && dir != self.data_path + { + let _ = fs::remove_dir(dir); + } + } + } + + /// Replaces field types in the in-memory schema (no merge): used when the schema + /// authority decided a different type than this node inferred. + pub fn overwrite_schema_fields(&self, fields: &[Arc]) { + let mut metadata = self.metadata.write().expect(LOCK_EXPECT); + for field in fields { + metadata.schema.insert(field.name().clone(), field.clone()); + } + } + /// Groups arrow files only from the specified inprocess folder fn group_single_inprocess_arrow_files( &self, @@ -1348,7 +1553,27 @@ impl Stream { let tmp_path = path.with_extension("schema.tmp"); let staging_schemas = self.get_schemas_if_present()?; - if !staging_schemas.is_empty() { + if self.schema_resolution_enabled() { + // Advertise only what the schema authority accepted: an unresolved type in the + // per-node schema file would surface the very conflict resolution prevents. + let view = self.resolution.view(); + let keep = |s: Schema| { + let fields: Vec> = s + .fields() + .iter() + .filter(|f| { + view.fields + .get(f.name()) + .is_some_and(|t| t == f.data_type()) + }) + .cloned() + .collect(); + Schema::new(fields) + }; + let mut schemas = vec![keep(schema)]; + schemas.extend(staging_schemas.into_iter().map(keep)); + schema = merge_schemas_tolerant(&self.stream_name, schemas); + } else if !staging_schemas.is_empty() { let mut staging_schemas = staging_schemas; staging_schemas.push(schema); schema = Schema::try_merge(staging_schemas)?; @@ -2002,8 +2227,13 @@ impl Stream { let now = SystemTime::now(); let group_minute = minute_from_system_time(now) - 1; - let staging_files = - self.arrow_files_grouped_exclude_time(now, group_minute, init_signal, shutdown_signal); + let staging_files = self.arrow_files_grouped_for_conversion( + now, + group_minute, + init_signal, + shutdown_signal, + tenant_id, + ); span.record("file_group_count", staging_files.len()); self.convert_arrow_file_groups_to_parquet( staging_files, @@ -2120,15 +2350,34 @@ impl Stream { ) -> Result, StagingError> { // Metrics sort every output row group explicitly, so reading their IPC // streams forward avoids reverse seeks and a full-column row reversal. + // Renames decided by the schema authority are applied per file before merging. + // The view only grows, so the current one is valid for every file it accepted. + let view = self.resolution.view(); let record_reader = - ConversionRecordReader::for_stream(&arrow_files, self.is_otel_metrics()); + ConversionRecordReader::for_stream(&arrow_files, self.is_otel_metrics(), &view); self.remove_invalid_arrow_files(record_reader.invalid_files(), tenant_id); + for file in record_reader.unrenameable_files() { + error!( + "Arrow file {} of stream {} cannot be renamed to the resolved schema; keeping it unconverted", + file.display(), + self.stream_name + ); + } let readable_arrow_files = record_reader.readable_files().to_vec(); if record_reader.reader_count() == 0 { return Ok(None); } - let merged_schema = record_reader.merged_schema(); + // A type conflict across files must never panic the conversion thread; keep the + // files for a later cycle instead. + let merged_schema = record_reader.merged_schema().map_err(|err| { + error!( + "Schema merge failed for {} of stream {}; arrow files kept for retry: {err}", + parquet_path.display(), + self.stream_name + ); + StagingError::Arrow(err) + })?; let props = self.parquet_writer_props(&merged_schema, time_partition, custom_partition); let schema = Arc::new(merged_schema.clone()); let mut part_path = parquet_path.clone(); @@ -2613,9 +2862,15 @@ impl Stream { return current_schema; } - let schema = record_reader.merged_schema(); + let Ok(schema) = record_reader.merged_schema() else { + warn!( + "staged arrow files of stream {} have conflicting schemas; keeping current schema", + self.stream_name + ); + return current_schema; + }; - Schema::try_merge(vec![schema, current_schema]).unwrap() + Schema::try_merge(vec![schema, current_schema.clone()]).unwrap_or(current_schema) } /// Stores the provided stream metadata in memory mapping @@ -3601,7 +3856,7 @@ mod tests { let record_reader = ConversionRecordReader::Forward(MergedForwardRecordReader::try_new(&arrow_files)); - let merged_schema = Arc::new(record_reader.merged_schema()); + let merged_schema = Arc::new(record_reader.merged_schema().unwrap()); let props = WriterProperties::builder() .set_max_row_group_row_count(Some(5)) .build(); diff --git a/src/rbac/role.rs b/src/rbac/role.rs index bbfc1a5d7..53be8c1e1 100644 --- a/src/rbac/role.rs +++ b/src/rbac/role.rs @@ -62,6 +62,8 @@ pub enum Action { ListCluster, ListClusterMetrics, DeleteNode, + /// Intra-cluster: ingestor asks the schema authority to resolve field types. + ResolveSchema, All, SuperAdmin, GetAnalytics, @@ -133,6 +135,7 @@ impl RoleBuilder { | Action::ListCluster | Action::ListClusterMetrics | Action::DeleteNode + | Action::ResolveSchema | Action::PutHotTierEnabled | Action::GetHotTierEnabled | Action::DeleteHotTierEnabled diff --git a/src/schema_registry/authority.rs b/src/schema_registry/authority.rs new file mode 100644 index 000000000..d1f203616 --- /dev/null +++ b/src/schema_registry/authority.rs @@ -0,0 +1,423 @@ +/* + * Parseable Server (C) 2022 - 2025 Parseable, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + */ + +//! The schema authority hosted by the querier (OSS) or Prism (enterprise). +//! +//! Exactly one authority runs per cluster. It keeps one write-once [`StreamRegistry`] +//! per (tenant, stream), serialises resolutions per stream, and durably writes the +//! stream-level `.schema` before replying. + +use std::{ + collections::{HashMap, HashSet}, + sync::{Arc, Mutex}, + time::{Duration, Instant}, +}; + +use actix_web::{HttpRequest, HttpResponse, ResponseError, http::StatusCode, web}; +use arrow_schema::{Field, Schema}; +use once_cell::sync::Lazy; +use tokio::sync::{Mutex as AsyncMutex, OwnedMutexGuard}; +use tracing::{info, warn}; + +use crate::{ + event::{ + DEFAULT_TIMESTAMP_KEY, FORMAT_KEY, FORMAT_VERIFY_KEY, SOURCE_IP_KEY, USER_AGENT_KEY, + commit_schema, + }, + handlers::http::modal::NodeType, + option::Mode, + parseable::{DEFAULT_TENANT, PARSEABLE}, + utils::get_tenant_id_from_request, +}; + +use super::{ + authoritative_schema_path, authority_node_type, merge_schemas_tolerant, + protocol::{ + ErrorCode, Outcome, PROTOCOL_VERSION, RegistryEntry, ResolveErrorBody, ResolveRequest, + ResolveResponse, + }, + registry::StreamRegistry, +}; + +/// Metadata key holding the registry version inside the persisted `.schema`. +pub const REGISTRY_VERSION_KEY: &str = "parseable.registry_version"; + +type Slot = Arc>>; + +const AUTHORITY_CHECK_TTL: Duration = Duration::from_secs(30); + +#[derive(Default)] +pub struct SchemaAuthority { + slots: Mutex>, + /// Cached "does the cluster have a Prism node" (querier mode only). + prism_present: Mutex>, +} + +pub static SCHEMA_AUTHORITY: Lazy = Lazy::new(SchemaAuthority::default); + +#[derive(Debug, thiserror::Error)] +pub enum AuthorityError { + #[error("{0}")] + BadRequest(String), + #[error("stream {0} not found")] + StreamNotFound(String), + #[error("stream {0} is being deleted")] + StreamDeleting(String), + #[error("unsupported schema resolution protocol version {0}")] + UnsupportedProtocol(u32), + #[error("schema authority unavailable: {0}")] + Unavailable(String), + #[error("this node is not the schema authority; the cluster's Prism node is")] + NotAuthority, +} + +impl AuthorityError { + fn code(&self) -> ErrorCode { + match self { + Self::BadRequest(_) => ErrorCode::BadRequest, + Self::StreamNotFound(_) => ErrorCode::StreamNotFound, + Self::StreamDeleting(_) => ErrorCode::StreamDeleting, + Self::UnsupportedProtocol(_) => ErrorCode::UnsupportedProtocol, + Self::Unavailable(_) => ErrorCode::Unavailable, + Self::NotAuthority => ErrorCode::NotAuthority, + } + } +} + +impl ResponseError for AuthorityError { + fn status_code(&self) -> StatusCode { + match self { + Self::BadRequest(_) => StatusCode::BAD_REQUEST, + Self::StreamNotFound(_) => StatusCode::NOT_FOUND, + Self::StreamDeleting(_) => StatusCode::CONFLICT, + Self::UnsupportedProtocol(_) => StatusCode::UPGRADE_REQUIRED, + Self::Unavailable(_) => StatusCode::SERVICE_UNAVAILABLE, + Self::NotAuthority => StatusCode::CONFLICT, + } + } + + fn error_response(&self) -> HttpResponse { + HttpResponse::build(self.status_code()).json(ResolveErrorBody { + protocol_version: PROTOCOL_VERSION, + code: self.code(), + message: self.to_string(), + }) + } +} + +fn slot_key(tenant_id: &Option, stream: &str) -> (String, String) { + ( + tenant_id.as_deref().unwrap_or(DEFAULT_TENANT).to_owned(), + stream.to_owned(), + ) +} + +impl SchemaAuthority { + fn slot(&self, tenant_id: &Option, stream: &str) -> Slot { + self.slots + .lock() + .expect("schema authority lock poisoned") + .entry(slot_key(tenant_id, stream)) + .or_default() + .clone() + } + + /// Takes the per-stream lock and drops the cached registry, so another writer of the + /// stream-level `.schema` is serialised with resolutions and the registry is re-read + /// from storage afterwards. + pub async fn lock_and_invalidate( + &self, + tenant_id: &Option, + stream: &str, + ) -> OwnedMutexGuard> { + let mut guard = self.slot(tenant_id, stream).lock_owned().await; + *guard = None; + guard + } + + /// Prism is the authority whenever the cluster has one; a querier then refuses, so there + /// is never more than one node deciding types. The answer is cached briefly. + async fn is_authority(&self, tenant_id: &Option) -> Result { + match PARSEABLE.options.mode { + Mode::Prism => return Ok(true), + Mode::Query => {} + _ => return Ok(false), + } + if let Some((at, prism_present)) = *self.prism_present.lock().expect("lock poisoned") + && at.elapsed() < AUTHORITY_CHECK_TTL + { + return Ok(!prism_present); + } + let prism_present = authority_node_type(tenant_id) + .await + .map_err(|e| AuthorityError::Unavailable(e.to_string()))? + == NodeType::Prism; + *self.prism_present.lock().expect("lock poisoned") = Some((Instant::now(), prism_present)); + Ok(!prism_present) + } + + /// Drops cached state, e.g. after the stream was deleted. + pub fn forget_stream(&self, tenant_id: &Option, stream: &str) { + self.slots + .lock() + .expect("schema authority lock poisoned") + .remove(&slot_key(tenant_id, stream)); + } + + pub async fn resolve(&self, req: ResolveRequest) -> Result { + if req.protocol_version != PROTOCOL_VERSION { + return Err(AuthorityError::UnsupportedProtocol(req.protocol_version)); + } + if !self.is_authority(&req.tenant_id).await? { + return Err(AuthorityError::NotAuthority); + } + if req.stream.is_empty() { + return Err(AuthorityError::BadRequest("stream name is empty".into())); + } + // Load before taking the slot lock: loading may rewrite `.schema` through + // `create_schema_from_metastore`, which takes the same lock. + if !PARSEABLE + .check_or_load_stream(&req.stream, &req.tenant_id) + .await + { + return Err(AuthorityError::StreamNotFound(req.stream)); + } + let stream = PARSEABLE + .get_stream(&req.stream, &req.tenant_id) + .map_err(|_| AuthorityError::StreamNotFound(req.stream.clone()))?; + if stream.is_deleting() { + return Err(AuthorityError::StreamDeleting(req.stream)); + } + + let mut guard = self.slot(&req.tenant_id, &req.stream).lock_owned().await; + if guard.is_none() { + let mut reserved: HashSet = [ + DEFAULT_TIMESTAMP_KEY, + USER_AGENT_KEY, + SOURCE_IP_KEY, + FORMAT_KEY, + FORMAT_VERIFY_KEY, + ] + .into_iter() + .map(str::to_owned) + .collect(); + reserved.extend(stream.get_time_partition()); + reserved.extend( + stream + .get_custom_partition() + .iter() + .flat_map(|p| p.split(',').map(|s| s.trim().to_owned())), + ); + *guard = Some(load_registry(&req.stream, &req.tenant_id, reserved).await?); + } + let registry = guard.as_mut().expect("registry loaded above"); + let version_before = registry.version; + let resolution = registry.resolve(&req.fields); + + if !resolution.registered.is_empty() { + let mut next = registry.clone(); + next.apply(&resolution.registered); + let schema = registry_schema(&next); + // Persist before replying: once an ingestor sees an answer it must never change. + PARSEABLE + .metastore + .put_schema(schema.clone(), &req.stream, &req.tenant_id) + .await + .map_err(|e| AuthorityError::Unavailable(e.to_string()))?; + *registry = next; + // Best effort: let this node's own queries see the new fields immediately. + let in_memory = Schema::new(schema.fields().clone()); + if let Err(err) = commit_schema(&req.stream, Arc::new(in_memory), &req.tenant_id) { + warn!( + stream = req.stream, + "in-memory schema update skipped: {err}" + ); + } + } + + for r in &resolution.resolutions { + match r.outcome { + Outcome::Renamed | Outcome::AlreadyRenamed => info!( + stream = req.stream, + node = req.node_id, + "schema conflict: field {} proposed as {} is registered as {:?}; writing it as {}", + r.original_name, + r.proposed_type, + r.registry_type, + r.resolved_name + ), + Outcome::Unresolvable => warn!( + stream = req.stream, + node = req.node_id, + "schema conflict on field {} ({}) cannot be resolved automatically", + r.original_name, + r.proposed_type + ), + Outcome::Accepted => {} + } + } + + let delta = if req.known_version < version_before { + let mut entries: Vec = registry + .fields + .iter() + .map(|(name, data_type)| RegistryEntry { + name: name.clone(), + data_type: data_type.clone(), + }) + .collect(); + entries.sort_by(|a, b| a.name.cmp(&b.name)); + entries + } else { + Vec::new() + }; + + Ok(ResolveResponse { + protocol_version: PROTOCOL_VERSION, + request_id: req.request_id, + version: registry.version, + resolutions: resolution.resolutions, + delta, + }) + } +} + +fn registry_schema(registry: &StreamRegistry) -> Schema { + let mut fields: Vec = registry + .fields + .iter() + .map(|(name, data_type)| Field::new(name, data_type.clone(), true)) + .collect(); + fields.sort_by(|a, b| a.name().cmp(b.name())); + Schema::new_with_metadata( + fields, + HashMap::from([( + REGISTRY_VERSION_KEY.to_owned(), + registry.version.to_string(), + )]), + ) +} + +/// Seeds a registry from the authoritative `.schema` first, then from every per-node +/// schema file (first type wins). Persists immediately when seeding added fields, so a +/// restart re-seeds to the same answer regardless of listing order. +async fn load_registry( + stream: &str, + tenant_id: &Option, + reserved: HashSet, +) -> Result { + let mut registry = StreamRegistry { + reserved, + ..Default::default() + }; + let storage = PARSEABLE.storage.get_object_store(); + let authoritative = match storage + .get_object(&authoritative_schema_path(stream, tenant_id), tenant_id) + .await + { + Ok(bytes) => serde_json::from_slice::(&bytes).ok(), + Err(_) => None, + }; + let mut persisted_fields = 0; + if let Some(schema) = &authoritative { + registry.version = schema + .metadata() + .get(REGISTRY_VERSION_KEY) + .and_then(|v| v.parse().ok()) + .unwrap_or(0); + registry.seed(schema.fields().iter().map(|f| f.as_ref())); + persisted_fields = registry.fields.len(); + } + + let others = PARSEABLE + .metastore + .get_all_schemas(stream, tenant_id) + .await + .map_err(|e| AuthorityError::Unavailable(e.to_string()))?; + let merged = merge_schemas_tolerant(stream, others); + for (name, kept, skipped) in registry.seed(merged.fields().iter().map(|f| f.as_ref())) { + warn!( + stream, + "legacy schema conflict on field {name}: keeping {kept}, ignoring {skipped}" + ); + } + + if registry.fields.len() != persisted_fields || authoritative.is_none() { + registry.version += 1; + PARSEABLE + .metastore + .put_schema(registry_schema(®istry), stream, tenant_id) + .await + .map_err(|e| AuthorityError::Unavailable(e.to_string()))?; + } + Ok(registry) +} + +/// `POST {base}/internal/schema/resolve` +pub async fn resolve_handler( + req: HttpRequest, + body: web::Json, +) -> Result { + let body = body.into_inner(); + let header_tenant = get_tenant_id_from_request(&req).filter(|t| t != DEFAULT_TENANT); + let body_tenant = body.tenant_id.clone().filter(|t| t != DEFAULT_TENANT); + if header_tenant.is_some() && header_tenant != body_tenant { + return Err(AuthorityError::BadRequest( + "tenant in request body does not match the authenticated tenant".into(), + )); + } + let mut body = body; + body.tenant_id = body_tenant; + let response = SCHEMA_AUTHORITY.resolve(body).await?; + Ok(HttpResponse::Ok().json(response)) +} + +#[cfg(test)] +mod tests { + use super::*; + use arrow_schema::DataType; + + #[test] + fn registry_schema_carries_version_and_sorted_fields() { + let mut r = StreamRegistry::default(); + r.fields.insert("b".into(), DataType::Utf8); + r.fields.insert("a".into(), DataType::Int64); + r.version = 7; + let s = registry_schema(&r); + assert_eq!(s.field(0).name(), "a"); + assert_eq!(s.metadata()[REGISTRY_VERSION_KEY], "7"); + let round: Schema = serde_json::from_slice(&serde_json::to_vec(&s).unwrap()).unwrap(); + assert_eq!(round.metadata()[REGISTRY_VERSION_KEY], "7"); + } + + #[test] + fn error_status_mapping() { + assert_eq!( + AuthorityError::StreamNotFound("s".into()).status_code(), + StatusCode::NOT_FOUND + ); + assert_eq!( + AuthorityError::StreamDeleting("s".into()).status_code(), + StatusCode::CONFLICT + ); + assert_eq!( + AuthorityError::UnsupportedProtocol(9).status_code(), + StatusCode::UPGRADE_REQUIRED + ); + } +} diff --git a/src/schema_registry/client.rs b/src/schema_registry/client.rs new file mode 100644 index 000000000..6d45d1704 --- /dev/null +++ b/src/schema_registry/client.rs @@ -0,0 +1,273 @@ +/* + * Parseable Server (C) 2022 - 2025 Parseable, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + */ + +//! Ingestor-side client of the schema authority. + +use std::{ + sync::Mutex, + time::{Duration, Instant}, +}; + +use once_cell::sync::Lazy; +use reqwest::{StatusCode, header}; +use tokio::{runtime::Runtime, sync::Semaphore}; +use tracing::warn; + +use crate::{ + INTRA_CLUSTER_CLIENT, + handlers::http::{ + base_path_without_preceding_slash, cluster::utils::check_liveness, + middleware::CLUSTER_SECRET, modal::NodeMetadata, + }, + parseable::{DEFAULT_TENANT, PARSEABLE}, +}; + +use super::protocol::{ + ErrorCode, PROTOCOL_HEADER, PROTOCOL_VERSION, RESOLVE_PATH, ResolveErrorBody, ResolveRequest, + ResolveResponse, +}; +use super::{authority_node_type, nodes_of_type}; + +const REQUEST_TIMEOUT: Duration = Duration::from_secs(10); +const MAX_ATTEMPTS: u32 = 3; +const BACKOFF_BASE: Duration = Duration::from_millis(200); +const AUTHORITY_CACHE_TTL: Duration = Duration::from_secs(30); + +#[derive(Debug, thiserror::Error)] +pub enum ResolveFailure { + /// Network error, timeout, overload or an authority that is not ready. Retry later. + #[error("schema authority unavailable: {0}")] + Transient(String), + /// The authority does not know this endpoint (older querier). + #[error("schema authority does not support schema resolution")] + AuthorityTooOld, + #[error("stream not found on the schema authority")] + StreamNotFound, + #[error("stream is being deleted")] + StreamDeleting, + /// Request rejected; retrying the same request will not help. + #[error("schema resolution rejected: {0}")] + Rejected(String), +} + +#[derive(Clone, Debug)] +struct Target { + /// Base URL ending with `/`. + domain: String, + authorization: String, +} + +static RESOLVER_RUNTIME: Lazy = Lazy::new(|| { + tokio::runtime::Builder::new_multi_thread() + .worker_threads(2) + .thread_name("schema-resolver") + .enable_all() + .build() + .expect("schema resolver runtime") +}); + +static PERMITS: Lazy = + Lazy::new(|| Semaphore::new(PARSEABLE.options.schema_resolve_concurrency as usize)); + +static AUTHORITY: Lazy>> = Lazy::new(|| Mutex::new(None)); + +fn invalidate_authority() { + *AUTHORITY.lock().expect("authority cache poisoned") = None; +} + +async fn find_authority(tenant_id: &Option) -> Result { + if let Some((at, target)) = AUTHORITY.lock().expect("authority cache poisoned").clone() + && at.elapsed() < AUTHORITY_CACHE_TTL + { + return Ok(target); + } + + // The authority (Prism if the cluster has one, else the querier) publishes its address + // in its node metadata. Pick deterministically (lowest live node id) so every ingestor + // talks to the same node. + let node_type = authority_node_type(tenant_id) + .await + .map_err(|e| ResolveFailure::Transient(e.to_string()))?; + let mut nodes: Vec = nodes_of_type(node_type.clone(), tenant_id) + .await + .map_err(|e| ResolveFailure::Transient(e.to_string()))?; + nodes.sort_by(|a, b| a.node_id.cmp(&b.node_id)); + let mut chosen = None; + for node in nodes { + if check_liveness(&node.domain_name).await { + chosen = Some(node); + break; + } + } + let q = chosen.ok_or_else(|| { + ResolveFailure::Transient(format!( + "no live {node_type:?} node found for schema resolution" + )) + })?; + let domain = if q.domain_name.ends_with('/') { + q.domain_name + } else { + format!("{}/", q.domain_name) + }; + let target = Target { + domain, + authorization: q.token, + }; + + *AUTHORITY.lock().expect("authority cache poisoned") = Some((Instant::now(), target.clone())); + Ok(target) +} + +fn headers(target: &Target, tenant_id: &Option) -> header::HeaderMap { + let mut map = header::HeaderMap::new(); + map.insert( + header::HeaderName::from_static(PROTOCOL_HEADER), + header::HeaderValue::from(PROTOCOL_VERSION), + ); + if let Some((_, hash)) = CLUSTER_SECRET.get() + && let Ok(hash) = header::HeaderValue::from_str(hash) + { + map.insert( + header::HeaderName::from_static( + crate::handlers::http::middleware::CLUSTER_SECRET_HEADER, + ), + hash, + ); + if let Ok(tenant) = + header::HeaderValue::from_str(tenant_id.as_deref().unwrap_or(DEFAULT_TENANT)) + { + map.insert( + header::HeaderName::from_static("intra-cluster-tenant"), + tenant, + ); + } + if let Ok(user) = header::HeaderValue::from_str(&PARSEABLE.options.username) { + map.insert( + header::HeaderName::from_static("intra-cluster-userid"), + user, + ); + } + } else if let Ok(auth) = header::HeaderValue::from_str(&target.authorization) { + map.insert(header::AUTHORIZATION, auth); + } + map +} + +enum Attempt { + Done(Result), + Retry(String), +} + +async fn attempt(req: &ResolveRequest) -> Attempt { + let target = match find_authority(&req.tenant_id).await { + Ok(t) => t, + Err(e) => return Attempt::Retry(e.to_string()), + }; + let url = format!( + "{}{}/{}", + target.domain, + base_path_without_preceding_slash(), + RESOLVE_PATH + ); + let response = INTRA_CLUSTER_CLIENT + .post(url) + .headers(headers(&target, &req.tenant_id)) + .timeout(REQUEST_TIMEOUT) + .json(req) + .send() + .await; + let response = match response { + Ok(r) => r, + Err(e) => { + invalidate_authority(); + return Attempt::Retry(e.to_string()); + } + }; + let status = response.status(); + let bytes = match response.bytes().await { + Ok(b) => b, + Err(e) => return Attempt::Retry(e.to_string()), + }; + if status.is_success() { + return Attempt::Done( + serde_json::from_slice::(&bytes) + .map_err(|e| ResolveFailure::Rejected(format!("invalid response: {e}"))), + ); + } + match serde_json::from_slice::(&bytes) { + Ok(body) => match body.code { + ErrorCode::StreamNotFound => Attempt::Done(Err(ResolveFailure::StreamNotFound)), + ErrorCode::StreamDeleting => Attempt::Done(Err(ResolveFailure::StreamDeleting)), + ErrorCode::UnsupportedProtocol => Attempt::Done(Err(ResolveFailure::AuthorityTooOld)), + ErrorCode::BadRequest => Attempt::Done(Err(ResolveFailure::Rejected(body.message))), + ErrorCode::NotAuthority => { + invalidate_authority(); + Attempt::Retry(body.message) + } + ErrorCode::Unavailable => Attempt::Retry(body.message), + }, + // Not one of our error bodies: an authority without the route answers a bare 404. + Err(_) => match status { + StatusCode::NOT_FOUND | StatusCode::METHOD_NOT_ALLOWED => { + Attempt::Done(Err(ResolveFailure::AuthorityTooOld)) + } + StatusCode::UNAUTHORIZED | StatusCode::FORBIDDEN => Attempt::Done(Err( + ResolveFailure::Rejected(format!("authority refused credentials ({status})")), + )), + _ => Attempt::Retry(format!("authority returned {status}")), + }, + } +} + +/// Resolves with a few short in-process retries. Cross-cycle retries are the caller's job. +pub async fn resolve(req: ResolveRequest) -> Result { + let _permit = PERMITS + .acquire() + .await + .map_err(|e| ResolveFailure::Transient(e.to_string()))?; + let mut last = String::new(); + for n in 0..MAX_ATTEMPTS { + match attempt(&req).await { + Attempt::Done(result) => return result, + Attempt::Retry(reason) => { + warn!( + stream = req.stream, + "schema resolution attempt {} failed: {reason}", + n + 1 + ); + last = reason; + if n + 1 < MAX_ATTEMPTS { + let jitter = rand::random::() % 100; + tokio::time::sleep(BACKOFF_BASE * 3u32.pow(n) + Duration::from_millis(jitter)) + .await; + } + } + } + } + Err(ResolveFailure::Transient(last)) +} + +/// Blocking wrapper for the conversion threads (which run on `spawn_blocking`). +pub fn resolve_blocking(req: ResolveRequest) -> Result { + let (tx, rx) = tokio::sync::oneshot::channel(); + RESOLVER_RUNTIME.spawn(async move { + let _ = tx.send(resolve(req).await); + }); + rx.blocking_recv() + .map_err(|_| ResolveFailure::Transient("resolver task dropped".into()))? +} diff --git a/src/schema_registry/ingestor.rs b/src/schema_registry/ingestor.rs new file mode 100644 index 000000000..6e888d098 --- /dev/null +++ b/src/schema_registry/ingestor.rs @@ -0,0 +1,581 @@ +/* + * Parseable Server (C) 2022 - 2025 Parseable, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + */ + +//! Per-stream resolution state on an ingestor. +//! +//! What is "unresolved" is derived from the schemas of the arrow files about to be +//! converted, so the state survives restarts without extra bookkeeping. The only +//! persisted piece is the [`ResolvedView`] (authority answers), kept in the stream's +//! staging directory. + +use std::{ + collections::{BTreeSet, HashMap, HashSet}, + fs::File, + hash::{Hash, Hasher}, + io::BufReader, + path::{Path, PathBuf}, + sync::{ + Arc, Mutex, RwLock, + atomic::{AtomicBool, AtomicU32, Ordering}, + }, +}; + +use arrow_ipc::reader::StreamReader; +use arrow_schema::{DataType, Field, Schema}; +use tracing::{error, warn}; +use ulid::Ulid; + +use crate::option::SchemaResolveMode; + +use super::{ + client::ResolveFailure, + protocol::{PROTOCOL_VERSION, ResolveRequest, ResolveResponse}, + view::{FieldState, ResolvedView}, +}; + +pub type Resolver<'a> = dyn Fn(ResolveRequest) -> Result + 'a; + +/// Everything `ensure_resolved` needs to know about the stream and the node. +pub struct ResolveContext<'a> { + pub stream: &'a str, + pub tenant_id: &'a Option, + pub node_id: String, + pub data_path: &'a Path, + pub mode: SchemaResolveMode, + pub max_deferred_cycles: u32, + pub resolver: &'a Resolver<'a>, +} + +pub struct ResolutionOutcome { + /// Files whose every `(name, type)` is resolved; safe to convert. + pub ready: Vec, + /// Files that must stay unconverted for now. + pub deferred: Vec, +} + +#[derive(Default)] +pub struct ResolutionState { + blocked: AtomicBool, + deferred_cycles: AtomicU32, + /// Serialises "classify -> RPC -> apply" for one stream. + resolve_lock: Mutex<()>, + view: RwLock>, + /// Request id reused for retries of the same unresolved field set. + pending_request: Mutex>, + last_error: Mutex>, +} + +impl ResolutionState { + pub fn load(data_path: &Path) -> Self { + Self { + view: RwLock::new(Arc::new(ResolvedView::load(data_path))), + ..Default::default() + } + } + + pub fn view(&self) -> Arc { + self.view.read().expect("view lock poisoned").clone() + } + + pub fn is_blocked(&self) -> bool { + self.blocked.load(Ordering::Acquire) + } + + pub fn last_error(&self) -> Option { + self.last_error.lock().expect("lock poisoned").clone() + } + + pub fn deferred_cycles(&self) -> u32 { + self.deferred_cycles.load(Ordering::Acquire) + } + + fn reset(&self) { + self.deferred_cycles.store(0, Ordering::Release); + self.blocked.store(false, Ordering::Release); + *self.last_error.lock().expect("lock poisoned") = None; + } + + fn record_failure(&self, ctx: &ResolveContext<'_>, reason: String) { + let cycles = self.deferred_cycles.fetch_add(1, Ordering::AcqRel) + 1; + *self.last_error.lock().expect("lock poisoned") = Some(reason.clone()); + if ctx.max_deferred_cycles > 0 && cycles >= ctx.max_deferred_cycles { + if !self.blocked.swap(true, Ordering::AcqRel) { + error!( + stream = ctx.stream, + "schema resolution failed for {cycles} cycles, rejecting ingestion until resolved: {reason}" + ); + } + } else { + warn!( + stream = ctx.stream, + "schema resolution deferred (cycle {cycles}): {reason}" + ); + } + } + + /// Splits `files` into convertible and deferred, asking the authority about any + /// field not yet resolved. `on_apply` runs after a new view was persisted and before + /// it becomes visible, so the stream can align its in-memory and staged schemas. + pub fn ensure_resolved( + &self, + ctx: &ResolveContext<'_>, + files: Vec, + on_apply: impl FnOnce(&ResolvedView), + ) -> ResolutionOutcome { + let _guard = self.resolve_lock.lock().expect("resolve lock poisoned"); + + let schemas: Vec<(PathBuf, Option>)> = files + .into_iter() + .map(|path| { + let schema = read_file_schema(&path); + (path, schema) + }) + .collect(); + + let view = self.view(); + let unresolved = unresolved_fields(&view, &schemas); + if unresolved.is_empty() { + return self.finish(ctx, &view, schemas); + } + + let fields: Vec = unresolved + .iter() + .map(|(name, data_type)| Field::new(name, data_type.clone(), true)) + .collect(); + let request_id = self.request_id(&fields); + let request = ResolveRequest { + protocol_version: PROTOCOL_VERSION, + request_id, + node_id: ctx.node_id.clone(), + tenant_id: ctx.tenant_id.clone(), + stream: ctx.stream.to_owned(), + known_version: view.registry_version, + fields, + }; + + match (ctx.resolver)(request) { + Ok(response) => { + let next = + view.with_resolutions(response.version, &response.resolutions, &response.delta); + if let Err(err) = next.persist(ctx.data_path) { + // Without a durable view a restart could convert with stale renames; + // the answer is idempotent, so just ask again next cycle. + self.record_failure(ctx, format!("could not persist resolved fields: {err}")); + return self.split(&view, schemas); + } + on_apply(&next); + let next = Arc::new(next); + *self.view.write().expect("view lock poisoned") = next.clone(); + *self.pending_request.lock().expect("lock poisoned") = None; + self.finish(ctx, &next, schemas) + } + Err(ResolveFailure::AuthorityTooOld) if ctx.mode == SchemaResolveMode::Prefer => { + warn!( + stream = ctx.stream, + "schema authority does not support resolution; converting with local types" + ); + self.reset(); + ResolutionOutcome { + ready: schemas.into_iter().map(|(p, _)| p).collect(), + deferred: Vec::new(), + } + } + Err(ResolveFailure::StreamDeleting) => { + // The stream is going away; nothing to resolve and nothing to block. + self.split(&view, schemas) + } + Err(err) => { + self.record_failure(ctx, err.to_string()); + self.split(&view, schemas) + } + } + } + + fn finish( + &self, + ctx: &ResolveContext<'_>, + view: &ResolvedView, + schemas: Vec<(PathBuf, Option>)>, + ) -> ResolutionOutcome { + let blocking: Vec = schemas + .iter() + .filter_map(|(_, s)| s.as_ref()) + .flat_map(|s| s.fields().iter()) + .filter(|f| view.field_state(f.name(), f.data_type()) == FieldState::Unresolvable) + .map(|f| format!("{} ({})", f.name(), f.data_type())) + .collect::>() + .into_iter() + .collect(); + let outcome = self.split(view, schemas); + if blocking.is_empty() { + self.reset(); + } else { + self.record_failure( + ctx, + format!( + "fields cannot be reconciled with the cluster schema automatically: {}", + blocking.join(", ") + ), + ); + } + outcome + } + + fn split( + &self, + view: &ResolvedView, + schemas: Vec<(PathBuf, Option>)>, + ) -> ResolutionOutcome { + let mut ready = Vec::new(); + let mut deferred = Vec::new(); + for (path, schema) in schemas { + let convertible = match &schema { + // Unreadable files are left to the conversion path, which already + // classifies and removes invalid arrow files. + None => true, + Some(schema) => schema + .fields() + .iter() + .all(|f| view.field_state(f.name(), f.data_type()) == FieldState::Resolved), + }; + if convertible { + ready.push(path); + } else { + deferred.push(path); + } + } + ResolutionOutcome { ready, deferred } + } + + fn request_id(&self, fields: &[Field]) -> Ulid { + let mut hasher = std::collections::hash_map::DefaultHasher::new(); + for f in fields { + f.name().hash(&mut hasher); + f.data_type().hash(&mut hasher); + } + let key = hasher.finish(); + let mut pending = self.pending_request.lock().expect("lock poisoned"); + match *pending { + Some((k, id)) if k == key => id, + _ => { + let id = Ulid::new(); + *pending = Some((key, id)); + id + } + } + } +} + +/// Reads only the schema message of an arrow IPC stream file. +fn read_file_schema(path: &Path) -> Option> { + let file = File::open(path).ok()?; + StreamReader::try_new(BufReader::new(file), None) + .ok() + .map(|r| r.schema()) +} + +/// Distinct unresolved `(name, type)` pairs, sorted for a deterministic request. +fn unresolved_fields( + view: &ResolvedView, + schemas: &[(PathBuf, Option>)], +) -> Vec<(String, DataType)> { + let mut seen = HashSet::new(); + let mut out = Vec::new(); + for schema in schemas.iter().filter_map(|(_, s)| s.as_ref()) { + for f in schema.fields() { + if view.field_state(f.name(), f.data_type()) == FieldState::Unresolved + && seen.insert((f.name().clone(), f.data_type().clone())) + { + out.push((f.name().clone(), f.data_type().clone())); + } + } + } + out.sort_by(|a, b| { + a.0.cmp(&b.0) + .then_with(|| format!("{:?}", a.1).cmp(&format!("{:?}", b.1))) + }); + out +} + +/// Types the in-memory schema must switch to after a resolution: for every field whose +/// in-memory type differs from the authority's, the authority wins. +pub fn schema_overrides( + view: &ResolvedView, + current: &HashMap>, +) -> Vec> { + let mut out = Vec::new(); + for (name, data_type) in &view.fields { + match current.get(name) { + Some(field) if field.data_type() != data_type => out.push(Arc::new( + field.as_ref().clone().with_data_type(data_type.clone()), + )), + None if view.renames.values().any(|to| to == name) => { + out.push(Arc::new(Field::new(name, data_type.clone(), true))) + } + _ => {} + } + } + out +} + +#[cfg(test)] +mod tests { + use std::{cell::RefCell, sync::Arc}; + + use arrow_array::{ArrayRef, Int64Array, RecordBatch, StringArray}; + use arrow_ipc::writer::StreamWriter; + + use super::*; + use crate::schema_registry::{ + protocol::{FieldResolution, Outcome}, + registry::StreamRegistry, + }; + + fn write_arrow(dir: &Path, name: &str, field: Field, array: ArrayRef) -> PathBuf { + let path = dir.join(name); + let schema = Arc::new(Schema::new(vec![field])); + let batch = RecordBatch::try_new(schema.clone(), vec![array]).unwrap(); + let mut w = StreamWriter::try_new(File::create(&path).unwrap(), &schema).unwrap(); + w.write(&batch).unwrap(); + w.finish().unwrap(); + path + } + + fn ctx<'a>( + dir: &'a Path, + resolver: &'a Resolver<'a>, + tenant: &'a Option, + ) -> ResolveContext<'a> { + ResolveContext { + stream: "s", + tenant_id: tenant, + node_id: "n".into(), + data_path: dir, + mode: SchemaResolveMode::Require, + max_deferred_cycles: 3, + resolver, + } + } + + /// A resolver backed by an in-process registry, like the querier. + fn registry_resolver( + registry: &RefCell, + ) -> impl Fn(ResolveRequest) -> Result + '_ { + move |req| { + let mut reg = registry.borrow_mut(); + let res = reg.resolve(&req.fields); + reg.apply(&res.registered); + Ok(ResolveResponse { + protocol_version: PROTOCOL_VERSION, + request_id: req.request_id, + version: reg.version, + resolutions: res.resolutions, + delta: Vec::new(), + }) + } + } + + #[test] + fn two_ingestors_converge_on_one_type() { + let registry = RefCell::new(StreamRegistry::default()); + let resolver = registry_resolver(®istry); + let tenant = None; + + let d1 = tempfile::tempdir().unwrap(); + let d2 = tempfile::tempdir().unwrap(); + let f1 = write_arrow( + d1.path(), + "a.arrows", + Field::new("a", DataType::Int64, true), + Arc::new(Int64Array::from(vec![1])), + ); + let f2 = write_arrow( + d2.path(), + "a.arrows", + Field::new("a", DataType::Utf8, true), + Arc::new(StringArray::from(vec!["x"])), + ); + + let s1 = ResolutionState::default(); + let s2 = ResolutionState::default(); + let o1 = s1.ensure_resolved( + &ctx(d1.path(), &resolver, &tenant), + vec![f1.clone()], + |_| {}, + ); + let o2 = s2.ensure_resolved( + &ctx(d2.path(), &resolver, &tenant), + vec![f2.clone()], + |_| {}, + ); + assert_eq!(o1.ready, vec![f1]); + assert_eq!(o2.ready, vec![f2.clone()]); + + let v2 = s2.view(); + let renamed = v2 + .rename_schema(&read_file_schema(&f2).unwrap()) + .unwrap() + .unwrap(); + assert_eq!(renamed.field(0).name(), "a_utf8"); + assert_eq!(v2.fields["a"], DataType::Int64); + // Persisted: a fresh state on the same dir needs no RPC. + assert_eq!( + ResolutionState::load(d2.path()).view().as_ref(), + v2.as_ref() + ); + } + + #[test] + fn failures_defer_then_block_then_recover() { + let dir = tempfile::tempdir().unwrap(); + let tenant = None; + let file = write_arrow( + dir.path(), + "a.arrows", + Field::new("a", DataType::Int64, true), + Arc::new(Int64Array::from(vec![1])), + ); + let failing = |_req: ResolveRequest| Err(ResolveFailure::Transient("down".into())); + let state = ResolutionState::default(); + for cycle in 1..=3 { + let out = state.ensure_resolved( + &ctx(dir.path(), &failing, &tenant), + vec![file.clone()], + |_| {}, + ); + assert!(out.ready.is_empty()); + assert_eq!(out.deferred, vec![file.clone()]); + assert_eq!(state.is_blocked(), cycle >= 3); + } + let registry = RefCell::new(StreamRegistry::default()); + let ok = registry_resolver(®istry); + let out = state.ensure_resolved(&ctx(dir.path(), &ok, &tenant), vec![file.clone()], |_| {}); + assert_eq!(out.ready, vec![file]); + assert!(!state.is_blocked()); + assert_eq!(state.deferred_cycles(), 0); + } + + #[test] + fn retries_reuse_request_id() { + let dir = tempfile::tempdir().unwrap(); + let tenant = None; + let file = write_arrow( + dir.path(), + "a.arrows", + Field::new("a", DataType::Int64, true), + Arc::new(Int64Array::from(vec![1])), + ); + let ids = RefCell::new(Vec::new()); + let failing = |req: ResolveRequest| { + ids.borrow_mut().push(req.request_id); + Err(ResolveFailure::Transient("down".into())) + }; + let state = ResolutionState::default(); + state.ensure_resolved( + &ctx(dir.path(), &failing, &tenant), + vec![file.clone()], + |_| {}, + ); + state.ensure_resolved(&ctx(dir.path(), &failing, &tenant), vec![file], |_| {}); + let ids = ids.borrow(); + assert_eq!(ids.len(), 2); + assert_eq!(ids[0], ids[1]); + } + + #[test] + fn unresolvable_field_blocks_its_files() { + let dir = tempfile::tempdir().unwrap(); + let tenant = None; + let file = write_arrow( + dir.path(), + "a.arrows", + Field::new("ts", DataType::Utf8, true), + Arc::new(StringArray::from(vec!["x"])), + ); + let resolver = |req: ResolveRequest| { + Ok(ResolveResponse { + protocol_version: PROTOCOL_VERSION, + request_id: req.request_id, + version: 1, + resolutions: vec![FieldResolution { + original_name: "ts".into(), + proposed_type: DataType::Utf8, + resolved_name: "ts".into(), + data_type: DataType::Utf8, + registry_type: None, + outcome: Outcome::Unresolvable, + }], + delta: Vec::new(), + }) + }; + let state = ResolutionState::default(); + let out = state.ensure_resolved( + &ctx(dir.path(), &resolver, &tenant), + vec![file.clone()], + |_| {}, + ); + assert_eq!(out.deferred, vec![file]); + assert_eq!(state.deferred_cycles(), 1); + } + + #[test] + fn too_old_authority_falls_back_in_prefer_mode() { + let dir = tempfile::tempdir().unwrap(); + let tenant = None; + let file = write_arrow( + dir.path(), + "a.arrows", + Field::new("a", DataType::Int64, true), + Arc::new(Int64Array::from(vec![1])), + ); + let old = |_req: ResolveRequest| Err(ResolveFailure::AuthorityTooOld); + let state = ResolutionState::default(); + let mut c = ctx(dir.path(), &old, &tenant); + c.mode = SchemaResolveMode::Prefer; + let out = state.ensure_resolved(&c, vec![file.clone()], |_| {}); + assert_eq!(out.ready, vec![file.clone()]); + c.mode = SchemaResolveMode::Require; + let out = state.ensure_resolved(&c, vec![file.clone()], |_| {}); + assert_eq!(out.deferred, vec![file]); + } + + #[test] + fn overrides_flip_loser_type_and_add_rename_target() { + let view = ResolvedView::default().with_resolutions( + 1, + &[FieldResolution { + original_name: "a".into(), + proposed_type: DataType::Utf8, + resolved_name: "a_utf8".into(), + data_type: DataType::Utf8, + registry_type: Some(DataType::Int64), + outcome: Outcome::Renamed, + }], + &[], + ); + let current: HashMap> = HashMap::from([( + "a".to_string(), + Arc::new(Field::new("a", DataType::Utf8, true)), + )]); + let mut o = schema_overrides(&view, ¤t); + o.sort_by(|a, b| a.name().cmp(b.name())); + assert_eq!(o.len(), 2); + assert_eq!(o[0].data_type(), &DataType::Int64); + assert_eq!(o[1].name(), "a_utf8"); + } +} diff --git a/src/schema_registry/mod.rs b/src/schema_registry/mod.rs new file mode 100644 index 000000000..8abb12972 --- /dev/null +++ b/src/schema_registry/mod.rs @@ -0,0 +1,151 @@ +/* + * Parseable Server (C) 2022 - 2025 Parseable, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + */ + +//! Cluster-wide schema resolution for distributed ingestion. +//! +//! Ingestors infer types independently. Before converting arrow files to parquet they ask +//! a single authority (the querier, or Prism) which type each new field has. The authority +//! keeps a write-once name registry per stream; a later conflicting type is written under +//! `_`, the same rule the ingest-time conflict path uses. +//! See `docs/schema-resolution/` for the design. + +pub mod authority; +pub mod client; +pub mod ingestor; +pub mod protocol; +pub mod registry; +pub mod view; + +use std::{collections::HashMap, sync::Arc}; + +use arrow_schema::{Field, Schema}; +use relative_path::RelativePathBuf; +use tracing::warn; + +use crate::storage::{SCHEMA_FILE_NAME, STREAM_ROOT_DIRECTORY}; + +/// Node metadata of every node of `node_type`, tenant-scoped first, then cluster-wide. +pub async fn nodes_of_type( + node_type: crate::handlers::http::modal::NodeType, + tenant_id: &Option, +) -> anyhow::Result> { + use crate::handlers::http::cluster::get_node_info; + let mut nodes = get_node_info(node_type.clone(), tenant_id).await?; + if nodes.is_empty() && tenant_id.is_some() { + nodes = get_node_info(node_type, &None).await?; + } + Ok(nodes) +} + +/// Which node type is the schema authority: Prism when the cluster has one registered +/// (enterprise), otherwise the querier. Decided from registered metadata, not liveness, so +/// a Prism that is briefly down never causes ingestors to fall back to a second authority. +pub async fn authority_node_type( + tenant_id: &Option, +) -> anyhow::Result { + use crate::handlers::http::modal::NodeType; + if nodes_of_type(NodeType::Prism, tenant_id).await?.is_empty() { + Ok(NodeType::Querier) + } else { + Ok(NodeType::Prism) + } +} + +/// Path of the stream-level (authority owned) schema, independent of this node's mode. +pub fn authoritative_schema_path(stream: &str, tenant_id: &Option) -> RelativePathBuf { + let tenant = tenant_id.as_deref().unwrap_or(""); + RelativePathBuf::from_iter([tenant, stream, STREAM_ROOT_DIRECTORY, SCHEMA_FILE_NAME]) +} + +/// Unions schemas without ever failing: the first type seen for a field name wins and +/// later conflicting types are skipped with a warning. Metadata of the first schema is kept. +pub fn merge_schemas_tolerant(stream: &str, schemas: impl IntoIterator) -> Schema { + let mut fields: Vec = Vec::new(); + let mut index: HashMap = HashMap::new(); + let mut metadata = None; + + for schema in schemas { + if metadata.is_none() { + metadata = Some(schema.metadata().clone()); + } + for field in schema.fields() { + match index.get(field.name()) { + None => { + index.insert(field.name().clone(), fields.len()); + fields.push(field.as_ref().clone()); + } + Some(&i) => { + let mut merged = fields[i].clone(); + if merged.try_merge(field).is_ok() { + fields[i] = merged; + } else { + warn!( + stream, + "schema conflict on field {}: keeping {}, skipping {}", + field.name(), + fields[i].data_type(), + field.data_type() + ); + } + } + } + } + } + + Schema::new_with_metadata( + fields.into_iter().map(Arc::new).collect::>(), + metadata.unwrap_or_default(), + ) +} + +#[cfg(test)] +mod tests { + use super::*; + use arrow_schema::DataType; + + #[test] + fn tolerant_merge_keeps_first_type() { + let a = Schema::new(vec![ + Field::new("a", DataType::Int64, true), + Field::new("b", DataType::Utf8, true), + ]); + let b = Schema::new(vec![ + Field::new("a", DataType::Utf8, true), + Field::new("c", DataType::Boolean, true), + ]); + let merged = merge_schemas_tolerant("s", [a, b]); + assert_eq!(merged.fields().len(), 3); + assert_eq!( + merged.field_with_name("a").unwrap().data_type(), + &DataType::Int64 + ); + assert!(merged.field_with_name("c").is_ok()); + } + + #[test] + fn authoritative_path_ignores_node_mode() { + assert_eq!( + authoritative_schema_path("s", &None).as_str(), + "s/.stream/.schema" + ); + assert_eq!( + authoritative_schema_path("s", &Some("t".into())).as_str(), + "t/s/.stream/.schema" + ); + } +} diff --git a/src/schema_registry/protocol.rs b/src/schema_registry/protocol.rs new file mode 100644 index 000000000..7fdf3872c --- /dev/null +++ b/src/schema_registry/protocol.rs @@ -0,0 +1,143 @@ +/* + * Parseable Server (C) 2022 - 2025 Parseable, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + */ + +//! Wire types for the ingestor -> querier schema resolution RPC. + +use arrow_schema::{DataType, Field}; +use serde::{Deserialize, Serialize}; +use ulid::Ulid; + +pub const PROTOCOL_VERSION: u32 = 1; +pub const PROTOCOL_HEADER: &str = "x-p-schema-proto"; +/// Path (relative to the API base path) of the resolve endpoint on the authority. +pub const RESOLVE_PATH: &str = "internal/schema/resolve"; + +#[derive(Serialize, Deserialize, Debug, Clone)] +pub struct ResolveRequest { + pub protocol_version: u32, + /// Idempotency / correlation key. Reused verbatim on every retry of the same field set. + pub request_id: Ulid, + pub node_id: String, + pub tenant_id: Option, + pub stream: String, + /// Last `registry_version` this node observed for the stream (0 = never). + pub known_version: u64, + /// Fields the node has not resolved yet, with the type it wrote them as. + pub fields: Vec, +} + +#[derive(Serialize, Deserialize, Debug, Clone, Copy, PartialEq, Eq)] +#[serde(rename_all = "snake_case")] +pub enum Outcome { + /// The proposal was registered (first writer) or matches the registered type. + Accepted, + /// The name is registered with another type; write the column as `resolved_name`. + Renamed, + /// Like `Renamed`, and the suffixed name was already registered by an earlier request. + AlreadyRenamed, + /// No safe name exists (reserved column, or the suffix chain is exhausted). + Unresolvable, +} + +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq)] +pub struct FieldResolution { + pub original_name: String, + /// Type the node proposed (as found in its arrow files). + pub proposed_type: DataType, + /// Name the node must write this column under. Equals `original_name` unless renamed. + pub resolved_name: String, + /// Authoritative type of `resolved_name`. + pub data_type: DataType, + /// Authoritative type of `original_name`, so a loser can flip its in-memory schema. + pub registry_type: Option, + pub outcome: Outcome, +} + +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq)] +pub struct RegistryEntry { + pub name: String, + pub data_type: DataType, +} + +#[derive(Serialize, Deserialize, Debug, Clone)] +pub struct ResolveResponse { + pub protocol_version: u32, + pub request_id: Ulid, + /// Registry version after this request (>= known_version). + pub version: u64, + /// One entry per proposed (non-null) field, in request order. + pub resolutions: Vec, + /// Full registry snapshot when the node is behind; lets it refresh its cache. + #[serde(default)] + pub delta: Vec, +} + +#[derive(Serialize, Deserialize, Debug, Clone, Copy, PartialEq, Eq)] +#[serde(rename_all = "SCREAMING_SNAKE_CASE")] +pub enum ErrorCode { + BadRequest, + StreamNotFound, + StreamDeleting, + Unavailable, + UnsupportedProtocol, + NotAuthority, +} + +#[derive(Serialize, Deserialize, Debug, Clone)] +pub struct ResolveErrorBody { + pub protocol_version: u32, + pub code: ErrorCode, + pub message: String, +} + +#[cfg(test)] +mod tests { + use super::*; + use arrow_schema::TimeUnit; + + #[test] + fn request_round_trips_arrow_types() { + let req = ResolveRequest { + protocol_version: PROTOCOL_VERSION, + request_id: Ulid::new(), + node_id: "n1".into(), + tenant_id: None, + stream: "s".into(), + known_version: 3, + fields: vec![ + Field::new("a", DataType::Int64, true), + Field::new("t", DataType::Timestamp(TimeUnit::Millisecond, None), true), + ], + }; + let json = serde_json::to_string(&req).unwrap(); + let back: ResolveRequest = serde_json::from_str(&json).unwrap(); + assert_eq!(back.fields, req.fields); + assert_eq!(back.request_id, req.request_id); + } + + #[test] + fn error_codes_are_screaming_snake_case() { + let body = ResolveErrorBody { + protocol_version: 1, + code: ErrorCode::StreamNotFound, + message: String::new(), + }; + let json = serde_json::to_value(&body).unwrap(); + assert_eq!(json["code"], "STREAM_NOT_FOUND"); + } +} diff --git a/src/schema_registry/registry.rs b/src/schema_registry/registry.rs new file mode 100644 index 000000000..23d4b3d49 --- /dev/null +++ b/src/schema_registry/registry.rs @@ -0,0 +1,314 @@ +/* + * Parseable Server (C) 2022 - 2025 Parseable, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + */ + +//! Pure (I/O free) write-once field registry and the resolution rule. + +use std::collections::{HashMap, HashSet}; + +use arrow_schema::{DataType, Field}; + +use crate::event::format::get_datatype_suffix; + +use super::protocol::{FieldResolution, Outcome}; + +/// Max number of `_` steps before a field is declared unresolvable. +pub const MAX_SUFFIX_DEPTH: usize = 3; + +/// Types that are the same column type for registry purposes. +pub fn normalize(data_type: &DataType) -> DataType { + match data_type { + DataType::LargeUtf8 | DataType::Utf8View => DataType::Utf8, + DataType::LargeBinary | DataType::BinaryView => DataType::Binary, + other => other.clone(), + } +} + +pub fn compatible(registered: &DataType, proposed: &DataType) -> bool { + normalize(registered) == normalize(proposed) +} + +/// Authoritative name -> type map of one stream. +#[derive(Debug, Default, Clone)] +pub struct StreamRegistry { + pub fields: HashMap, + pub version: u64, + /// Columns that must never be renamed (time partition, custom partitions, p_* columns). + pub reserved: HashSet, +} + +/// Result of resolving one request against a registry snapshot. +#[derive(Debug, Default)] +pub struct Resolution { + pub resolutions: Vec, + /// Names first registered by this request, in registration order. + pub registered: Vec<(String, DataType)>, +} + +impl StreamRegistry { + /// Resolves `proposals` without mutating `self`. Deterministic: the answer is a pure + /// function of the registry contents and the proposals, so retries are idempotent. + pub fn resolve(&self, proposals: &[Field]) -> Resolution { + let mut overlay: HashMap = HashMap::new(); + let mut out = Resolution::default(); + + for proposal in proposals { + let proposed = proposal.data_type(); + if proposed.is_null() { + continue; + } + let original = proposal.name().to_string(); + let lookup = |name: &str, overlay: &HashMap| { + overlay.get(name).or_else(|| self.fields.get(name)).cloned() + }; + let registry_type = lookup(&original, &overlay); + + let mut name = original.clone(); + let mut depth = 0; + let resolution = loop { + match lookup(&name, &overlay) { + None => { + overlay.insert(name.clone(), proposed.clone()); + out.registered.push((name.clone(), proposed.clone())); + break FieldResolution { + original_name: original.clone(), + proposed_type: proposed.clone(), + resolved_name: name.clone(), + data_type: proposed.clone(), + registry_type: Some(if depth == 0 { + proposed.clone() + } else { + registry_type.clone().expect("conflict implies registered") + }), + outcome: if depth == 0 { + Outcome::Accepted + } else { + Outcome::Renamed + }, + }; + } + Some(existing) if compatible(&existing, proposed) => { + break FieldResolution { + original_name: original.clone(), + proposed_type: proposed.clone(), + resolved_name: name.clone(), + data_type: existing, + registry_type: registry_type.clone(), + outcome: if depth == 0 { + Outcome::Accepted + } else { + Outcome::AlreadyRenamed + }, + }; + } + Some(_) => { + depth += 1; + if self.reserved.contains(&original) || depth > MAX_SUFFIX_DEPTH { + break FieldResolution { + original_name: original.clone(), + proposed_type: proposed.clone(), + resolved_name: original.clone(), + data_type: proposed.clone(), + registry_type: registry_type.clone(), + outcome: Outcome::Unresolvable, + }; + } + name = format!("{name}_{}", get_datatype_suffix(proposed)); + } + } + }; + out.resolutions.push(resolution); + } + out + } + + pub fn apply(&mut self, registered: &[(String, DataType)]) { + if registered.is_empty() { + return; + } + for (name, data_type) in registered { + self.fields + .entry(name.clone()) + .or_insert_with(|| data_type.clone()); + } + self.version += 1; + } + + /// Seeds from an existing schema. The first type seen for a name wins; later conflicting + /// types are returned so the caller can log them. + pub fn seed<'a>( + &mut self, + fields: impl IntoIterator, + ) -> Vec<(String, DataType, DataType)> { + let mut conflicts = Vec::new(); + for field in fields { + if field.data_type().is_null() { + continue; + } + match self.fields.get(field.name()) { + None => { + self.fields + .insert(field.name().clone(), field.data_type().clone()); + } + Some(existing) if compatible(existing, field.data_type()) => {} + Some(existing) => conflicts.push(( + field.name().clone(), + existing.clone(), + field.data_type().clone(), + )), + } + } + conflicts + } +} + +#[cfg(test)] +mod tests { + use super::*; + use arrow_schema::TimeUnit; + + fn f(name: &str, t: DataType) -> Field { + Field::new(name, t, true) + } + + fn reg(fields: &[(&str, DataType)]) -> StreamRegistry { + StreamRegistry { + fields: fields + .iter() + .map(|(n, t)| (n.to_string(), t.clone())) + .collect(), + version: 1, + reserved: HashSet::new(), + } + } + + #[test] + fn unknown_field_is_registered() { + let r = reg(&[]); + let res = r.resolve(&[f("a", DataType::Int64)]); + assert_eq!(res.resolutions[0].outcome, Outcome::Accepted); + assert_eq!(res.resolutions[0].resolved_name, "a"); + assert_eq!(res.registered, vec![("a".to_string(), DataType::Int64)]); + } + + #[test] + fn same_type_is_accepted_without_registration() { + let r = reg(&[("a", DataType::Int64)]); + let res = r.resolve(&[f("a", DataType::Int64)]); + assert_eq!(res.resolutions[0].outcome, Outcome::Accepted); + assert!(res.registered.is_empty()); + } + + #[test] + fn conflicting_type_is_renamed_with_local_suffix_rule() { + let r = reg(&[("a", DataType::Int64)]); + let res = r.resolve(&[f("a", DataType::Utf8)]); + let fr = &res.resolutions[0]; + assert_eq!(fr.outcome, Outcome::Renamed); + assert_eq!(fr.resolved_name, "a_utf8"); + assert_eq!(fr.registry_type, Some(DataType::Int64)); + assert_eq!(res.registered, vec![("a_utf8".to_string(), DataType::Utf8)]); + } + + #[test] + fn existing_rename_target_is_reused() { + let r = reg(&[("a", DataType::Int64), ("a_utf8", DataType::Utf8)]); + let res = r.resolve(&[f("a", DataType::Utf8)]); + assert_eq!(res.resolutions[0].outcome, Outcome::AlreadyRenamed); + assert_eq!(res.resolutions[0].resolved_name, "a_utf8"); + assert!(res.registered.is_empty()); + } + + #[test] + fn rename_target_taken_by_other_type_chains_suffix() { + let r = reg(&[("a", DataType::Int64), ("a_utf8", DataType::Boolean)]); + let res = r.resolve(&[f("a", DataType::Utf8)]); + assert_eq!(res.resolutions[0].resolved_name, "a_utf8_utf8"); + assert_eq!(res.resolutions[0].outcome, Outcome::Renamed); + } + + #[test] + fn suffix_chain_is_bounded() { + let r = reg(&[ + ("a", DataType::Int64), + ("a_utf8", DataType::Int64), + ("a_utf8_utf8", DataType::Int64), + ("a_utf8_utf8_utf8", DataType::Int64), + ]); + let res = r.resolve(&[f("a", DataType::Utf8)]); + assert_eq!(res.resolutions[0].outcome, Outcome::Unresolvable); + assert!(res.registered.is_empty()); + } + + #[test] + fn reserved_columns_are_never_renamed() { + let mut r = reg(&[("ts", DataType::Timestamp(TimeUnit::Millisecond, None))]); + r.reserved.insert("ts".into()); + let res = r.resolve(&[f("ts", DataType::Utf8)]); + assert_eq!(res.resolutions[0].outcome, Outcome::Unresolvable); + } + + #[test] + fn null_proposals_are_ignored() { + let r = reg(&[]); + let res = r.resolve(&[f("a", DataType::Null)]); + assert!(res.resolutions.is_empty()); + assert!(res.registered.is_empty()); + } + + #[test] + fn large_utf8_is_compatible_with_utf8() { + let r = reg(&[("a", DataType::Utf8)]); + let res = r.resolve(&[f("a", DataType::LargeUtf8)]); + assert_eq!(res.resolutions[0].outcome, Outcome::Accepted); + } + + #[test] + fn same_request_conflict_uses_overlay() { + let r = reg(&[]); + let res = r.resolve(&[f("a", DataType::Int64), f("a", DataType::Utf8)]); + assert_eq!(res.resolutions[0].resolved_name, "a"); + assert_eq!(res.resolutions[1].resolved_name, "a_utf8"); + assert_eq!(res.registered.len(), 2); + } + + #[test] + fn retry_after_apply_is_idempotent() { + // Ingestor 1 registers a:Int64, ingestor 2 gets a_utf8, ingestor 1 retries. + let mut r = reg(&[]); + let first = r.resolve(&[f("a", DataType::Int64)]); + r.apply(&first.registered); + let second = r.resolve(&[f("a", DataType::Utf8)]); + r.apply(&second.registered); + let retry = r.resolve(&[f("a", DataType::Int64)]); + assert_eq!(retry.resolutions[0].resolved_name, "a"); + assert_eq!(retry.resolutions[0].outcome, Outcome::Accepted); + let retry2 = r.resolve(&[f("a", DataType::Utf8)]); + assert_eq!(retry2.resolutions[0].resolved_name, "a_utf8"); + assert_eq!(r.version, 3); + } + + #[test] + fn seed_keeps_first_type_and_reports_conflicts() { + let mut r = StreamRegistry::default(); + let a = f("a", DataType::Int64); + let b = f("a", DataType::Utf8); + let conflicts = r.seed([&a, &b]); + assert_eq!(r.fields["a"], DataType::Int64); + assert_eq!(conflicts.len(), 1); + } +} diff --git a/src/schema_registry/view.rs b/src/schema_registry/view.rs new file mode 100644 index 000000000..067a224b4 --- /dev/null +++ b/src/schema_registry/view.rs @@ -0,0 +1,332 @@ +/* + * Parseable Server (C) 2022 - 2025 Parseable, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + */ + +//! Ingestor-side cache of what the authority decided for a stream. + +use std::{ + collections::{HashMap, HashSet}, + path::Path, + sync::Arc, +}; + +use arrow_array::RecordBatch; +use arrow_schema::{ArrowError, DataType, Field, Schema}; +use serde::{Deserialize, Serialize}; + +use super::{ + protocol::{FieldResolution, Outcome, RegistryEntry}, + registry::compatible, +}; + +pub const VIEW_FILE_NAME: &str = ".field_resolution.json"; + +/// Resolved fields and renames for one stream on one ingestor. +/// +/// Write-once by construction (it mirrors a write-once registry), so an older snapshot is +/// always a subset of a newer one. +#[derive(Debug, Default, Clone, PartialEq)] +pub struct ResolvedView { + pub registry_version: u64, + /// Authoritative type per column name, including rename targets. + pub fields: HashMap, + /// (name, type written in the arrow file) -> column name to write to parquet. + pub renames: HashMap<(String, DataType), String>, + /// (name, type) pairs the authority could not place. Files carrying them are not converted. + pub unresolvable: HashSet<(String, DataType)>, +} + +#[derive(Serialize, Deserialize)] +struct RenameEntry { + from: String, + from_type: DataType, + to: String, +} + +#[derive(Serialize, Deserialize, Default)] +struct ViewFile { + registry_version: u64, + fields: Vec, + renames: Vec, +} + +/// Outcome of checking one `(name, type)` against the view. +#[derive(Debug, PartialEq, Eq)] +pub enum FieldState { + Resolved, + Unresolved, + Unresolvable, +} + +impl ResolvedView { + pub fn is_empty(&self) -> bool { + self.fields.is_empty() && self.renames.is_empty() + } + + pub fn field_state(&self, name: &str, data_type: &DataType) -> FieldState { + if data_type.is_null() { + return FieldState::Resolved; + } + if self + .fields + .get(name) + .is_some_and(|t| compatible(t, data_type)) + || self + .renames + .contains_key(&(name.to_string(), data_type.clone())) + { + FieldState::Resolved + } else if self + .unresolvable + .contains(&(name.to_string(), data_type.clone())) + { + FieldState::Unresolvable + } else { + FieldState::Unresolved + } + } + + /// Folds an authority answer into a new view. + pub fn with_resolutions( + &self, + version: u64, + resolutions: &[FieldResolution], + delta: &[RegistryEntry], + ) -> Self { + let mut next = self.clone(); + for entry in delta { + next.fields + .insert(entry.name.clone(), entry.data_type.clone()); + } + for r in resolutions { + let key = (r.original_name.clone(), r.proposed_type.clone()); + match r.outcome { + Outcome::Unresolvable => { + next.unresolvable.insert(key); + if let Some(t) = &r.registry_type { + next.fields.insert(r.original_name.clone(), t.clone()); + } + } + Outcome::Accepted | Outcome::Renamed | Outcome::AlreadyRenamed => { + next.unresolvable.remove(&key); + next.fields + .insert(r.resolved_name.clone(), r.data_type.clone()); + if let Some(t) = &r.registry_type { + next.fields.insert(r.original_name.clone(), t.clone()); + } + if r.resolved_name != r.original_name { + next.renames.insert(key, r.resolved_name.clone()); + } + } + } + } + next.registry_version = next.registry_version.max(version); + next + } + + /// Returns the renamed schema of an arrow file, or `None` when no column is renamed. + pub fn rename_schema(&self, schema: &Schema) -> Result>, ArrowError> { + if self.renames.is_empty() { + return Ok(None); + } + let mut changed = false; + let fields: Vec> = schema + .fields() + .iter() + .map( + |f| match self.renames.get(&(f.name().clone(), f.data_type().clone())) { + Some(to) => { + changed = true; + Arc::new(f.as_ref().clone().with_name(to)) + } + None => f.clone(), + }, + ) + .collect(); + if !changed { + return Ok(None); + } + let mut seen = HashSet::with_capacity(fields.len()); + for f in &fields { + if !seen.insert(f.name().as_str()) { + return Err(ArrowError::SchemaError(format!( + "renaming produced duplicate column {}", + f.name() + ))); + } + } + Ok(Some(Arc::new(Schema::new_with_metadata( + fields, + schema.metadata().clone(), + )))) + } + + pub fn load(dir: &Path) -> Self { + let path = dir.join(VIEW_FILE_NAME); + let Ok(bytes) = std::fs::read(&path) else { + return Self::default(); + }; + match serde_json::from_slice::(&bytes) { + Ok(file) => Self { + registry_version: file.registry_version, + fields: file + .fields + .into_iter() + .map(|e| (e.name, e.data_type)) + .collect(), + renames: file + .renames + .into_iter() + .map(|r| ((r.from, r.from_type), r.to)) + .collect(), + unresolvable: HashSet::new(), + }, + Err(err) => { + tracing::warn!("ignoring unreadable {}: {err}", path.display()); + Self::default() + } + } + } + + /// Atomically persists the view (tmp file + rename). + pub fn persist(&self, dir: &Path) -> std::io::Result<()> { + std::fs::create_dir_all(dir)?; + let mut fields: Vec = self + .fields + .iter() + .map(|(name, data_type)| RegistryEntry { + name: name.clone(), + data_type: data_type.clone(), + }) + .collect(); + fields.sort_by(|a, b| a.name.cmp(&b.name)); + let mut renames: Vec = self + .renames + .iter() + .map(|((from, from_type), to)| RenameEntry { + from: from.clone(), + from_type: from_type.clone(), + to: to.clone(), + }) + .collect(); + renames.sort_by(|a, b| a.from.cmp(&b.from).then(a.to.cmp(&b.to))); + let file = ViewFile { + registry_version: self.registry_version, + fields, + renames, + }; + let bytes = serde_json::to_vec(&file).map_err(std::io::Error::other)?; + let tmp = dir.join(format!("{VIEW_FILE_NAME}.tmp")); + std::fs::write(&tmp, bytes)?; + std::fs::rename(&tmp, dir.join(VIEW_FILE_NAME)) + } +} + +/// Zero-copy rename of a batch to `schema` (same column order, new names). +pub fn rename_batch( + schema: &Option>, + batch: RecordBatch, +) -> Result { + match schema { + Some(schema) => RecordBatch::try_new(schema.clone(), batch.columns().to_vec()), + None => Ok(batch), + } +} + +#[cfg(test)] +mod tests { + use super::*; + use arrow_array::{ArrayRef, Int64Array, StringArray}; + + fn renamed_view() -> ResolvedView { + ResolvedView::default().with_resolutions( + 2, + &[FieldResolution { + original_name: "a".into(), + proposed_type: DataType::Utf8, + resolved_name: "a_utf8".into(), + data_type: DataType::Utf8, + registry_type: Some(DataType::Int64), + outcome: Outcome::Renamed, + }], + &[], + ) + } + + #[test] + fn renamed_field_is_resolved_and_winner_type_recorded() { + let v = renamed_view(); + assert_eq!(v.field_state("a", &DataType::Utf8), FieldState::Resolved); + assert_eq!(v.field_state("a", &DataType::Int64), FieldState::Resolved); + assert_eq!(v.field_state("b", &DataType::Int64), FieldState::Unresolved); + assert_eq!(v.fields["a"], DataType::Int64); + assert_eq!(v.fields["a_utf8"], DataType::Utf8); + assert_eq!(v.registry_version, 2); + } + + #[test] + fn rename_is_zero_copy() { + let v = renamed_view(); + let schema = Schema::new(vec![ + Field::new("a", DataType::Utf8, true), + Field::new("b", DataType::Int64, true), + ]); + let renamed = v.rename_schema(&schema).unwrap().expect("renamed"); + assert_eq!(renamed.field(0).name(), "a_utf8"); + assert_eq!(renamed.field(1).name(), "b"); + + let a: ArrayRef = Arc::new(StringArray::from(vec!["x"])); + let b: ArrayRef = Arc::new(Int64Array::from(vec![1])); + let batch = RecordBatch::try_new(Arc::new(schema), vec![a.clone(), b.clone()]).unwrap(); + let out = rename_batch(&Some(renamed), batch).unwrap(); + assert!(Arc::ptr_eq(out.column(0), &a)); + assert_eq!(out.schema().field(0).name(), "a_utf8"); + } + + #[test] + fn winner_typed_column_is_not_renamed() { + let v = renamed_view(); + let schema = Schema::new(vec![Field::new("a", DataType::Int64, true)]); + assert!(v.rename_schema(&schema).unwrap().is_none()); + } + + #[test] + fn duplicate_after_rename_is_an_error() { + let v = renamed_view(); + let schema = Schema::new(vec![ + Field::new("a", DataType::Utf8, true), + Field::new("a_utf8", DataType::Utf8, true), + ]); + assert!(v.rename_schema(&schema).is_err()); + } + + #[test] + fn persist_and_load_round_trip() { + let dir = tempfile::tempdir().unwrap(); + let v = renamed_view(); + v.persist(dir.path()).unwrap(); + assert_eq!(ResolvedView::load(dir.path()), v); + } + + #[test] + fn missing_or_corrupt_file_is_empty_view() { + let dir = tempfile::tempdir().unwrap(); + assert!(ResolvedView::load(dir.path()).is_empty()); + std::fs::write(dir.path().join(VIEW_FILE_NAME), b"{not json").unwrap(); + assert!(ResolvedView::load(dir.path()).is_empty()); + } +} diff --git a/src/storage/object_storage.rs b/src/storage/object_storage.rs index 1a6f7320a..32d6cc804 100644 --- a/src/storage/object_storage.rs +++ b/src/storage/object_storage.rs @@ -861,6 +861,17 @@ pub trait ObjectStorage: Debug + Send + Sync + 'static { stream_name: &str, tenant_id: &Option, ) -> Result { + // On the schema authority, serialise this writer of the stream-level `.schema` + // with resolutions, and make the authority re-read it afterwards. + let _authority_guard = if matches!(PARSEABLE.options.mode, Mode::Query | Mode::Prism) { + Some( + crate::schema_registry::authority::SCHEMA_AUTHORITY + .lock_and_invalidate(tenant_id, stream_name) + .await, + ) + } else { + None + }; let schema = fetch_schema(stream_name, tenant_id).await?; let schema_bytes = Bytes::from(serde_json::to_vec(&schema)?); // convert to bytes @@ -1813,11 +1824,17 @@ pub async fn commit_schema_to_storage( .await .map_err(|e| ObjectStorageError::MetastoreError(Box::new(e.to_detail())))?; - let new_schema = Schema::try_merge(vec![ - schema, - serde_json::from_slice::(&stream_schema)?, - ]) - .map_err(|e| ObjectStorageError::Custom(e.to_string()))?; + let stored = serde_json::from_slice::(&stream_schema)?; + let new_schema = if PARSEABLE.options.mode == Mode::Ingest + && PARSEABLE.options.schema_resolve_mode != crate::option::SchemaResolveMode::Off + { + // This node's own file: a type resolved by the schema authority replaces the type + // this node uploaded before the resolution. + crate::schema_registry::merge_schemas_tolerant(stream_name, [schema, stored]) + } else { + Schema::try_merge(vec![schema, stored]) + .map_err(|e| ObjectStorageError::Custom(e.to_string()))? + }; PARSEABLE .metastore diff --git a/src/utils/mod.rs b/src/utils/mod.rs index bbf2de6e2..300858eb6 100644 --- a/src/utils/mod.rs +++ b/src/utils/mod.rs @@ -211,6 +211,7 @@ pub async fn user_auth_for_datasets( Action::CreateUserGroup, Action::DeleteUserGroup, Action::DeleteNode, + Action::ResolveSchema, ] .contains(action) => { From 8b9f3be48ac72a572d9a8ebe8ebc082295606779 Mon Sep 17 00:00:00 2001 From: Anant Vindal Date: Wed, 11 Nov 2026 14:23:29 +0530 Subject: [PATCH 02/13] fix: harden distributed schema resolution - kafka: a chunk rejected because its stream is blocked by schema resolution is retried every 60s and its offsets are committed only after it succeeds. Previously the error was logged and the offsets committed, silently dropping the records. - authority: clear the cached registry once a stream deletion completes, so a stream recreated under the same name starts from an empty registry instead of inheriting the old field types. - registry: a rename target can no longer be a column name carried by the same request. A file holding both `a` and `a_utf8` previously mapped both to `a_utf8`, failing the rename and leaving the file unconverted in staging. - authority: drop the querier-side Prism check and NOT_AUTHORITY. Only the OSS querier or Prism mounts the resolve route, so the cluster can only ever have one authority. - client: clear the cached authority on a 404/405, so an ingestor that picked a node without the route (an enterprise querier chosen before Prism registered) finds the right node on its next attempt. - document that the querier/Prism must be upgraded before ingestors in `require` mode --- src/connectors/kafka/processor.rs | 53 +++++++++++++++---- src/schema_registry/authority.rs | 88 +++++++++++-------------------- src/schema_registry/client.rs | 7 ++- src/schema_registry/ingestor.rs | 4 ++ src/schema_registry/protocol.rs | 1 - src/schema_registry/registry.rs | 32 +++++++++-- src/storage/object_storage.rs | 4 ++ 7 files changed, 114 insertions(+), 75 deletions(-) diff --git a/src/connectors/kafka/processor.rs b/src/connectors/kafka/processor.rs index a892691b2..8f63479f6 100644 --- a/src/connectors/kafka/processor.rs +++ b/src/connectors/kafka/processor.rs @@ -20,6 +20,7 @@ use crate::{ connectors::common::processor::Processor, event::{ USER_AGENT_KEY, + error::EventError, format::{EventFormat, LogSourceEntry, json}, }, handlers::TelemetryType, @@ -35,8 +36,9 @@ use rdkafka::consumer::{CommitMode, Consumer}; use serde_json::Value; use std::collections::HashMap; use std::sync::Arc; +use std::time::Duration; use tokio_stream::wrappers::ReceiverStream; -use tracing::{debug, error}; +use tracing::{debug, error, warn}; use super::{ConsumerRecord, StreamConsumer, TopicPartition, config::BufferConfig}; @@ -93,7 +95,7 @@ struct StreamMeta { /// poll/commit/recv), same reasoning as the rayon offload used for otel and /// prometheus remote-write ingestion. fn build_and_process_event( - records: Vec, + records: &[ConsumerRecord], stream_name: String, tenant_id: Option, meta: StreamMeta, @@ -131,6 +133,16 @@ fn build_and_process_event( Ok(()) } +/// Matches the ingest-side `Retry-After` for a stream blocked by schema resolution. +const SCHEMA_BLOCKED_RETRY: Duration = Duration::from_secs(60); + +fn is_schema_resolution_blocked(err: &anyhow::Error) -> bool { + matches!( + err.downcast_ref::(), + Some(EventError::SchemaResolutionBlocked(_, _)) + ) +} + #[async_trait] impl Processor, ()> for ParseableSinkProcessor { async fn process(&self, records: Vec) -> anyhow::Result<()> { @@ -140,14 +152,35 @@ impl Processor, ()> for ParseableSinkProcessor { let stream_name = records.first().map(|r| r.topic.clone()).unwrap_or_default(); let tenant_id = records.first().and_then(|r| r.tenant_id.clone()); - let meta = self.prepare_stream(&stream_name, &tenant_id).await?; - - let (tx, rx) = tokio::sync::oneshot::channel(); - rayon::spawn(move || { - let res = build_and_process_event(records, stream_name, tenant_id, meta); - let _ = tx.send(res); - }); - rx.await??; + let mut records = records; + loop { + let meta = self.prepare_stream(&stream_name, &tenant_id).await?; + let (tx, rx) = tokio::sync::oneshot::channel(); + let (name, tenant) = (stream_name.clone(), tenant_id.clone()); + rayon::spawn(move || { + let res = build_and_process_event(&records, name, tenant, meta); + let _ = tx.send((records, res)); + }); + let (returned, res) = rx.await?; + match res { + // The stream rejects writes until its schema is reconciled with the cluster. + // Committing now would drop these records, so hold the chunk (and its + // offsets) and retry once resolution had a chance to run. + Err(e) if is_schema_resolution_blocked(&e) => { + warn!( + stream = stream_name, + "ingestion blocked by schema resolution, retrying {len} records in {}s: {e}", + SCHEMA_BLOCKED_RETRY.as_secs() + ); + records = returned; + tokio::time::sleep(SCHEMA_BLOCKED_RETRY).await; + } + other => { + other?; + break; + } + } + } debug!("Processed {len} records"); Ok(()) diff --git a/src/schema_registry/authority.rs b/src/schema_registry/authority.rs index d1f203616..256025ef9 100644 --- a/src/schema_registry/authority.rs +++ b/src/schema_registry/authority.rs @@ -25,7 +25,6 @@ use std::{ collections::{HashMap, HashSet}, sync::{Arc, Mutex}, - time::{Duration, Instant}, }; use actix_web::{HttpRequest, HttpResponse, ResponseError, http::StatusCode, web}; @@ -39,14 +38,12 @@ use crate::{ DEFAULT_TIMESTAMP_KEY, FORMAT_KEY, FORMAT_VERIFY_KEY, SOURCE_IP_KEY, USER_AGENT_KEY, commit_schema, }, - handlers::http::modal::NodeType, - option::Mode, - parseable::{DEFAULT_TENANT, PARSEABLE}, + parseable::{DEFAULT_TENANT, PARSEABLE, Stream}, utils::get_tenant_id_from_request, }; use super::{ - authoritative_schema_path, authority_node_type, merge_schemas_tolerant, + authoritative_schema_path, merge_schemas_tolerant, protocol::{ ErrorCode, Outcome, PROTOCOL_VERSION, RegistryEntry, ResolveErrorBody, ResolveRequest, ResolveResponse, @@ -59,13 +56,9 @@ pub const REGISTRY_VERSION_KEY: &str = "parseable.registry_version"; type Slot = Arc>>; -const AUTHORITY_CHECK_TTL: Duration = Duration::from_secs(30); - #[derive(Default)] pub struct SchemaAuthority { slots: Mutex>, - /// Cached "does the cluster have a Prism node" (querier mode only). - prism_present: Mutex>, } pub static SCHEMA_AUTHORITY: Lazy = Lazy::new(SchemaAuthority::default); @@ -82,8 +75,6 @@ pub enum AuthorityError { UnsupportedProtocol(u32), #[error("schema authority unavailable: {0}")] Unavailable(String), - #[error("this node is not the schema authority; the cluster's Prism node is")] - NotAuthority, } impl AuthorityError { @@ -94,7 +85,6 @@ impl AuthorityError { Self::StreamDeleting(_) => ErrorCode::StreamDeleting, Self::UnsupportedProtocol(_) => ErrorCode::UnsupportedProtocol, Self::Unavailable(_) => ErrorCode::Unavailable, - Self::NotAuthority => ErrorCode::NotAuthority, } } } @@ -107,7 +97,6 @@ impl ResponseError for AuthorityError { Self::StreamDeleting(_) => StatusCode::CONFLICT, Self::UnsupportedProtocol(_) => StatusCode::UPGRADE_REQUIRED, Self::Unavailable(_) => StatusCode::SERVICE_UNAVAILABLE, - Self::NotAuthority => StatusCode::CONFLICT, } } @@ -150,27 +139,6 @@ impl SchemaAuthority { guard } - /// Prism is the authority whenever the cluster has one; a querier then refuses, so there - /// is never more than one node deciding types. The answer is cached briefly. - async fn is_authority(&self, tenant_id: &Option) -> Result { - match PARSEABLE.options.mode { - Mode::Prism => return Ok(true), - Mode::Query => {} - _ => return Ok(false), - } - if let Some((at, prism_present)) = *self.prism_present.lock().expect("lock poisoned") - && at.elapsed() < AUTHORITY_CHECK_TTL - { - return Ok(!prism_present); - } - let prism_present = authority_node_type(tenant_id) - .await - .map_err(|e| AuthorityError::Unavailable(e.to_string()))? - == NodeType::Prism; - *self.prism_present.lock().expect("lock poisoned") = Some((Instant::now(), prism_present)); - Ok(!prism_present) - } - /// Drops cached state, e.g. after the stream was deleted. pub fn forget_stream(&self, tenant_id: &Option, stream: &str) { self.slots @@ -183,9 +151,6 @@ impl SchemaAuthority { if req.protocol_version != PROTOCOL_VERSION { return Err(AuthorityError::UnsupportedProtocol(req.protocol_version)); } - if !self.is_authority(&req.tenant_id).await? { - return Err(AuthorityError::NotAuthority); - } if req.stream.is_empty() { return Err(AuthorityError::BadRequest("stream name is empty".into())); } @@ -204,31 +169,18 @@ impl SchemaAuthority { return Err(AuthorityError::StreamDeleting(req.stream)); } + let reserved = reserved_fields(&stream); let mut guard = self.slot(&req.tenant_id, &req.stream).lock_owned().await; if guard.is_none() { - let mut reserved: HashSet = [ - DEFAULT_TIMESTAMP_KEY, - USER_AGENT_KEY, - SOURCE_IP_KEY, - FORMAT_KEY, - FORMAT_VERIFY_KEY, - ] - .into_iter() - .map(str::to_owned) - .collect(); - reserved.extend(stream.get_time_partition()); - reserved.extend( - stream - .get_custom_partition() - .iter() - .flat_map(|p| p.split(',').map(|s| s.trim().to_owned())), - ); *guard = Some(load_registry(&req.stream, &req.tenant_id, reserved).await?); } - let registry = guard.as_mut().expect("registry loaded above"); - let version_before = registry.version; - let resolution = registry.resolve(&req.fields); + let version_before = guard.as_ref().expect("registry loaded above").version; + let resolution = guard + .as_ref() + .expect("registry loaded above") + .resolve(&req.fields); + let registry = guard.as_mut().expect("registry loaded above"); if !resolution.registered.is_empty() { let mut next = registry.clone(); next.apply(&resolution.registered); @@ -297,6 +249,28 @@ impl SchemaAuthority { } } +/// Columns that are never renamed: the server-set `p_*` fields and the partition columns. +fn reserved_fields(stream: &Stream) -> HashSet { + let mut reserved: HashSet = [ + DEFAULT_TIMESTAMP_KEY, + USER_AGENT_KEY, + SOURCE_IP_KEY, + FORMAT_KEY, + FORMAT_VERIFY_KEY, + ] + .into_iter() + .map(str::to_owned) + .collect(); + reserved.extend(stream.get_time_partition()); + reserved.extend( + stream + .get_custom_partition() + .iter() + .flat_map(|p| p.split(',').map(|s| s.trim().to_owned())), + ); + reserved +} + fn registry_schema(registry: &StreamRegistry) -> Schema { let mut fields: Vec = registry .fields diff --git a/src/schema_registry/client.rs b/src/schema_registry/client.rs index 6d45d1704..611fd481d 100644 --- a/src/schema_registry/client.rs +++ b/src/schema_registry/client.rs @@ -215,15 +215,14 @@ async fn attempt(req: &ResolveRequest) -> Attempt { ErrorCode::StreamDeleting => Attempt::Done(Err(ResolveFailure::StreamDeleting)), ErrorCode::UnsupportedProtocol => Attempt::Done(Err(ResolveFailure::AuthorityTooOld)), ErrorCode::BadRequest => Attempt::Done(Err(ResolveFailure::Rejected(body.message))), - ErrorCode::NotAuthority => { - invalidate_authority(); - Attempt::Retry(body.message) - } ErrorCode::Unavailable => Attempt::Retry(body.message), }, // Not one of our error bodies: an authority without the route answers a bare 404. Err(_) => match status { StatusCode::NOT_FOUND | StatusCode::METHOD_NOT_ALLOWED => { + // Also a node that never serves resolution (an enterprise querier picked + // before Prism registered): re-discover on the next attempt. + invalidate_authority(); Attempt::Done(Err(ResolveFailure::AuthorityTooOld)) } StatusCode::UNAUTHORIZED | StatusCode::FORBIDDEN => Attempt::Done(Err( diff --git a/src/schema_registry/ingestor.rs b/src/schema_registry/ingestor.rs index 6e888d098..53de42702 100644 --- a/src/schema_registry/ingestor.rs +++ b/src/schema_registry/ingestor.rs @@ -184,6 +184,10 @@ impl ResolutionState { *self.pending_request.lock().expect("lock poisoned") = None; self.finish(ctx, &next, schemas) } + // Rolling upgrades: in `require` mode an authority without the resolve endpoint + // counts as a failure, so upgrade the querier/Prism before the ingestors. + // Ingestors upgraded first block streams with new fields after + // `max_deferred_cycles` until the authority is upgraded. Err(ResolveFailure::AuthorityTooOld) if ctx.mode == SchemaResolveMode::Prefer => { warn!( stream = ctx.stream, diff --git a/src/schema_registry/protocol.rs b/src/schema_registry/protocol.rs index 7fdf3872c..bc2ea2f9e 100644 --- a/src/schema_registry/protocol.rs +++ b/src/schema_registry/protocol.rs @@ -95,7 +95,6 @@ pub enum ErrorCode { StreamDeleting, Unavailable, UnsupportedProtocol, - NotAuthority, } #[derive(Serialize, Deserialize, Debug, Clone)] diff --git a/src/schema_registry/registry.rs b/src/schema_registry/registry.rs index 23d4b3d49..3ef56794d 100644 --- a/src/schema_registry/registry.rs +++ b/src/schema_registry/registry.rs @@ -65,6 +65,9 @@ impl StreamRegistry { pub fn resolve(&self, proposals: &[Field]) -> Resolution { let mut overlay: HashMap = HashMap::new(); let mut out = Resolution::default(); + // A rename target must not be a column the same files already carry, or renaming + // would leave two columns with one name in a file (e.g. `a` and `a_utf8` together). + let proposed_names: HashSet<&str> = proposals.iter().map(|f| f.name().as_str()).collect(); for proposal in proposals { let proposed = proposal.data_type(); @@ -80,8 +83,9 @@ impl StreamRegistry { let mut name = original.clone(); let mut depth = 0; let resolution = loop { + let taken_by_request = depth > 0 && proposed_names.contains(name.as_str()); match lookup(&name, &overlay) { - None => { + None if !taken_by_request => { overlay.insert(name.clone(), proposed.clone()); out.registered.push((name.clone(), proposed.clone())); break FieldResolution { @@ -101,7 +105,7 @@ impl StreamRegistry { }, }; } - Some(existing) if compatible(&existing, proposed) => { + Some(existing) if !taken_by_request && compatible(&existing, proposed) => { break FieldResolution { original_name: original.clone(), proposed_type: proposed.clone(), @@ -115,7 +119,7 @@ impl StreamRegistry { }, }; } - Some(_) => { + _ => { depth += 1; if self.reserved.contains(&original) || depth > MAX_SUFFIX_DEPTH { break FieldResolution { @@ -311,4 +315,26 @@ mod tests { assert_eq!(r.fields["a"], DataType::Int64); assert_eq!(conflicts.len(), 1); } + + #[test] + fn rename_skips_names_carried_by_the_same_request() { + // A file carrying both `a: Utf8` and `a_utf8: Utf8` must not map both to `a_utf8`. + for order in [ + [f("a", DataType::Utf8), f("a_utf8", DataType::Utf8)], + [f("a_utf8", DataType::Utf8), f("a", DataType::Utf8)], + ] { + let r = reg(&[("a", DataType::Int64)]); + let res = r.resolve(&order); + let by_name = |n: &str| { + res.resolutions + .iter() + .find(|x| x.original_name == n) + .unwrap() + .resolved_name + .clone() + }; + assert_eq!(by_name("a"), "a_utf8_utf8"); + assert_eq!(by_name("a_utf8"), "a_utf8"); + } + } } diff --git a/src/storage/object_storage.rs b/src/storage/object_storage.rs index 32d6cc804..abab5c890 100644 --- a/src/storage/object_storage.rs +++ b/src/storage/object_storage.rs @@ -1559,6 +1559,10 @@ async fn run_stream_deletion( // Evict and reset stats before clearing the tombstone: once the // marker is gone, a replacement can be created under this name. streams.delete_if_still_deleting(stream_name, tenant_id); + // The authority's cached registry belongs to the deleted stream; a stream + // recreated under this name must start from an empty registry. + crate::schema_registry::authority::SCHEMA_AUTHORITY + .forget_stream(tenant_id, stream_name); if let Err(e) = stats::delete_stats_for_mode(options.mode, stream_name, "json", tenant_id) { From 1b20cc08eb41f693ccf863ccd540447109af2fdb Mon Sep 17 00:00:00 2001 From: Anant Vindal Date: Wed, 11 Nov 2026 22:56:54 +0530 Subject: [PATCH 03/13] refactor: make schema resolution on/off, scope tenants to cluster secret - replace P_SCHEMA_RESOLVE_MODE (off/prefer/require) with P_SCHEMA_RESOLUTION (default true); drop the prefer fallback to local types, which reintroduced the conflict. An authority without the endpoint is a failed cycle: upgrade the querier/Prism before ingestors. - re-resolve a file whose cached renames would collide with a column it carries, instead of leaving it unconverted in staging - the authority takes the tenant only from authentication: verified intra-cluster-tenant with P_CLUSTER_SECRET, else the caller's own tenant; ingestors refuse to resolve tenant streams without the secret - address DeepSource findings --- src/cli.rs | 11 ++- src/option.rs | 25 ------ src/parseable/staging/reader.rs | 7 +- src/parseable/streams.rs | 8 +- src/schema_registry/authority.rs | 43 +++++++--- src/schema_registry/client.rs | 29 +++++-- src/schema_registry/ingestor.rs | 136 +++++++++++++++++++++---------- src/schema_registry/protocol.rs | 2 +- src/storage/object_storage.rs | 19 ++--- 9 files changed, 172 insertions(+), 108 deletions(-) diff --git a/src/cli.rs b/src/cli.rs index e46d62b71..936a978fb 100644 --- a/src/cli.rs +++ b/src/cli.rs @@ -26,7 +26,7 @@ use crate::connectors::kafka::config::KafkaConfig; use crate::{ oidc::{self, OpenidConfig}, - option::{Compression, Mode, SchemaResolveMode, validation}, + option::{Compression, Mode, validation}, storage::{AzureBlobConfig, FSConfig, GcsConfig, S3Config}, }; @@ -560,12 +560,11 @@ pub struct Options { #[arg( long, - env = "P_SCHEMA_RESOLVE_MODE", - default_value = "require", - value_parser = validation::schema_resolve_mode, - help = "Distributed ingestors: reconcile new field types with the querier before writing parquet (off, prefer, require)" + env = "P_SCHEMA_RESOLUTION", + default_value = "true", + help = "Distributed ingestors: reconcile new field types with the schema authority (querier or Prism) before writing parquet. Upgrade the authority before the ingestors." )] - pub schema_resolve_mode: SchemaResolveMode, + pub schema_resolution: bool, #[arg( long, diff --git a/src/option.rs b/src/option.rs index 42b15c97b..53eb39fdc 100644 --- a/src/option.rs +++ b/src/option.rs @@ -30,19 +30,6 @@ pub enum Mode { All, } -/// How an ingestor reconciles new field types with the schema authority. -/// Only distributed ingestors (`P_MODE=ingest`) use it; standalone mode has one schema. -#[derive(Debug, Default, Clone, Copy, PartialEq, Eq)] -pub enum SchemaResolveMode { - /// Escape hatch: every ingestor keeps its locally inferred types (legacy behaviour). - Off, - /// Resolve with the authority; fall back to local types only if the authority is too old. - Prefer, - /// Resolve with the authority; never convert unresolved fields (default). - #[default] - Require, -} - #[derive(Debug, thiserror::Error)] #[error( "Starting Standalone Mode is not permitted when Distributed Mode is enabled. Please restart the server with Distributed Mode enabled." @@ -161,18 +148,6 @@ pub mod validation { } } - pub fn schema_resolve_mode(s: &str) -> Result { - use super::SchemaResolveMode; - match s { - "off" => Ok(SchemaResolveMode::Off), - "prefer" => Ok(SchemaResolveMode::Prefer), - "require" => Ok(SchemaResolveMode::Require), - _ => Err( - "Invalid schema resolve mode, expected one of: off, prefer, require".to_string(), - ), - } - } - pub fn compression(s: &str) -> Result { match s { "uncompressed" => Ok(Compression::Uncompressed), diff --git a/src/parseable/staging/reader.rs b/src/parseable/staging/reader.rs index 8d084bcce..2e1365cf6 100644 --- a/src/parseable/staging/reader.rs +++ b/src/parseable/staging/reader.rs @@ -87,7 +87,9 @@ impl ForwardFilePlan { } fn effective_schema(&self) -> &Schema { - self.renamed.as_deref().unwrap_or(self.schema.as_ref()) + self.renamed + .as_deref() + .unwrap_or_else(|| self.schema.as_ref()) } /// Opens independent virtual Arrow streams over striped record batches. @@ -347,8 +349,7 @@ impl MergedReverseRecordReader { .map(|(reader, renamed)| { renamed .as_ref() - .map(|s| s.as_ref().clone()) - .unwrap_or_else(|| reader.schema().as_ref().clone()) + .map_or_else(|| reader.schema().as_ref().clone(), |s| s.as_ref().clone()) }), ) } diff --git a/src/parseable/streams.rs b/src/parseable/streams.rs index 61e41348f..5f907a2d5 100644 --- a/src/parseable/streams.rs +++ b/src/parseable/streams.rs @@ -81,7 +81,7 @@ use crate::{ hottier::StreamHotTier, metadata::{LogStreamMetadata, SchemaVersion}, metrics, - option::{Mode, SchemaResolveMode}, + option::Mode, parseable::{DEFAULT_TENANT, PARSEABLE}, schema_registry::{ client::resolve_blocking, @@ -776,8 +776,7 @@ impl Stream { ) -> StreamRef { let stream_name = stream_name.into(); let data_path = options.local_stream_data_path(&stream_name, tenant_id); - let resolving = - options.mode == Mode::Ingest && options.schema_resolve_mode != SchemaResolveMode::Off; + let resolving = options.mode == Mode::Ingest && options.schema_resolution; let resolution = if resolving { ResolutionState::load(&data_path) } else { @@ -1192,7 +1191,7 @@ impl Stream { /// True when this stream reconciles new field types with the cluster's schema authority. pub fn schema_resolution_enabled(&self) -> bool { self.options.mode == Mode::Ingest - && self.options.schema_resolve_mode != SchemaResolveMode::Off + && self.options.schema_resolution && !self.get_static_schema_flag() && self.get_stream_type() != StreamType::Internal } @@ -1227,7 +1226,6 @@ impl Stream { tenant_id, node_id, data_path: &self.data_path, - mode: self.options.schema_resolve_mode, max_deferred_cycles: self.options.schema_resolve_max_deferred_cycles, resolver: &resolver, }; diff --git a/src/schema_registry/authority.rs b/src/schema_registry/authority.rs index 256025ef9..51f9e9572 100644 --- a/src/schema_registry/authority.rs +++ b/src/schema_registry/authority.rs @@ -342,21 +342,31 @@ async fn load_registry( Ok(registry) } +/// The tenant comes from authentication, never from the body: with `P_CLUSTER_SECRET` the +/// route only admits verified intra-cluster requests and the tenant is their +/// `intra-cluster-tenant`; without it, the caller's own tenant. The body may only repeat it, +/// so no caller can name another tenant. Returns `None` for the default tenant. +fn authorized_tenant( + header: Option, + body: Option, +) -> Result, AuthorityError> { + let header = header.filter(|t| t != DEFAULT_TENANT); + let body = body.filter(|t| t != DEFAULT_TENANT); + if header != body { + return Err(AuthorityError::BadRequest( + "tenant in request body does not match the authenticated tenant".into(), + )); + } + Ok(body) +} + /// `POST {base}/internal/schema/resolve` pub async fn resolve_handler( req: HttpRequest, body: web::Json, ) -> Result { - let body = body.into_inner(); - let header_tenant = get_tenant_id_from_request(&req).filter(|t| t != DEFAULT_TENANT); - let body_tenant = body.tenant_id.clone().filter(|t| t != DEFAULT_TENANT); - if header_tenant.is_some() && header_tenant != body_tenant { - return Err(AuthorityError::BadRequest( - "tenant in request body does not match the authenticated tenant".into(), - )); - } - let mut body = body; - body.tenant_id = body_tenant; + let mut body = body.into_inner(); + body.tenant_id = authorized_tenant(get_tenant_id_from_request(&req), body.tenant_id)?; let response = SCHEMA_AUTHORITY.resolve(body).await?; Ok(HttpResponse::Ok().json(response)) } @@ -379,6 +389,19 @@ mod tests { assert_eq!(round.metadata()[REGISTRY_VERSION_KEY], "7"); } + #[test] + fn tenant_must_match_the_authenticated_one() { + let t = |s: &str| Some(s.to_string()); + assert_eq!(authorized_tenant(None, None).unwrap(), None); + assert_eq!(authorized_tenant(t(DEFAULT_TENANT), None).unwrap(), None); + assert_eq!(authorized_tenant(t("acme"), t("acme")).unwrap(), t("acme")); + // A default-tenant caller cannot reach another tenant, nor a tenant another one. + assert!(authorized_tenant(None, t("acme")).is_err()); + assert!(authorized_tenant(t(DEFAULT_TENANT), t("acme")).is_err()); + assert!(authorized_tenant(t("acme"), t("other")).is_err()); + assert!(authorized_tenant(t("acme"), None).is_err()); + } + #[test] fn error_status_mapping() { assert_eq!( diff --git a/src/schema_registry/client.rs b/src/schema_registry/client.rs index 611fd481d..f7bddc423 100644 --- a/src/schema_registry/client.rs +++ b/src/schema_registry/client.rs @@ -48,14 +48,16 @@ const MAX_ATTEMPTS: u32 = 3; const BACKOFF_BASE: Duration = Duration::from_millis(200); const AUTHORITY_CACHE_TTL: Duration = Duration::from_secs(30); +/// An authority without the resolve endpoint (or protocol) blocks streams with new fields +/// until it is upgraded, so upgrade the querier/Prism before the ingestors. +const UPGRADE_AUTHORITY: &str = + "schema authority does not support this schema resolution protocol; upgrade the querier/Prism"; + #[derive(Debug, thiserror::Error)] pub enum ResolveFailure { /// Network error, timeout, overload or an authority that is not ready. Retry later. #[error("schema authority unavailable: {0}")] Transient(String), - /// The authority does not know this endpoint (older querier). - #[error("schema authority does not support schema resolution")] - AuthorityTooOld, #[error("stream not found on the schema authority")] StreamNotFound, #[error("stream is being deleted")] @@ -213,7 +215,9 @@ async fn attempt(req: &ResolveRequest) -> Attempt { Ok(body) => match body.code { ErrorCode::StreamNotFound => Attempt::Done(Err(ResolveFailure::StreamNotFound)), ErrorCode::StreamDeleting => Attempt::Done(Err(ResolveFailure::StreamDeleting)), - ErrorCode::UnsupportedProtocol => Attempt::Done(Err(ResolveFailure::AuthorityTooOld)), + ErrorCode::UnsupportedProtocol => { + Attempt::Done(Err(ResolveFailure::Rejected(UPGRADE_AUTHORITY.into()))) + } ErrorCode::BadRequest => Attempt::Done(Err(ResolveFailure::Rejected(body.message))), ErrorCode::Unavailable => Attempt::Retry(body.message), }, @@ -223,7 +227,7 @@ async fn attempt(req: &ResolveRequest) -> Attempt { // Also a node that never serves resolution (an enterprise querier picked // before Prism registered): re-discover on the next attempt. invalidate_authority(); - Attempt::Done(Err(ResolveFailure::AuthorityTooOld)) + Attempt::Done(Err(ResolveFailure::Rejected(UPGRADE_AUTHORITY.into()))) } StatusCode::UNAUTHORIZED | StatusCode::FORBIDDEN => Attempt::Done(Err( ResolveFailure::Rejected(format!("authority refused credentials ({status})")), @@ -235,11 +239,24 @@ async fn attempt(req: &ResolveRequest) -> Attempt { /// Resolves with a few short in-process retries. Cross-cycle retries are the caller's job. pub async fn resolve(req: ResolveRequest) -> Result { + // Only a cluster-secret request carries a tenant other than the caller's own (the + // authority takes it from `intra-cluster-tenant`); plain credentials resolve the + // default tenant only. + if CLUSTER_SECRET.get().is_none() + && req + .tenant_id + .as_deref() + .is_some_and(|t| t != crate::parseable::DEFAULT_TENANT) + { + return Err(ResolveFailure::Rejected( + "schema resolution for a tenant stream requires P_CLUSTER_SECRET".into(), + )); + } let _permit = PERMITS .acquire() .await .map_err(|e| ResolveFailure::Transient(e.to_string()))?; - let mut last = String::new(); + let mut last = String::default(); for n in 0..MAX_ATTEMPTS { match attempt(&req).await { Attempt::Done(result) => return result, diff --git a/src/schema_registry/ingestor.rs b/src/schema_registry/ingestor.rs index 53de42702..280dcd617 100644 --- a/src/schema_registry/ingestor.rs +++ b/src/schema_registry/ingestor.rs @@ -40,8 +40,6 @@ use arrow_schema::{DataType, Field, Schema}; use tracing::{error, warn}; use ulid::Ulid; -use crate::option::SchemaResolveMode; - use super::{ client::ResolveFailure, protocol::{PROTOCOL_VERSION, ResolveRequest, ResolveResponse}, @@ -56,7 +54,6 @@ pub struct ResolveContext<'a> { pub tenant_id: &'a Option, pub node_id: String, pub data_path: &'a Path, - pub mode: SchemaResolveMode, pub max_deferred_cycles: u32, pub resolver: &'a Resolver<'a>, } @@ -180,25 +177,13 @@ impl ResolutionState { } on_apply(&next); let next = Arc::new(next); - *self.view.write().expect("view lock poisoned") = next.clone(); + self.view + .write() + .expect("view lock poisoned") + .clone_from(&next); *self.pending_request.lock().expect("lock poisoned") = None; self.finish(ctx, &next, schemas) } - // Rolling upgrades: in `require` mode an authority without the resolve endpoint - // counts as a failure, so upgrade the querier/Prism before the ingestors. - // Ingestors upgraded first block streams with new fields after - // `max_deferred_cycles` until the authority is upgraded. - Err(ResolveFailure::AuthorityTooOld) if ctx.mode == SchemaResolveMode::Prefer => { - warn!( - stream = ctx.stream, - "schema authority does not support resolution; converting with local types" - ); - self.reset(); - ResolutionOutcome { - ready: schemas.into_iter().map(|(p, _)| p).collect(), - deferred: Vec::new(), - } - } Err(ResolveFailure::StreamDeleting) => { // The stream is going away; nothing to resolve and nothing to block. self.split(&view, schemas) @@ -216,15 +201,21 @@ impl ResolutionState { view: &ResolvedView, schemas: Vec<(PathBuf, Option>)>, ) -> ResolutionOutcome { - let blocking: Vec = schemas + let mut blocking: BTreeSet = schemas .iter() .filter_map(|(_, s)| s.as_ref()) .flat_map(|s| s.fields().iter()) .filter(|f| view.field_state(f.name(), f.data_type()) == FieldState::Unresolvable) .map(|f| format!("{} ({})", f.name(), f.data_type())) - .collect::>() - .into_iter() .collect(); + blocking.extend( + schemas + .iter() + .filter_map(|(_, s)| s.as_ref()) + .filter_map(|s| view.rename_schema(s).err()) + .map(|e| e.to_string()), + ); + let blocking: Vec = blocking.into_iter().collect(); let outcome = self.split(view, schemas); if blocking.is_empty() { self.reset(); @@ -248,15 +239,17 @@ impl ResolutionState { let mut ready = Vec::new(); let mut deferred = Vec::new(); for (path, schema) in schemas { - let convertible = match &schema { - // Unreadable files are left to the conversion path, which already - // classifies and removes invalid arrow files. - None => true, - Some(schema) => schema - .fields() - .iter() - .all(|f| view.field_state(f.name(), f.data_type()) == FieldState::Resolved), - }; + let convertible = + match &schema { + // Unreadable files are left to the conversion path, which already + // classifies and removes invalid arrow files. + None => true, + Some(schema) => { + schema.fields().iter().all(|f| { + view.field_state(f.name(), f.data_type()) == FieldState::Resolved + }) && view.rename_schema(schema).is_ok() + } + }; if convertible { ready.push(path); } else { @@ -267,7 +260,7 @@ impl ResolutionState { } fn request_id(&self, fields: &[Field]) -> Ulid { - let mut hasher = std::collections::hash_map::DefaultHasher::new(); + let mut hasher = std::collections::hash_map::DefaultHasher::default(); for f in fields { f.name().hash(&mut hasher); f.data_type().hash(&mut hasher); @@ -301,8 +294,15 @@ fn unresolved_fields( let mut seen = HashSet::new(); let mut out = Vec::new(); for schema in schemas.iter().filter_map(|(_, s)| s.as_ref()) { + // A cached rename can land on a column this file already carries (`a` -> `a_utf8` + // next to a literal `a_utf8`). Send every column of such a file, so the authority + // sees the names in use and picks a free target. + let collides = view.rename_schema(schema).is_err(); for f in schema.fields() { - if view.field_state(f.name(), f.data_type()) == FieldState::Unresolved + if f.data_type().is_null() { + continue; + } + if (collides || view.field_state(f.name(), f.data_type()) == FieldState::Unresolved) && seen.insert((f.name().clone(), f.data_type().clone())) { out.push((f.name().clone(), f.data_type().clone())); @@ -370,7 +370,6 @@ mod tests { tenant_id: tenant, node_id: "n".into(), data_path: dir, - mode: SchemaResolveMode::Require, max_deferred_cycles: 3, resolver, } @@ -444,6 +443,62 @@ mod tests { ); } + #[test] + fn cached_rename_colliding_with_a_column_is_re_resolved() { + let registry = RefCell::new(StreamRegistry::default()); + let resolver = registry_resolver(®istry); + let tenant = None; + let dir = tempfile::tempdir().unwrap(); + let state = ResolutionState::default(); + + // Another node owns `a: Int64`; this node's first file maps `a: Utf8` to `a_utf8`. + registry + .borrow_mut() + .apply(&[("a".to_string(), DataType::Int64)]); + let first = write_arrow( + dir.path(), + "1.arrows", + Field::new("a", DataType::Utf8, true), + Arc::new(StringArray::from(vec!["x"])), + ); + let out = state.ensure_resolved(&ctx(dir.path(), &resolver, &tenant), vec![first], |_| {}); + assert_eq!(out.deferred.len(), 0); + assert_eq!( + state.view().renames[&("a".to_string(), DataType::Utf8)], + "a_utf8" + ); + + // A later file carries both `a: Utf8` and a literal `a_utf8`: both count as resolved, + // but the cached rename would duplicate `a_utf8`. + let path = dir.path().join("2.arrows"); + let schema = Arc::new(Schema::new(vec![ + Field::new("a", DataType::Utf8, true), + Field::new("a_utf8", DataType::Utf8, true), + ])); + let batch = RecordBatch::try_new( + schema.clone(), + vec![ + Arc::new(StringArray::from(vec!["x"])) as ArrayRef, + Arc::new(StringArray::from(vec!["y"])) as ArrayRef, + ], + ) + .unwrap(); + let mut w = StreamWriter::try_new(File::create(&path).unwrap(), &schema).unwrap(); + w.write(&batch).unwrap(); + w.finish().unwrap(); + + let out = state.ensure_resolved( + &ctx(dir.path(), &resolver, &tenant), + vec![path.clone()], + |_| {}, + ); + assert_eq!(out.ready, vec![path]); + assert!(!state.is_blocked()); + let renamed = state.view().rename_schema(&schema).unwrap().unwrap(); + let names: Vec<_> = renamed.fields().iter().map(|f| f.name().as_str()).collect(); + assert_eq!(names, ["a_utf8_utf8", "a_utf8"]); + } + #[test] fn failures_defer_then_block_then_recover() { let dir = tempfile::tempdir().unwrap(); @@ -538,7 +593,7 @@ mod tests { } #[test] - fn too_old_authority_falls_back_in_prefer_mode() { + fn authority_without_resolution_defers() { let dir = tempfile::tempdir().unwrap(); let tenant = None; let file = write_arrow( @@ -547,15 +602,12 @@ mod tests { Field::new("a", DataType::Int64, true), Arc::new(Int64Array::from(vec![1])), ); - let old = |_req: ResolveRequest| Err(ResolveFailure::AuthorityTooOld); + let old = |_req: ResolveRequest| Err(ResolveFailure::Rejected("upgrade".into())); let state = ResolutionState::default(); - let mut c = ctx(dir.path(), &old, &tenant); - c.mode = SchemaResolveMode::Prefer; - let out = state.ensure_resolved(&c, vec![file.clone()], |_| {}); - assert_eq!(out.ready, vec![file.clone()]); - c.mode = SchemaResolveMode::Require; - let out = state.ensure_resolved(&c, vec![file.clone()], |_| {}); + let out = + state.ensure_resolved(&ctx(dir.path(), &old, &tenant), vec![file.clone()], |_| {}); assert_eq!(out.deferred, vec![file]); + assert_eq!(state.deferred_cycles(), 1); } #[test] diff --git a/src/schema_registry/protocol.rs b/src/schema_registry/protocol.rs index bc2ea2f9e..4c9fcc0e5 100644 --- a/src/schema_registry/protocol.rs +++ b/src/schema_registry/protocol.rs @@ -134,7 +134,7 @@ mod tests { let body = ResolveErrorBody { protocol_version: 1, code: ErrorCode::StreamNotFound, - message: String::new(), + message: String::default(), }; let json = serde_json::to_value(&body).unwrap(); assert_eq!(json["code"], "STREAM_NOT_FOUND"); diff --git a/src/storage/object_storage.rs b/src/storage/object_storage.rs index abab5c890..d906d0104 100644 --- a/src/storage/object_storage.rs +++ b/src/storage/object_storage.rs @@ -1829,16 +1829,15 @@ pub async fn commit_schema_to_storage( .map_err(|e| ObjectStorageError::MetastoreError(Box::new(e.to_detail())))?; let stored = serde_json::from_slice::(&stream_schema)?; - let new_schema = if PARSEABLE.options.mode == Mode::Ingest - && PARSEABLE.options.schema_resolve_mode != crate::option::SchemaResolveMode::Off - { - // This node's own file: a type resolved by the schema authority replaces the type - // this node uploaded before the resolution. - crate::schema_registry::merge_schemas_tolerant(stream_name, [schema, stored]) - } else { - Schema::try_merge(vec![schema, stored]) - .map_err(|e| ObjectStorageError::Custom(e.to_string()))? - }; + let new_schema = + if PARSEABLE.options.mode == Mode::Ingest && PARSEABLE.options.schema_resolution { + // This node's own file: a type resolved by the schema authority replaces the type + // this node uploaded before the resolution. + crate::schema_registry::merge_schemas_tolerant(stream_name, [schema, stored]) + } else { + Schema::try_merge(vec![schema, stored]) + .map_err(|e| ObjectStorageError::Custom(e.to_string()))? + }; PARSEABLE .metastore From 457ddf69df40be4e50ee1c6b557722d64344797a Mon Sep 17 00:00:00 2001 From: Anant Vindal Date: Thu, 12 Nov 2026 07:49:16 +0530 Subject: [PATCH 04/13] fix: keep cached field renames stable across resolutions A file whose cached rename collided with one of its own columns was re-resolved, and the new answer replaced the global (name, type) rename. Files already accepted under the old mapping then failed conversion with duplicate columns and stayed in processing_* until restart, the mapping could flip back and forth between cycles, and the same field split across two columns. Rename targets are now append-only per (name, type). Each file takes the first target not used by another of its columns, so a file converts the same way under any later view and files without a clash keep the first target. The view file keeps its entry format, one entry per target in grant order. --- src/parseable/streams.rs | 3 +- src/schema_registry/ingestor.rs | 10 +- src/schema_registry/view.rs | 157 ++++++++++++++++++++++++-------- 3 files changed, 129 insertions(+), 41 deletions(-) diff --git a/src/parseable/streams.rs b/src/parseable/streams.rs index 5f907a2d5..867f6674d 100644 --- a/src/parseable/streams.rs +++ b/src/parseable/streams.rs @@ -2349,7 +2349,8 @@ impl Stream { // Metrics sort every output row group explicitly, so reading their IPC // streams forward avoids reverse seeks and a full-column row reversal. // Renames decided by the schema authority are applied per file before merging. - // The view only grows, so the current one is valid for every file it accepted. + // Renames are append-only, so every file converts under the current view exactly + // as it did under the view that accepted it. let view = self.resolution.view(); let record_reader = ConversionRecordReader::for_stream(&arrow_files, self.is_otel_metrics(), &view); diff --git a/src/schema_registry/ingestor.rs b/src/schema_registry/ingestor.rs index 280dcd617..c0cffb6f7 100644 --- a/src/schema_registry/ingestor.rs +++ b/src/schema_registry/ingestor.rs @@ -328,7 +328,7 @@ pub fn schema_overrides( Some(field) if field.data_type() != data_type => out.push(Arc::new( field.as_ref().clone().with_data_type(data_type.clone()), )), - None if view.renames.values().any(|to| to == name) => { + None if view.renames.values().flatten().any(|to| to == name) => { out.push(Arc::new(Field::new(name, data_type.clone(), true))) } _ => {} @@ -465,7 +465,7 @@ mod tests { assert_eq!(out.deferred.len(), 0); assert_eq!( state.view().renames[&("a".to_string(), DataType::Utf8)], - "a_utf8" + vec!["a_utf8".to_string()] ); // A later file carries both `a: Utf8` and a literal `a_utf8`: both count as resolved, @@ -497,6 +497,12 @@ mod tests { let renamed = state.view().rename_schema(&schema).unwrap().unwrap(); let names: Vec<_> = renamed.fields().iter().map(|f| f.name().as_str()).collect(); assert_eq!(names, ["a_utf8_utf8", "a_utf8"]); + + // The new target is appended: files carrying only `a: Utf8` keep `a_utf8`, so a file + // accepted before this answer still converts as it was accepted. + let only_a = Schema::new(vec![Field::new("a", DataType::Utf8, true)]); + let renamed = state.view().rename_schema(&only_a).unwrap().unwrap(); + assert_eq!(renamed.field(0).name(), "a_utf8"); } #[test] diff --git a/src/schema_registry/view.rs b/src/schema_registry/view.rs index 067a224b4..996bca47d 100644 --- a/src/schema_registry/view.rs +++ b/src/schema_registry/view.rs @@ -37,15 +37,18 @@ pub const VIEW_FILE_NAME: &str = ".field_resolution.json"; /// Resolved fields and renames for one stream on one ingestor. /// -/// Write-once by construction (it mirrors a write-once registry), so an older snapshot is -/// always a subset of a newer one. +/// Append-only by construction (it mirrors a write-once registry), so an older snapshot is +/// always a prefix-preserving subset of a newer one: a file converts the same way under any +/// later view. #[derive(Debug, Default, Clone, PartialEq)] pub struct ResolvedView { pub registry_version: u64, /// Authoritative type per column name, including rename targets. pub fields: HashMap, - /// (name, type written in the arrow file) -> column name to write to parquet. - pub renames: HashMap<(String, DataType), String>, + /// (name, type written in the arrow file) -> registered columns it may be written to, in + /// the order the authority granted them. Append-only: a file whose earlier target is + /// taken by another of its columns uses the next one, and existing entries never change. + pub renames: HashMap<(String, DataType), Vec>, /// (name, type) pairs the authority could not place. Files carrying them are not converted. pub unresolvable: HashSet<(String, DataType)>, } @@ -129,7 +132,10 @@ impl ResolvedView { next.fields.insert(r.original_name.clone(), t.clone()); } if r.resolved_name != r.original_name { - next.renames.insert(key, r.resolved_name.clone()); + let targets = next.renames.entry(key).or_default(); + if !targets.contains(&r.resolved_name) { + targets.push(r.resolved_name.clone()); + } } } } @@ -139,35 +145,42 @@ impl ResolvedView { } /// Returns the renamed schema of an arrow file, or `None` when no column is renamed. + /// + /// Each renamed column takes its first target not used by another column of the file. + /// Fails when every target is taken; the file then needs a new target from the authority. pub fn rename_schema(&self, schema: &Schema) -> Result>, ArrowError> { if self.renames.is_empty() { return Ok(None); } - let mut changed = false; - let fields: Vec> = schema + let targets = |f: &Field| { + self.renames + .get(&(f.name().clone(), f.data_type().clone())) + .filter(|t| !t.is_empty()) + }; + if !schema.fields().iter().any(|f| targets(f).is_some()) { + return Ok(None); + } + let mut taken: HashSet<&str> = schema .fields() .iter() - .map( - |f| match self.renames.get(&(f.name().clone(), f.data_type().clone())) { - Some(to) => { - changed = true; - Arc::new(f.as_ref().clone().with_name(to)) - } - None => f.clone(), - }, - ) + .filter(|f| targets(f).is_none()) + .map(|f| f.name().as_str()) .collect(); - if !changed { - return Ok(None); - } - let mut seen = HashSet::with_capacity(fields.len()); - for f in &fields { - if !seen.insert(f.name().as_str()) { + let mut fields = Vec::with_capacity(schema.fields().len()); + for f in schema.fields() { + let Some(candidates) = targets(f) else { + fields.push(f.clone()); + continue; + }; + let Some(to) = candidates.iter().find(|t| !taken.contains(t.as_str())) else { return Err(ArrowError::SchemaError(format!( - "renaming produced duplicate column {}", - f.name() + "renaming {} produced a duplicate column: {} already taken", + f.name(), + candidates.join(", ") ))); - } + }; + taken.insert(to); + fields.push(Arc::new(f.as_ref().clone().with_name(to))); } Ok(Some(Arc::new(Schema::new_with_metadata( fields, @@ -188,11 +201,13 @@ impl ResolvedView { .into_iter() .map(|e| (e.name, e.data_type)) .collect(), - renames: file - .renames - .into_iter() - .map(|r| ((r.from, r.from_type), r.to)) - .collect(), + renames: file.renames.into_iter().fold( + HashMap::<_, Vec>::new(), + |mut acc, r| { + acc.entry((r.from, r.from_type)).or_default().push(r.to); + acc + }, + ), unresolvable: HashSet::new(), }, Err(err) => { @@ -214,16 +229,19 @@ impl ResolvedView { }) .collect(); fields.sort_by(|a, b| a.name.cmp(&b.name)); - let mut renames: Vec = self - .renames - .iter() - .map(|((from, from_type), to)| RenameEntry { - from: from.clone(), - from_type: from_type.clone(), - to: to.clone(), + // One entry per target, in grant order: `load` rebuilds each list in file order. + let mut keys: Vec<&(String, DataType)> = self.renames.keys().collect(); + keys.sort_by_cached_key(|(from, from_type)| (from.clone(), format!("{from_type:?}"))); + let renames: Vec = keys + .into_iter() + .flat_map(|key| { + self.renames[key].iter().map(|to| RenameEntry { + from: key.0.clone(), + from_type: key.1.clone(), + to: to.clone(), + }) }) .collect(); - renames.sort_by(|a, b| a.from.cmp(&b.from).then(a.to.cmp(&b.to))); let file = ViewFile { registry_version: self.registry_version, fields, @@ -314,12 +332,75 @@ mod tests { assert!(v.rename_schema(&schema).is_err()); } + /// `a: Utf8` granted `a_utf8`, then `a_utf8_utf8` for a file that also carries `a_utf8`. + fn two_target_view() -> ResolvedView { + renamed_view().with_resolutions( + 3, + &[FieldResolution { + original_name: "a".into(), + proposed_type: DataType::Utf8, + resolved_name: "a_utf8_utf8".into(), + data_type: DataType::Utf8, + registry_type: Some(DataType::Int64), + outcome: Outcome::Renamed, + }], + &[], + ) + } + + fn renamed_names(v: &ResolvedView, fields: &[&str]) -> Vec { + let schema = Schema::new( + fields + .iter() + .map(|n| Field::new(*n, DataType::Utf8, true)) + .collect::>(), + ); + v.rename_schema(&schema) + .unwrap() + .expect("renamed") + .fields() + .iter() + .map(|f| f.name().clone()) + .collect() + } + + #[test] + fn later_targets_never_change_earlier_renames() { + let v = two_target_view(); + assert_eq!( + v.renames[&("a".to_string(), DataType::Utf8)], + vec!["a_utf8".to_string(), "a_utf8_utf8".to_string()] + ); + // Files that converted under the first answer still convert the same way. + assert_eq!(renamed_names(&v, &["a"]), vec!["a_utf8"]); + assert_eq!( + renamed_names(&v, &["a", "a_utf8_utf8"]), + vec!["a_utf8", "a_utf8_utf8"] + ); + // Only a file whose first target is taken moves to the next one. + assert_eq!( + renamed_names(&v, &["a", "a_utf8"]), + vec!["a_utf8_utf8", "a_utf8"] + ); + // Every target taken: needs a new one from the authority. + let all = Schema::new( + ["a", "a_utf8", "a_utf8_utf8"] + .iter() + .map(|n| Field::new(*n, DataType::Utf8, true)) + .collect::>(), + ); + assert!(v.rename_schema(&all).is_err()); + } + #[test] fn persist_and_load_round_trip() { let dir = tempfile::tempdir().unwrap(); let v = renamed_view(); v.persist(dir.path()).unwrap(); assert_eq!(ResolvedView::load(dir.path()), v); + let v = two_target_view(); + v.persist(dir.path()).unwrap(); + assert_eq!(ResolvedView::load(dir.path()), v); } #[test] From 6277d9fb3b01a992044188c23b528b8d185b2ca5 Mon Sep 17 00:00:00 2001 From: Anant Vindal Date: Thu, 12 Nov 2026 10:28:10 +0530 Subject: [PATCH 05/13] refactor: simplify authority discovery, drop suffix depth limit - look up the schema authority on every request and take the first live node of the authority type (Prism if registered, else the querier). The cluster runs one such node, so ordering by node_id and the 30s discovery cache added nothing; requests only happen for fields a node has not resolved yet. - remove MAX_SUFFIX_DEPTH. Types that share a suffix (every list is `list`) chained one level per type, so a fifth list type for a field became unresolvable and blocked the stream. The chain now continues until it reaches a free or compatible name; it always ends because only finitely many names are registered or proposed. Only a conflict on a reserved column is unresolvable. --- src/schema_registry/client.rs | 43 +++++++-------------------------- src/schema_registry/protocol.rs | 3 ++- src/schema_registry/registry.rs | 38 +++++++++++++++++++---------- 3 files changed, 36 insertions(+), 48 deletions(-) diff --git a/src/schema_registry/client.rs b/src/schema_registry/client.rs index f7bddc423..05e00d25c 100644 --- a/src/schema_registry/client.rs +++ b/src/schema_registry/client.rs @@ -18,10 +18,7 @@ //! Ingestor-side client of the schema authority. -use std::{ - sync::Mutex, - time::{Duration, Instant}, -}; +use std::time::Duration; use once_cell::sync::Lazy; use reqwest::{StatusCode, header}; @@ -46,7 +43,6 @@ use super::{authority_node_type, nodes_of_type}; const REQUEST_TIMEOUT: Duration = Duration::from_secs(10); const MAX_ATTEMPTS: u32 = 3; const BACKOFF_BASE: Duration = Duration::from_millis(200); -const AUTHORITY_CACHE_TTL: Duration = Duration::from_secs(30); /// An authority without the resolve endpoint (or protocol) blocks streams with new fields /// until it is upgraded, so upgrade the querier/Prism before the ingestors. @@ -86,29 +82,17 @@ static RESOLVER_RUNTIME: Lazy = Lazy::new(|| { static PERMITS: Lazy = Lazy::new(|| Semaphore::new(PARSEABLE.options.schema_resolve_concurrency as usize)); -static AUTHORITY: Lazy>> = Lazy::new(|| Mutex::new(None)); - -fn invalidate_authority() { - *AUTHORITY.lock().expect("authority cache poisoned") = None; -} - +/// The authority (Prism if the cluster has one, else the querier) publishes its address in +/// its node metadata. A cluster runs one such node; the liveness check only skips metadata +/// left behind by a node that is gone. Looked up on every request: requests are rare (only +/// for fields a node has not resolved yet), and a fresh lookup follows a restarted authority. async fn find_authority(tenant_id: &Option) -> Result { - if let Some((at, target)) = AUTHORITY.lock().expect("authority cache poisoned").clone() - && at.elapsed() < AUTHORITY_CACHE_TTL - { - return Ok(target); - } - - // The authority (Prism if the cluster has one, else the querier) publishes its address - // in its node metadata. Pick deterministically (lowest live node id) so every ingestor - // talks to the same node. let node_type = authority_node_type(tenant_id) .await .map_err(|e| ResolveFailure::Transient(e.to_string()))?; - let mut nodes: Vec = nodes_of_type(node_type.clone(), tenant_id) + let nodes: Vec = nodes_of_type(node_type.clone(), tenant_id) .await .map_err(|e| ResolveFailure::Transient(e.to_string()))?; - nodes.sort_by(|a, b| a.node_id.cmp(&b.node_id)); let mut chosen = None; for node in nodes { if check_liveness(&node.domain_name).await { @@ -126,13 +110,10 @@ async fn find_authority(tenant_id: &Option) -> Result) -> header::HeaderMap { @@ -195,10 +176,7 @@ async fn attempt(req: &ResolveRequest) -> Attempt { .await; let response = match response { Ok(r) => r, - Err(e) => { - invalidate_authority(); - return Attempt::Retry(e.to_string()); - } + Err(e) => return Attempt::Retry(e.to_string()), }; let status = response.status(); let bytes = match response.bytes().await { @@ -224,9 +202,6 @@ async fn attempt(req: &ResolveRequest) -> Attempt { // Not one of our error bodies: an authority without the route answers a bare 404. Err(_) => match status { StatusCode::NOT_FOUND | StatusCode::METHOD_NOT_ALLOWED => { - // Also a node that never serves resolution (an enterprise querier picked - // before Prism registered): re-discover on the next attempt. - invalidate_authority(); Attempt::Done(Err(ResolveFailure::Rejected(UPGRADE_AUTHORITY.into()))) } StatusCode::UNAUTHORIZED | StatusCode::FORBIDDEN => Attempt::Done(Err( diff --git a/src/schema_registry/protocol.rs b/src/schema_registry/protocol.rs index 4c9fcc0e5..90b074703 100644 --- a/src/schema_registry/protocol.rs +++ b/src/schema_registry/protocol.rs @@ -50,7 +50,8 @@ pub enum Outcome { Renamed, /// Like `Renamed`, and the suffixed name was already registered by an earlier request. AlreadyRenamed, - /// No safe name exists (reserved column, or the suffix chain is exhausted). + /// No safe name exists: the field is a reserved column (`p_*` or a partition column) + /// registered with another type. Unresolvable, } diff --git a/src/schema_registry/registry.rs b/src/schema_registry/registry.rs index 3ef56794d..6993ed792 100644 --- a/src/schema_registry/registry.rs +++ b/src/schema_registry/registry.rs @@ -26,9 +26,6 @@ use crate::event::format::get_datatype_suffix; use super::protocol::{FieldResolution, Outcome}; -/// Max number of `_` steps before a field is declared unresolvable. -pub const MAX_SUFFIX_DEPTH: usize = 3; - /// Types that are the same column type for registry purposes. pub fn normalize(data_type: &DataType) -> DataType { match data_type { @@ -119,9 +116,11 @@ impl StreamRegistry { }, }; } + // No depth limit: every step makes a longer name, and only finitely many + // names are registered or proposed, so the chain always reaches a free one. + // Types sharing a suffix (every list is `list`) chain one level per type. _ => { - depth += 1; - if self.reserved.contains(&original) || depth > MAX_SUFFIX_DEPTH { + if self.reserved.contains(&original) { break FieldResolution { original_name: original.clone(), proposed_type: proposed.clone(), @@ -131,6 +130,7 @@ impl StreamRegistry { outcome: Outcome::Unresolvable, }; } + depth += 1; name = format!("{name}_{}", get_datatype_suffix(proposed)); } } @@ -182,6 +182,8 @@ impl StreamRegistry { #[cfg(test)] mod tests { + use std::sync::Arc; + use super::*; use arrow_schema::TimeUnit; @@ -246,16 +248,26 @@ mod tests { } #[test] - fn suffix_chain_is_bounded() { + fn suffix_chain_continues_until_a_free_name() { + // Every list type shares the `list` suffix, so each new list type goes one level deeper. + let list = |t: DataType| DataType::List(Arc::new(Field::new_list_field(t, true))); let r = reg(&[ - ("a", DataType::Int64), - ("a_utf8", DataType::Int64), - ("a_utf8_utf8", DataType::Int64), - ("a_utf8_utf8_utf8", DataType::Int64), + ("a", list(DataType::Int64)), + ("a_list", list(DataType::Float64)), + ("a_list_list", list(DataType::Utf8)), + ("a_list_list_list", list(DataType::Boolean)), ]); - let res = r.resolve(&[f("a", DataType::Utf8)]); - assert_eq!(res.resolutions[0].outcome, Outcome::Unresolvable); - assert!(res.registered.is_empty()); + let res = r.resolve(&[f("a", list(DataType::Date32))]); + assert_eq!(res.resolutions[0].resolved_name, "a_list_list_list_list"); + assert_eq!(res.resolutions[0].outcome, Outcome::Renamed); + assert_eq!( + res.registered, + vec![("a_list_list_list_list".to_string(), list(DataType::Date32))] + ); + // An already registered list type still resolves to its existing column. + let res = r.resolve(&[f("a", list(DataType::Utf8))]); + assert_eq!(res.resolutions[0].resolved_name, "a_list_list"); + assert_eq!(res.resolutions[0].outcome, Outcome::AlreadyRenamed); } #[test] From c793108a84bcfd9e124bfe702cfb9b326c6b8379 Mon Sep 17 00:00:00 2001 From: Anant Vindal Date: Thu, 12 Nov 2026 18:40:48 +0530 Subject: [PATCH 06/13] fix: name conflict-renamed columns after their stored type Ingest-time conflict renaming took the column suffix from the raw JSON type, while schema V1 stores the new column under its own overrides: every number as Float64, and date strings in time-named fields as Timestamp. So V1 integers landed in `x_int64` columns typed Float64, and date strings in `x_utf8` columns typed Timestamp. In distributed mode the schema authority names the same data after the stored type (`x_float64`, `x_timestamp_ms`), so one field's values split across two columns depending on which path renamed them. Both rename paths (batch-level and per-record) now take the suffix from the type the new column is stored as: the same inference and V1 overrides run on the values being moved. V0 streams keep `_int64`, since V0 stores integers as Int64. Streams that already have `x_int64` or `x_utf8` siblings keep that data. New conflicts go to the correctly named column. --- src/event/format/json.rs | 136 +++++++++++++++++++++++++++++++++++++++ src/event/format/mod.rs | 128 +++++++++++++++++++++++++++--------- 2 files changed, 235 insertions(+), 29 deletions(-) diff --git a/src/event/format/json.rs b/src/event/format/json.rs index a0127fe41..1f3698dc5 100644 --- a/src/event/format/json.rs +++ b/src/event/format/json.rs @@ -113,6 +113,7 @@ impl EventFormat for Event { stream_schema, &value_arr, schema_version, + infer_timestamp, ); // If there are conflicts, rename the fields in JSON values @@ -137,6 +138,7 @@ impl EventFormat for Event { &raw_inferred_schema, stream_schema, schema_version, + infer_timestamp, ) }; @@ -587,6 +589,7 @@ mod tests { use std::str::FromStr; use arrow_array::UInt64Array; + use arrow_schema::TimeUnit; use serde_json::json; use super::*; @@ -690,4 +693,137 @@ mod tests { assert!(parsed.is_err()); } + + /// Runs `to_data` on `body` against a stream that already has `existing`, decodes the + /// result, and returns the columns the event adds to the stream. + fn new_columns( + existing: &[(&str, DataType)], + body: Value, + schema_version: SchemaVersion, + infer_timestamp: bool, + ) -> Vec<(String, DataType)> { + let schema: HashMap> = existing + .iter() + .map(|(n, t)| (n.to_string(), Arc::new(Field::new(*n, t.clone(), true)))) + .collect(); + let (data, fields, _) = Event::new(body, Utc::now()) + .to_data(&schema, None, schema_version, false, infer_timestamp) + .unwrap(); + Event::decode(data, Arc::new(Schema::new(fields.clone()))).unwrap(); + fields + .iter() + .filter(|f| !schema.contains_key(f.name())) + .map(|f| (f.name().clone(), f.data_type().clone())) + .collect() + } + + /// A conflict rename names the new column after the type it is stored as, so the + /// suffix always matches the column's type (and the schema authority's naming). + #[test] + fn conflict_rename_suffix_matches_stored_type() { + let ts = DataType::Timestamp(TimeUnit::Millisecond, None); + let v1 = SchemaVersion::V1; + // (existing columns, event, schema version, infer_timestamp, expected new column) + type Case = ( + Vec<(&'static str, DataType)>, + Value, + SchemaVersion, + bool, + (&'static str, DataType), + ); + let cases: Vec = vec![ + // V1 stores every number as Float64, integers included. + ( + vec![("g", DataType::Utf8)], + json!({"g": 1000}), + v1, + true, + ("g_float64", DataType::Float64), + ), + ( + vec![("g", DataType::Utf8)], + json!({"g": 1.5}), + v1, + true, + ("g_float64", DataType::Float64), + ), + // Per-record path: mixed types in one batch. + ( + vec![("g", DataType::Utf8)], + json!([{"g": "a"}, {"g": 1000}]), + v1, + true, + ("g_float64", DataType::Float64), + ), + // A date string in a time-named field is stored as Timestamp. + ( + vec![("event_time", DataType::Float64)], + json!({"event_time": "2025-01-01T00:00:00Z"}), + v1, + true, + ("event_time_timestamp_ms", ts.clone()), + ), + ( + vec![("event_time", DataType::Float64)], + json!([{"event_time": 5}, {"event_time": "2025-01-01T00:00:00Z"}]), + v1, + true, + ("event_time_timestamp_ms", ts.clone()), + ), + // ... but not with timestamp inference off, + ( + vec![("event_time", DataType::Float64)], + json!({"event_time": "2025-01-01T00:00:00Z"}), + v1, + false, + ("event_time_utf8", DataType::Utf8), + ), + // nor when the string is not a date, + ( + vec![("event_time", DataType::Float64)], + json!({"event_time": "abc"}), + v1, + true, + ("event_time_utf8", DataType::Utf8), + ), + // nor in a field whose name is not time-like. + ( + vec![("g", DataType::Float64)], + json!({"g": "2025-01-01T00:00:00Z"}), + v1, + true, + ("g_utf8", DataType::Utf8), + ), + // Numbers into a Timestamp field. + ( + vec![("event_time", ts.clone())], + json!({"event_time": 5}), + v1, + true, + ("event_time_float64", DataType::Float64), + ), + // V0 keeps integers as Int64. + ( + vec![("g", DataType::Utf8)], + json!({"g": 1000}), + SchemaVersion::V0, + true, + ("g_int64", DataType::Int64), + ), + ]; + for (existing, body, version, infer_timestamp, (name, data_type)) in cases { + let label = + format!("{existing:?} + {body} ({version:?}, infer_timestamp={infer_timestamp})"); + let added = new_columns(&existing, body, version, infer_timestamp); + assert_eq!( + added, + vec![(name.to_string(), data_type.clone())], + "{label}" + ); + assert!( + name.ends_with(super::super::get_datatype_suffix(&data_type)), + "{label}" + ); + } + } } diff --git a/src/event/format/mod.rs b/src/event/format/mod.rs index af6b4f7d5..3aa99bec6 100644 --- a/src/event/format/mod.rs +++ b/src/event/format/mod.rs @@ -504,11 +504,13 @@ fn value_compatible_with_type( /// Returns a HashMap mapping original field names to new field names (with datatype suffix) /// for fields that have conflicting types. /// Takes JSON values to check if values are actually compatible with existing types. +/// The suffix is the type the new column is stored as (see [`renamed_column_type`]). pub fn detect_schema_conflicts( inferred_schema: &Schema, existing_schema: &HashMap>, values: &[Value], schema_version: SchemaVersion, + infer_timestamp: bool, ) -> HashMap { let mut conflicts = HashMap::new(); @@ -516,26 +518,34 @@ pub fn detect_schema_conflicts( if let Some(existing_field) = existing_schema.get(field.name()) { // Check if data types are different (potential conflict) if existing_field.data_type() != field.data_type() && !field.data_type().is_null() { - // Before declaring conflict, check if all values can be coerced to existing type - let all_values_compatible = values.iter().all(|v| { - if let Some(field_value) = v.get(field.name()) { - if field_value.is_null() { - return true; // null is compatible with any type - } - value_compatible_with_type( - field_value, - existing_field.data_type(), - schema_version, - ) - } else { - true // field not present in this value, no conflict - } - }); + // The values that cannot be coerced to the existing type: exactly the ones + // `rename_conflicting_fields_in_json` moves to the new column. Nulls and + // records without the field never conflict. + let moved: Vec<&Value> = values + .iter() + .filter_map(|v| v.get(field.name())) + .filter(|v| { + !v.is_null() + && !value_compatible_with_type( + v, + existing_field.data_type(), + schema_version, + ) + }) + .collect(); // Only mark as conflict if values cannot be coerced to existing type - if !all_values_compatible { - let suffix = get_datatype_suffix(field.data_type()); - let new_name = format!("{}_{}", field.name(), suffix); + if !moved.is_empty() { + let data_type = match renamed_column_type( + field.name(), + moved, + schema_version, + infer_timestamp, + ) { + DataType::Null => field.data_type().clone(), + data_type => data_type, + }; + let new_name = format!("{}_{}", field.name(), get_datatype_suffix(&data_type)); conflicts.insert(field.name().to_string(), new_name); } } @@ -545,6 +555,44 @@ pub fn detect_schema_conflicts( conflicts } +/// The type of the new column a conflict rename creates for `values` (the values of field +/// `name` that move to it): what ingestion infers for them as a new column, after the schema +/// version's overrides (V1 stores every number as Float64, and date strings in a time-named +/// field as Timestamp). Suffixing with this type rather than the raw JSON type keeps the +/// column's name in step with its type, which is also the name the distributed schema +/// authority gives the same data. +fn renamed_column_type<'a>( + name: &str, + values: impl IntoIterator, + schema_version: SchemaVersion, + infer_timestamp: bool, +) -> DataType { + // Evaluated under a stand-in for the new name `_`. It contains `name`, so + // the time-name rule matches it exactly when it matches the new name (only the + // `timestamp_*` suffixes contain a time part, and they are chosen only for timestamps). + // It is never the series-hash column, whose name alone decides that column's type. + let column = format!("{name}_"); + let values: Vec<&Value> = values.into_iter().filter(|v| !v.is_null()).collect(); + let records: Vec = values + .iter() + .map(|v| Value::Object(serde_json::Map::from_iter([(column.clone(), (*v).clone())]))) + .collect(); + let Ok(inferred) = arrow_json::reader::infer_json_schema_from_iterator(records.iter().map(Ok)) + else { + return values + .first() + .map_or(DataType::Null, |v| datatype_for_value(v)); + }; + // Same per-record pass as `update_field_type_in_schema`, which types the new column. + let schema = records.iter().fold(Arc::new(inferred), |schema, record| { + override_data_type(schema, record.clone(), schema_version, infer_timestamp) + }); + schema + .fields() + .first() + .map_or(DataType::Null, |f| f.data_type().clone()) +} + /// Renames fields in JSON values according to the provided field mapping. /// Used to resolve schema conflicts by renaming fields with conflicting types. /// Renames conflicting JSON fields to typed sibling columns, but only for @@ -606,8 +654,9 @@ pub fn rename_conflicting_fields_in_json( /// /// For each record, scan each non-null field. If the value doesn't satisfy /// the storage type, rename that field in this specific record to -/// `_` so it routes to (or creates) a typed -/// sibling column instead of crashing the batch. +/// `_`, the suffix being the type the value is stored as +/// (see `renamed_column_type`), so it routes to (or creates) a typed sibling +/// column instead of crashing the batch. /// /// This is a no-op (and skips the per-record loop entirely) when: /// - the batch has at most one record — for a single record arrow's @@ -621,6 +670,7 @@ pub fn rename_per_record_type_mismatches( inferred_schema: &Schema, existing_schema: &HashMap>, schema_version: SchemaVersion, + infer_timestamp: bool, ) -> Vec { if values.len() <= 1 { return values; @@ -697,8 +747,9 @@ pub fn rename_per_record_type_mismatches( if value_compatible_with_type(&val, target_type, schema_version) { return (key, val); } - let suffix = get_datatype_suffix(&datatype_for_value(&val)); - let new_key = format!("{key}_{suffix}"); + let data_type = + renamed_column_type(&key, [&val], schema_version, infer_timestamp); + let new_key = format!("{key}_{}", get_datatype_suffix(&data_type)); (new_key, val) }) .collect(); @@ -707,8 +758,8 @@ pub fn rename_per_record_type_mismatches( .collect() } -/// Best-effort mapping from a JSON value to its arrow DataType. Used only to -/// pick a type-suffix for the per-record rename (e.g. "bool", "i64", "f64"). +/// Best-effort mapping from a JSON value to its arrow DataType. Used only as the +/// fallback in `renamed_column_type` when schema inference fails. fn datatype_for_value(value: &Value) -> DataType { match value { Value::Null => DataType::Null, @@ -774,6 +825,7 @@ mod tests { &existing_schema, &values, SchemaVersion::V1, + true, ); // Should detect conflict for body_timestamp (timestamp string can't be parsed as Float64) @@ -804,6 +856,7 @@ mod tests { &existing_schema, &values, SchemaVersion::V1, + true, ); // No conflicts - types match for existing field @@ -913,6 +966,7 @@ mod tests { &existing_schema, &values, SchemaVersion::V1, + true, ); // Should detect conflict and rename to body_timestamp_utf8 @@ -947,6 +1001,7 @@ mod tests { &existing_schema, &values, SchemaVersion::V1, + true, ); // Should NOT detect conflict because the value can be parsed as timestamp @@ -973,6 +1028,7 @@ mod tests { &existing_schema, &values, SchemaVersion::V1, + true, ); assert_eq!(conflicts.get("date"), Some(&"date_utf8".to_string())); @@ -999,6 +1055,7 @@ mod tests { &existing_schema, &values, SchemaVersion::V1, + true, ); // Should detect conflict because number cannot be stored in Utf8 field @@ -1030,6 +1087,7 @@ mod tests { &existing_schema, &values, SchemaVersion::V0, + true, ); // Should detect conflict because integer doesn't match strict Float64 validation @@ -1061,6 +1119,7 @@ mod tests { &existing_schema, &values, SchemaVersion::V1, + true, ); // Should NOT detect conflict because V1 allows any number for Float64 @@ -1227,8 +1286,13 @@ mod tests { json!({"escaped": true}), json!({"escaped": null}), ]; - let renamed = - rename_per_record_type_mismatches(value_arr, &inferred, &storage, SchemaVersion::V1); + let renamed = rename_per_record_type_mismatches( + value_arr, + &inferred, + &storage, + SchemaVersion::V1, + true, + ); assert!(renamed[0].as_object().unwrap().contains_key("escaped")); // bool record routes to escaped_bool @@ -1252,6 +1316,7 @@ mod tests { &inferred, &storage, SchemaVersion::V1, + true, ); for v in &renamed { assert!(v.as_object().unwrap().contains_key("amount")); @@ -1268,12 +1333,13 @@ mod tests { &inferred, &storage, SchemaVersion::V1, + true, ); assert!(renamed[0].as_object().unwrap().contains_key("new_field")); } #[test] - fn rename_per_record_number_into_utf8_uses_int64_suffix() { + fn rename_per_record_number_into_utf8_uses_stored_float64_suffix() { let mut storage: HashMap> = HashMap::new(); storage.insert( "request_body".to_string(), @@ -1282,7 +1348,8 @@ mod tests { let inferred = Schema::new(vec![Field::new("request_body", DataType::Utf8, true)]); // Need >= 2 records for the per-record pass to run; second record's // string keeps inferred type at Utf8 (matches storage), unblocking - // the precheck so the int record gets renamed. + // the precheck so the int record gets renamed. V1 stores every number + // as Float64, so the sibling is named for that, not for the JSON integer. let renamed = rename_per_record_type_mismatches( vec![ json!({"request_body": 200}), @@ -1291,12 +1358,13 @@ mod tests { &inferred, &storage, SchemaVersion::V1, + true, ); assert!( renamed[0] .as_object() .unwrap() - .contains_key("request_body_int64") + .contains_key("request_body_float64") ); } @@ -1317,6 +1385,7 @@ mod tests { &inferred, &storage, SchemaVersion::V1, + true, ); // No rename because the loop is skipped for single-record batches. assert!(renamed[0].as_object().unwrap().contains_key("escaped")); @@ -1336,6 +1405,7 @@ mod tests { &inferred, &storage, SchemaVersion::V1, + true, ); assert!(renamed[0].as_object().unwrap().contains_key("escaped")); assert!(renamed[1].as_object().unwrap().contains_key("escaped_bool")); From bf092982104c8ffe4e322fdcbfc6bf0beab439da Mon Sep 17 00:00:00 2001 From: Anant Vindal Date: Sat, 14 Nov 2026 07:24:24 +0530 Subject: [PATCH 07/13] fix: keep both values when a conflict rename hits an existing key A record carrying a conflicting `f` and a literal `f_utf8` lost one of the two values: the ingest-time rename moved `f` onto `f_utf8` inside the same JSON map, so one value overwrote the other before any file was written. Both rename passes now build each record through one helper. Kept keys claim their names first; a moved value whose target is already a non-null key of the record, or a stream column the value does not fit, chains one more suffix (`f_utf8` -> `f_utf8_utf8`) until the name is free. This is the same chain the schema authority uses, so ingestors and the authority agree on the name. A null at the target is overwritten, since null and absent mean the same. Renaming onto an existing sibling column of another type used to fail the event; it now moves to the next free name. --- src/event/format/json.rs | 17 +++ src/event/format/mod.rs | 289 +++++++++++++++++++++++++++++---------- 2 files changed, 234 insertions(+), 72 deletions(-) diff --git a/src/event/format/json.rs b/src/event/format/json.rs index 1f3698dc5..0146ff536 100644 --- a/src/event/format/json.rs +++ b/src/event/format/json.rs @@ -826,4 +826,21 @@ mod tests { ); } } + + /// An event carrying both a conflicting `f` and a literal `f_utf8` keeps both values + /// end to end: the moved value lands in `f_utf8_utf8`, the literal one in `f_utf8`. + #[test] + fn conflict_rename_keeps_literal_key_of_the_same_name() { + let existing = [("f", DataType::Float64), ("f_utf8", DataType::Utf8)]; + for body in [ + json!({"f": "moved", "f_utf8": "literal"}), + json!([{"f": 1.5}, {"f": "moved", "f_utf8": "literal"}]), + ] { + assert_eq!( + new_columns(&existing, body.clone(), SchemaVersion::V1, true), + vec![("f_utf8_utf8".to_string(), DataType::Utf8)], + "{body}" + ); + } + } } diff --git a/src/event/format/mod.rs b/src/event/format/mod.rs index 3aa99bec6..7dfd63181 100644 --- a/src/event/format/mod.rs +++ b/src/event/format/mod.rs @@ -618,31 +618,87 @@ pub fn rename_conflicting_fields_in_json( values .into_iter() .map(|value| { - if let Value::Object(map) = value { - let new_map: serde_json::Map = map - .into_iter() - .map(|(key, val)| { - if let Some(new_key) = field_mapping.get(&key) { - if existing_schema.get(&key).is_some_and(|field| { - value_compatible_with_type(&val, field.data_type(), schema_version) - }) { - (key, val) - } else { - (new_key.clone(), val) - } - } else { - (key, val) - } - }) - .collect(); - Value::Object(new_map) - } else { - value - } + let Value::Object(map) = value else { + return value; + }; + let entries = map + .into_iter() + .map(|(key, val)| { + let Some(new_key) = field_mapping.get(&key) else { + return (key, val, None); + }; + if existing_schema.get(&key).is_some_and(|field| { + value_compatible_with_type(&val, field.data_type(), schema_version) + }) { + return (key, val, None); + } + // `new_key` is `_`; a collision chains the same suffix. + let suffix = new_key + .strip_prefix(key.as_str()) + .and_then(|rest| rest.strip_prefix('_')) + .unwrap_or(new_key); + let target = (new_key.clone(), suffix); + (key, val, Some(target)) + }) + .collect(); + Value::Object(place_record_values( + entries, + existing_schema, + schema_version, + )) }) .collect() } +/// One key of a record, its value, and where the value moves: the target name +/// `_` and the suffix, or `None` to keep the key. +type RecordEntry<'a> = (String, Value, Option<(String, &'a str)>); + +/// Builds one record from its `(key, value, move)` entries, where `move` is the typed +/// sibling name `_` a value moves to plus that suffix (`None` keeps the key). +/// +/// Kept keys claim their names first. A moved value whose target is already a non-null key +/// of the record, or a stream column the value does not fit, chains one more suffix +/// (`a_utf8` → `a_utf8_utf8`) until the name is free, the way the schema authority chains. +/// So a record carrying both `a` and a literal `a_utf8` keeps both values instead of one +/// overwriting the other. A null in the target is dropped: null and absent mean the same. +fn place_record_values( + entries: Vec>, + existing_schema: &HashMap>, + schema_version: SchemaVersion, +) -> serde_json::Map { + let mut record = serde_json::Map::with_capacity(entries.len()); + let mut moved = Vec::new(); + for (key, val, target) in entries { + match target { + None => { + record.insert(key, val); + } + Some((name, suffix)) => moved.push((name, suffix, val)), + } + } + for (mut name, suffix, val) in moved { + // Terminates: each step makes a longer name, and the record and stream are finite. + while record.get(&name).is_some_and(|v| !v.is_null()) + || existing_schema + .get(&name) + .is_some_and(|f| !value_fits_type(&val, f.data_type(), schema_version)) + { + name = format!("{name}_{suffix}"); + } + record.insert(name, val); + } + record +} + +/// Whether `value` can be stored in a column of `data_type`. Nested values are left to +/// arrow when the column is nested too: `value_compatible_with_type` cannot judge them. +fn value_fits_type(value: &Value, data_type: &DataType, schema_version: SchemaVersion) -> bool { + ((value.is_array() || value.is_object()) + && (data_type.is_list() || matches!(data_type, DataType::Struct(_) | DataType::Map(_, _)))) + || value_compatible_with_type(value, data_type, schema_version) +} + /// Per-record fallback when batch-level conflict detection misses a type /// mismatch. This happens when a single batch contains records with /// incompatible JSON types for the same field (e.g. record A has @@ -694,66 +750,40 @@ pub fn rename_per_record_type_mismatches( let Value::Object(map) = value else { return value; }; - let needs_rename = map.iter().any(|(key, val)| { - if val.is_null() { - return false; - } - let target_type = existing_schema - .get(key) - .map(|f| f.data_type()) - .or_else(|| inferred_types.get(key.as_str()).copied()); - let Some(target_type) = target_type else { - return false; - }; - if (val.is_array() || val.is_object()) - && (target_type.is_list() - || matches!(target_type, DataType::Struct(_) | DataType::Map(_, _))) - { - return false; - } - !value_compatible_with_type(val, target_type, schema_version) - }); - if !needs_rename { + // Prefer storage's declared type; fall back to the inferred type so + // within-batch mismatches on new fields are caught. Nested values on a + // nested column are left to arrow (see `value_fits_type`); scalars still + // flow through the check, so an array on e.g. a Utf8 column is routed to a + // typed sibling rather than crashing decode. + let mismatched = |key: &str, val: &Value| { + !val.is_null() + && existing_schema + .get(key) + .map(|f| f.data_type()) + .or_else(|| inferred_types.get(key).copied()) + .is_some_and(|t| !value_fits_type(val, t, schema_version)) + }; + if !map.iter().any(|(key, val)| mismatched(key, val)) { return Value::Object(map); } - let new_map: serde_json::Map = map + let entries = map .into_iter() .map(|(key, val)| { - if val.is_null() { - return (key, val); - } - // Prefer storage's declared type; fall back to the inferred - // type so within-batch mismatches on new fields are caught. - let target_type = existing_schema - .get(&key) - .map(|f| f.data_type()) - .or_else(|| inferred_types.get(key.as_str()).copied()); - let Some(target_type) = target_type else { - return (key, val); - }; - // When the resolved target is a structural arrow type - // (list/struct/map), arrow validates conformance at decode - // time and value_compatible_with_type can't reliably judge - // arrays/objects — skip to avoid spurious renames of valid - // nested values. Scalars still flow through the check, so - // an array landing on e.g. a Utf8 column is still routed - // to a typed sibling rather than crashing decode. - if (val.is_array() || val.is_object()) - && (target_type.is_list() - || matches!(target_type, DataType::Struct(_) | DataType::Map(_, _))) - { - return (key, val); - } - if value_compatible_with_type(&val, target_type, schema_version) { - return (key, val); + if !mismatched(&key, &val) { + return (key, val, None); } let data_type = renamed_column_type(&key, [&val], schema_version, infer_timestamp); - let new_key = format!("{key}_{}", get_datatype_suffix(&data_type)); - (new_key, val) + let suffix = get_datatype_suffix(&data_type); + let target = (format!("{key}_{suffix}"), suffix); + (key, val, Some(target)) }) .collect(); - Value::Object(new_map) + Value::Object(place_record_values( + entries, + existing_schema, + schema_version, + )) }) .collect() } @@ -1436,4 +1466,119 @@ mod tests { &DataType::Timestamp(TimeUnit::Millisecond, None) ); } + + fn storage(fields: &[(&str, DataType)]) -> HashMap> { + fields + .iter() + .map(|(n, t)| (n.to_string(), Arc::new(Field::new(*n, t.clone(), true)))) + .collect() + } + + /// A record carrying both `f` (moving to `f_utf8`) and a literal `f_utf8` keeps both + /// values: the moved one chains to `f_utf8_utf8`, as the schema authority names it. + #[test] + fn rename_into_key_the_record_already_has_chains_suffix() { + let existing = storage(&[("f", DataType::Float64), ("f_utf8", DataType::Utf8)]); + let mapping = HashMap::from([("f".to_string(), "f_utf8".to_string())]); + let renamed = rename_conflicting_fields_in_json( + vec![ + json!({"f": "moved", "f_utf8": "literal"}), + json!({"f": "alone"}), + ], + &mapping, + &existing, + SchemaVersion::V1, + ); + assert_eq!( + renamed, + vec![ + json!({"f_utf8": "literal", "f_utf8_utf8": "moved"}), + json!({"f_utf8": "alone"}), + ] + ); + } + + #[test] + fn per_record_rename_into_key_the_record_already_has_chains_suffix() { + let existing = storage(&[("f", DataType::Float64), ("f_utf8", DataType::Utf8)]); + let inferred = Schema::new(vec![ + Field::new("f", DataType::Float64, true), + Field::new("f_utf8", DataType::Utf8, true), + ]); + let renamed = rename_per_record_type_mismatches( + vec![ + json!({"f": 1.5}), + json!({"f": "moved", "f_utf8": "literal"}), + ], + &inferred, + &existing, + SchemaVersion::V1, + true, + ); + assert_eq!(renamed[0], json!({"f": 1.5})); + assert_eq!( + renamed[1], + json!({"f_utf8": "literal", "f_utf8_utf8": "moved"}) + ); + } + + /// A target that is a stream column of another type is skipped the same way. + #[test] + fn rename_into_column_of_other_type_chains_suffix() { + let existing = storage(&[("f", DataType::Float64), ("f_utf8", DataType::Boolean)]); + let mapping = HashMap::from([("f".to_string(), "f_utf8".to_string())]); + let renamed = rename_conflicting_fields_in_json( + vec![json!({"f": "x"})], + &mapping, + &existing, + SchemaVersion::V1, + ); + assert_eq!(renamed, vec![json!({"f_utf8_utf8": "x"})]); + } + + /// The chain continues past every taken name, and a literal key whose own value moves + /// away frees its name for the value moving in. + #[test] + fn rename_chain_skips_every_taken_name_and_reuses_freed_ones() { + let existing = storage(&[("f", DataType::Float64), ("f_utf8", DataType::Utf8)]); + let mapping = HashMap::from([("f".to_string(), "f_utf8".to_string())]); + let renamed = rename_conflicting_fields_in_json( + vec![json!({"f": "m", "f_utf8": "a", "f_utf8_utf8": "b"})], + &mapping, + &existing, + SchemaVersion::V1, + ); + assert_eq!( + renamed, + vec![json!({"f_utf8": "a", "f_utf8_utf8": "b", "f_utf8_utf8_utf8": "m"})] + ); + + // `f_utf8: true` cannot stay in the Utf8 `f_utf8` and moves to `f_utf8_bool`. + let inferred = Schema::new(vec![ + Field::new("f", DataType::Utf8, true), + Field::new("f_utf8", DataType::Utf8, true), + ]); + let renamed = rename_per_record_type_mismatches( + vec![json!({"f": 1.5}), json!({"f": "m", "f_utf8": true})], + &inferred, + &existing, + SchemaVersion::V1, + true, + ); + assert_eq!(renamed[1], json!({"f_utf8": "m", "f_utf8_bool": true})); + } + + /// A null literal holds no value, so the moved value takes its name. + #[test] + fn rename_target_holding_null_is_reused() { + let existing = storage(&[("f", DataType::Float64), ("f_utf8", DataType::Utf8)]); + let mapping = HashMap::from([("f".to_string(), "f_utf8".to_string())]); + let renamed = rename_conflicting_fields_in_json( + vec![json!({"f": "x", "f_utf8": null})], + &mapping, + &existing, + SchemaVersion::V1, + ); + assert_eq!(renamed, vec![json!({"f_utf8": "x"})]); + } } From 885bf3e6868356bf160ddf8932acde030fa35935 Mon Sep 17 00:00:00 2001 From: Anant Vindal Date: Sat, 14 Nov 2026 11:37:26 +0530 Subject: [PATCH 08/13] fix: address review findings in schema resolution - kafka: bound chunks in flight per partition. A chunk blocked by schema resolution keeps its records until it succeeds; with unbounded concurrency the partition kept being read into ever more held chunks for the length of the block. Now blocked chunks stop the reads and the consumer waits. - authority: fence resolutions against stream deletion. Deletion takes the stream's authority lock before removing its files and holds it until they are gone, and resolve re-checks the deleting flag under that lock, so a resolution in flight can no longer write `.schema` back for a deleted stream or over its replacement. - ingestor: publish the resolved view before moving the in-memory schema to the resolved types, and read the view under the schema lock in commit_schema. An event built against the old type no longer sees the new type without the rename that allows it and is not rejected. The per-node schema file now also advertises the newly resolved fields right away. --- src/connectors/kafka/processor.rs | 8 ++++- src/event/mod.rs | 38 ++++++++++---------- src/handlers/http/middleware.rs | 34 +++++++++--------- src/migration/mod.rs | 34 +++++++++--------- src/migration/schema_migration.rs | 34 +++++++++--------- src/migration/stream_metadata_migration.rs | 34 +++++++++--------- src/rbac/role.rs | 34 +++++++++--------- src/schema_registry/authority.rs | 6 ++++ src/schema_registry/ingestor.rs | 42 ++++++++++++++++++++-- src/storage/object_storage.rs | 6 ++++ src/utils/actix.rs | 34 +++++++++--------- 11 files changed, 180 insertions(+), 124 deletions(-) diff --git a/src/connectors/kafka/processor.rs b/src/connectors/kafka/processor.rs index 8f63479f6..9bc9f26fb 100644 --- a/src/connectors/kafka/processor.rs +++ b/src/connectors/kafka/processor.rs @@ -136,6 +136,12 @@ fn build_and_process_event( /// Matches the ingest-side `Retry-After` for a stream blocked by schema resolution. const SCHEMA_BLOCKED_RETRY: Duration = Duration::from_secs(60); +/// Chunks of one partition processed at once. A chunk blocked by schema resolution keeps +/// its records until it succeeds, so an unbounded count would keep reading the partition +/// into ever more held chunks for as long as the block lasts. With a bound, blocked chunks +/// stop the reads instead: the partition's queue fills and the consumer waits. +const MAX_CHUNKS_IN_FLIGHT: Option = Some(4); + fn is_schema_resolution_blocked(err: &anyhow::Error) -> bool { matches!( err.downcast_ref::(), @@ -228,7 +234,7 @@ where ); chunked_stream - .for_each_concurrent(None, |records| async { + .for_each_concurrent(MAX_CHUNKS_IN_FLIGHT, |records| async { if let Some(last_record) = records.iter().max_by_key(|r| r.offset) { let tpl = last_record.offset_to_commit().unwrap(); diff --git a/src/event/mod.rs b/src/event/mod.rs index 42fb0260f..8b36363f9 100644 --- a/src/event/mod.rs +++ b/src/event/mod.rs @@ -1,21 +1,21 @@ /* -* Parseable Server (C) 2022 - 2025 Parseable, Inc. -* -* This program is free software: you can redistribute it and/or modify -* it under the terms of the GNU Affero General Public License as -* published by the Free Software Foundation, either version 3 of the -* License, or (at your option) any later version. -* -* This program is distributed in the hope that it will be useful, -* but WITHOUT ANY WARRANTY; without even the implied warranty of -* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -* GNU Affero General Public License for more details. -* -* You should have received a copy of the GNU Affero General Public License -* along with this program. If not, see . -* -* -*/ + * Parseable Server (C) 2022 - 2025 Parseable, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * + */ pub mod format; @@ -175,8 +175,10 @@ pub fn commit_schema( // global lock while waiting for the per-stream metadata lock can deadlock // concurrent ingestion that needs the map to finish processing an event. let stream = PARSEABLE.get_stream(stream_name, tenant_id)?; - let view = stream.resolution.view(); let mut metadata = stream.metadata.write().expect(LOCK_EXPECT); + // Read under the schema lock: a resolution publishes its view before it changes the + // schema, so a resolved type seen here always comes with the renames it allows. + let view = stream.resolution.view(); let map = &mut metadata.schema; // A batch built against the schema from before a resolved type change can still carry // the old type. Its arrow file is renamed at conversion, so it must not fail here. diff --git a/src/handlers/http/middleware.rs b/src/handlers/http/middleware.rs index 13332968d..c4fe2ecc0 100644 --- a/src/handlers/http/middleware.rs +++ b/src/handlers/http/middleware.rs @@ -1,21 +1,21 @@ /* -* Parseable Server (C) 2022 - 2025 Parseable, Inc. -* -* This program is free software: you can redistribute it and/or modify -* it under the terms of the GNU Affero General Public License as -* published by the Free Software Foundation, either version 3 of the -* License, or (at your option) any later version. -* -* This program is distributed in the hope that it will be useful, -* but WITHOUT ANY WARRANTY; without even the implied warranty of -* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -* GNU Affero General Public License for more details. -* -* You should have received a copy of the GNU Affero General Public License -* along with this program. If not, see . -* -* -*/ + * Parseable Server (C) 2022 - 2025 Parseable, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * + */ use std::future::{Ready, ready}; diff --git a/src/migration/mod.rs b/src/migration/mod.rs index 492521f16..835913856 100644 --- a/src/migration/mod.rs +++ b/src/migration/mod.rs @@ -1,21 +1,21 @@ /* -* Parseable Server (C) 2022 - 2025 Parseable, Inc. -* -* This program is free software: you can redistribute it and/or modify -* it under the terms of the GNU Affero General Public License as -* published by the Free Software Foundation, either version 3 of the -* License, or (at your option) any later version. -* -* This program is distributed in the hope that it will be useful, -* but WITHOUT ANY WARRANTY; without even the implied warranty of -* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -* GNU Affero General Public License for more details. -* -* You should have received a copy of the GNU Affero General Public License -* along with this program. If not, see . -* -* -*/ + * Parseable Server (C) 2022 - 2025 Parseable, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * + */ pub mod metadata_migration; mod schema_migration; diff --git a/src/migration/schema_migration.rs b/src/migration/schema_migration.rs index 822f261d4..bfe1ee070 100644 --- a/src/migration/schema_migration.rs +++ b/src/migration/schema_migration.rs @@ -1,21 +1,21 @@ /* -* Parseable Server (C) 2022 - 2025 Parseable, Inc. -* -* This program is free software: you can redistribute it and/or modify -* it under the terms of the GNU Affero General Public License as -* published by the Free Software Foundation, either version 3 of the -* License, or (at your option) any later version. -* -* This program is distributed in the hope that it will be useful, -* but WITHOUT ANY WARRANTY; without even the implied warranty of -* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -* GNU Affero General Public License for more details. -* -* You should have received a copy of the GNU Affero General Public License -* along with this program. If not, see . -* -* -*/ + * Parseable Server (C) 2022 - 2025 Parseable, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * + */ use std::collections::HashMap; diff --git a/src/migration/stream_metadata_migration.rs b/src/migration/stream_metadata_migration.rs index 05017e8f9..5cb534e51 100644 --- a/src/migration/stream_metadata_migration.rs +++ b/src/migration/stream_metadata_migration.rs @@ -1,21 +1,21 @@ /* -* Parseable Server (C) 2022 - 2025 Parseable, Inc. -* -* This program is free software: you can redistribute it and/or modify -* it under the terms of the GNU Affero General Public License as -* published by the Free Software Foundation, either version 3 of the -* License, or (at your option) any later version. -* -* This program is distributed in the hope that it will be useful, -* but WITHOUT ANY WARRANTY; without even the implied warranty of -* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -* GNU Affero General Public License for more details. -* -* You should have received a copy of the GNU Affero General Public License -* along with this program. If not, see . -* -* -*/ + * Parseable Server (C) 2022 - 2025 Parseable, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * + */ use crate::{ catalog::snapshot::CURRENT_SNAPSHOT_VERSION, diff --git a/src/rbac/role.rs b/src/rbac/role.rs index 53be8c1e1..a7f97813d 100644 --- a/src/rbac/role.rs +++ b/src/rbac/role.rs @@ -1,21 +1,21 @@ /* -* Parseable Server (C) 2022 - 2025 Parseable, Inc. -* -* This program is free software: you can redistribute it and/or modify -* it under the terms of the GNU Affero General Public License as -* published by the Free Software Foundation, either version 3 of the -* License, or (at your option) any later version. -* -* This program is distributed in the hope that it will be useful, -* but WITHOUT ANY WARRANTY; without even the implied warranty of -* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -* GNU Affero General Public License for more details. -* -* You should have received a copy of the GNU Affero General Public License -* along with this program. If not, see . -* -* -*/ + * Parseable Server (C) 2022 - 2025 Parseable, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * + */ // Represents actions that corresponds to an api #[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] diff --git a/src/schema_registry/authority.rs b/src/schema_registry/authority.rs index 51f9e9572..303e102c0 100644 --- a/src/schema_registry/authority.rs +++ b/src/schema_registry/authority.rs @@ -171,6 +171,12 @@ impl SchemaAuthority { let reserved = reserved_fields(&stream); let mut guard = self.slot(&req.tenant_id, &req.stream).lock_owned().await; + // Deletion marks the stream, then takes this lock before removing its files and + // holds it until they are gone. Checking again under the lock means a resolution + // that waited behind deletion never writes `.schema` for the deleted stream. + if stream.is_deleting() { + return Err(AuthorityError::StreamDeleting(req.stream)); + } if guard.is_none() { *guard = Some(load_registry(&req.stream, &req.tenant_id, reserved).await?); } diff --git a/src/schema_registry/ingestor.rs b/src/schema_registry/ingestor.rs index c0cffb6f7..4779b13f6 100644 --- a/src/schema_registry/ingestor.rs +++ b/src/schema_registry/ingestor.rs @@ -126,8 +126,8 @@ impl ResolutionState { } /// Splits `files` into convertible and deferred, asking the authority about any - /// field not yet resolved. `on_apply` runs after a new view was persisted and before - /// it becomes visible, so the stream can align its in-memory and staged schemas. + /// field not yet resolved. `on_apply` runs after a new view was persisted and made + /// visible, so the stream can align its in-memory and staged schemas with it. pub fn ensure_resolved( &self, ctx: &ResolveContext<'_>, @@ -175,12 +175,15 @@ impl ResolutionState { self.record_failure(ctx, format!("could not persist resolved fields: {err}")); return self.split(&view, schemas); } - on_apply(&next); + // Publish the view before `on_apply` moves the in-memory schema to the + // resolved types: `commit_schema` reads the view under the schema lock, so + // whoever sees a resolved type also sees the rename that allows the old one. let next = Arc::new(next); self.view .write() .expect("view lock poisoned") .clone_from(&next); + on_apply(&next); *self.pending_request.lock().expect("lock poisoned") = None; self.finish(ctx, &next, schemas) } @@ -443,6 +446,39 @@ mod tests { ); } + /// `on_apply` moves the in-memory schema to the resolved types, so the view carrying + /// the matching renames must already be visible to other threads when it runs. + #[test] + fn resolved_view_is_visible_before_on_apply() { + let registry = RefCell::new(StreamRegistry::default()); + registry + .borrow_mut() + .apply(&[("a".to_string(), DataType::Int64)]); + let resolver = registry_resolver(®istry); + let tenant = None; + let dir = tempfile::tempdir().unwrap(); + let file = write_arrow( + dir.path(), + "a.arrows", + Field::new("a", DataType::Utf8, true), + Arc::new(StringArray::from(vec!["x"])), + ); + + let state = ResolutionState::default(); + let applied = RefCell::new(false); + state.ensure_resolved(&ctx(dir.path(), &resolver, &tenant), vec![file], |next| { + let visible = state.view(); + assert_eq!(visible.registry_version, next.registry_version); + assert!( + visible + .renames + .contains_key(&("a".to_string(), DataType::Utf8)) + ); + *applied.borrow_mut() = true; + }); + assert!(*applied.borrow()); + } + #[test] fn cached_rename_colliding_with_a_column_is_re_resolved() { let registry = RefCell::new(StreamRegistry::default()); diff --git a/src/storage/object_storage.rs b/src/storage/object_storage.rs index d906d0104..9f3b7edc4 100644 --- a/src/storage/object_storage.rs +++ b/src/storage/object_storage.rs @@ -1554,6 +1554,12 @@ async fn run_stream_deletion( return; } } + // Wait for schema resolutions of this stream in flight, and keep new ones out until its + // files are gone: one that already passed its deletion check would otherwise write + // `.schema` back after the prefix is removed. Held to the end of this function. + let _resolutions = crate::schema_registry::authority::SCHEMA_AUTHORITY + .lock_and_invalidate(tenant_id, stream_name) + .await; match storage.delete_stream(stream_name, tenant_id).await { Ok(()) => { // Evict and reset stats before clearing the tombstone: once the diff --git a/src/utils/actix.rs b/src/utils/actix.rs index ac76075a8..a28189d44 100644 --- a/src/utils/actix.rs +++ b/src/utils/actix.rs @@ -1,21 +1,21 @@ /* -* Parseable Server (C) 2022 - 2025 Parseable, Inc. -* -* This program is free software: you can redistribute it and/or modify -* it under the terms of the GNU Affero General Public License as -* published by the Free Software Foundation, either version 3 of the -* License, or (at your option) any later version. -* -* This program is distributed in the hope that it will be useful, -* but WITHOUT ANY WARRANTY; without even the implied warranty of -* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -* GNU Affero General Public License for more details. -* -* You should have received a copy of the GNU Affero General Public License -* along with this program. If not, see . -* -* -*/ + * Parseable Server (C) 2022 - 2025 Parseable, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * + */ use actix_web::{ Error, FromRequest, HttpMessage, HttpRequest, From 4cb599bf2c8eb845ad5c52eb7740377595f0f8cd Mon Sep 17 00:00:00 2001 From: Anant Vindal Date: Sat, 14 Nov 2026 12:14:33 +0530 Subject: [PATCH 09/13] fix(kafka): pause only the partition blocked by schema resolution A chunk whose stream rejects writes until its schema is reconciled is held and retried, with its offsets uncommitted. Capping chunks in flight to bound those held records stopped draining the partition's queue; once full, the consumer loop shared by every partition waited on it, stalling all topics for as long as the block lasted. Now the first blocked chunk pauses its partition and the last one to go through resumes it. Chunk processing is unbounded again, so blocked chunks drain the queue instead of filling it, and the pause bounds what a blocked partition holds. The processor reports a block as `SchemaBlocked` carrying the records back, and the retry moves to the partition worker, which owns the consumer. --- src/connectors/kafka/processor.rs | 171 +++++++++++++++++++++++------- 1 file changed, 130 insertions(+), 41 deletions(-) diff --git a/src/connectors/kafka/processor.rs b/src/connectors/kafka/processor.rs index 9bc9f26fb..680a34e1b 100644 --- a/src/connectors/kafka/processor.rs +++ b/src/connectors/kafka/processor.rs @@ -38,7 +38,7 @@ use std::collections::HashMap; use std::sync::Arc; use std::time::Duration; use tokio_stream::wrappers::ReceiverStream; -use tracing::{debug, error, warn}; +use tracing::{debug, error, info, warn}; use super::{ConsumerRecord, StreamConsumer, TopicPartition, config::BufferConfig}; @@ -136,21 +136,28 @@ fn build_and_process_event( /// Matches the ingest-side `Retry-After` for a stream blocked by schema resolution. const SCHEMA_BLOCKED_RETRY: Duration = Duration::from_secs(60); -/// Chunks of one partition processed at once. A chunk blocked by schema resolution keeps -/// its records until it succeeds, so an unbounded count would keep reading the partition -/// into ever more held chunks for as long as the block lasts. With a bound, blocked chunks -/// stop the reads instead: the partition's queue fills and the consumer waits. -const MAX_CHUNKS_IN_FLIGHT: Option = Some(4); - -fn is_schema_resolution_blocked(err: &anyhow::Error) -> bool { - matches!( - err.downcast_ref::(), - Some(EventError::SchemaResolutionBlocked(_, _)) - ) +/// A chunk whose stream rejects writes until its schema is reconciled with the cluster. +/// Carries the records back, so the chunk can be held (and its offsets left uncommitted) +/// and retried instead of dropped. +#[derive(thiserror::Error)] +#[error("ingestion blocked by schema resolution: {reason}")] +pub struct SchemaBlocked { + pub records: Vec, + reason: String, +} + +impl std::fmt::Debug for SchemaBlocked { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("SchemaBlocked") + .field("records", &self.records.len()) + .field("reason", &self.reason) + .finish() + } } #[async_trait] impl Processor, ()> for ParseableSinkProcessor { + /// Fails with [`SchemaBlocked`] while the stream rejects writes. async fn process(&self, records: Vec) -> anyhow::Result<()> { let len = records.len(); debug!("Processing {len} records"); @@ -158,34 +165,21 @@ impl Processor, ()> for ParseableSinkProcessor { let stream_name = records.first().map(|r| r.topic.clone()).unwrap_or_default(); let tenant_id = records.first().and_then(|r| r.tenant_id.clone()); - let mut records = records; - loop { - let meta = self.prepare_stream(&stream_name, &tenant_id).await?; - let (tx, rx) = tokio::sync::oneshot::channel(); - let (name, tenant) = (stream_name.clone(), tenant_id.clone()); - rayon::spawn(move || { - let res = build_and_process_event(&records, name, tenant, meta); - let _ = tx.send((records, res)); - }); - let (returned, res) = rx.await?; - match res { - // The stream rejects writes until its schema is reconciled with the cluster. - // Committing now would drop these records, so hold the chunk (and its - // offsets) and retry once resolution had a chance to run. - Err(e) if is_schema_resolution_blocked(&e) => { - warn!( - stream = stream_name, - "ingestion blocked by schema resolution, retrying {len} records in {}s: {e}", - SCHEMA_BLOCKED_RETRY.as_secs() - ); - records = returned; - tokio::time::sleep(SCHEMA_BLOCKED_RETRY).await; + let meta = self.prepare_stream(&stream_name, &tenant_id).await?; + let (tx, rx) = tokio::sync::oneshot::channel(); + rayon::spawn(move || { + let res = build_and_process_event(&records, stream_name, tenant_id, meta); + let _ = tx.send((records, res)); + }); + let (records, res) = rx.await?; + if let Err(err) = res { + return Err(match err.downcast::() { + Ok(EventError::SchemaResolutionBlocked(_, reason)) => { + SchemaBlocked { records, reason }.into() } - other => { - other?; - break; - } - } + Ok(other) => other.into(), + Err(other) => other, + }); } debug!("Processed {len} records"); @@ -193,6 +187,58 @@ impl Processor, ()> for ParseableSinkProcessor { } } +/// Pauses fetching for one partition while any of its chunks is blocked by schema +/// resolution, and resumes it when the last one goes through. +/// +/// All partitions share one consumer loop, which waits whenever a partition's queue is +/// full. Pausing keeps a blocked partition from filling its queue, so the loop keeps +/// serving the other partitions; the blocked chunks hold at most what was already fetched. +/// A resumed partition fetches again from the last record handed to the application, so +/// nothing is skipped or read twice. +struct PartitionPause<'a> { + consumer: &'a StreamConsumer, + tp: &'a TopicPartition, + blocked_chunks: std::sync::Mutex, +} + +impl<'a> PartitionPause<'a> { + fn new(consumer: &'a StreamConsumer, tp: &'a TopicPartition) -> Self { + Self { + consumer, + tp, + blocked_chunks: std::sync::Mutex::new(0), + } + } + + fn tpl(&self) -> rdkafka::TopicPartitionList { + let mut tpl = rdkafka::TopicPartitionList::new(); + tpl.add_partition(&self.tp.topic, self.tp.partition); + tpl + } + + fn hold(&self) { + let mut blocked = self.blocked_chunks.lock().expect("pause lock poisoned"); + if *blocked == 0 { + match self.consumer.pause(&self.tpl()) { + Ok(()) => warn!("Paused {:?}: its stream rejects writes", self.tp), + Err(e) => error!(error = %e, "Failed to pause {:?}", self.tp), + } + } + *blocked += 1; + } + + fn release(&self) { + let mut blocked = self.blocked_chunks.lock().expect("pause lock poisoned"); + *blocked -= 1; + if *blocked == 0 { + match self.consumer.resume(&self.tpl()) { + Ok(()) => info!("Resumed {:?}", self.tp), + Err(e) => error!(error = %e, "Failed to resume {:?}", self.tp), + } + } + } +} + #[derive(Clone)] pub struct StreamWorker

where @@ -222,6 +268,45 @@ where } } + /// Processes one chunk, holding it while its stream is blocked by schema resolution: + /// committing then would drop its records. Returns once the chunk went through or + /// failed for another reason. + async fn process_chunk( + &self, + tp: &TopicPartition, + pause: &PartitionPause<'_>, + mut records: Vec, + ) -> anyhow::Result<()> { + let mut held = false; + let result = loop { + match self.processor.process(records).await { + Err(err) => match err.downcast::() { + Ok(blocked) => { + if !held { + pause.hold(); + held = true; + } + warn!( + "{:?}: {}; retrying {} records in {}s", + tp, + blocked.reason, + blocked.records.len(), + SCHEMA_BLOCKED_RETRY.as_secs() + ); + records = blocked.records; + tokio::time::sleep(SCHEMA_BLOCKED_RETRY).await; + } + Err(err) => break Err(err), + }, + Ok(()) => break Ok(()), + } + }; + if held { + pause.release(); + } + result + } + pub async fn process_partition( &self, tp: TopicPartition, @@ -232,13 +317,17 @@ where self.buffer_config.buffer_size, self.buffer_config.buffer_timeout, ); + let pause = PartitionPause::new(&self.consumer, &tp); + // Unbounded on purpose: blocked chunks must keep draining the partition's queue, or + // a full queue would stall the consumer loop shared by every partition. The pause + // bounds what a blocked partition holds. chunked_stream - .for_each_concurrent(MAX_CHUNKS_IN_FLIGHT, |records| async { + .for_each_concurrent(None, |records| async { if let Some(last_record) = records.iter().max_by_key(|r| r.offset) { let tpl = last_record.offset_to_commit().unwrap(); - if let Err(e) = self.processor.process(records).await { + if let Err(e) = self.process_chunk(&tp, &pause, records).await { error!("Failed to process records for {:?}: {:?}", tp, e); } From dee867a943c916dc465da917cf5741eb77ea0920 Mon Sep 17 00:00:00 2001 From: Anant Vindal Date: Sat, 14 Nov 2026 19:24:43 +0530 Subject: [PATCH 10/13] fix: accept stale-typed events after any resolved type change; order Kafka commits A stream switches a field to the authority's type whenever it learns it: from a rename, from the registry snapshot sent to a lagging node, or at restart. commit_schema tolerated events still carrying the old type only when a rename existed, so a type learned from the snapshot rejected valid events built just before the switch. It now tolerates the old type whenever the stream's type is the authority's; the event's arrow file is resolved at conversion like any other unresolved type. Kafka chunks of a partition finish out of order: one can wait out a schema block while later ones go through. Each committed its own offset, so a later chunk could commit past records still waiting, and a restart or rebalance would skip them. Offsets are now committed only through the last consecutive completed chunk, under a lock so the committed offset never moves back. A partition revoked while paused is resumed when its worker is dropped. --- src/connectors/kafka/processor.rs | 141 ++++++++++++++++++++++++++---- src/event/mod.rs | 13 ++- src/schema_registry/view.rs | 47 ++++++++++ 3 files changed, 175 insertions(+), 26 deletions(-) diff --git a/src/connectors/kafka/processor.rs b/src/connectors/kafka/processor.rs index 680a34e1b..8d914038e 100644 --- a/src/connectors/kafka/processor.rs +++ b/src/connectors/kafka/processor.rs @@ -34,7 +34,7 @@ use chrono::Utc; use futures_util::StreamExt; use rdkafka::consumer::{CommitMode, Consumer}; use serde_json::Value; -use std::collections::HashMap; +use std::collections::{BTreeMap, HashMap}; use std::sync::Arc; use std::time::Duration; use tokio_stream::wrappers::ReceiverStream; @@ -231,10 +231,75 @@ impl<'a> PartitionPause<'a> { let mut blocked = self.blocked_chunks.lock().expect("pause lock poisoned"); *blocked -= 1; if *blocked == 0 { - match self.consumer.resume(&self.tpl()) { - Ok(()) => info!("Resumed {:?}", self.tp), - Err(e) => error!(error = %e, "Failed to resume {:?}", self.tp), - } + self.resume(); + } + } + + fn resume(&self) { + match self.consumer.resume(&self.tpl()) { + Ok(()) => info!("Resumed {:?}", self.tp), + Err(e) => error!(error = %e, "Failed to resume {:?}", self.tp), + } + } +} + +impl Drop for PartitionPause<'_> { + /// A revoked partition drops its worker with chunks still held. Resume it anyway, so a + /// later assignment of the partition does not start paused with nothing to resume it. + fn drop(&mut self) { + let blocked = self.blocked_chunks.get_mut().map_or(0, |b| *b); + if blocked > 0 { + self.resume(); + } + } +} + +/// Commits one partition's chunks in order. +/// +/// Chunks finish out of order: one can wait out a schema block while later ones go +/// through. Committing each chunk's own offset would then move the committed offset past +/// records not written yet, and a restart or rebalance would skip them. A chunk's offset +/// is committed only once every earlier chunk is done, so the committed offset never +/// passes an unwritten record. Records written after it may be read again after a restart. +#[derive(Default)] +struct OrderedCommits { + state: std::sync::Mutex>, +} + +struct CommitState { + /// Sequence number of the earliest chunk not done yet. + next: usize, + /// Done chunks after `next`, with the offsets each would commit. + done: BTreeMap>, +} + +impl Default for CommitState { + fn default() -> Self { + Self { + next: 0, + done: BTreeMap::new(), + } + } +} + +impl OrderedCommits { + /// Marks chunk `seq` (numbered from 0 in stream order) done and calls `commit` with the + /// offsets of the latest chunk that every earlier chunk is done before. Runs `commit` + /// under the lock, so commits never go backwards. + fn complete(&self, seq: usize, offsets: Option, commit: impl FnOnce(T)) { + let mut state = self.state.lock().expect("commit lock poisoned"); + state.done.insert(seq, offsets); + let mut latest = None; + loop { + let next = state.next; + let Some(offsets) = state.done.remove(&next) else { + break; + }; + state.next += 1; + latest = offsets.or(latest); + } + if let Some(offsets) = latest { + commit(offsets); } } } @@ -318,27 +383,36 @@ where self.buffer_config.buffer_timeout, ); let pause = PartitionPause::new(&self.consumer, &tp); + let commits = OrderedCommits::default(); // Unbounded on purpose: blocked chunks must keep draining the partition's queue, or // a full queue would stall the consumer loop shared by every partition. The pause // bounds what a blocked partition holds. chunked_stream - .for_each_concurrent(None, |records| async { - if let Some(last_record) = records.iter().max_by_key(|r| r.offset) { - let tpl = last_record.offset_to_commit().unwrap(); - - if let Err(e) = self.process_chunk(&tp, &pause, records).await { + .enumerate() + .for_each_concurrent(None, |(seq, records)| { + let (tp, pause, commits) = (&tp, &pause, &commits); + async move { + let tpl = records + .iter() + .max_by_key(|r| r.offset) + .map(|last| last.offset_to_commit().unwrap()); + if !records.is_empty() + && let Err(e) = self.process_chunk(tp, pause, records).await + { error!("Failed to process records for {:?}: {:?}", tp, e); } - //CommitMode::Async race condition. - //@see https://github.com/confluentinc/librdkafka/issues/4534 - //@see https://github.com/confluentinc/librdkafka/issues/4059 - if let Err(e) = self.consumer.commit(&tpl, CommitMode::Sync) { - error!(error = %e, "Failed to commit offsets for {:?}", tpl); - } else { - debug!("Committed offsets for {:?}", tpl); - } + commits.complete(seq, tpl, |tpl| { + //CommitMode::Async race condition. + //@see https://github.com/confluentinc/librdkafka/issues/4534 + //@see https://github.com/confluentinc/librdkafka/issues/4059 + if let Err(e) = self.consumer.commit(&tpl, CommitMode::Sync) { + error!(error = %e, "Failed to commit offsets for {:?}", tpl); + } else { + debug!("Committed offsets for {:?}", tpl); + } + }); } }) .await; @@ -348,3 +422,34 @@ where Ok(()) } } + +#[cfg(test)] +mod tests { + use super::*; + + fn complete(commits: &OrderedCommits, seq: usize, offset: Option) -> Option { + let mut committed = None; + commits.complete(seq, offset, |o| committed = Some(o)); + committed + } + + #[test] + fn later_chunk_waits_for_earlier_ones() { + let commits = OrderedCommits::::default(); + // Chunk 0 is held (schema block) while 1 and 2 go through: nothing committed. + assert_eq!(complete(&commits, 1, Some(20)), None); + assert_eq!(complete(&commits, 2, Some(30)), None); + // Chunk 0 done: commits through the furthest consecutive chunk. + assert_eq!(complete(&commits, 0, Some(10)), Some(30)); + assert_eq!(complete(&commits, 3, Some(40)), Some(40)); + } + + #[test] + fn chunk_without_offsets_still_advances() { + let commits = OrderedCommits::::default(); + assert_eq!(complete(&commits, 1, Some(20)), None); + assert_eq!(complete(&commits, 0, None), Some(20)); + assert_eq!(complete(&commits, 2, None), None); + assert_eq!(complete(&commits, 3, Some(40)), Some(40)); + } +} diff --git a/src/event/mod.rs b/src/event/mod.rs index 8b36363f9..e15cda157 100644 --- a/src/event/mod.rs +++ b/src/event/mod.rs @@ -177,21 +177,18 @@ pub fn commit_schema( let stream = PARSEABLE.get_stream(stream_name, tenant_id)?; let mut metadata = stream.metadata.write().expect(LOCK_EXPECT); // Read under the schema lock: a resolution publishes its view before it changes the - // schema, so a resolved type seen here always comes with the renames it allows. + // schema, so a type the authority set is always in the view seen here. let view = stream.resolution.view(); let map = &mut metadata.schema; // A batch built against the schema from before a resolved type change can still carry - // the old type. Its arrow file is renamed at conversion, so it must not fail here. + // the old type. Its arrow file is resolved at conversion, so it must not fail here. let incoming: Vec> = schema .fields() .iter() .filter(|f| { - !(map - .get(f.name()) - .is_some_and(|cur| cur.data_type() != f.data_type()) - && view - .renames - .contains_key(&(f.name().clone(), f.data_type().clone()))) + !map.get(f.name()).is_some_and(|cur| { + view.predates_resolved_type(f.name(), cur.data_type(), f.data_type()) + }) }) .cloned() .collect(); diff --git a/src/schema_registry/view.rs b/src/schema_registry/view.rs index 996bca47d..f04d3c99d 100644 --- a/src/schema_registry/view.rs +++ b/src/schema_registry/view.rs @@ -103,6 +103,27 @@ impl ResolvedView { } } + /// Whether a batch field typed `incoming` predates a type the authority set for `name`: + /// the stream's type `current` is the authority's, and `incoming` is another one. + /// + /// Such a batch was built against the schema from before a resolution changed the + /// type (a stream switches to the authority's type whatever told it so: a rename, a + /// registry snapshot or a restart). It must still be accepted: its arrow file carries + /// `incoming`, which conversion resolves like any other unresolved type, renaming it or + /// holding the file back. + pub fn predates_resolved_type( + &self, + name: &str, + current: &DataType, + incoming: &DataType, + ) -> bool { + current != incoming + && self + .fields + .get(name) + .is_some_and(|resolved| compatible(resolved, current)) + } + /// Folds an authority answer into a new view. pub fn with_resolutions( &self, @@ -403,6 +424,32 @@ mod tests { assert_eq!(ResolvedView::load(dir.path()), v); } + /// A type learned only from a registry snapshot carries no rename, yet the stream + /// switches to it; a batch still carrying the local type must be accepted. + #[test] + fn snapshot_type_tolerates_batches_typed_before_it() { + let v = ResolvedView::default().with_resolutions( + 4, + &[], + &[RegistryEntry { + name: "b".into(), + data_type: DataType::Float64, + }], + ); + assert!(v.renames.is_empty()); + // The stream switched to the authority's type: an older Utf8 batch predates it. + assert!(v.predates_resolved_type("b", &DataType::Float64, &DataType::Utf8)); + // Same type: nothing to tolerate. + assert!(!v.predates_resolved_type("b", &DataType::Float64, &DataType::Float64)); + // The stream's type is not the authority's: a real conflict, not a stale batch. + assert!(!v.predates_resolved_type("b", &DataType::Utf8, &DataType::Float64)); + // Nothing resolved for the name. + assert!(!v.predates_resolved_type("c", &DataType::Float64, &DataType::Utf8)); + // A renamed type is covered the same way. + let v = renamed_view(); + assert!(v.predates_resolved_type("a", &DataType::Int64, &DataType::Utf8)); + } + #[test] fn missing_or_corrupt_file_is_empty_view() { let dir = tempfile::tempdir().unwrap(); From a801ac9caf6b2b440b8a574874865af71d266337 Mon Sep 17 00:00:00 2001 From: Anant Vindal Date: Sun, 15 Nov 2026 08:26:38 +0530 Subject: [PATCH 11/13] fix(kafka): share partition pause across workers A revoked partition's worker is dropped without the revoke waiting for it, so its Drop could resume the partition after a new worker for the same partition had already paused it. The new worker's later blocked chunks did not pause again, and the resumed partition kept filling with sleeping chunks. The count of blocked chunks now belongs to the partition and is shared by all its workers; the partition is paused exactly while the count is above zero. A dropped worker gives back only its own chunks. Each worker also pauses on its own first blocked chunk, so it does not depend on an older worker's pause having survived the reassignment. --- src/connectors/kafka/processor.rs | 133 +++++++++++++++++++++++++----- 1 file changed, 113 insertions(+), 20 deletions(-) diff --git a/src/connectors/kafka/processor.rs b/src/connectors/kafka/processor.rs index 8d914038e..b8a64411b 100644 --- a/src/connectors/kafka/processor.rs +++ b/src/connectors/kafka/processor.rs @@ -195,18 +195,30 @@ impl Processor, ()> for ParseableSinkProcessor { /// serving the other partitions; the blocked chunks hold at most what was already fetched. /// A resumed partition fetches again from the last record handed to the application, so /// nothing is skipped or read twice. +/// +/// The pause belongs to the partition, not to one worker: it outlives a revoke, and a +/// revoked worker can still be shutting down when a reassignment starts a new one. The +/// count of blocked chunks is therefore shared by every worker of the partition, and the +/// partition is paused exactly while it is above zero. struct PartitionPause<'a> { consumer: &'a StreamConsumer, tp: &'a TopicPartition, - blocked_chunks: std::sync::Mutex, + blocked: &'a BlockedChunks, + /// This worker's share of the partition's blocked chunks. + held: std::sync::atomic::AtomicUsize, } impl<'a> PartitionPause<'a> { - fn new(consumer: &'a StreamConsumer, tp: &'a TopicPartition) -> Self { + fn new( + consumer: &'a StreamConsumer, + tp: &'a TopicPartition, + blocked: &'a BlockedChunks, + ) -> Self { Self { consumer, tp, - blocked_chunks: std::sync::Mutex::new(0), + blocked, + held: std::sync::atomic::AtomicUsize::default(), } } @@ -217,22 +229,20 @@ impl<'a> PartitionPause<'a> { } fn hold(&self) { - let mut blocked = self.blocked_chunks.lock().expect("pause lock poisoned"); - if *blocked == 0 { - match self.consumer.pause(&self.tpl()) { + // A worker pauses on its first blocked chunk even if an older worker's pause should + // still be in effect: pausing again is harmless, and the new worker does not have + // to rely on that pause having survived the reassignment. + let first = self.held.fetch_add(1, std::sync::atomic::Ordering::Relaxed) == 0; + self.blocked + .add(self.tp, first, || match self.consumer.pause(&self.tpl()) { Ok(()) => warn!("Paused {:?}: its stream rejects writes", self.tp), Err(e) => error!(error = %e, "Failed to pause {:?}", self.tp), - } - } - *blocked += 1; + }); } fn release(&self) { - let mut blocked = self.blocked_chunks.lock().expect("pause lock poisoned"); - *blocked -= 1; - if *blocked == 0 { - self.resume(); - } + self.held.fetch_sub(1, std::sync::atomic::Ordering::Relaxed); + self.blocked.remove(self.tp, 1, || self.resume()); } fn resume(&self) { @@ -244,14 +254,57 @@ impl<'a> PartitionPause<'a> { } impl Drop for PartitionPause<'_> { - /// A revoked partition drops its worker with chunks still held. Resume it anyway, so a - /// later assignment of the partition does not start paused with nothing to resume it. + /// A revoked partition drops its worker with chunks still held. Give them back, so a + /// later assignment of the partition does not stay paused with nothing to resume it, + /// while a newer worker's blocked chunks keep it paused. fn drop(&mut self) { - let blocked = self.blocked_chunks.get_mut().map_or(0, |b| *b); - if blocked > 0 { - self.resume(); + let held = *self.held.get_mut(); + if held > 0 { + self.blocked.remove(self.tp, held, || self.resume()); + } + } +} + +/// Blocked chunks per partition, across every worker of one consumer. +#[derive(Default)] +struct BlockedChunks { + counts: std::sync::Mutex>, +} + +impl BlockedChunks { + /// Adds one blocked chunk of `tp`, calling `pause` if it is the first or `force` is + /// set. Runs `pause` under the lock, so pauses and resumes reach the consumer in count + /// order. + fn add(&self, tp: &K, force: bool, pause: impl FnOnce()) { + let mut counts = self.lock(); + let count = counts.entry(tp.clone()).or_default(); + if *count == 0 || force { + pause(); + } + *count += 1; + } + + /// Removes `n` blocked chunks of `tp`, calling `resume` (under the lock) if none is + /// left. + fn remove(&self, tp: &K, n: usize, resume: impl FnOnce()) { + let mut counts = self.lock(); + let Some(count) = counts.get_mut(tp) else { + return; + }; + *count = count.saturating_sub(n); + if *count == 0 { + counts.remove(tp); + resume(); } } + + /// Also used from `Drop`, which must not panic: the counts stay consistent even if a + /// holder panicked, since every update is a single step. + fn lock(&self) -> std::sync::MutexGuard<'_, HashMap> { + self.counts + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + } } /// Commits one partition's chunks in order. @@ -312,6 +365,7 @@ where processor: Arc

, consumer: Arc, buffer_config: BufferConfig, + blocked: Arc, } impl

StreamWorker

@@ -330,6 +384,7 @@ where processor, consumer, buffer_config, + blocked: Arc::default(), } } @@ -382,7 +437,7 @@ where self.buffer_config.buffer_size, self.buffer_config.buffer_timeout, ); - let pause = PartitionPause::new(&self.consumer, &tp); + let pause = PartitionPause::new(&self.consumer, &tp, &self.blocked); let commits = OrderedCommits::default(); // Unbounded on purpose: blocked chunks must keep draining the partition's queue, or @@ -444,6 +499,44 @@ mod tests { assert_eq!(complete(&commits, 3, Some(40)), Some(40)); } + /// Applies one change to the counts and reports the pause or resume it made, if any. + fn hold(blocked: &BlockedChunks, first_of_worker: bool) -> Option<&'static str> { + let mut call = None; + blocked.add(&0, first_of_worker, || call = Some("pause")); + call + } + + fn release(blocked: &BlockedChunks, n: usize) -> Option<&'static str> { + let mut call = None; + blocked.remove(&0, n, || call = Some("resume")); + call + } + + #[test] + fn partition_paused_while_any_chunk_is_blocked() { + let blocked = BlockedChunks::::default(); + assert_eq!(hold(&blocked, true), Some("pause")); + assert_eq!(hold(&blocked, false), None); + assert_eq!(release(&blocked, 1), None); + assert_eq!(release(&blocked, 1), Some("resume")); + // Nothing left to resume. + assert_eq!(release(&blocked, 1), None); + } + + #[test] + fn revoked_worker_keeps_newer_workers_pause() { + let blocked = BlockedChunks::::default(); + // The old worker pauses the partition with two blocked chunks, then is revoked. + assert_eq!(hold(&blocked, true), Some("pause")); + assert_eq!(hold(&blocked, false), None); + // The partition is reassigned and a new worker blocks before the old one is dropped. + assert_eq!(hold(&blocked, true), Some("pause")); + // Dropping the old worker gives back its two chunks without resuming. + assert_eq!(release(&blocked, 2), None); + // The new worker's chunk goes through and resumes the partition. + assert_eq!(release(&blocked, 1), Some("resume")); + } + #[test] fn chunk_without_offsets_still_advances() { let commits = OrderedCommits::::default(); From a11e9e60372eba9b4132f0cca38220bdd0b47e29 Mon Sep 17 00:00:00 2001 From: Anant Vindal Date: Tue, 17 Nov 2026 11:41:04 +0530 Subject: [PATCH 12/13] fix: let cluster nodes resolve schemas for any tenant With P_CLUSTER_SECRET set, an ingestor resolving a schema for a non-default tenant sent the root username as intra-cluster-userid. That user only exists in the default tenant, so IntraClusterRequest created no session and the route's ResolveSchema check rejected the call. New fields stayed unconverted and ingestion fell back to 503s. The resolve route now authorises in the handler. A verified cluster-secret caller is a node acting for the tenant it names in intra-cluster-tenant; no user of that tenant is needed. Without the secret the caller must still hold ResolveSchema and acts for its own tenant. The body still cannot name another tenant. Also share the intra-cluster-tenant header name as a constant. --- src/handlers/http/middleware.rs | 4 ++- src/handlers/http/modal/query_server.rs | 3 +- src/schema_registry/authority.rs | 44 ++++++++++++++++++++----- src/schema_registry/client.rs | 4 ++- 4 files changed, 44 insertions(+), 11 deletions(-) diff --git a/src/handlers/http/middleware.rs b/src/handlers/http/middleware.rs index c4fe2ecc0..262349c6b 100644 --- a/src/handlers/http/middleware.rs +++ b/src/handlers/http/middleware.rs @@ -57,6 +57,8 @@ use crate::{ use serde::{Deserialize, Serialize}; pub const CLUSTER_SECRET_HEADER: &str = "x-p-cluster-secret"; +/// Tenant an intra-cluster request acts for, trusted once the cluster secret is verified. +pub const INTRA_CLUSTER_TENANT_HEADER: &str = "intra-cluster-tenant"; pub static CLUSTER_SECRET: OnceCell<(String, String)> = OnceCell::new(); #[derive(Serialize, Deserialize, Debug)] @@ -719,7 +721,7 @@ where fn call(&self, mut req: ServiceRequest) -> Self::Future { let (err, id) = if let Some((secret, _)) = CLUSTER_SECRET.get() { if let Some(header) = req.headers().get(CLUSTER_SECRET_HEADER) - && let Some(tenant) = req.headers().get("intra-cluster-tenant") + && let Some(tenant) = req.headers().get(INTRA_CLUSTER_TENANT_HEADER) && let Some(userid) = req.headers().get("intra-cluster-userid") && let Ok(incoming_secret) = header.to_str() && let Ok(tenant) = tenant.to_str() diff --git a/src/handlers/http/modal/query_server.rs b/src/handlers/http/modal/query_server.rs index 40c4f508d..e5da5156b 100644 --- a/src/handlers/http/modal/query_server.rs +++ b/src/handlers/http/modal/query_server.rs @@ -261,9 +261,10 @@ impl QueryServer { web::scope("/internal/schema").service( web::resource("/resolve") .route( + // Authorised in the handler: a cluster-secret caller is a node, not a + // user (see `authority::caller_tenant`). web::post() .to(schema_registry::authority::resolve_handler) - .authorize(Action::ResolveSchema) .wrap(IntraClusterRequest), ) .app_data(web::JsonConfig::default().limit(1 << 20)), diff --git a/src/schema_registry/authority.rs b/src/schema_registry/authority.rs index 303e102c0..24f79faec 100644 --- a/src/schema_registry/authority.rs +++ b/src/schema_registry/authority.rs @@ -27,7 +27,9 @@ use std::{ sync::{Arc, Mutex}, }; -use actix_web::{HttpRequest, HttpResponse, ResponseError, http::StatusCode, web}; +use actix_web::{ + HttpRequest, HttpResponse, ResponseError, error::ErrorUnauthorized, http::StatusCode, web, +}; use arrow_schema::{Field, Schema}; use once_cell::sync::Lazy; use tokio::sync::{Mutex as AsyncMutex, OwnedMutexGuard}; @@ -38,8 +40,10 @@ use crate::{ DEFAULT_TIMESTAMP_KEY, FORMAT_KEY, FORMAT_VERIFY_KEY, SOURCE_IP_KEY, USER_AGENT_KEY, commit_schema, }, + handlers::http::middleware::{CLUSTER_SECRET, INTRA_CLUSTER_TENANT_HEADER}, parseable::{DEFAULT_TENANT, PARSEABLE, Stream}, - utils::get_tenant_id_from_request, + rbac::{self, Users, role::Action}, + utils::{actix::extract_session_key_from_req, get_tenant_id_from_key}, }; use super::{ @@ -348,10 +352,34 @@ async fn load_registry( Ok(registry) } -/// The tenant comes from authentication, never from the body: with `P_CLUSTER_SECRET` the -/// route only admits verified intra-cluster requests and the tenant is their -/// `intra-cluster-tenant`; without it, the caller's own tenant. The body may only repeat it, -/// so no caller can name another tenant. Returns `None` for the default tenant. +/// Authenticates the caller and returns the tenant it acts for. +/// +/// With `P_CLUSTER_SECRET`, the route's `IntraClusterRequest` has already verified the +/// secret, which only cluster nodes hold. The caller is that node, acting for the tenant it +/// names in `intra-cluster-tenant`: an ingestor resolves on its own behalf, so no user of +/// that tenant is involved, and none need exist under the user id it sends. The secret +/// already lets a node act as any user of any tenant, so this grants nothing more. +/// +/// Without the secret the caller is a user, who must be allowed `ResolveSchema` and acts +/// for their own tenant. +fn caller_tenant(req: &HttpRequest) -> Result, actix_web::Error> { + if CLUSTER_SECRET.get().is_some() { + return Ok(req + .headers() + .get(INTRA_CLUSTER_TENANT_HEADER) + .and_then(|tenant| tenant.to_str().ok()) + .map(String::from)); + } + let key = extract_session_key_from_req(req)?; + match Users.authorize(key.clone(), Action::ResolveSchema, None, None) { + rbac::Response::Authorized => Ok(get_tenant_id_from_key(&key)), + _ => Err(ErrorUnauthorized("not allowed to resolve schemas")), + } +} + +/// The tenant comes from authentication (see [`caller_tenant`]), never from the body. The +/// body may only repeat it, so no caller can name another tenant. Returns `None` for the +/// default tenant. fn authorized_tenant( header: Option, body: Option, @@ -370,9 +398,9 @@ fn authorized_tenant( pub async fn resolve_handler( req: HttpRequest, body: web::Json, -) -> Result { +) -> Result { let mut body = body.into_inner(); - body.tenant_id = authorized_tenant(get_tenant_id_from_request(&req), body.tenant_id)?; + body.tenant_id = authorized_tenant(caller_tenant(&req)?, body.tenant_id)?; let response = SCHEMA_AUTHORITY.resolve(body).await?; Ok(HttpResponse::Ok().json(response)) } diff --git a/src/schema_registry/client.rs b/src/schema_registry/client.rs index 05e00d25c..b0717fa5c 100644 --- a/src/schema_registry/client.rs +++ b/src/schema_registry/client.rs @@ -135,7 +135,9 @@ fn headers(target: &Target, tenant_id: &Option) -> header::HeaderMap { header::HeaderValue::from_str(tenant_id.as_deref().unwrap_or(DEFAULT_TENANT)) { map.insert( - header::HeaderName::from_static("intra-cluster-tenant"), + header::HeaderName::from_static( + crate::handlers::http::middleware::INTRA_CLUSTER_TENANT_HEADER, + ), tenant, ); } From 69baaf6c840c9a98a78f423f4b91ec73ed890445 Mon Sep 17 00:00:00 2001 From: Anant Vindal Date: Tue, 17 Nov 2026 19:33:25 +0530 Subject: [PATCH 13/13] fix: keep rename targets distinct and never reseed a saved registry Name list renames after their element type. get_datatype_suffix gave every list type the same `list` suffix, so ingest-time renaming sent a second list type of a field into the first one's column and rejected the batch. Across ingestors this happened all the time: after a resolution an ingestor holds another ingestor's list type under `f_list`, and its own lists were rejected. Suffixes now carry the element type (`list_utf8`, `list_list_int64`), and ingestion chains past a column of another nested type like the registry does, so both sides pick the same column for the same type. Never grant one rename target to two proposals of a request. A file carrying `f` and a literal `f_utf8` of the same type, next to an existing `f_utf8_utf8`, got that target for both. The file could never be renamed, and the stream blocked after the deferral cycles with the authority up. Seed a new registry only when the saved one does not exist. Any failed read or undecodable file was treated as missing, so the authority rebuilt the registry from per-node schemas in listing order and could overwrite types it had already granted. It now returns an error, and ingestors retry next cycle. --- src/event/format/json.rs | 52 +++++++++++++++++- src/event/format/mod.rs | 34 +++++++++--- src/schema_registry/authority.rs | 65 +++++++++++++++++++--- src/schema_registry/registry.rs | 92 ++++++++++++++++++++++++++++---- 4 files changed, 218 insertions(+), 25 deletions(-) diff --git a/src/event/format/json.rs b/src/event/format/json.rs index 0146ff536..e8f39676d 100644 --- a/src/event/format/json.rs +++ b/src/event/format/json.rs @@ -821,12 +821,62 @@ mod tests { "{label}" ); assert!( - name.ends_with(super::super::get_datatype_suffix(&data_type)), + name.ends_with(&*super::super::get_datatype_suffix(&data_type)), "{label}" ); } } + /// A conflicting list goes to a column named after its element type, so a field that + /// changes list type (`[1.5]`, then `["a"]`) gets a second column instead of the first + /// list column rejecting the batch. + #[test] + fn list_types_of_one_field_get_their_own_columns() { + let list = |t: DataType| DataType::List(Arc::new(Field::new_list_field(t, true))); + let existing = [ + ("x", DataType::Float64), + ("x_list_float64", list(DataType::Float64)), + ]; + assert_eq!( + new_columns(&existing, json!({"x": ["a"]}), SchemaVersion::V1, true), + vec![("x_list_utf8".to_string(), list(DataType::Utf8))] + ); + } + + /// After a resolution an ingestor holds another ingestor's list type in a typed column + /// (`f_list_utf8`). Its own list type still lands in a column of its type. + #[test] + fn nested_list_lands_next_to_another_ingestors_list_column() { + let list = |t: DataType| DataType::List(Arc::new(Field::new_list_field(t, true))); + let existing = [ + ("f", DataType::Float64), + ("f_list_utf8", list(DataType::Utf8)), + ]; + let added = new_columns(&existing, json!({"f": [[1]]}), SchemaVersion::V1, true); + assert_eq!(added.len(), 1, "{added:?}"); + let (name, data_type) = &added[0]; + assert!(matches!(data_type, DataType::List(item) if item.data_type().is_list())); + assert_eq!( + *name, + format!("f_{}", super::super::get_datatype_suffix(data_type)) + ); + } + + /// A list whose typed name is taken by a list of another type (a literal column of + /// that name) chains, like a scalar, instead of being decoded into the wrong list. + #[test] + fn list_rename_chains_past_a_list_column_of_another_type() { + let list = |t: DataType| DataType::List(Arc::new(Field::new_list_field(t, true))); + let existing = [ + ("f", DataType::Float64), + ("f_list_utf8", list(DataType::Float64)), + ]; + assert_eq!( + new_columns(&existing, json!({"f": ["a"]}), SchemaVersion::V1, true), + vec![("f_list_utf8_list_utf8".to_string(), list(DataType::Utf8))] + ); + } + /// An event carrying both a conflicting `f` and a literal `f_utf8` keeps both values /// end to end: the moved value lands in `f_utf8_utf8`, the literal one in `f_utf8`. #[test] diff --git a/src/event/format/mod.rs b/src/event/format/mod.rs index 7dfd63181..905ec9d55 100644 --- a/src/event/format/mod.rs +++ b/src/event/format/mod.rs @@ -18,6 +18,7 @@ */ use std::{ + borrow::Cow, collections::{HashMap, HashSet}, fmt::Display, sync::Arc, @@ -423,7 +424,20 @@ pub fn override_data_type( /// Returns a short suffix string for a given DataType to be used in field renaming /// when schema conflicts occur. -pub fn get_datatype_suffix(data_type: &DataType) -> &'static str { +/// +/// Lists carry their element type (`list_utf8`, `list_list_int64`), so two list types of +/// one field get different columns. Ingestion and the schema authority both name renames +/// with this function, so they pick the same column for the same type. +pub fn get_datatype_suffix(data_type: &DataType) -> Cow<'static, str> { + match data_type { + DataType::List(item) | DataType::LargeList(item) | DataType::FixedSizeList(item, _) => { + Cow::Owned(format!("list_{}", get_datatype_suffix(item.data_type()))) + } + other => Cow::Borrowed(scalar_datatype_suffix(other)), + } +} + +fn scalar_datatype_suffix(data_type: &DataType) -> &'static str { match data_type { DataType::Null => "null", DataType::Boolean => "bool", @@ -446,7 +460,6 @@ pub fn get_datatype_suffix(data_type: &DataType) -> &'static str { DataType::Timestamp(TimeUnit::Microsecond, _) => "timestamp_us", DataType::Timestamp(TimeUnit::Nanosecond, _) => "timestamp_ns", DataType::Time32(_) | DataType::Time64(_) => "time", - DataType::List(_) | DataType::LargeList(_) => "list", DataType::Struct(_) => "struct", DataType::Map(_, _) => "map", _ => "unknown", @@ -637,7 +650,7 @@ pub fn rename_conflicting_fields_in_json( .strip_prefix(key.as_str()) .and_then(|rest| rest.strip_prefix('_')) .unwrap_or(new_key); - let target = (new_key.clone(), suffix); + let target = (new_key.clone(), Cow::Borrowed(suffix)); (key, val, Some(target)) }) .collect(); @@ -652,7 +665,7 @@ pub fn rename_conflicting_fields_in_json( /// One key of a record, its value, and where the value moves: the target name /// `_` and the suffix, or `None` to keep the key. -type RecordEntry<'a> = (String, Value, Option<(String, &'a str)>); +type RecordEntry<'a> = (String, Value, Option<(String, Cow<'a, str>)>); /// Builds one record from its `(key, value, move)` entries, where `move` is the typed /// sibling name `_` a value moves to plus that suffix (`None` keeps the key). @@ -678,11 +691,18 @@ fn place_record_values( } } for (mut name, suffix, val) in moved { + // A nested value's type is its suffix (`list_utf8`): a column of another nested + // type is taken even though arrays "fit" any list column. Scalars go by value. + let taken_by = |f: &Field| { + if val.is_array() || val.is_object() { + get_datatype_suffix(f.data_type()) != suffix + } else { + !value_fits_type(&val, f.data_type(), schema_version) + } + }; // Terminates: each step makes a longer name, and the record and stream are finite. while record.get(&name).is_some_and(|v| !v.is_null()) - || existing_schema - .get(&name) - .is_some_and(|f| !value_fits_type(&val, f.data_type(), schema_version)) + || existing_schema.get(&name).is_some_and(|f| taken_by(f)) { name = format!("{name}_{suffix}"); } diff --git a/src/schema_registry/authority.rs b/src/schema_registry/authority.rs index 24f79faec..3a763ce7f 100644 --- a/src/schema_registry/authority.rs +++ b/src/schema_registry/authority.rs @@ -31,6 +31,7 @@ use actix_web::{ HttpRequest, HttpResponse, ResponseError, error::ErrorUnauthorized, http::StatusCode, web, }; use arrow_schema::{Field, Schema}; +use bytes::Bytes; use once_cell::sync::Lazy; use tokio::sync::{Mutex as AsyncMutex, OwnedMutexGuard}; use tracing::{info, warn}; @@ -43,6 +44,7 @@ use crate::{ handlers::http::middleware::{CLUSTER_SECRET, INTRA_CLUSTER_TENANT_HEADER}, parseable::{DEFAULT_TENANT, PARSEABLE, Stream}, rbac::{self, Users, role::Action}, + storage::ObjectStorageError, utils::{actix::extract_session_key_from_req, get_tenant_id_from_key}, }; @@ -297,6 +299,31 @@ fn registry_schema(registry: &StreamRegistry) -> Schema { ) } +/// The saved registry, or `None` only when it does not exist yet. +/// +/// Any other read failure, or a file that does not decode, is an error: seeding a new +/// registry from the per-node schemas instead could change types already granted (the +/// first type in listing order wins), and saving it would make that permanent while +/// ingestors keep writing the old types. +fn authoritative_schema( + stream: &str, + read: Result, +) -> Result, AuthorityError> { + match read { + Ok(bytes) => serde_json::from_slice::(&bytes) + .map(Some) + .map_err(|e| { + AuthorityError::Unavailable(format!( + "schema registry of stream {stream} is unreadable: {e}" + )) + }), + Err(ObjectStorageError::NoSuchKey(_)) => Ok(None), + Err(e) => Err(AuthorityError::Unavailable(format!( + "could not read schema registry of stream {stream}: {e}" + ))), + } +} + /// Seeds a registry from the authoritative `.schema` first, then from every per-node /// schema file (first type wins). Persists immediately when seeding added fields, so a /// restart re-seeds to the same answer regardless of listing order. @@ -310,13 +337,12 @@ async fn load_registry( ..Default::default() }; let storage = PARSEABLE.storage.get_object_store(); - let authoritative = match storage - .get_object(&authoritative_schema_path(stream, tenant_id), tenant_id) - .await - { - Ok(bytes) => serde_json::from_slice::(&bytes).ok(), - Err(_) => None, - }; + let authoritative = authoritative_schema( + stream, + storage + .get_object(&authoritative_schema_path(stream, tenant_id), tenant_id) + .await, + )?; let mut persisted_fields = 0; if let Some(schema) = &authoritative { registry.version = schema @@ -410,6 +436,31 @@ mod tests { use super::*; use arrow_schema::DataType; + #[test] + fn registry_is_seeded_only_when_the_saved_one_is_missing() { + let mut r = StreamRegistry::default(); + r.fields.insert("a".into(), DataType::Int64); + let saved = Bytes::from(serde_json::to_vec(®istry_schema(&r)).unwrap()); + let schema = authoritative_schema("s", Ok(saved)).unwrap().unwrap(); + assert_eq!(schema.field(0).name(), "a"); + + assert!( + authoritative_schema("s", Err(ObjectStorageError::NoSuchKey("k".into()))) + .unwrap() + .is_none() + ); + // A failed read or a corrupt file must not look like a missing registry: the + // caller would re-seed and could change granted types. + assert!(matches!( + authoritative_schema("s", Err(ObjectStorageError::Custom("timeout".into()))), + Err(AuthorityError::Unavailable(_)) + )); + assert!(matches!( + authoritative_schema("s", Ok(Bytes::from_static(b"{not json"))), + Err(AuthorityError::Unavailable(_)) + )); + } + #[test] fn registry_schema_carries_version_and_sorted_fields() { let mut r = StreamRegistry::default(); diff --git a/src/schema_registry/registry.rs b/src/schema_registry/registry.rs index 6993ed792..d479a96e4 100644 --- a/src/schema_registry/registry.rs +++ b/src/schema_registry/registry.rs @@ -65,6 +65,10 @@ impl StreamRegistry { // A rename target must not be a column the same files already carry, or renaming // would leave two columns with one name in a file (e.g. `a` and `a_utf8` together). let proposed_names: HashSet<&str> = proposals.iter().map(|f| f.name().as_str()).collect(); + // Nor may two proposals be renamed to the same column: a file carrying `a` and + // `a_utf8` (both Utf8) next to registered `a_utf8: Boolean` would otherwise send + // both to an existing `a_utf8_utf8`. + let mut granted: HashSet = HashSet::new(); for proposal in proposals { let proposed = proposal.data_type(); @@ -80,7 +84,8 @@ impl StreamRegistry { let mut name = original.clone(); let mut depth = 0; let resolution = loop { - let taken_by_request = depth > 0 && proposed_names.contains(name.as_str()); + let taken_by_request = depth > 0 + && (proposed_names.contains(name.as_str()) || granted.contains(&name)); match lookup(&name, &overlay) { None if !taken_by_request => { overlay.insert(name.clone(), proposed.clone()); @@ -135,6 +140,9 @@ impl StreamRegistry { } } }; + if resolution.resolved_name != resolution.original_name { + granted.insert(resolution.resolved_name.clone()); + } out.resolutions.push(resolution); } out @@ -249,24 +257,56 @@ mod tests { #[test] fn suffix_chain_continues_until_a_free_name() { - // Every list type shares the `list` suffix, so each new list type goes one level deeper. + // The typed name is taken by another type twice over: the chain goes on until free. let list = |t: DataType| DataType::List(Arc::new(Field::new_list_field(t, true))); let r = reg(&[ ("a", list(DataType::Int64)), - ("a_list", list(DataType::Float64)), - ("a_list_list", list(DataType::Utf8)), - ("a_list_list_list", list(DataType::Boolean)), + ("a_list_date", DataType::Boolean), + ("a_list_date_list_date", DataType::Utf8), ]); let res = r.resolve(&[f("a", list(DataType::Date32))]); - assert_eq!(res.resolutions[0].resolved_name, "a_list_list_list_list"); + assert_eq!( + res.resolutions[0].resolved_name, + "a_list_date_list_date_list_date" + ); assert_eq!(res.resolutions[0].outcome, Outcome::Renamed); assert_eq!( res.registered, - vec![("a_list_list_list_list".to_string(), list(DataType::Date32))] + vec![( + "a_list_date_list_date_list_date".to_string(), + list(DataType::Date32) + )] + ); + } + + #[test] + fn each_list_type_gets_its_own_column() { + // The suffix names the element type, so list types of one field never share a + // column, however many there are, and an ingestor renaming locally lands on the + // same name. An old `a_list` column (named before suffixes carried the element + // type) is left as it is: new data of that type goes to the typed name. + let list = |t: DataType| DataType::List(Arc::new(Field::new_list_field(t, true))); + let r = reg(&[("a", DataType::Float64), ("a_list", list(DataType::Utf8))]); + let res = r.resolve(&[ + f("a", list(DataType::Utf8)), + f("a", list(list(DataType::Int64))), + ]); + let names: Vec<&str> = res + .resolutions + .iter() + .map(|x| x.resolved_name.as_str()) + .collect(); + assert_eq!(names, ["a_list_utf8", "a_list_list_int64"]); + assert!( + res.resolutions + .iter() + .all(|x| x.outcome == Outcome::Renamed) ); - // An already registered list type still resolves to its existing column. - let res = r.resolve(&[f("a", list(DataType::Utf8))]); - assert_eq!(res.resolutions[0].resolved_name, "a_list_list"); + // Asked again, the same type resolves to its column. + let mut r = r; + r.apply(&res.registered); + let res = r.resolve(&[f("a", list(list(DataType::Int64)))]); + assert_eq!(res.resolutions[0].resolved_name, "a_list_list_int64"); assert_eq!(res.resolutions[0].outcome, Outcome::AlreadyRenamed); } @@ -349,4 +389,36 @@ mod tests { assert_eq!(by_name("a_utf8"), "a_utf8"); } } + + #[test] + fn two_proposals_never_share_an_existing_rename_target() { + // Registered: `a` and `a_utf8` hold other types, `a_utf8_utf8` already holds Utf8 + // (an earlier rename). A file carrying `a: Utf8` and `a_utf8: Utf8` must get two + // different targets, whatever the order of the proposals. + for order in [ + [f("a", DataType::Utf8), f("a_utf8", DataType::Utf8)], + [f("a_utf8", DataType::Utf8), f("a", DataType::Utf8)], + ] { + let r = reg(&[ + ("a", DataType::Float64), + ("a_utf8", DataType::Boolean), + ("a_utf8_utf8", DataType::Utf8), + ]); + let res = r.resolve(&order); + let targets: Vec<&str> = res + .resolutions + .iter() + .map(|x| x.resolved_name.as_str()) + .collect(); + assert_eq!(targets.len(), 2); + assert_ne!(targets[0], targets[1], "{targets:?}"); + assert!( + res.resolutions + .iter() + .all(|x| x.data_type == DataType::Utf8 && x.resolved_name != x.original_name), + "{:?}", + res.resolutions + ); + } + } }