diff --git a/src/cli.rs b/src/cli.rs index 440769263..936a978fb 100644 --- a/src/cli.rs +++ b/src/cli.rs @@ -558,6 +558,31 @@ pub struct Options { )] pub querier_endpoint: String, + #[arg( + long, + env = "P_SCHEMA_RESOLUTION", + default_value = "true", + help = "Distributed ingestors: reconcile new field types with the schema authority (querier or Prism) before writing parquet. Upgrade the authority before the ingestors." + )] + pub schema_resolution: bool, + + #[arg( + long, + env = "P_SCHEMA_RESOLVE_MAX_DEFERRED_CYCLES", + default_value = "3", + help = "Failed resolution cycles after which a stream rejects ingestion with 503 (0 = never)" + )] + pub schema_resolve_max_deferred_cycles: u32, + + #[arg( + long, + env = "P_SCHEMA_RESOLVE_CONCURRENCY", + default_value = "8", + value_parser = value_parser!(u16).range(1..), + help = "Maximum concurrent schema resolution requests from one ingestor" + )] + pub schema_resolve_concurrency: u16, + #[arg( long, env = "P_PRISM_ENDPOINT", diff --git a/src/connectors/kafka/processor.rs b/src/connectors/kafka/processor.rs index a892691b2..b8a64411b 100644 --- a/src/connectors/kafka/processor.rs +++ b/src/connectors/kafka/processor.rs @@ -20,6 +20,7 @@ use crate::{ connectors::common::processor::Processor, event::{ USER_AGENT_KEY, + error::EventError, format::{EventFormat, LogSourceEntry, json}, }, handlers::TelemetryType, @@ -33,10 +34,11 @@ use chrono::Utc; use futures_util::StreamExt; use rdkafka::consumer::{CommitMode, Consumer}; use serde_json::Value; -use std::collections::HashMap; +use std::collections::{BTreeMap, HashMap}; use std::sync::Arc; +use std::time::Duration; use tokio_stream::wrappers::ReceiverStream; -use tracing::{debug, error}; +use tracing::{debug, error, info, warn}; use super::{ConsumerRecord, StreamConsumer, TopicPartition, config::BufferConfig}; @@ -93,7 +95,7 @@ struct StreamMeta { /// poll/commit/recv), same reasoning as the rayon offload used for otel and /// prometheus remote-write ingestion. fn build_and_process_event( - records: Vec, + records: &[ConsumerRecord], stream_name: String, tenant_id: Option, meta: StreamMeta, @@ -131,8 +133,31 @@ fn build_and_process_event( Ok(()) } +/// Matches the ingest-side `Retry-After` for a stream blocked by schema resolution. +const SCHEMA_BLOCKED_RETRY: Duration = Duration::from_secs(60); + +/// A chunk whose stream rejects writes until its schema is reconciled with the cluster. +/// Carries the records back, so the chunk can be held (and its offsets left uncommitted) +/// and retried instead of dropped. +#[derive(thiserror::Error)] +#[error("ingestion blocked by schema resolution: {reason}")] +pub struct SchemaBlocked { + pub records: Vec, + reason: String, +} + +impl std::fmt::Debug for SchemaBlocked { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("SchemaBlocked") + .field("records", &self.records.len()) + .field("reason", &self.reason) + .finish() + } +} + #[async_trait] impl Processor, ()> for ParseableSinkProcessor { + /// Fails with [`SchemaBlocked`] while the stream rejects writes. async fn process(&self, records: Vec) -> anyhow::Result<()> { let len = records.len(); debug!("Processing {len} records"); @@ -141,19 +166,197 @@ impl Processor, ()> for ParseableSinkProcessor { let tenant_id = records.first().and_then(|r| r.tenant_id.clone()); let meta = self.prepare_stream(&stream_name, &tenant_id).await?; - let (tx, rx) = tokio::sync::oneshot::channel(); rayon::spawn(move || { - let res = build_and_process_event(records, stream_name, tenant_id, meta); - let _ = tx.send(res); + let res = build_and_process_event(&records, stream_name, tenant_id, meta); + let _ = tx.send((records, res)); }); - rx.await??; + let (records, res) = rx.await?; + if let Err(err) = res { + return Err(match err.downcast::() { + Ok(EventError::SchemaResolutionBlocked(_, reason)) => { + SchemaBlocked { records, reason }.into() + } + Ok(other) => other.into(), + Err(other) => other, + }); + } debug!("Processed {len} records"); Ok(()) } } +/// Pauses fetching for one partition while any of its chunks is blocked by schema +/// resolution, and resumes it when the last one goes through. +/// +/// All partitions share one consumer loop, which waits whenever a partition's queue is +/// full. Pausing keeps a blocked partition from filling its queue, so the loop keeps +/// serving the other partitions; the blocked chunks hold at most what was already fetched. +/// A resumed partition fetches again from the last record handed to the application, so +/// nothing is skipped or read twice. +/// +/// The pause belongs to the partition, not to one worker: it outlives a revoke, and a +/// revoked worker can still be shutting down when a reassignment starts a new one. The +/// count of blocked chunks is therefore shared by every worker of the partition, and the +/// partition is paused exactly while it is above zero. +struct PartitionPause<'a> { + consumer: &'a StreamConsumer, + tp: &'a TopicPartition, + blocked: &'a BlockedChunks, + /// This worker's share of the partition's blocked chunks. + held: std::sync::atomic::AtomicUsize, +} + +impl<'a> PartitionPause<'a> { + fn new( + consumer: &'a StreamConsumer, + tp: &'a TopicPartition, + blocked: &'a BlockedChunks, + ) -> Self { + Self { + consumer, + tp, + blocked, + held: std::sync::atomic::AtomicUsize::default(), + } + } + + fn tpl(&self) -> rdkafka::TopicPartitionList { + let mut tpl = rdkafka::TopicPartitionList::new(); + tpl.add_partition(&self.tp.topic, self.tp.partition); + tpl + } + + fn hold(&self) { + // A worker pauses on its first blocked chunk even if an older worker's pause should + // still be in effect: pausing again is harmless, and the new worker does not have + // to rely on that pause having survived the reassignment. + let first = self.held.fetch_add(1, std::sync::atomic::Ordering::Relaxed) == 0; + self.blocked + .add(self.tp, first, || match self.consumer.pause(&self.tpl()) { + Ok(()) => warn!("Paused {:?}: its stream rejects writes", self.tp), + Err(e) => error!(error = %e, "Failed to pause {:?}", self.tp), + }); + } + + fn release(&self) { + self.held.fetch_sub(1, std::sync::atomic::Ordering::Relaxed); + self.blocked.remove(self.tp, 1, || self.resume()); + } + + fn resume(&self) { + match self.consumer.resume(&self.tpl()) { + Ok(()) => info!("Resumed {:?}", self.tp), + Err(e) => error!(error = %e, "Failed to resume {:?}", self.tp), + } + } +} + +impl Drop for PartitionPause<'_> { + /// A revoked partition drops its worker with chunks still held. Give them back, so a + /// later assignment of the partition does not stay paused with nothing to resume it, + /// while a newer worker's blocked chunks keep it paused. + fn drop(&mut self) { + let held = *self.held.get_mut(); + if held > 0 { + self.blocked.remove(self.tp, held, || self.resume()); + } + } +} + +/// Blocked chunks per partition, across every worker of one consumer. +#[derive(Default)] +struct BlockedChunks { + counts: std::sync::Mutex>, +} + +impl BlockedChunks { + /// Adds one blocked chunk of `tp`, calling `pause` if it is the first or `force` is + /// set. Runs `pause` under the lock, so pauses and resumes reach the consumer in count + /// order. + fn add(&self, tp: &K, force: bool, pause: impl FnOnce()) { + let mut counts = self.lock(); + let count = counts.entry(tp.clone()).or_default(); + if *count == 0 || force { + pause(); + } + *count += 1; + } + + /// Removes `n` blocked chunks of `tp`, calling `resume` (under the lock) if none is + /// left. + fn remove(&self, tp: &K, n: usize, resume: impl FnOnce()) { + let mut counts = self.lock(); + let Some(count) = counts.get_mut(tp) else { + return; + }; + *count = count.saturating_sub(n); + if *count == 0 { + counts.remove(tp); + resume(); + } + } + + /// Also used from `Drop`, which must not panic: the counts stay consistent even if a + /// holder panicked, since every update is a single step. + fn lock(&self) -> std::sync::MutexGuard<'_, HashMap> { + self.counts + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + } +} + +/// Commits one partition's chunks in order. +/// +/// Chunks finish out of order: one can wait out a schema block while later ones go +/// through. Committing each chunk's own offset would then move the committed offset past +/// records not written yet, and a restart or rebalance would skip them. A chunk's offset +/// is committed only once every earlier chunk is done, so the committed offset never +/// passes an unwritten record. Records written after it may be read again after a restart. +#[derive(Default)] +struct OrderedCommits { + state: std::sync::Mutex>, +} + +struct CommitState { + /// Sequence number of the earliest chunk not done yet. + next: usize, + /// Done chunks after `next`, with the offsets each would commit. + done: BTreeMap>, +} + +impl Default for CommitState { + fn default() -> Self { + Self { + next: 0, + done: BTreeMap::new(), + } + } +} + +impl OrderedCommits { + /// Marks chunk `seq` (numbered from 0 in stream order) done and calls `commit` with the + /// offsets of the latest chunk that every earlier chunk is done before. Runs `commit` + /// under the lock, so commits never go backwards. + fn complete(&self, seq: usize, offsets: Option, commit: impl FnOnce(T)) { + let mut state = self.state.lock().expect("commit lock poisoned"); + state.done.insert(seq, offsets); + let mut latest = None; + loop { + let next = state.next; + let Some(offsets) = state.done.remove(&next) else { + break; + }; + state.next += 1; + latest = offsets.or(latest); + } + if let Some(offsets) = latest { + commit(offsets); + } + } +} + #[derive(Clone)] pub struct StreamWorker

where @@ -162,6 +365,7 @@ where processor: Arc

, consumer: Arc, buffer_config: BufferConfig, + blocked: Arc, } impl

StreamWorker

@@ -180,9 +384,49 @@ where processor, consumer, buffer_config, + blocked: Arc::default(), } } + /// Processes one chunk, holding it while its stream is blocked by schema resolution: + /// committing then would drop its records. Returns once the chunk went through or + /// failed for another reason. + async fn process_chunk( + &self, + tp: &TopicPartition, + pause: &PartitionPause<'_>, + mut records: Vec, + ) -> anyhow::Result<()> { + let mut held = false; + let result = loop { + match self.processor.process(records).await { + Err(err) => match err.downcast::() { + Ok(blocked) => { + if !held { + pause.hold(); + held = true; + } + warn!( + "{:?}: {}; retrying {} records in {}s", + tp, + blocked.reason, + blocked.records.len(), + SCHEMA_BLOCKED_RETRY.as_secs() + ); + records = blocked.records; + tokio::time::sleep(SCHEMA_BLOCKED_RETRY).await; + } + Err(err) => break Err(err), + }, + Ok(()) => break Ok(()), + } + }; + if held { + pause.release(); + } + result + } + pub async fn process_partition( &self, tp: TopicPartition, @@ -193,24 +437,37 @@ where self.buffer_config.buffer_size, self.buffer_config.buffer_timeout, ); + let pause = PartitionPause::new(&self.consumer, &tp, &self.blocked); + let commits = OrderedCommits::default(); + // Unbounded on purpose: blocked chunks must keep draining the partition's queue, or + // a full queue would stall the consumer loop shared by every partition. The pause + // bounds what a blocked partition holds. chunked_stream - .for_each_concurrent(None, |records| async { - if let Some(last_record) = records.iter().max_by_key(|r| r.offset) { - let tpl = last_record.offset_to_commit().unwrap(); - - if let Err(e) = self.processor.process(records).await { + .enumerate() + .for_each_concurrent(None, |(seq, records)| { + let (tp, pause, commits) = (&tp, &pause, &commits); + async move { + let tpl = records + .iter() + .max_by_key(|r| r.offset) + .map(|last| last.offset_to_commit().unwrap()); + if !records.is_empty() + && let Err(e) = self.process_chunk(tp, pause, records).await + { error!("Failed to process records for {:?}: {:?}", tp, e); } - //CommitMode::Async race condition. - //@see https://github.com/confluentinc/librdkafka/issues/4534 - //@see https://github.com/confluentinc/librdkafka/issues/4059 - if let Err(e) = self.consumer.commit(&tpl, CommitMode::Sync) { - error!(error = %e, "Failed to commit offsets for {:?}", tpl); - } else { - debug!("Committed offsets for {:?}", tpl); - } + commits.complete(seq, tpl, |tpl| { + //CommitMode::Async race condition. + //@see https://github.com/confluentinc/librdkafka/issues/4534 + //@see https://github.com/confluentinc/librdkafka/issues/4059 + if let Err(e) = self.consumer.commit(&tpl, CommitMode::Sync) { + error!(error = %e, "Failed to commit offsets for {:?}", tpl); + } else { + debug!("Committed offsets for {:?}", tpl); + } + }); } }) .await; @@ -220,3 +477,72 @@ where Ok(()) } } + +#[cfg(test)] +mod tests { + use super::*; + + fn complete(commits: &OrderedCommits, seq: usize, offset: Option) -> Option { + let mut committed = None; + commits.complete(seq, offset, |o| committed = Some(o)); + committed + } + + #[test] + fn later_chunk_waits_for_earlier_ones() { + let commits = OrderedCommits::::default(); + // Chunk 0 is held (schema block) while 1 and 2 go through: nothing committed. + assert_eq!(complete(&commits, 1, Some(20)), None); + assert_eq!(complete(&commits, 2, Some(30)), None); + // Chunk 0 done: commits through the furthest consecutive chunk. + assert_eq!(complete(&commits, 0, Some(10)), Some(30)); + assert_eq!(complete(&commits, 3, Some(40)), Some(40)); + } + + /// Applies one change to the counts and reports the pause or resume it made, if any. + fn hold(blocked: &BlockedChunks, first_of_worker: bool) -> Option<&'static str> { + let mut call = None; + blocked.add(&0, first_of_worker, || call = Some("pause")); + call + } + + fn release(blocked: &BlockedChunks, n: usize) -> Option<&'static str> { + let mut call = None; + blocked.remove(&0, n, || call = Some("resume")); + call + } + + #[test] + fn partition_paused_while_any_chunk_is_blocked() { + let blocked = BlockedChunks::::default(); + assert_eq!(hold(&blocked, true), Some("pause")); + assert_eq!(hold(&blocked, false), None); + assert_eq!(release(&blocked, 1), None); + assert_eq!(release(&blocked, 1), Some("resume")); + // Nothing left to resume. + assert_eq!(release(&blocked, 1), None); + } + + #[test] + fn revoked_worker_keeps_newer_workers_pause() { + let blocked = BlockedChunks::::default(); + // The old worker pauses the partition with two blocked chunks, then is revoked. + assert_eq!(hold(&blocked, true), Some("pause")); + assert_eq!(hold(&blocked, false), None); + // The partition is reassigned and a new worker blocks before the old one is dropped. + assert_eq!(hold(&blocked, true), Some("pause")); + // Dropping the old worker gives back its two chunks without resuming. + assert_eq!(release(&blocked, 2), None); + // The new worker's chunk goes through and resumes the partition. + assert_eq!(release(&blocked, 1), Some("resume")); + } + + #[test] + fn chunk_without_offsets_still_advances() { + let commits = OrderedCommits::::default(); + assert_eq!(complete(&commits, 1, Some(20)), None); + assert_eq!(complete(&commits, 0, None), Some(20)); + assert_eq!(complete(&commits, 2, None), None); + assert_eq!(complete(&commits, 3, Some(40)), Some(40)); + } +} diff --git a/src/event/format/json.rs b/src/event/format/json.rs index a0127fe41..e8f39676d 100644 --- a/src/event/format/json.rs +++ b/src/event/format/json.rs @@ -113,6 +113,7 @@ impl EventFormat for Event { stream_schema, &value_arr, schema_version, + infer_timestamp, ); // If there are conflicts, rename the fields in JSON values @@ -137,6 +138,7 @@ impl EventFormat for Event { &raw_inferred_schema, stream_schema, schema_version, + infer_timestamp, ) }; @@ -587,6 +589,7 @@ mod tests { use std::str::FromStr; use arrow_array::UInt64Array; + use arrow_schema::TimeUnit; use serde_json::json; use super::*; @@ -690,4 +693,204 @@ mod tests { assert!(parsed.is_err()); } + + /// Runs `to_data` on `body` against a stream that already has `existing`, decodes the + /// result, and returns the columns the event adds to the stream. + fn new_columns( + existing: &[(&str, DataType)], + body: Value, + schema_version: SchemaVersion, + infer_timestamp: bool, + ) -> Vec<(String, DataType)> { + let schema: HashMap> = existing + .iter() + .map(|(n, t)| (n.to_string(), Arc::new(Field::new(*n, t.clone(), true)))) + .collect(); + let (data, fields, _) = Event::new(body, Utc::now()) + .to_data(&schema, None, schema_version, false, infer_timestamp) + .unwrap(); + Event::decode(data, Arc::new(Schema::new(fields.clone()))).unwrap(); + fields + .iter() + .filter(|f| !schema.contains_key(f.name())) + .map(|f| (f.name().clone(), f.data_type().clone())) + .collect() + } + + /// A conflict rename names the new column after the type it is stored as, so the + /// suffix always matches the column's type (and the schema authority's naming). + #[test] + fn conflict_rename_suffix_matches_stored_type() { + let ts = DataType::Timestamp(TimeUnit::Millisecond, None); + let v1 = SchemaVersion::V1; + // (existing columns, event, schema version, infer_timestamp, expected new column) + type Case = ( + Vec<(&'static str, DataType)>, + Value, + SchemaVersion, + bool, + (&'static str, DataType), + ); + let cases: Vec = vec![ + // V1 stores every number as Float64, integers included. + ( + vec![("g", DataType::Utf8)], + json!({"g": 1000}), + v1, + true, + ("g_float64", DataType::Float64), + ), + ( + vec![("g", DataType::Utf8)], + json!({"g": 1.5}), + v1, + true, + ("g_float64", DataType::Float64), + ), + // Per-record path: mixed types in one batch. + ( + vec![("g", DataType::Utf8)], + json!([{"g": "a"}, {"g": 1000}]), + v1, + true, + ("g_float64", DataType::Float64), + ), + // A date string in a time-named field is stored as Timestamp. + ( + vec![("event_time", DataType::Float64)], + json!({"event_time": "2025-01-01T00:00:00Z"}), + v1, + true, + ("event_time_timestamp_ms", ts.clone()), + ), + ( + vec![("event_time", DataType::Float64)], + json!([{"event_time": 5}, {"event_time": "2025-01-01T00:00:00Z"}]), + v1, + true, + ("event_time_timestamp_ms", ts.clone()), + ), + // ... but not with timestamp inference off, + ( + vec![("event_time", DataType::Float64)], + json!({"event_time": "2025-01-01T00:00:00Z"}), + v1, + false, + ("event_time_utf8", DataType::Utf8), + ), + // nor when the string is not a date, + ( + vec![("event_time", DataType::Float64)], + json!({"event_time": "abc"}), + v1, + true, + ("event_time_utf8", DataType::Utf8), + ), + // nor in a field whose name is not time-like. + ( + vec![("g", DataType::Float64)], + json!({"g": "2025-01-01T00:00:00Z"}), + v1, + true, + ("g_utf8", DataType::Utf8), + ), + // Numbers into a Timestamp field. + ( + vec![("event_time", ts.clone())], + json!({"event_time": 5}), + v1, + true, + ("event_time_float64", DataType::Float64), + ), + // V0 keeps integers as Int64. + ( + vec![("g", DataType::Utf8)], + json!({"g": 1000}), + SchemaVersion::V0, + true, + ("g_int64", DataType::Int64), + ), + ]; + for (existing, body, version, infer_timestamp, (name, data_type)) in cases { + let label = + format!("{existing:?} + {body} ({version:?}, infer_timestamp={infer_timestamp})"); + let added = new_columns(&existing, body, version, infer_timestamp); + assert_eq!( + added, + vec![(name.to_string(), data_type.clone())], + "{label}" + ); + assert!( + name.ends_with(&*super::super::get_datatype_suffix(&data_type)), + "{label}" + ); + } + } + + /// A conflicting list goes to a column named after its element type, so a field that + /// changes list type (`[1.5]`, then `["a"]`) gets a second column instead of the first + /// list column rejecting the batch. + #[test] + fn list_types_of_one_field_get_their_own_columns() { + let list = |t: DataType| DataType::List(Arc::new(Field::new_list_field(t, true))); + let existing = [ + ("x", DataType::Float64), + ("x_list_float64", list(DataType::Float64)), + ]; + assert_eq!( + new_columns(&existing, json!({"x": ["a"]}), SchemaVersion::V1, true), + vec![("x_list_utf8".to_string(), list(DataType::Utf8))] + ); + } + + /// After a resolution an ingestor holds another ingestor's list type in a typed column + /// (`f_list_utf8`). Its own list type still lands in a column of its type. + #[test] + fn nested_list_lands_next_to_another_ingestors_list_column() { + let list = |t: DataType| DataType::List(Arc::new(Field::new_list_field(t, true))); + let existing = [ + ("f", DataType::Float64), + ("f_list_utf8", list(DataType::Utf8)), + ]; + let added = new_columns(&existing, json!({"f": [[1]]}), SchemaVersion::V1, true); + assert_eq!(added.len(), 1, "{added:?}"); + let (name, data_type) = &added[0]; + assert!(matches!(data_type, DataType::List(item) if item.data_type().is_list())); + assert_eq!( + *name, + format!("f_{}", super::super::get_datatype_suffix(data_type)) + ); + } + + /// A list whose typed name is taken by a list of another type (a literal column of + /// that name) chains, like a scalar, instead of being decoded into the wrong list. + #[test] + fn list_rename_chains_past_a_list_column_of_another_type() { + let list = |t: DataType| DataType::List(Arc::new(Field::new_list_field(t, true))); + let existing = [ + ("f", DataType::Float64), + ("f_list_utf8", list(DataType::Float64)), + ]; + assert_eq!( + new_columns(&existing, json!({"f": ["a"]}), SchemaVersion::V1, true), + vec![("f_list_utf8_list_utf8".to_string(), list(DataType::Utf8))] + ); + } + + /// An event carrying both a conflicting `f` and a literal `f_utf8` keeps both values + /// end to end: the moved value lands in `f_utf8_utf8`, the literal one in `f_utf8`. + #[test] + fn conflict_rename_keeps_literal_key_of_the_same_name() { + let existing = [("f", DataType::Float64), ("f_utf8", DataType::Utf8)]; + for body in [ + json!({"f": "moved", "f_utf8": "literal"}), + json!([{"f": 1.5}, {"f": "moved", "f_utf8": "literal"}]), + ] { + assert_eq!( + new_columns(&existing, body.clone(), SchemaVersion::V1, true), + vec![("f_utf8_utf8".to_string(), DataType::Utf8)], + "{body}" + ); + } + } } diff --git a/src/event/format/mod.rs b/src/event/format/mod.rs index af6b4f7d5..905ec9d55 100644 --- a/src/event/format/mod.rs +++ b/src/event/format/mod.rs @@ -18,6 +18,7 @@ */ use std::{ + borrow::Cow, collections::{HashMap, HashSet}, fmt::Display, sync::Arc, @@ -423,7 +424,20 @@ pub fn override_data_type( /// Returns a short suffix string for a given DataType to be used in field renaming /// when schema conflicts occur. -pub fn get_datatype_suffix(data_type: &DataType) -> &'static str { +/// +/// Lists carry their element type (`list_utf8`, `list_list_int64`), so two list types of +/// one field get different columns. Ingestion and the schema authority both name renames +/// with this function, so they pick the same column for the same type. +pub fn get_datatype_suffix(data_type: &DataType) -> Cow<'static, str> { + match data_type { + DataType::List(item) | DataType::LargeList(item) | DataType::FixedSizeList(item, _) => { + Cow::Owned(format!("list_{}", get_datatype_suffix(item.data_type()))) + } + other => Cow::Borrowed(scalar_datatype_suffix(other)), + } +} + +fn scalar_datatype_suffix(data_type: &DataType) -> &'static str { match data_type { DataType::Null => "null", DataType::Boolean => "bool", @@ -446,7 +460,6 @@ pub fn get_datatype_suffix(data_type: &DataType) -> &'static str { DataType::Timestamp(TimeUnit::Microsecond, _) => "timestamp_us", DataType::Timestamp(TimeUnit::Nanosecond, _) => "timestamp_ns", DataType::Time32(_) | DataType::Time64(_) => "time", - DataType::List(_) | DataType::LargeList(_) => "list", DataType::Struct(_) => "struct", DataType::Map(_, _) => "map", _ => "unknown", @@ -504,11 +517,13 @@ fn value_compatible_with_type( /// Returns a HashMap mapping original field names to new field names (with datatype suffix) /// for fields that have conflicting types. /// Takes JSON values to check if values are actually compatible with existing types. +/// The suffix is the type the new column is stored as (see [`renamed_column_type`]). pub fn detect_schema_conflicts( inferred_schema: &Schema, existing_schema: &HashMap>, values: &[Value], schema_version: SchemaVersion, + infer_timestamp: bool, ) -> HashMap { let mut conflicts = HashMap::new(); @@ -516,26 +531,34 @@ pub fn detect_schema_conflicts( if let Some(existing_field) = existing_schema.get(field.name()) { // Check if data types are different (potential conflict) if existing_field.data_type() != field.data_type() && !field.data_type().is_null() { - // Before declaring conflict, check if all values can be coerced to existing type - let all_values_compatible = values.iter().all(|v| { - if let Some(field_value) = v.get(field.name()) { - if field_value.is_null() { - return true; // null is compatible with any type - } - value_compatible_with_type( - field_value, - existing_field.data_type(), - schema_version, - ) - } else { - true // field not present in this value, no conflict - } - }); + // The values that cannot be coerced to the existing type: exactly the ones + // `rename_conflicting_fields_in_json` moves to the new column. Nulls and + // records without the field never conflict. + let moved: Vec<&Value> = values + .iter() + .filter_map(|v| v.get(field.name())) + .filter(|v| { + !v.is_null() + && !value_compatible_with_type( + v, + existing_field.data_type(), + schema_version, + ) + }) + .collect(); // Only mark as conflict if values cannot be coerced to existing type - if !all_values_compatible { - let suffix = get_datatype_suffix(field.data_type()); - let new_name = format!("{}_{}", field.name(), suffix); + if !moved.is_empty() { + let data_type = match renamed_column_type( + field.name(), + moved, + schema_version, + infer_timestamp, + ) { + DataType::Null => field.data_type().clone(), + data_type => data_type, + }; + let new_name = format!("{}_{}", field.name(), get_datatype_suffix(&data_type)); conflicts.insert(field.name().to_string(), new_name); } } @@ -545,6 +568,44 @@ pub fn detect_schema_conflicts( conflicts } +/// The type of the new column a conflict rename creates for `values` (the values of field +/// `name` that move to it): what ingestion infers for them as a new column, after the schema +/// version's overrides (V1 stores every number as Float64, and date strings in a time-named +/// field as Timestamp). Suffixing with this type rather than the raw JSON type keeps the +/// column's name in step with its type, which is also the name the distributed schema +/// authority gives the same data. +fn renamed_column_type<'a>( + name: &str, + values: impl IntoIterator, + schema_version: SchemaVersion, + infer_timestamp: bool, +) -> DataType { + // Evaluated under a stand-in for the new name `_`. It contains `name`, so + // the time-name rule matches it exactly when it matches the new name (only the + // `timestamp_*` suffixes contain a time part, and they are chosen only for timestamps). + // It is never the series-hash column, whose name alone decides that column's type. + let column = format!("{name}_"); + let values: Vec<&Value> = values.into_iter().filter(|v| !v.is_null()).collect(); + let records: Vec = values + .iter() + .map(|v| Value::Object(serde_json::Map::from_iter([(column.clone(), (*v).clone())]))) + .collect(); + let Ok(inferred) = arrow_json::reader::infer_json_schema_from_iterator(records.iter().map(Ok)) + else { + return values + .first() + .map_or(DataType::Null, |v| datatype_for_value(v)); + }; + // Same per-record pass as `update_field_type_in_schema`, which types the new column. + let schema = records.iter().fold(Arc::new(inferred), |schema, record| { + override_data_type(schema, record.clone(), schema_version, infer_timestamp) + }); + schema + .fields() + .first() + .map_or(DataType::Null, |f| f.data_type().clone()) +} + /// Renames fields in JSON values according to the provided field mapping. /// Used to resolve schema conflicts by renaming fields with conflicting types. /// Renames conflicting JSON fields to typed sibling columns, but only for @@ -570,31 +631,94 @@ pub fn rename_conflicting_fields_in_json( values .into_iter() .map(|value| { - if let Value::Object(map) = value { - let new_map: serde_json::Map = map - .into_iter() - .map(|(key, val)| { - if let Some(new_key) = field_mapping.get(&key) { - if existing_schema.get(&key).is_some_and(|field| { - value_compatible_with_type(&val, field.data_type(), schema_version) - }) { - (key, val) - } else { - (new_key.clone(), val) - } - } else { - (key, val) - } - }) - .collect(); - Value::Object(new_map) - } else { - value - } + let Value::Object(map) = value else { + return value; + }; + let entries = map + .into_iter() + .map(|(key, val)| { + let Some(new_key) = field_mapping.get(&key) else { + return (key, val, None); + }; + if existing_schema.get(&key).is_some_and(|field| { + value_compatible_with_type(&val, field.data_type(), schema_version) + }) { + return (key, val, None); + } + // `new_key` is `_`; a collision chains the same suffix. + let suffix = new_key + .strip_prefix(key.as_str()) + .and_then(|rest| rest.strip_prefix('_')) + .unwrap_or(new_key); + let target = (new_key.clone(), Cow::Borrowed(suffix)); + (key, val, Some(target)) + }) + .collect(); + Value::Object(place_record_values( + entries, + existing_schema, + schema_version, + )) }) .collect() } +/// One key of a record, its value, and where the value moves: the target name +/// `_` and the suffix, or `None` to keep the key. +type RecordEntry<'a> = (String, Value, Option<(String, Cow<'a, str>)>); + +/// Builds one record from its `(key, value, move)` entries, where `move` is the typed +/// sibling name `_` a value moves to plus that suffix (`None` keeps the key). +/// +/// Kept keys claim their names first. A moved value whose target is already a non-null key +/// of the record, or a stream column the value does not fit, chains one more suffix +/// (`a_utf8` → `a_utf8_utf8`) until the name is free, the way the schema authority chains. +/// So a record carrying both `a` and a literal `a_utf8` keeps both values instead of one +/// overwriting the other. A null in the target is dropped: null and absent mean the same. +fn place_record_values( + entries: Vec>, + existing_schema: &HashMap>, + schema_version: SchemaVersion, +) -> serde_json::Map { + let mut record = serde_json::Map::with_capacity(entries.len()); + let mut moved = Vec::new(); + for (key, val, target) in entries { + match target { + None => { + record.insert(key, val); + } + Some((name, suffix)) => moved.push((name, suffix, val)), + } + } + for (mut name, suffix, val) in moved { + // A nested value's type is its suffix (`list_utf8`): a column of another nested + // type is taken even though arrays "fit" any list column. Scalars go by value. + let taken_by = |f: &Field| { + if val.is_array() || val.is_object() { + get_datatype_suffix(f.data_type()) != suffix + } else { + !value_fits_type(&val, f.data_type(), schema_version) + } + }; + // Terminates: each step makes a longer name, and the record and stream are finite. + while record.get(&name).is_some_and(|v| !v.is_null()) + || existing_schema.get(&name).is_some_and(|f| taken_by(f)) + { + name = format!("{name}_{suffix}"); + } + record.insert(name, val); + } + record +} + +/// Whether `value` can be stored in a column of `data_type`. Nested values are left to +/// arrow when the column is nested too: `value_compatible_with_type` cannot judge them. +fn value_fits_type(value: &Value, data_type: &DataType, schema_version: SchemaVersion) -> bool { + ((value.is_array() || value.is_object()) + && (data_type.is_list() || matches!(data_type, DataType::Struct(_) | DataType::Map(_, _)))) + || value_compatible_with_type(value, data_type, schema_version) +} + /// Per-record fallback when batch-level conflict detection misses a type /// mismatch. This happens when a single batch contains records with /// incompatible JSON types for the same field (e.g. record A has @@ -606,8 +730,9 @@ pub fn rename_conflicting_fields_in_json( /// /// For each record, scan each non-null field. If the value doesn't satisfy /// the storage type, rename that field in this specific record to -/// `_` so it routes to (or creates) a typed -/// sibling column instead of crashing the batch. +/// `_`, the suffix being the type the value is stored as +/// (see `renamed_column_type`), so it routes to (or creates) a typed sibling +/// column instead of crashing the batch. /// /// This is a no-op (and skips the per-record loop entirely) when: /// - the batch has at most one record — for a single record arrow's @@ -621,6 +746,7 @@ pub fn rename_per_record_type_mismatches( inferred_schema: &Schema, existing_schema: &HashMap>, schema_version: SchemaVersion, + infer_timestamp: bool, ) -> Vec { if values.len() <= 1 { return values; @@ -644,71 +770,46 @@ pub fn rename_per_record_type_mismatches( let Value::Object(map) = value else { return value; }; - let needs_rename = map.iter().any(|(key, val)| { - if val.is_null() { - return false; - } - let target_type = existing_schema - .get(key) - .map(|f| f.data_type()) - .or_else(|| inferred_types.get(key.as_str()).copied()); - let Some(target_type) = target_type else { - return false; - }; - if (val.is_array() || val.is_object()) - && (target_type.is_list() - || matches!(target_type, DataType::Struct(_) | DataType::Map(_, _))) - { - return false; - } - !value_compatible_with_type(val, target_type, schema_version) - }); - if !needs_rename { + // Prefer storage's declared type; fall back to the inferred type so + // within-batch mismatches on new fields are caught. Nested values on a + // nested column are left to arrow (see `value_fits_type`); scalars still + // flow through the check, so an array on e.g. a Utf8 column is routed to a + // typed sibling rather than crashing decode. + let mismatched = |key: &str, val: &Value| { + !val.is_null() + && existing_schema + .get(key) + .map(|f| f.data_type()) + .or_else(|| inferred_types.get(key).copied()) + .is_some_and(|t| !value_fits_type(val, t, schema_version)) + }; + if !map.iter().any(|(key, val)| mismatched(key, val)) { return Value::Object(map); } - let new_map: serde_json::Map = map + let entries = map .into_iter() .map(|(key, val)| { - if val.is_null() { - return (key, val); - } - // Prefer storage's declared type; fall back to the inferred - // type so within-batch mismatches on new fields are caught. - let target_type = existing_schema - .get(&key) - .map(|f| f.data_type()) - .or_else(|| inferred_types.get(key.as_str()).copied()); - let Some(target_type) = target_type else { - return (key, val); - }; - // When the resolved target is a structural arrow type - // (list/struct/map), arrow validates conformance at decode - // time and value_compatible_with_type can't reliably judge - // arrays/objects — skip to avoid spurious renames of valid - // nested values. Scalars still flow through the check, so - // an array landing on e.g. a Utf8 column is still routed - // to a typed sibling rather than crashing decode. - if (val.is_array() || val.is_object()) - && (target_type.is_list() - || matches!(target_type, DataType::Struct(_) | DataType::Map(_, _))) - { - return (key, val); + if !mismatched(&key, &val) { + return (key, val, None); } - if value_compatible_with_type(&val, target_type, schema_version) { - return (key, val); - } - let suffix = get_datatype_suffix(&datatype_for_value(&val)); - let new_key = format!("{key}_{suffix}"); - (new_key, val) + let data_type = + renamed_column_type(&key, [&val], schema_version, infer_timestamp); + let suffix = get_datatype_suffix(&data_type); + let target = (format!("{key}_{suffix}"), suffix); + (key, val, Some(target)) }) .collect(); - Value::Object(new_map) + Value::Object(place_record_values( + entries, + existing_schema, + schema_version, + )) }) .collect() } -/// Best-effort mapping from a JSON value to its arrow DataType. Used only to -/// pick a type-suffix for the per-record rename (e.g. "bool", "i64", "f64"). +/// Best-effort mapping from a JSON value to its arrow DataType. Used only as the +/// fallback in `renamed_column_type` when schema inference fails. fn datatype_for_value(value: &Value) -> DataType { match value { Value::Null => DataType::Null, @@ -774,6 +875,7 @@ mod tests { &existing_schema, &values, SchemaVersion::V1, + true, ); // Should detect conflict for body_timestamp (timestamp string can't be parsed as Float64) @@ -804,6 +906,7 @@ mod tests { &existing_schema, &values, SchemaVersion::V1, + true, ); // No conflicts - types match for existing field @@ -913,6 +1016,7 @@ mod tests { &existing_schema, &values, SchemaVersion::V1, + true, ); // Should detect conflict and rename to body_timestamp_utf8 @@ -947,6 +1051,7 @@ mod tests { &existing_schema, &values, SchemaVersion::V1, + true, ); // Should NOT detect conflict because the value can be parsed as timestamp @@ -973,6 +1078,7 @@ mod tests { &existing_schema, &values, SchemaVersion::V1, + true, ); assert_eq!(conflicts.get("date"), Some(&"date_utf8".to_string())); @@ -999,6 +1105,7 @@ mod tests { &existing_schema, &values, SchemaVersion::V1, + true, ); // Should detect conflict because number cannot be stored in Utf8 field @@ -1030,6 +1137,7 @@ mod tests { &existing_schema, &values, SchemaVersion::V0, + true, ); // Should detect conflict because integer doesn't match strict Float64 validation @@ -1061,6 +1169,7 @@ mod tests { &existing_schema, &values, SchemaVersion::V1, + true, ); // Should NOT detect conflict because V1 allows any number for Float64 @@ -1227,8 +1336,13 @@ mod tests { json!({"escaped": true}), json!({"escaped": null}), ]; - let renamed = - rename_per_record_type_mismatches(value_arr, &inferred, &storage, SchemaVersion::V1); + let renamed = rename_per_record_type_mismatches( + value_arr, + &inferred, + &storage, + SchemaVersion::V1, + true, + ); assert!(renamed[0].as_object().unwrap().contains_key("escaped")); // bool record routes to escaped_bool @@ -1252,6 +1366,7 @@ mod tests { &inferred, &storage, SchemaVersion::V1, + true, ); for v in &renamed { assert!(v.as_object().unwrap().contains_key("amount")); @@ -1268,12 +1383,13 @@ mod tests { &inferred, &storage, SchemaVersion::V1, + true, ); assert!(renamed[0].as_object().unwrap().contains_key("new_field")); } #[test] - fn rename_per_record_number_into_utf8_uses_int64_suffix() { + fn rename_per_record_number_into_utf8_uses_stored_float64_suffix() { let mut storage: HashMap> = HashMap::new(); storage.insert( "request_body".to_string(), @@ -1282,7 +1398,8 @@ mod tests { let inferred = Schema::new(vec![Field::new("request_body", DataType::Utf8, true)]); // Need >= 2 records for the per-record pass to run; second record's // string keeps inferred type at Utf8 (matches storage), unblocking - // the precheck so the int record gets renamed. + // the precheck so the int record gets renamed. V1 stores every number + // as Float64, so the sibling is named for that, not for the JSON integer. let renamed = rename_per_record_type_mismatches( vec![ json!({"request_body": 200}), @@ -1291,12 +1408,13 @@ mod tests { &inferred, &storage, SchemaVersion::V1, + true, ); assert!( renamed[0] .as_object() .unwrap() - .contains_key("request_body_int64") + .contains_key("request_body_float64") ); } @@ -1317,6 +1435,7 @@ mod tests { &inferred, &storage, SchemaVersion::V1, + true, ); // No rename because the loop is skipped for single-record batches. assert!(renamed[0].as_object().unwrap().contains_key("escaped")); @@ -1336,6 +1455,7 @@ mod tests { &inferred, &storage, SchemaVersion::V1, + true, ); assert!(renamed[0].as_object().unwrap().contains_key("escaped")); assert!(renamed[1].as_object().unwrap().contains_key("escaped_bool")); @@ -1366,4 +1486,119 @@ mod tests { &DataType::Timestamp(TimeUnit::Millisecond, None) ); } + + fn storage(fields: &[(&str, DataType)]) -> HashMap> { + fields + .iter() + .map(|(n, t)| (n.to_string(), Arc::new(Field::new(*n, t.clone(), true)))) + .collect() + } + + /// A record carrying both `f` (moving to `f_utf8`) and a literal `f_utf8` keeps both + /// values: the moved one chains to `f_utf8_utf8`, as the schema authority names it. + #[test] + fn rename_into_key_the_record_already_has_chains_suffix() { + let existing = storage(&[("f", DataType::Float64), ("f_utf8", DataType::Utf8)]); + let mapping = HashMap::from([("f".to_string(), "f_utf8".to_string())]); + let renamed = rename_conflicting_fields_in_json( + vec![ + json!({"f": "moved", "f_utf8": "literal"}), + json!({"f": "alone"}), + ], + &mapping, + &existing, + SchemaVersion::V1, + ); + assert_eq!( + renamed, + vec![ + json!({"f_utf8": "literal", "f_utf8_utf8": "moved"}), + json!({"f_utf8": "alone"}), + ] + ); + } + + #[test] + fn per_record_rename_into_key_the_record_already_has_chains_suffix() { + let existing = storage(&[("f", DataType::Float64), ("f_utf8", DataType::Utf8)]); + let inferred = Schema::new(vec![ + Field::new("f", DataType::Float64, true), + Field::new("f_utf8", DataType::Utf8, true), + ]); + let renamed = rename_per_record_type_mismatches( + vec![ + json!({"f": 1.5}), + json!({"f": "moved", "f_utf8": "literal"}), + ], + &inferred, + &existing, + SchemaVersion::V1, + true, + ); + assert_eq!(renamed[0], json!({"f": 1.5})); + assert_eq!( + renamed[1], + json!({"f_utf8": "literal", "f_utf8_utf8": "moved"}) + ); + } + + /// A target that is a stream column of another type is skipped the same way. + #[test] + fn rename_into_column_of_other_type_chains_suffix() { + let existing = storage(&[("f", DataType::Float64), ("f_utf8", DataType::Boolean)]); + let mapping = HashMap::from([("f".to_string(), "f_utf8".to_string())]); + let renamed = rename_conflicting_fields_in_json( + vec![json!({"f": "x"})], + &mapping, + &existing, + SchemaVersion::V1, + ); + assert_eq!(renamed, vec![json!({"f_utf8_utf8": "x"})]); + } + + /// The chain continues past every taken name, and a literal key whose own value moves + /// away frees its name for the value moving in. + #[test] + fn rename_chain_skips_every_taken_name_and_reuses_freed_ones() { + let existing = storage(&[("f", DataType::Float64), ("f_utf8", DataType::Utf8)]); + let mapping = HashMap::from([("f".to_string(), "f_utf8".to_string())]); + let renamed = rename_conflicting_fields_in_json( + vec![json!({"f": "m", "f_utf8": "a", "f_utf8_utf8": "b"})], + &mapping, + &existing, + SchemaVersion::V1, + ); + assert_eq!( + renamed, + vec![json!({"f_utf8": "a", "f_utf8_utf8": "b", "f_utf8_utf8_utf8": "m"})] + ); + + // `f_utf8: true` cannot stay in the Utf8 `f_utf8` and moves to `f_utf8_bool`. + let inferred = Schema::new(vec![ + Field::new("f", DataType::Utf8, true), + Field::new("f_utf8", DataType::Utf8, true), + ]); + let renamed = rename_per_record_type_mismatches( + vec![json!({"f": 1.5}), json!({"f": "m", "f_utf8": true})], + &inferred, + &existing, + SchemaVersion::V1, + true, + ); + assert_eq!(renamed[1], json!({"f_utf8": "m", "f_utf8_bool": true})); + } + + /// A null literal holds no value, so the moved value takes its name. + #[test] + fn rename_target_holding_null_is_reused() { + let existing = storage(&[("f", DataType::Float64), ("f_utf8", DataType::Utf8)]); + let mapping = HashMap::from([("f".to_string(), "f_utf8".to_string())]); + let renamed = rename_conflicting_fields_in_json( + vec![json!({"f": "x", "f_utf8": null})], + &mapping, + &existing, + SchemaVersion::V1, + ); + assert_eq!(renamed, vec![json!({"f_utf8": "x"})]); + } } diff --git a/src/event/mod.rs b/src/event/mod.rs index f6d0c435e..e15cda157 100644 --- a/src/event/mod.rs +++ b/src/event/mod.rs @@ -1,21 +1,21 @@ /* -* Parseable Server (C) 2022 - 2025 Parseable, Inc. -* -* This program is free software: you can redistribute it and/or modify -* it under the terms of the GNU Affero General Public License as -* published by the Free Software Foundation, either version 3 of the -* License, or (at your option) any later version. -* -* This program is distributed in the hope that it will be useful, -* but WITHOUT ANY WARRANTY; without even the implied warranty of -* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -* GNU Affero General Public License for more details. -* -* You should have received a copy of the GNU Affero General Public License -* along with this program. If not, see . -* -* -*/ + * Parseable Server (C) 2022 - 2025 Parseable, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * + */ pub mod format; @@ -87,6 +87,9 @@ impl Event { let stream = PARSEABLE.get_stream(&self.stream_name, &self.tenant_id)?; let _ingestion_guard = stream.ingestion_guard()?; + stream.check_schema_resolution().map_err(|reason| { + EventError::SchemaResolutionBlocked(self.stream_name.clone(), reason) + })?; if self.is_first_event { commit_schema(&self.stream_name, self.rb.schema(), &self.tenant_id)?; @@ -145,13 +148,20 @@ impl Event { } } +/// Key of the arrow file a batch is written to. It covers names *and* types: a batch whose +/// field changed type (after schema resolution) must never be appended to a file whose IPC +/// header declares the old type. Each name/type is framed so `["ab","c"]` != `["a","bc"]`. pub fn get_schema_key(fields: &[Arc]) -> String { + use std::hash::{Hash, Hasher}; // Fields must be sorted let mut hasher = xxhash_rust::xxh3::Xxh3::new(); for field in fields.iter().sorted_by_key(|v| v.name()) { hasher.update(field.name().as_bytes()); + hasher.update(&[0xFF]); + field.data_type().hash(&mut hasher); + hasher.update(&[0xFE]); } - let hash = hasher.digest(); + let hash = hasher.finish(); format!("{hash:x}") } @@ -166,9 +176,24 @@ pub fn commit_schema( // concurrent ingestion that needs the map to finish processing an event. let stream = PARSEABLE.get_stream(stream_name, tenant_id)?; let mut metadata = stream.metadata.write().expect(LOCK_EXPECT); + // Read under the schema lock: a resolution publishes its view before it changes the + // schema, so a type the authority set is always in the view seen here. + let view = stream.resolution.view(); let map = &mut metadata.schema; + // A batch built against the schema from before a resolved type change can still carry + // the old type. Its arrow file is resolved at conversion, so it must not fail here. + let incoming: Vec> = schema + .fields() + .iter() + .filter(|f| { + !map.get(f.name()).is_some_and(|cur| { + view.predates_resolved_type(f.name(), cur.data_type(), f.data_type()) + }) + }) + .cloned() + .collect(); let current_schema = Schema::new(map.values().cloned().collect::()); - let schema = Schema::try_merge(vec![current_schema, schema.as_ref().clone()])?; + let schema = Schema::try_merge(vec![current_schema, Schema::new(incoming)])?; map.clear(); map.extend(schema.fields.iter().map(|f| (f.name().clone(), f.clone()))); Ok(()) @@ -188,5 +213,9 @@ pub mod error { ObjectStorage(#[from] ObjectStorageError), #[error(transparent)] StreamNotFound(#[from] StreamNotFound), + #[error( + "Stream {0} is temporarily not accepting data: its schema could not be reconciled with the cluster ({1}). Retry later" + )] + SchemaResolutionBlocked(String, String), } } diff --git a/src/handlers/http/ingest.rs b/src/handlers/http/ingest.rs index d808ea575..80ec36eb7 100644 --- a/src/handlers/http/ingest.rs +++ b/src/handlers/http/ingest.rs @@ -582,6 +582,8 @@ impl actix_web::ResponseError for PostError { StreamNotFound(_) | Event(EventError::StreamNotFound(_)) => StatusCode::NOT_FOUND, + Event(EventError::SchemaResolutionBlocked(_, _)) => StatusCode::SERVICE_UNAVAILABLE, + Event(_) | CreateStream(_) | CustomError(_) @@ -605,6 +607,12 @@ impl actix_web::ResponseError for PostError { .insert_header(ContentType::json()) .json(metastore_error.to_detail()) } + PostError::Event(EventError::SchemaResolutionBlocked(_, _)) => { + actix_web::HttpResponse::build(self.status_code()) + .insert_header(ContentType::plaintext()) + .insert_header((actix_web::http::header::RETRY_AFTER, "60")) + .body(self.to_string()) + } _ => actix_web::HttpResponse::build(self.status_code()) .insert_header(ContentType::plaintext()) .body(self.to_string()), diff --git a/src/handlers/http/middleware.rs b/src/handlers/http/middleware.rs index 13332968d..262349c6b 100644 --- a/src/handlers/http/middleware.rs +++ b/src/handlers/http/middleware.rs @@ -1,21 +1,21 @@ /* -* Parseable Server (C) 2022 - 2025 Parseable, Inc. -* -* This program is free software: you can redistribute it and/or modify -* it under the terms of the GNU Affero General Public License as -* published by the Free Software Foundation, either version 3 of the -* License, or (at your option) any later version. -* -* This program is distributed in the hope that it will be useful, -* but WITHOUT ANY WARRANTY; without even the implied warranty of -* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -* GNU Affero General Public License for more details. -* -* You should have received a copy of the GNU Affero General Public License -* along with this program. If not, see . -* -* -*/ + * Parseable Server (C) 2022 - 2025 Parseable, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * + */ use std::future::{Ready, ready}; @@ -57,6 +57,8 @@ use crate::{ use serde::{Deserialize, Serialize}; pub const CLUSTER_SECRET_HEADER: &str = "x-p-cluster-secret"; +/// Tenant an intra-cluster request acts for, trusted once the cluster secret is verified. +pub const INTRA_CLUSTER_TENANT_HEADER: &str = "intra-cluster-tenant"; pub static CLUSTER_SECRET: OnceCell<(String, String)> = OnceCell::new(); #[derive(Serialize, Deserialize, Debug)] @@ -719,7 +721,7 @@ where fn call(&self, mut req: ServiceRequest) -> Self::Future { let (err, id) = if let Some((secret, _)) = CLUSTER_SECRET.get() { if let Some(header) = req.headers().get(CLUSTER_SECRET_HEADER) - && let Some(tenant) = req.headers().get("intra-cluster-tenant") + && let Some(tenant) = req.headers().get(INTRA_CLUSTER_TENANT_HEADER) && let Some(userid) = req.headers().get("intra-cluster-userid") && let Ok(incoming_secret) = header.to_str() && let Ok(tenant) = tenant.to_str() diff --git a/src/handlers/http/mod.rs b/src/handlers/http/mod.rs index 6bf903095..441d6aa34 100644 --- a/src/handlers/http/mod.rs +++ b/src/handlers/http/mod.rs @@ -144,6 +144,8 @@ pub async fn fetch_schema( .get_all_schemas(stream_name, tenant_id) .await?; + // Strict on purpose: a type conflict must surface as an error, never be papered over + // by picking one type, which would make the query engine cast data at read time. let new_schema = Schema::try_merge(res)?; Ok(new_schema) } diff --git a/src/handlers/http/modal/query_server.rs b/src/handlers/http/modal/query_server.rs index 22dbbfd03..e5da5156b 100644 --- a/src/handlers/http/modal/query_server.rs +++ b/src/handlers/http/modal/query_server.rs @@ -22,7 +22,7 @@ use crate::handlers::airplane; use crate::handlers::http::cluster; use crate::handlers::http::logstream; use crate::handlers::http::max_event_payload_size; -use crate::handlers::http::middleware::{DisAllowRootUser, RouteExt}; +use crate::handlers::http::middleware::{DisAllowRootUser, IntraClusterRequest, RouteExt}; use crate::handlers::http::modal::initialize_hot_tier_metadata_on_startup; use crate::handlers::http::rbac::patch_user; use crate::handlers::http::{base_path, prism_base_path}; @@ -32,7 +32,7 @@ use crate::hottier::HotTierManager; use crate::hottier::HotTierMessage; use crate::hottier::hottier_runtime; use crate::rbac::role::Action; -use crate::{analytics, migration, storage}; +use crate::{analytics, migration, schema_registry, storage}; use actix_web::web::{ServiceConfig, resource}; use actix_web::{Scope, web}; use actix_web_prometheus::PrometheusMetrics; @@ -74,6 +74,7 @@ impl ParseableServer for QueryServer { .service(Server::get_alerts_webscope()) .service(Server::get_targets_webscope()) .service(Self::get_cluster_web_scope()) + .service(Self::get_schema_authority_scope()) .service(Server::get_demo_data_webscope()) .service(Server::get_alert_target_policy()), ) @@ -255,6 +256,21 @@ impl QueryServer { ) } + /// Intra-cluster schema authority: ingestors resolve new field types here. + pub fn get_schema_authority_scope() -> Scope { + web::scope("/internal/schema").service( + web::resource("/resolve") + .route( + // Authorised in the handler: a cluster-secret caller is a node, not a + // user (see `authority::caller_tenant`). + web::post() + .to(schema_registry::authority::resolve_handler) + .wrap(IntraClusterRequest), + ) + .app_data(web::JsonConfig::default().limit(1 << 20)), + ) + } + // get the logstream web scope pub fn get_logstream_webscope() -> Scope { web::scope("/logstream") diff --git a/src/handlers/http/modal/utils/ingest_utils.rs b/src/handlers/http/modal/utils/ingest_utils.rs index e428977b2..10ab755b5 100644 --- a/src/handlers/http/modal/utils/ingest_utils.rs +++ b/src/handlers/http/modal/utils/ingest_utils.rs @@ -28,6 +28,7 @@ use std::collections::HashMap; use tokio::sync::oneshot; use tracing::{error, instrument, warn}; +use crate::event::error::EventError; use crate::{ LOCK_EXPECT, event::{ @@ -540,6 +541,14 @@ pub fn validate_stream_for_ingestion( return Err(PostError::StreamBeingDeleted(stream_name.to_string())); } + // Reject early while the stream's schema cannot be reconciled with the cluster. + stream.check_schema_resolution().map_err(|reason| { + PostError::Event(EventError::SchemaResolutionBlocked( + stream_name.to_string(), + reason, + )) + })?; + // Validate that the stream's log source is compatible stream .get_log_source() diff --git a/src/lib.rs b/src/lib.rs index 58abee54d..929d51965 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -51,6 +51,7 @@ pub mod prism; pub mod query; pub mod rbac; mod response; +pub mod schema_registry; pub mod sse; mod static_schema; mod stats; diff --git a/src/metastore/metastores/object_store_metastore.rs b/src/metastore/metastores/object_store_metastore.rs index c8d43a4c7..7af9ab20c 100644 --- a/src/metastore/metastores/object_store_metastore.rs +++ b/src/metastore/metastores/object_store_metastore.rs @@ -1189,10 +1189,14 @@ impl Metastore for ObjectStoreMetastore { ) .await? .iter() - // we should be able to unwrap as we know the data is valid schema - .map(|byte_obj| { - serde_json::from_slice(byte_obj) - .unwrap_or_else(|_| panic!("got an invalid schema for stream: {stream_name}")) + .filter_map(|byte_obj| match serde_json::from_slice(byte_obj) { + Ok(schema) => Some(schema), + Err(err) => { + tracing::warn!( + "skipping unreadable schema file of stream {stream_name}: {err}" + ); + None + } }) .collect()) } diff --git a/src/migration/mod.rs b/src/migration/mod.rs index 492521f16..835913856 100644 --- a/src/migration/mod.rs +++ b/src/migration/mod.rs @@ -1,21 +1,21 @@ /* -* Parseable Server (C) 2022 - 2025 Parseable, Inc. -* -* This program is free software: you can redistribute it and/or modify -* it under the terms of the GNU Affero General Public License as -* published by the Free Software Foundation, either version 3 of the -* License, or (at your option) any later version. -* -* This program is distributed in the hope that it will be useful, -* but WITHOUT ANY WARRANTY; without even the implied warranty of -* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -* GNU Affero General Public License for more details. -* -* You should have received a copy of the GNU Affero General Public License -* along with this program. If not, see . -* -* -*/ + * Parseable Server (C) 2022 - 2025 Parseable, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * + */ pub mod metadata_migration; mod schema_migration; diff --git a/src/migration/schema_migration.rs b/src/migration/schema_migration.rs index 822f261d4..bfe1ee070 100644 --- a/src/migration/schema_migration.rs +++ b/src/migration/schema_migration.rs @@ -1,21 +1,21 @@ /* -* Parseable Server (C) 2022 - 2025 Parseable, Inc. -* -* This program is free software: you can redistribute it and/or modify -* it under the terms of the GNU Affero General Public License as -* published by the Free Software Foundation, either version 3 of the -* License, or (at your option) any later version. -* -* This program is distributed in the hope that it will be useful, -* but WITHOUT ANY WARRANTY; without even the implied warranty of -* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -* GNU Affero General Public License for more details. -* -* You should have received a copy of the GNU Affero General Public License -* along with this program. If not, see . -* -* -*/ + * Parseable Server (C) 2022 - 2025 Parseable, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * + */ use std::collections::HashMap; diff --git a/src/migration/stream_metadata_migration.rs b/src/migration/stream_metadata_migration.rs index 05017e8f9..5cb534e51 100644 --- a/src/migration/stream_metadata_migration.rs +++ b/src/migration/stream_metadata_migration.rs @@ -1,21 +1,21 @@ /* -* Parseable Server (C) 2022 - 2025 Parseable, Inc. -* -* This program is free software: you can redistribute it and/or modify -* it under the terms of the GNU Affero General Public License as -* published by the Free Software Foundation, either version 3 of the -* License, or (at your option) any later version. -* -* This program is distributed in the hope that it will be useful, -* but WITHOUT ANY WARRANTY; without even the implied warranty of -* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -* GNU Affero General Public License for more details. -* -* You should have received a copy of the GNU Affero General Public License -* along with this program. If not, see . -* -* -*/ + * Parseable Server (C) 2022 - 2025 Parseable, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * + */ use crate::{ catalog::snapshot::CURRENT_SNAPSHOT_VERSION, diff --git a/src/parseable/staging/reader.rs b/src/parseable/staging/reader.rs index 00cd399da..2e1365cf6 100644 --- a/src/parseable/staging/reader.rs +++ b/src/parseable/staging/reader.rs @@ -37,6 +37,7 @@ use tracing::{error, info_span}; use crate::{ event::DEFAULT_TIMESTAMP_KEY, + schema_registry::view::{ResolvedView, rename_batch}, utils::arrow::{adapt_batch, reverse}, }; @@ -44,8 +45,12 @@ use crate::{ /// Merges reverse readers while retaining the source-file validation result. pub struct MergedReverseRecordReader { pub readers: Vec>>>, + /// Resolved-name schema per reader (None = no rename needed). + renamed: Vec>>, pub readable_files: Vec, pub invalid_files: Vec, + /// Valid files whose columns cannot be renamed to the resolved schema. Left untouched. + pub unrenameable_files: Vec, } #[derive(Debug)] @@ -55,12 +60,16 @@ pub struct MergedForwardRecordReader { file_plans: Vec, pub readable_files: Vec, pub invalid_files: Vec, + /// Valid files whose columns cannot be renamed to the resolved schema. Left untouched. + pub unrenameable_files: Vec, } #[derive(Debug)] pub(crate) struct ForwardFilePlan { path: PathBuf, schema: Arc, + /// Resolved-name schema (None = no rename needed). + renamed: Option>, messages: Vec<(MessageHeader, usize, usize)>, record_batches: usize, } @@ -73,6 +82,16 @@ pub(crate) struct ForwardReaderLane { } impl ForwardFilePlan { + pub(crate) fn renamed_schema(&self) -> Option> { + self.renamed.clone() + } + + fn effective_schema(&self) -> &Schema { + self.renamed + .as_deref() + .unwrap_or_else(|| self.schema.as_ref()) + } + /// Opens independent virtual Arrow streams over striped record batches. /// Every stream begins with the original schema and receives every /// dictionary update, preserving Arrow IPC decoder state. @@ -140,11 +159,18 @@ impl ForwardFilePlan { impl MergedForwardRecordReader { /// Opens valid Arrow files and excludes any crash-truncated tail message. + #[cfg(test)] pub fn try_new(file_paths: &[PathBuf]) -> Self { + Self::try_new_with_view(file_paths, &ResolvedView::default()) + } + + /// Like [`Self::try_new`], renaming columns as decided by the schema authority. + pub fn try_new_with_view(file_paths: &[PathBuf], view: &ResolvedView) -> Self { let _span = info_span!("open_forward_arrow_files", file_count = file_paths.len()).entered(); let mut file_plans = Vec::with_capacity(file_paths.len()); let mut readable_files = Vec::with_capacity(file_paths.len()); let mut invalid_files = Vec::new(); + let mut unrenameable_files = Vec::new(); for path in file_paths { match File::open(path) { Err(err) => { @@ -179,9 +205,19 @@ impl MergedForwardRecordReader { .iter() .filter(|(header, _, _)| *header == MessageHeader::RecordBatch) .count(); + let schema = schema_reader.schema(); + let renamed = match view.rename_schema(&schema) { + Ok(renamed) => renamed, + Err(err) => { + error!("Cannot apply resolved schema to {path:?}: {err}"); + unrenameable_files.push(path.clone()); + continue; + } + }; file_plans.push(ForwardFilePlan { path: path.clone(), - schema: schema_reader.schema(), + schema, + renamed, messages, record_batches, }); @@ -194,6 +230,7 @@ impl MergedForwardRecordReader { file_plans, readable_files, invalid_files, + unrenameable_files, } } @@ -205,24 +242,30 @@ impl MergedForwardRecordReader { self.file_plans.into_iter() } - /// Returns the union of schemas exposed by all readable Arrow streams. - pub fn merged_schema(&self) -> Schema { + /// Returns the union of (renamed) schemas exposed by all readable Arrow streams. + pub fn merged_schema(&self) -> Result { Schema::try_merge( self.file_plans .iter() - .map(|plan| plan.schema.as_ref().clone()), + .map(|plan| plan.effective_schema().clone()), ) - .unwrap() } } impl MergedReverseRecordReader { /// Opens valid Arrow files and separates files that fail structural validation. pub fn try_new(file_paths: &[PathBuf]) -> Self { + Self::try_new_with_view(file_paths, &ResolvedView::default()) + } + + /// Like [`Self::try_new`], renaming columns as decided by the schema authority. + pub fn try_new_with_view(file_paths: &[PathBuf], view: &ResolvedView) -> Self { let _span = info_span!("open_arrow_files", file_count = file_paths.len()).entered(); let mut readers = Vec::with_capacity(file_paths.len()); + let mut renamed = Vec::with_capacity(file_paths.len()); let mut readable_files = Vec::with_capacity(file_paths.len()); let mut invalid_files = Vec::new(); + let mut unrenameable_files = Vec::new(); for path in file_paths { match File::open(path) { Err(err) => { @@ -238,7 +281,16 @@ impl MergedReverseRecordReader { continue; } }; + let renamed_schema = match view.rename_schema(&reader.schema()) { + Ok(renamed_schema) => renamed_schema, + Err(err) => { + error!("Cannot apply resolved schema to {path:?}: {err}"); + unrenameable_files.push(path.clone()); + continue; + } + }; readers.push(reader); + renamed.push(renamed_schema); readable_files.push(path.clone()); } } @@ -246,8 +298,10 @@ impl MergedReverseRecordReader { Self { readers, + renamed, readable_files, invalid_files, + unrenameable_files, } } @@ -257,7 +311,15 @@ impl MergedReverseRecordReader { schema: Arc, time_partition: Option, ) -> impl Iterator> { - let adapted_readers = self.readers; + // Rename before adapting: `adapt_batch` matches columns by name and would + // otherwise fill a renamed column with nulls. + let adapted_readers = + self.readers + .into_iter() + .zip(self.renamed) + .map(|(reader, renamed)| { + reader.map(move |batch| batch.and_then(|b| rename_batch(&renamed, b))) + }); kmerge_by( adapted_readers, move |a: &Result, b: &Result| { @@ -278,14 +340,18 @@ impl MergedReverseRecordReader { .map(move |batch| batch.map(|batch| adapt_batch(schema.clone(), &batch))) } - /// Returns the union of schemas exposed by all readable Arrow streams. - pub fn merged_schema(&self) -> Schema { + /// Returns the union of (renamed) schemas exposed by all readable Arrow streams. + pub fn merged_schema(&self) -> Result { Schema::try_merge( self.readers .iter() - .map(|reader| reader.schema().as_ref().clone()), + .zip(&self.renamed) + .map(|(reader, renamed)| { + renamed + .as_ref() + .map_or_else(|| reader.schema().as_ref().clone(), |s| s.as_ref().clone()) + }), ) - .unwrap() } } diff --git a/src/parseable/staging/writer.rs b/src/parseable/staging/writer.rs index 919e78c23..5c4f51e3b 100644 --- a/src/parseable/staging/writer.rs +++ b/src/parseable/staging/writer.rs @@ -74,6 +74,16 @@ pub struct Writer { pub disk: HashMap, } +impl Writer { + /// Disk-only writer, for nodes that never serve queries from staging. + pub fn without_memory() -> Self { + Self { + mem: None, + disk: HashMap::new(), + } + } +} + impl Default for Writer { fn default() -> Self { let mem = if *ENABLE_MEMORY_STAGING { diff --git a/src/parseable/streams.rs b/src/parseable/streams.rs index 66d722196..867f6674d 100644 --- a/src/parseable/streams.rs +++ b/src/parseable/streams.rs @@ -83,6 +83,12 @@ use crate::{ metrics, option::Mode, parseable::{DEFAULT_TENANT, PARSEABLE}, + schema_registry::{ + client::resolve_blocking, + ingestor::{ResolutionOutcome, ResolutionState, ResolveContext, schema_overrides}, + merge_schemas_tolerant, + view::{ResolvedView, rename_batch}, + }, storage::{StreamType, object_storage::to_bytes, retention::Retention}, sync::FLUSH_AND_CONVERT_RUNTIME, utils::{ @@ -415,6 +421,8 @@ pub struct Stream { // Ingestion/local sync hold read access; staging cleanup holds write access. // Acquire before schema_writer or writer when an operation needs both. staging_activity: RwLock<()>, + /// Cluster-wide schema resolution state (distributed ingestors only). + pub resolution: ResolutionState, } #[derive(Default)] @@ -475,10 +483,13 @@ struct ActiveMetricForwardFile { lanes: Vec, next_batch: usize, total_batches: usize, + /// Schema with resolved column names, applied to every batch before adapting it. + renamed: Option>, } impl ActiveMetricForwardFile { fn try_new(plan: ForwardFilePlan, readers_per_file: usize) -> Result { + let renamed = plan.renamed_schema(); let readers = plan .open_lanes(readers_per_file) .map_err(|err| ArrowError::IoError(err.to_string(), err))?; @@ -501,6 +512,7 @@ impl ActiveMetricForwardFile { lanes, next_batch: 0, total_batches, + renamed, }) } @@ -624,7 +636,12 @@ impl Iterator for MetricForwardRecordIterator { match &mut queued.state { MetricForwardFileState::Ready(active) => { if let Some(batch) = active.next_batch() { - return Some(batch.map(|batch| adapt_batch(self.schema.clone(), &batch))); + let renamed = &active.renamed; + return Some( + batch + .and_then(|batch| rename_batch(renamed, batch)) + .map(|batch| adapt_batch(self.schema.clone(), &batch)), + ); } } MetricForwardFileState::Failed(_) => {} @@ -648,11 +665,22 @@ impl Iterator for MetricForwardRecordIterator { } impl ConversionRecordReader { - fn for_stream(file_paths: &[PathBuf], forward: bool) -> Self { + fn for_stream(file_paths: &[PathBuf], forward: bool, view: &ResolvedView) -> Self { if forward { - Self::Forward(MergedForwardRecordReader::try_new(file_paths)) + Self::Forward(MergedForwardRecordReader::try_new_with_view( + file_paths, view, + )) } else { - Self::Reverse(MergedReverseRecordReader::try_new(file_paths)) + Self::Reverse(MergedReverseRecordReader::try_new_with_view( + file_paths, view, + )) + } + } + + fn unrenameable_files(&self) -> &[PathBuf] { + match self { + Self::Forward(reader) => &reader.unrenameable_files, + Self::Reverse(reader) => &reader.unrenameable_files, } } @@ -677,7 +705,7 @@ impl ConversionRecordReader { } } - fn merged_schema(&self) -> Schema { + fn merged_schema(&self) -> Result { match self { Self::Forward(reader) => reader.merged_schema(), Self::Reverse(reader) => reader.merged_schema(), @@ -724,8 +752,11 @@ impl StartupSyncPlan { let time_partition = self.stream.get_time_partition(); let custom_partition = self.stream.get_custom_partition(); + let staging_files = self + .stream + .resolve_inprocess_groups(self.staging_files, &self.tenant_id); let outcome = self.stream.convert_arrow_file_groups_to_parquet( - self.staging_files, + staging_files, time_partition.as_ref(), custom_partition.as_ref(), &self.tenant_id, @@ -745,13 +776,34 @@ impl Stream { ) -> StreamRef { let stream_name = stream_name.into(); let data_path = options.local_stream_data_path(&stream_name, tenant_id); + let resolving = options.mode == Mode::Ingest && options.schema_resolution; + let resolution = if resolving { + ResolutionState::load(&data_path) + } else { + ResolutionState::default() + }; + let mut metadata = metadata; + // Re-apply decisions made before a restart, so stale local types are not used + // for conflict detection of new events. + for field in schema_overrides(&resolution.view(), &metadata.schema) { + if metadata.schema.contains_key(field.name()) { + metadata.schema.insert(field.name().clone(), field); + } + } + // The in-memory staging buffer only serves queries, which ingestors never run, and + // it cannot merge batches across a resolved type change. + let writer = if resolving { + Writer::without_memory() + } else { + Writer::default() + }; Arc::new(Self { stream_name: stream_name.clone(), metadata: RwLock::new(metadata), data_path, options, - writer: Mutex::new(Writer::default()), + writer: Mutex::new(writer), push_queue: Mutex::new(PushQueue::default()), push_queue_empty: Condvar::default(), #[cfg(test)] @@ -759,6 +811,7 @@ impl Stream { schema_writer: Mutex::new(()), ingestor_id, staging_activity: RwLock::new(()), + resolution, }) } @@ -1070,17 +1123,45 @@ impl Stream { group_minute: u128, init_signal: bool, shutdown_signal: bool, + ) -> Vec<(PathBuf, Vec)> { + self.arrow_files_grouped_for_conversion( + exclude, + group_minute, + init_signal, + shutdown_signal, + &None, + ) + } + + fn arrow_files_grouped_for_conversion( + &self, + exclude: SystemTime, + group_minute: u128, + init_signal: bool, + shutdown_signal: bool, + tenant_id: &Option, ) -> Vec<(PathBuf, Vec)> { if init_signal { // Startup owns only this one-time snapshot of processing files. // Root Arrow files are left for regular sync so startup never // races a live writer or a periodic cycle for the same file. - self.group_inprocess_arrow_files(&Ulid::new().to_string()) + let groups = self.group_inprocess_arrow_files(&Ulid::new().to_string()); + self.resolve_inprocess_groups(groups, tenant_id) } else { let arrow_files = self.fetch_arrow_files_for_conversion(exclude, shutdown_signal); if arrow_files.is_empty() { return Vec::new(); } + // Resolve before moving: deferred files must stay in the root directory so + // the next periodic cycle picks them up again. + let arrow_files = if self.schema_resolution_enabled() { + self.resolve_arrow_files(arrow_files, tenant_id).ready + } else { + arrow_files + }; + if arrow_files.is_empty() { + return Vec::new(); + } // Never reuse a directory that may belong to the immutable startup // snapshot (possible when a process restarts within the same minute). @@ -1107,6 +1188,128 @@ impl Stream { } } + /// True when this stream reconciles new field types with the cluster's schema authority. + pub fn schema_resolution_enabled(&self) -> bool { + self.options.mode == Mode::Ingest + && self.options.schema_resolution + && !self.get_static_schema_flag() + && self.get_stream_type() != StreamType::Internal + } + + /// Rejects ingestion while the stream's schema cannot be reconciled with the cluster. + pub fn check_schema_resolution(&self) -> Result<(), String> { + if self.resolution.is_blocked() { + Err(self + .resolution + .last_error() + .unwrap_or_else(|| "schema resolution pending".to_owned())) + } else { + Ok(()) + } + } + + /// Asks the schema authority about fields in `files` it has not resolved yet and + /// aligns the in-memory and staged schemas with the answer. Blocking: call only from + /// conversion (blocking) threads. + fn resolve_arrow_files( + &self, + files: Vec, + tenant_id: &Option, + ) -> ResolutionOutcome { + let resolver = |req| resolve_blocking(req); + let node_id = self + .ingestor_id + .clone() + .unwrap_or_else(|| PARSEABLE.options.address.clone()); + let ctx = ResolveContext { + stream: &self.stream_name, + tenant_id, + node_id, + data_path: &self.data_path, + max_deferred_cycles: self.options.schema_resolve_max_deferred_cycles, + resolver: &resolver, + }; + self.resolution.ensure_resolved(&ctx, files, |view| { + let overrides = schema_overrides(view, &self.get_schema_raw()); + if !overrides.is_empty() { + self.overwrite_schema_fields(&overrides); + } + if let Err(err) = self.stage_schema_file(self.get_schema().as_ref().clone()) { + warn!( + "failed to stage resolved schema for stream {}: {err}", + self.stream_name + ); + } + }) + } + + /// Startup path: files are already inside processing directories, so deferred ones + /// are moved back to the root directory for the periodic cycle to retry. + fn resolve_inprocess_groups( + &self, + groups: Vec<(PathBuf, Vec)>, + tenant_id: &Option, + ) -> Vec<(PathBuf, Vec)> { + if groups.is_empty() || !self.schema_resolution_enabled() { + return groups; + } + let files: Vec = groups.iter().flat_map(|(_, f)| f.clone()).collect(); + let outcome = self.resolve_arrow_files(files, tenant_id); + if outcome.deferred.is_empty() { + return groups; + } + self.return_files_to_root(&outcome.deferred); + let ready: HashSet = outcome.ready.into_iter().collect(); + groups + .into_iter() + .map(|(parquet, files)| { + ( + parquet, + files.into_iter().filter(|f| ready.contains(f)).collect(), + ) + }) + .filter(|(_, files): &(PathBuf, Vec)| !files.is_empty()) + .collect() + } + + fn return_files_to_root(&self, files: &[PathBuf]) { + for file in files { + let Some(name) = file.file_name() else { + continue; + }; + let mut target = self.data_path.join(name); + if target.exists() + && let Some(stem) = file.file_stem().and_then(|s| s.to_str()) + { + target = self + .data_path + .join(format!("{stem}.{}.{ARROW_FILE_EXTENSION}", Ulid::new())); + } + if let Err(err) = fs::rename(file, &target) { + error!( + "Failed to return deferred arrow file {} to {}: {err}", + file.display(), + target.display() + ); + continue; + } + if let Some(dir) = file.parent() + && dir != self.data_path + { + let _ = fs::remove_dir(dir); + } + } + } + + /// Replaces field types in the in-memory schema (no merge): used when the schema + /// authority decided a different type than this node inferred. + pub fn overwrite_schema_fields(&self, fields: &[Arc]) { + let mut metadata = self.metadata.write().expect(LOCK_EXPECT); + for field in fields { + metadata.schema.insert(field.name().clone(), field.clone()); + } + } + /// Groups arrow files only from the specified inprocess folder fn group_single_inprocess_arrow_files( &self, @@ -1348,7 +1551,27 @@ impl Stream { let tmp_path = path.with_extension("schema.tmp"); let staging_schemas = self.get_schemas_if_present()?; - if !staging_schemas.is_empty() { + if self.schema_resolution_enabled() { + // Advertise only what the schema authority accepted: an unresolved type in the + // per-node schema file would surface the very conflict resolution prevents. + let view = self.resolution.view(); + let keep = |s: Schema| { + let fields: Vec> = s + .fields() + .iter() + .filter(|f| { + view.fields + .get(f.name()) + .is_some_and(|t| t == f.data_type()) + }) + .cloned() + .collect(); + Schema::new(fields) + }; + let mut schemas = vec![keep(schema)]; + schemas.extend(staging_schemas.into_iter().map(keep)); + schema = merge_schemas_tolerant(&self.stream_name, schemas); + } else if !staging_schemas.is_empty() { let mut staging_schemas = staging_schemas; staging_schemas.push(schema); schema = Schema::try_merge(staging_schemas)?; @@ -2002,8 +2225,13 @@ impl Stream { let now = SystemTime::now(); let group_minute = minute_from_system_time(now) - 1; - let staging_files = - self.arrow_files_grouped_exclude_time(now, group_minute, init_signal, shutdown_signal); + let staging_files = self.arrow_files_grouped_for_conversion( + now, + group_minute, + init_signal, + shutdown_signal, + tenant_id, + ); span.record("file_group_count", staging_files.len()); self.convert_arrow_file_groups_to_parquet( staging_files, @@ -2120,15 +2348,35 @@ impl Stream { ) -> Result, StagingError> { // Metrics sort every output row group explicitly, so reading their IPC // streams forward avoids reverse seeks and a full-column row reversal. + // Renames decided by the schema authority are applied per file before merging. + // Renames are append-only, so every file converts under the current view exactly + // as it did under the view that accepted it. + let view = self.resolution.view(); let record_reader = - ConversionRecordReader::for_stream(&arrow_files, self.is_otel_metrics()); + ConversionRecordReader::for_stream(&arrow_files, self.is_otel_metrics(), &view); self.remove_invalid_arrow_files(record_reader.invalid_files(), tenant_id); + for file in record_reader.unrenameable_files() { + error!( + "Arrow file {} of stream {} cannot be renamed to the resolved schema; keeping it unconverted", + file.display(), + self.stream_name + ); + } let readable_arrow_files = record_reader.readable_files().to_vec(); if record_reader.reader_count() == 0 { return Ok(None); } - let merged_schema = record_reader.merged_schema(); + // A type conflict across files must never panic the conversion thread; keep the + // files for a later cycle instead. + let merged_schema = record_reader.merged_schema().map_err(|err| { + error!( + "Schema merge failed for {} of stream {}; arrow files kept for retry: {err}", + parquet_path.display(), + self.stream_name + ); + StagingError::Arrow(err) + })?; let props = self.parquet_writer_props(&merged_schema, time_partition, custom_partition); let schema = Arc::new(merged_schema.clone()); let mut part_path = parquet_path.clone(); @@ -2613,9 +2861,15 @@ impl Stream { return current_schema; } - let schema = record_reader.merged_schema(); + let Ok(schema) = record_reader.merged_schema() else { + warn!( + "staged arrow files of stream {} have conflicting schemas; keeping current schema", + self.stream_name + ); + return current_schema; + }; - Schema::try_merge(vec![schema, current_schema]).unwrap() + Schema::try_merge(vec![schema, current_schema.clone()]).unwrap_or(current_schema) } /// Stores the provided stream metadata in memory mapping @@ -3601,7 +3855,7 @@ mod tests { let record_reader = ConversionRecordReader::Forward(MergedForwardRecordReader::try_new(&arrow_files)); - let merged_schema = Arc::new(record_reader.merged_schema()); + let merged_schema = Arc::new(record_reader.merged_schema().unwrap()); let props = WriterProperties::builder() .set_max_row_group_row_count(Some(5)) .build(); diff --git a/src/rbac/role.rs b/src/rbac/role.rs index bbfc1a5d7..a7f97813d 100644 --- a/src/rbac/role.rs +++ b/src/rbac/role.rs @@ -1,21 +1,21 @@ /* -* Parseable Server (C) 2022 - 2025 Parseable, Inc. -* -* This program is free software: you can redistribute it and/or modify -* it under the terms of the GNU Affero General Public License as -* published by the Free Software Foundation, either version 3 of the -* License, or (at your option) any later version. -* -* This program is distributed in the hope that it will be useful, -* but WITHOUT ANY WARRANTY; without even the implied warranty of -* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -* GNU Affero General Public License for more details. -* -* You should have received a copy of the GNU Affero General Public License -* along with this program. If not, see . -* -* -*/ + * Parseable Server (C) 2022 - 2025 Parseable, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * + */ // Represents actions that corresponds to an api #[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] @@ -62,6 +62,8 @@ pub enum Action { ListCluster, ListClusterMetrics, DeleteNode, + /// Intra-cluster: ingestor asks the schema authority to resolve field types. + ResolveSchema, All, SuperAdmin, GetAnalytics, @@ -133,6 +135,7 @@ impl RoleBuilder { | Action::ListCluster | Action::ListClusterMetrics | Action::DeleteNode + | Action::ResolveSchema | Action::PutHotTierEnabled | Action::GetHotTierEnabled | Action::DeleteHotTierEnabled diff --git a/src/schema_registry/authority.rs b/src/schema_registry/authority.rs new file mode 100644 index 000000000..3a763ce7f --- /dev/null +++ b/src/schema_registry/authority.rs @@ -0,0 +1,505 @@ +/* + * Parseable Server (C) 2022 - 2025 Parseable, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + */ + +//! The schema authority hosted by the querier (OSS) or Prism (enterprise). +//! +//! Exactly one authority runs per cluster. It keeps one write-once [`StreamRegistry`] +//! per (tenant, stream), serialises resolutions per stream, and durably writes the +//! stream-level `.schema` before replying. + +use std::{ + collections::{HashMap, HashSet}, + sync::{Arc, Mutex}, +}; + +use actix_web::{ + HttpRequest, HttpResponse, ResponseError, error::ErrorUnauthorized, http::StatusCode, web, +}; +use arrow_schema::{Field, Schema}; +use bytes::Bytes; +use once_cell::sync::Lazy; +use tokio::sync::{Mutex as AsyncMutex, OwnedMutexGuard}; +use tracing::{info, warn}; + +use crate::{ + event::{ + DEFAULT_TIMESTAMP_KEY, FORMAT_KEY, FORMAT_VERIFY_KEY, SOURCE_IP_KEY, USER_AGENT_KEY, + commit_schema, + }, + handlers::http::middleware::{CLUSTER_SECRET, INTRA_CLUSTER_TENANT_HEADER}, + parseable::{DEFAULT_TENANT, PARSEABLE, Stream}, + rbac::{self, Users, role::Action}, + storage::ObjectStorageError, + utils::{actix::extract_session_key_from_req, get_tenant_id_from_key}, +}; + +use super::{ + authoritative_schema_path, merge_schemas_tolerant, + protocol::{ + ErrorCode, Outcome, PROTOCOL_VERSION, RegistryEntry, ResolveErrorBody, ResolveRequest, + ResolveResponse, + }, + registry::StreamRegistry, +}; + +/// Metadata key holding the registry version inside the persisted `.schema`. +pub const REGISTRY_VERSION_KEY: &str = "parseable.registry_version"; + +type Slot = Arc>>; + +#[derive(Default)] +pub struct SchemaAuthority { + slots: Mutex>, +} + +pub static SCHEMA_AUTHORITY: Lazy = Lazy::new(SchemaAuthority::default); + +#[derive(Debug, thiserror::Error)] +pub enum AuthorityError { + #[error("{0}")] + BadRequest(String), + #[error("stream {0} not found")] + StreamNotFound(String), + #[error("stream {0} is being deleted")] + StreamDeleting(String), + #[error("unsupported schema resolution protocol version {0}")] + UnsupportedProtocol(u32), + #[error("schema authority unavailable: {0}")] + Unavailable(String), +} + +impl AuthorityError { + fn code(&self) -> ErrorCode { + match self { + Self::BadRequest(_) => ErrorCode::BadRequest, + Self::StreamNotFound(_) => ErrorCode::StreamNotFound, + Self::StreamDeleting(_) => ErrorCode::StreamDeleting, + Self::UnsupportedProtocol(_) => ErrorCode::UnsupportedProtocol, + Self::Unavailable(_) => ErrorCode::Unavailable, + } + } +} + +impl ResponseError for AuthorityError { + fn status_code(&self) -> StatusCode { + match self { + Self::BadRequest(_) => StatusCode::BAD_REQUEST, + Self::StreamNotFound(_) => StatusCode::NOT_FOUND, + Self::StreamDeleting(_) => StatusCode::CONFLICT, + Self::UnsupportedProtocol(_) => StatusCode::UPGRADE_REQUIRED, + Self::Unavailable(_) => StatusCode::SERVICE_UNAVAILABLE, + } + } + + fn error_response(&self) -> HttpResponse { + HttpResponse::build(self.status_code()).json(ResolveErrorBody { + protocol_version: PROTOCOL_VERSION, + code: self.code(), + message: self.to_string(), + }) + } +} + +fn slot_key(tenant_id: &Option, stream: &str) -> (String, String) { + ( + tenant_id.as_deref().unwrap_or(DEFAULT_TENANT).to_owned(), + stream.to_owned(), + ) +} + +impl SchemaAuthority { + fn slot(&self, tenant_id: &Option, stream: &str) -> Slot { + self.slots + .lock() + .expect("schema authority lock poisoned") + .entry(slot_key(tenant_id, stream)) + .or_default() + .clone() + } + + /// Takes the per-stream lock and drops the cached registry, so another writer of the + /// stream-level `.schema` is serialised with resolutions and the registry is re-read + /// from storage afterwards. + pub async fn lock_and_invalidate( + &self, + tenant_id: &Option, + stream: &str, + ) -> OwnedMutexGuard> { + let mut guard = self.slot(tenant_id, stream).lock_owned().await; + *guard = None; + guard + } + + /// Drops cached state, e.g. after the stream was deleted. + pub fn forget_stream(&self, tenant_id: &Option, stream: &str) { + self.slots + .lock() + .expect("schema authority lock poisoned") + .remove(&slot_key(tenant_id, stream)); + } + + pub async fn resolve(&self, req: ResolveRequest) -> Result { + if req.protocol_version != PROTOCOL_VERSION { + return Err(AuthorityError::UnsupportedProtocol(req.protocol_version)); + } + if req.stream.is_empty() { + return Err(AuthorityError::BadRequest("stream name is empty".into())); + } + // Load before taking the slot lock: loading may rewrite `.schema` through + // `create_schema_from_metastore`, which takes the same lock. + if !PARSEABLE + .check_or_load_stream(&req.stream, &req.tenant_id) + .await + { + return Err(AuthorityError::StreamNotFound(req.stream)); + } + let stream = PARSEABLE + .get_stream(&req.stream, &req.tenant_id) + .map_err(|_| AuthorityError::StreamNotFound(req.stream.clone()))?; + if stream.is_deleting() { + return Err(AuthorityError::StreamDeleting(req.stream)); + } + + let reserved = reserved_fields(&stream); + let mut guard = self.slot(&req.tenant_id, &req.stream).lock_owned().await; + // Deletion marks the stream, then takes this lock before removing its files and + // holds it until they are gone. Checking again under the lock means a resolution + // that waited behind deletion never writes `.schema` for the deleted stream. + if stream.is_deleting() { + return Err(AuthorityError::StreamDeleting(req.stream)); + } + if guard.is_none() { + *guard = Some(load_registry(&req.stream, &req.tenant_id, reserved).await?); + } + let version_before = guard.as_ref().expect("registry loaded above").version; + let resolution = guard + .as_ref() + .expect("registry loaded above") + .resolve(&req.fields); + + let registry = guard.as_mut().expect("registry loaded above"); + if !resolution.registered.is_empty() { + let mut next = registry.clone(); + next.apply(&resolution.registered); + let schema = registry_schema(&next); + // Persist before replying: once an ingestor sees an answer it must never change. + PARSEABLE + .metastore + .put_schema(schema.clone(), &req.stream, &req.tenant_id) + .await + .map_err(|e| AuthorityError::Unavailable(e.to_string()))?; + *registry = next; + // Best effort: let this node's own queries see the new fields immediately. + let in_memory = Schema::new(schema.fields().clone()); + if let Err(err) = commit_schema(&req.stream, Arc::new(in_memory), &req.tenant_id) { + warn!( + stream = req.stream, + "in-memory schema update skipped: {err}" + ); + } + } + + for r in &resolution.resolutions { + match r.outcome { + Outcome::Renamed | Outcome::AlreadyRenamed => info!( + stream = req.stream, + node = req.node_id, + "schema conflict: field {} proposed as {} is registered as {:?}; writing it as {}", + r.original_name, + r.proposed_type, + r.registry_type, + r.resolved_name + ), + Outcome::Unresolvable => warn!( + stream = req.stream, + node = req.node_id, + "schema conflict on field {} ({}) cannot be resolved automatically", + r.original_name, + r.proposed_type + ), + Outcome::Accepted => {} + } + } + + let delta = if req.known_version < version_before { + let mut entries: Vec = registry + .fields + .iter() + .map(|(name, data_type)| RegistryEntry { + name: name.clone(), + data_type: data_type.clone(), + }) + .collect(); + entries.sort_by(|a, b| a.name.cmp(&b.name)); + entries + } else { + Vec::new() + }; + + Ok(ResolveResponse { + protocol_version: PROTOCOL_VERSION, + request_id: req.request_id, + version: registry.version, + resolutions: resolution.resolutions, + delta, + }) + } +} + +/// Columns that are never renamed: the server-set `p_*` fields and the partition columns. +fn reserved_fields(stream: &Stream) -> HashSet { + let mut reserved: HashSet = [ + DEFAULT_TIMESTAMP_KEY, + USER_AGENT_KEY, + SOURCE_IP_KEY, + FORMAT_KEY, + FORMAT_VERIFY_KEY, + ] + .into_iter() + .map(str::to_owned) + .collect(); + reserved.extend(stream.get_time_partition()); + reserved.extend( + stream + .get_custom_partition() + .iter() + .flat_map(|p| p.split(',').map(|s| s.trim().to_owned())), + ); + reserved +} + +fn registry_schema(registry: &StreamRegistry) -> Schema { + let mut fields: Vec = registry + .fields + .iter() + .map(|(name, data_type)| Field::new(name, data_type.clone(), true)) + .collect(); + fields.sort_by(|a, b| a.name().cmp(b.name())); + Schema::new_with_metadata( + fields, + HashMap::from([( + REGISTRY_VERSION_KEY.to_owned(), + registry.version.to_string(), + )]), + ) +} + +/// The saved registry, or `None` only when it does not exist yet. +/// +/// Any other read failure, or a file that does not decode, is an error: seeding a new +/// registry from the per-node schemas instead could change types already granted (the +/// first type in listing order wins), and saving it would make that permanent while +/// ingestors keep writing the old types. +fn authoritative_schema( + stream: &str, + read: Result, +) -> Result, AuthorityError> { + match read { + Ok(bytes) => serde_json::from_slice::(&bytes) + .map(Some) + .map_err(|e| { + AuthorityError::Unavailable(format!( + "schema registry of stream {stream} is unreadable: {e}" + )) + }), + Err(ObjectStorageError::NoSuchKey(_)) => Ok(None), + Err(e) => Err(AuthorityError::Unavailable(format!( + "could not read schema registry of stream {stream}: {e}" + ))), + } +} + +/// Seeds a registry from the authoritative `.schema` first, then from every per-node +/// schema file (first type wins). Persists immediately when seeding added fields, so a +/// restart re-seeds to the same answer regardless of listing order. +async fn load_registry( + stream: &str, + tenant_id: &Option, + reserved: HashSet, +) -> Result { + let mut registry = StreamRegistry { + reserved, + ..Default::default() + }; + let storage = PARSEABLE.storage.get_object_store(); + let authoritative = authoritative_schema( + stream, + storage + .get_object(&authoritative_schema_path(stream, tenant_id), tenant_id) + .await, + )?; + let mut persisted_fields = 0; + if let Some(schema) = &authoritative { + registry.version = schema + .metadata() + .get(REGISTRY_VERSION_KEY) + .and_then(|v| v.parse().ok()) + .unwrap_or(0); + registry.seed(schema.fields().iter().map(|f| f.as_ref())); + persisted_fields = registry.fields.len(); + } + + let others = PARSEABLE + .metastore + .get_all_schemas(stream, tenant_id) + .await + .map_err(|e| AuthorityError::Unavailable(e.to_string()))?; + let merged = merge_schemas_tolerant(stream, others); + for (name, kept, skipped) in registry.seed(merged.fields().iter().map(|f| f.as_ref())) { + warn!( + stream, + "legacy schema conflict on field {name}: keeping {kept}, ignoring {skipped}" + ); + } + + if registry.fields.len() != persisted_fields || authoritative.is_none() { + registry.version += 1; + PARSEABLE + .metastore + .put_schema(registry_schema(®istry), stream, tenant_id) + .await + .map_err(|e| AuthorityError::Unavailable(e.to_string()))?; + } + Ok(registry) +} + +/// Authenticates the caller and returns the tenant it acts for. +/// +/// With `P_CLUSTER_SECRET`, the route's `IntraClusterRequest` has already verified the +/// secret, which only cluster nodes hold. The caller is that node, acting for the tenant it +/// names in `intra-cluster-tenant`: an ingestor resolves on its own behalf, so no user of +/// that tenant is involved, and none need exist under the user id it sends. The secret +/// already lets a node act as any user of any tenant, so this grants nothing more. +/// +/// Without the secret the caller is a user, who must be allowed `ResolveSchema` and acts +/// for their own tenant. +fn caller_tenant(req: &HttpRequest) -> Result, actix_web::Error> { + if CLUSTER_SECRET.get().is_some() { + return Ok(req + .headers() + .get(INTRA_CLUSTER_TENANT_HEADER) + .and_then(|tenant| tenant.to_str().ok()) + .map(String::from)); + } + let key = extract_session_key_from_req(req)?; + match Users.authorize(key.clone(), Action::ResolveSchema, None, None) { + rbac::Response::Authorized => Ok(get_tenant_id_from_key(&key)), + _ => Err(ErrorUnauthorized("not allowed to resolve schemas")), + } +} + +/// The tenant comes from authentication (see [`caller_tenant`]), never from the body. The +/// body may only repeat it, so no caller can name another tenant. Returns `None` for the +/// default tenant. +fn authorized_tenant( + header: Option, + body: Option, +) -> Result, AuthorityError> { + let header = header.filter(|t| t != DEFAULT_TENANT); + let body = body.filter(|t| t != DEFAULT_TENANT); + if header != body { + return Err(AuthorityError::BadRequest( + "tenant in request body does not match the authenticated tenant".into(), + )); + } + Ok(body) +} + +/// `POST {base}/internal/schema/resolve` +pub async fn resolve_handler( + req: HttpRequest, + body: web::Json, +) -> Result { + let mut body = body.into_inner(); + body.tenant_id = authorized_tenant(caller_tenant(&req)?, body.tenant_id)?; + let response = SCHEMA_AUTHORITY.resolve(body).await?; + Ok(HttpResponse::Ok().json(response)) +} + +#[cfg(test)] +mod tests { + use super::*; + use arrow_schema::DataType; + + #[test] + fn registry_is_seeded_only_when_the_saved_one_is_missing() { + let mut r = StreamRegistry::default(); + r.fields.insert("a".into(), DataType::Int64); + let saved = Bytes::from(serde_json::to_vec(®istry_schema(&r)).unwrap()); + let schema = authoritative_schema("s", Ok(saved)).unwrap().unwrap(); + assert_eq!(schema.field(0).name(), "a"); + + assert!( + authoritative_schema("s", Err(ObjectStorageError::NoSuchKey("k".into()))) + .unwrap() + .is_none() + ); + // A failed read or a corrupt file must not look like a missing registry: the + // caller would re-seed and could change granted types. + assert!(matches!( + authoritative_schema("s", Err(ObjectStorageError::Custom("timeout".into()))), + Err(AuthorityError::Unavailable(_)) + )); + assert!(matches!( + authoritative_schema("s", Ok(Bytes::from_static(b"{not json"))), + Err(AuthorityError::Unavailable(_)) + )); + } + + #[test] + fn registry_schema_carries_version_and_sorted_fields() { + let mut r = StreamRegistry::default(); + r.fields.insert("b".into(), DataType::Utf8); + r.fields.insert("a".into(), DataType::Int64); + r.version = 7; + let s = registry_schema(&r); + assert_eq!(s.field(0).name(), "a"); + assert_eq!(s.metadata()[REGISTRY_VERSION_KEY], "7"); + let round: Schema = serde_json::from_slice(&serde_json::to_vec(&s).unwrap()).unwrap(); + assert_eq!(round.metadata()[REGISTRY_VERSION_KEY], "7"); + } + + #[test] + fn tenant_must_match_the_authenticated_one() { + let t = |s: &str| Some(s.to_string()); + assert_eq!(authorized_tenant(None, None).unwrap(), None); + assert_eq!(authorized_tenant(t(DEFAULT_TENANT), None).unwrap(), None); + assert_eq!(authorized_tenant(t("acme"), t("acme")).unwrap(), t("acme")); + // A default-tenant caller cannot reach another tenant, nor a tenant another one. + assert!(authorized_tenant(None, t("acme")).is_err()); + assert!(authorized_tenant(t(DEFAULT_TENANT), t("acme")).is_err()); + assert!(authorized_tenant(t("acme"), t("other")).is_err()); + assert!(authorized_tenant(t("acme"), None).is_err()); + } + + #[test] + fn error_status_mapping() { + assert_eq!( + AuthorityError::StreamNotFound("s".into()).status_code(), + StatusCode::NOT_FOUND + ); + assert_eq!( + AuthorityError::StreamDeleting("s".into()).status_code(), + StatusCode::CONFLICT + ); + assert_eq!( + AuthorityError::UnsupportedProtocol(9).status_code(), + StatusCode::UPGRADE_REQUIRED + ); + } +} diff --git a/src/schema_registry/client.rs b/src/schema_registry/client.rs new file mode 100644 index 000000000..b0717fa5c --- /dev/null +++ b/src/schema_registry/client.rs @@ -0,0 +1,266 @@ +/* + * Parseable Server (C) 2022 - 2025 Parseable, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + */ + +//! Ingestor-side client of the schema authority. + +use std::time::Duration; + +use once_cell::sync::Lazy; +use reqwest::{StatusCode, header}; +use tokio::{runtime::Runtime, sync::Semaphore}; +use tracing::warn; + +use crate::{ + INTRA_CLUSTER_CLIENT, + handlers::http::{ + base_path_without_preceding_slash, cluster::utils::check_liveness, + middleware::CLUSTER_SECRET, modal::NodeMetadata, + }, + parseable::{DEFAULT_TENANT, PARSEABLE}, +}; + +use super::protocol::{ + ErrorCode, PROTOCOL_HEADER, PROTOCOL_VERSION, RESOLVE_PATH, ResolveErrorBody, ResolveRequest, + ResolveResponse, +}; +use super::{authority_node_type, nodes_of_type}; + +const REQUEST_TIMEOUT: Duration = Duration::from_secs(10); +const MAX_ATTEMPTS: u32 = 3; +const BACKOFF_BASE: Duration = Duration::from_millis(200); + +/// An authority without the resolve endpoint (or protocol) blocks streams with new fields +/// until it is upgraded, so upgrade the querier/Prism before the ingestors. +const UPGRADE_AUTHORITY: &str = + "schema authority does not support this schema resolution protocol; upgrade the querier/Prism"; + +#[derive(Debug, thiserror::Error)] +pub enum ResolveFailure { + /// Network error, timeout, overload or an authority that is not ready. Retry later. + #[error("schema authority unavailable: {0}")] + Transient(String), + #[error("stream not found on the schema authority")] + StreamNotFound, + #[error("stream is being deleted")] + StreamDeleting, + /// Request rejected; retrying the same request will not help. + #[error("schema resolution rejected: {0}")] + Rejected(String), +} + +#[derive(Clone, Debug)] +struct Target { + /// Base URL ending with `/`. + domain: String, + authorization: String, +} + +static RESOLVER_RUNTIME: Lazy = Lazy::new(|| { + tokio::runtime::Builder::new_multi_thread() + .worker_threads(2) + .thread_name("schema-resolver") + .enable_all() + .build() + .expect("schema resolver runtime") +}); + +static PERMITS: Lazy = + Lazy::new(|| Semaphore::new(PARSEABLE.options.schema_resolve_concurrency as usize)); + +/// The authority (Prism if the cluster has one, else the querier) publishes its address in +/// its node metadata. A cluster runs one such node; the liveness check only skips metadata +/// left behind by a node that is gone. Looked up on every request: requests are rare (only +/// for fields a node has not resolved yet), and a fresh lookup follows a restarted authority. +async fn find_authority(tenant_id: &Option) -> Result { + let node_type = authority_node_type(tenant_id) + .await + .map_err(|e| ResolveFailure::Transient(e.to_string()))?; + let nodes: Vec = nodes_of_type(node_type.clone(), tenant_id) + .await + .map_err(|e| ResolveFailure::Transient(e.to_string()))?; + let mut chosen = None; + for node in nodes { + if check_liveness(&node.domain_name).await { + chosen = Some(node); + break; + } + } + let q = chosen.ok_or_else(|| { + ResolveFailure::Transient(format!( + "no live {node_type:?} node found for schema resolution" + )) + })?; + let domain = if q.domain_name.ends_with('/') { + q.domain_name + } else { + format!("{}/", q.domain_name) + }; + Ok(Target { + domain, + authorization: q.token, + }) +} + +fn headers(target: &Target, tenant_id: &Option) -> header::HeaderMap { + let mut map = header::HeaderMap::new(); + map.insert( + header::HeaderName::from_static(PROTOCOL_HEADER), + header::HeaderValue::from(PROTOCOL_VERSION), + ); + if let Some((_, hash)) = CLUSTER_SECRET.get() + && let Ok(hash) = header::HeaderValue::from_str(hash) + { + map.insert( + header::HeaderName::from_static( + crate::handlers::http::middleware::CLUSTER_SECRET_HEADER, + ), + hash, + ); + if let Ok(tenant) = + header::HeaderValue::from_str(tenant_id.as_deref().unwrap_or(DEFAULT_TENANT)) + { + map.insert( + header::HeaderName::from_static( + crate::handlers::http::middleware::INTRA_CLUSTER_TENANT_HEADER, + ), + tenant, + ); + } + if let Ok(user) = header::HeaderValue::from_str(&PARSEABLE.options.username) { + map.insert( + header::HeaderName::from_static("intra-cluster-userid"), + user, + ); + } + } else if let Ok(auth) = header::HeaderValue::from_str(&target.authorization) { + map.insert(header::AUTHORIZATION, auth); + } + map +} + +enum Attempt { + Done(Result), + Retry(String), +} + +async fn attempt(req: &ResolveRequest) -> Attempt { + let target = match find_authority(&req.tenant_id).await { + Ok(t) => t, + Err(e) => return Attempt::Retry(e.to_string()), + }; + let url = format!( + "{}{}/{}", + target.domain, + base_path_without_preceding_slash(), + RESOLVE_PATH + ); + let response = INTRA_CLUSTER_CLIENT + .post(url) + .headers(headers(&target, &req.tenant_id)) + .timeout(REQUEST_TIMEOUT) + .json(req) + .send() + .await; + let response = match response { + Ok(r) => r, + Err(e) => return Attempt::Retry(e.to_string()), + }; + let status = response.status(); + let bytes = match response.bytes().await { + Ok(b) => b, + Err(e) => return Attempt::Retry(e.to_string()), + }; + if status.is_success() { + return Attempt::Done( + serde_json::from_slice::(&bytes) + .map_err(|e| ResolveFailure::Rejected(format!("invalid response: {e}"))), + ); + } + match serde_json::from_slice::(&bytes) { + Ok(body) => match body.code { + ErrorCode::StreamNotFound => Attempt::Done(Err(ResolveFailure::StreamNotFound)), + ErrorCode::StreamDeleting => Attempt::Done(Err(ResolveFailure::StreamDeleting)), + ErrorCode::UnsupportedProtocol => { + Attempt::Done(Err(ResolveFailure::Rejected(UPGRADE_AUTHORITY.into()))) + } + ErrorCode::BadRequest => Attempt::Done(Err(ResolveFailure::Rejected(body.message))), + ErrorCode::Unavailable => Attempt::Retry(body.message), + }, + // Not one of our error bodies: an authority without the route answers a bare 404. + Err(_) => match status { + StatusCode::NOT_FOUND | StatusCode::METHOD_NOT_ALLOWED => { + Attempt::Done(Err(ResolveFailure::Rejected(UPGRADE_AUTHORITY.into()))) + } + StatusCode::UNAUTHORIZED | StatusCode::FORBIDDEN => Attempt::Done(Err( + ResolveFailure::Rejected(format!("authority refused credentials ({status})")), + )), + _ => Attempt::Retry(format!("authority returned {status}")), + }, + } +} + +/// Resolves with a few short in-process retries. Cross-cycle retries are the caller's job. +pub async fn resolve(req: ResolveRequest) -> Result { + // Only a cluster-secret request carries a tenant other than the caller's own (the + // authority takes it from `intra-cluster-tenant`); plain credentials resolve the + // default tenant only. + if CLUSTER_SECRET.get().is_none() + && req + .tenant_id + .as_deref() + .is_some_and(|t| t != crate::parseable::DEFAULT_TENANT) + { + return Err(ResolveFailure::Rejected( + "schema resolution for a tenant stream requires P_CLUSTER_SECRET".into(), + )); + } + let _permit = PERMITS + .acquire() + .await + .map_err(|e| ResolveFailure::Transient(e.to_string()))?; + let mut last = String::default(); + for n in 0..MAX_ATTEMPTS { + match attempt(&req).await { + Attempt::Done(result) => return result, + Attempt::Retry(reason) => { + warn!( + stream = req.stream, + "schema resolution attempt {} failed: {reason}", + n + 1 + ); + last = reason; + if n + 1 < MAX_ATTEMPTS { + let jitter = rand::random::() % 100; + tokio::time::sleep(BACKOFF_BASE * 3u32.pow(n) + Duration::from_millis(jitter)) + .await; + } + } + } + } + Err(ResolveFailure::Transient(last)) +} + +/// Blocking wrapper for the conversion threads (which run on `spawn_blocking`). +pub fn resolve_blocking(req: ResolveRequest) -> Result { + let (tx, rx) = tokio::sync::oneshot::channel(); + RESOLVER_RUNTIME.spawn(async move { + let _ = tx.send(resolve(req).await); + }); + rx.blocking_recv() + .map_err(|_| ResolveFailure::Transient("resolver task dropped".into()))? +} diff --git a/src/schema_registry/ingestor.rs b/src/schema_registry/ingestor.rs new file mode 100644 index 000000000..4779b13f6 --- /dev/null +++ b/src/schema_registry/ingestor.rs @@ -0,0 +1,679 @@ +/* + * Parseable Server (C) 2022 - 2025 Parseable, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + */ + +//! Per-stream resolution state on an ingestor. +//! +//! What is "unresolved" is derived from the schemas of the arrow files about to be +//! converted, so the state survives restarts without extra bookkeeping. The only +//! persisted piece is the [`ResolvedView`] (authority answers), kept in the stream's +//! staging directory. + +use std::{ + collections::{BTreeSet, HashMap, HashSet}, + fs::File, + hash::{Hash, Hasher}, + io::BufReader, + path::{Path, PathBuf}, + sync::{ + Arc, Mutex, RwLock, + atomic::{AtomicBool, AtomicU32, Ordering}, + }, +}; + +use arrow_ipc::reader::StreamReader; +use arrow_schema::{DataType, Field, Schema}; +use tracing::{error, warn}; +use ulid::Ulid; + +use super::{ + client::ResolveFailure, + protocol::{PROTOCOL_VERSION, ResolveRequest, ResolveResponse}, + view::{FieldState, ResolvedView}, +}; + +pub type Resolver<'a> = dyn Fn(ResolveRequest) -> Result + 'a; + +/// Everything `ensure_resolved` needs to know about the stream and the node. +pub struct ResolveContext<'a> { + pub stream: &'a str, + pub tenant_id: &'a Option, + pub node_id: String, + pub data_path: &'a Path, + pub max_deferred_cycles: u32, + pub resolver: &'a Resolver<'a>, +} + +pub struct ResolutionOutcome { + /// Files whose every `(name, type)` is resolved; safe to convert. + pub ready: Vec, + /// Files that must stay unconverted for now. + pub deferred: Vec, +} + +#[derive(Default)] +pub struct ResolutionState { + blocked: AtomicBool, + deferred_cycles: AtomicU32, + /// Serialises "classify -> RPC -> apply" for one stream. + resolve_lock: Mutex<()>, + view: RwLock>, + /// Request id reused for retries of the same unresolved field set. + pending_request: Mutex>, + last_error: Mutex>, +} + +impl ResolutionState { + pub fn load(data_path: &Path) -> Self { + Self { + view: RwLock::new(Arc::new(ResolvedView::load(data_path))), + ..Default::default() + } + } + + pub fn view(&self) -> Arc { + self.view.read().expect("view lock poisoned").clone() + } + + pub fn is_blocked(&self) -> bool { + self.blocked.load(Ordering::Acquire) + } + + pub fn last_error(&self) -> Option { + self.last_error.lock().expect("lock poisoned").clone() + } + + pub fn deferred_cycles(&self) -> u32 { + self.deferred_cycles.load(Ordering::Acquire) + } + + fn reset(&self) { + self.deferred_cycles.store(0, Ordering::Release); + self.blocked.store(false, Ordering::Release); + *self.last_error.lock().expect("lock poisoned") = None; + } + + fn record_failure(&self, ctx: &ResolveContext<'_>, reason: String) { + let cycles = self.deferred_cycles.fetch_add(1, Ordering::AcqRel) + 1; + *self.last_error.lock().expect("lock poisoned") = Some(reason.clone()); + if ctx.max_deferred_cycles > 0 && cycles >= ctx.max_deferred_cycles { + if !self.blocked.swap(true, Ordering::AcqRel) { + error!( + stream = ctx.stream, + "schema resolution failed for {cycles} cycles, rejecting ingestion until resolved: {reason}" + ); + } + } else { + warn!( + stream = ctx.stream, + "schema resolution deferred (cycle {cycles}): {reason}" + ); + } + } + + /// Splits `files` into convertible and deferred, asking the authority about any + /// field not yet resolved. `on_apply` runs after a new view was persisted and made + /// visible, so the stream can align its in-memory and staged schemas with it. + pub fn ensure_resolved( + &self, + ctx: &ResolveContext<'_>, + files: Vec, + on_apply: impl FnOnce(&ResolvedView), + ) -> ResolutionOutcome { + let _guard = self.resolve_lock.lock().expect("resolve lock poisoned"); + + let schemas: Vec<(PathBuf, Option>)> = files + .into_iter() + .map(|path| { + let schema = read_file_schema(&path); + (path, schema) + }) + .collect(); + + let view = self.view(); + let unresolved = unresolved_fields(&view, &schemas); + if unresolved.is_empty() { + return self.finish(ctx, &view, schemas); + } + + let fields: Vec = unresolved + .iter() + .map(|(name, data_type)| Field::new(name, data_type.clone(), true)) + .collect(); + let request_id = self.request_id(&fields); + let request = ResolveRequest { + protocol_version: PROTOCOL_VERSION, + request_id, + node_id: ctx.node_id.clone(), + tenant_id: ctx.tenant_id.clone(), + stream: ctx.stream.to_owned(), + known_version: view.registry_version, + fields, + }; + + match (ctx.resolver)(request) { + Ok(response) => { + let next = + view.with_resolutions(response.version, &response.resolutions, &response.delta); + if let Err(err) = next.persist(ctx.data_path) { + // Without a durable view a restart could convert with stale renames; + // the answer is idempotent, so just ask again next cycle. + self.record_failure(ctx, format!("could not persist resolved fields: {err}")); + return self.split(&view, schemas); + } + // Publish the view before `on_apply` moves the in-memory schema to the + // resolved types: `commit_schema` reads the view under the schema lock, so + // whoever sees a resolved type also sees the rename that allows the old one. + let next = Arc::new(next); + self.view + .write() + .expect("view lock poisoned") + .clone_from(&next); + on_apply(&next); + *self.pending_request.lock().expect("lock poisoned") = None; + self.finish(ctx, &next, schemas) + } + Err(ResolveFailure::StreamDeleting) => { + // The stream is going away; nothing to resolve and nothing to block. + self.split(&view, schemas) + } + Err(err) => { + self.record_failure(ctx, err.to_string()); + self.split(&view, schemas) + } + } + } + + fn finish( + &self, + ctx: &ResolveContext<'_>, + view: &ResolvedView, + schemas: Vec<(PathBuf, Option>)>, + ) -> ResolutionOutcome { + let mut blocking: BTreeSet = schemas + .iter() + .filter_map(|(_, s)| s.as_ref()) + .flat_map(|s| s.fields().iter()) + .filter(|f| view.field_state(f.name(), f.data_type()) == FieldState::Unresolvable) + .map(|f| format!("{} ({})", f.name(), f.data_type())) + .collect(); + blocking.extend( + schemas + .iter() + .filter_map(|(_, s)| s.as_ref()) + .filter_map(|s| view.rename_schema(s).err()) + .map(|e| e.to_string()), + ); + let blocking: Vec = blocking.into_iter().collect(); + let outcome = self.split(view, schemas); + if blocking.is_empty() { + self.reset(); + } else { + self.record_failure( + ctx, + format!( + "fields cannot be reconciled with the cluster schema automatically: {}", + blocking.join(", ") + ), + ); + } + outcome + } + + fn split( + &self, + view: &ResolvedView, + schemas: Vec<(PathBuf, Option>)>, + ) -> ResolutionOutcome { + let mut ready = Vec::new(); + let mut deferred = Vec::new(); + for (path, schema) in schemas { + let convertible = + match &schema { + // Unreadable files are left to the conversion path, which already + // classifies and removes invalid arrow files. + None => true, + Some(schema) => { + schema.fields().iter().all(|f| { + view.field_state(f.name(), f.data_type()) == FieldState::Resolved + }) && view.rename_schema(schema).is_ok() + } + }; + if convertible { + ready.push(path); + } else { + deferred.push(path); + } + } + ResolutionOutcome { ready, deferred } + } + + fn request_id(&self, fields: &[Field]) -> Ulid { + let mut hasher = std::collections::hash_map::DefaultHasher::default(); + for f in fields { + f.name().hash(&mut hasher); + f.data_type().hash(&mut hasher); + } + let key = hasher.finish(); + let mut pending = self.pending_request.lock().expect("lock poisoned"); + match *pending { + Some((k, id)) if k == key => id, + _ => { + let id = Ulid::new(); + *pending = Some((key, id)); + id + } + } + } +} + +/// Reads only the schema message of an arrow IPC stream file. +fn read_file_schema(path: &Path) -> Option> { + let file = File::open(path).ok()?; + StreamReader::try_new(BufReader::new(file), None) + .ok() + .map(|r| r.schema()) +} + +/// Distinct unresolved `(name, type)` pairs, sorted for a deterministic request. +fn unresolved_fields( + view: &ResolvedView, + schemas: &[(PathBuf, Option>)], +) -> Vec<(String, DataType)> { + let mut seen = HashSet::new(); + let mut out = Vec::new(); + for schema in schemas.iter().filter_map(|(_, s)| s.as_ref()) { + // A cached rename can land on a column this file already carries (`a` -> `a_utf8` + // next to a literal `a_utf8`). Send every column of such a file, so the authority + // sees the names in use and picks a free target. + let collides = view.rename_schema(schema).is_err(); + for f in schema.fields() { + if f.data_type().is_null() { + continue; + } + if (collides || view.field_state(f.name(), f.data_type()) == FieldState::Unresolved) + && seen.insert((f.name().clone(), f.data_type().clone())) + { + out.push((f.name().clone(), f.data_type().clone())); + } + } + } + out.sort_by(|a, b| { + a.0.cmp(&b.0) + .then_with(|| format!("{:?}", a.1).cmp(&format!("{:?}", b.1))) + }); + out +} + +/// Types the in-memory schema must switch to after a resolution: for every field whose +/// in-memory type differs from the authority's, the authority wins. +pub fn schema_overrides( + view: &ResolvedView, + current: &HashMap>, +) -> Vec> { + let mut out = Vec::new(); + for (name, data_type) in &view.fields { + match current.get(name) { + Some(field) if field.data_type() != data_type => out.push(Arc::new( + field.as_ref().clone().with_data_type(data_type.clone()), + )), + None if view.renames.values().flatten().any(|to| to == name) => { + out.push(Arc::new(Field::new(name, data_type.clone(), true))) + } + _ => {} + } + } + out +} + +#[cfg(test)] +mod tests { + use std::{cell::RefCell, sync::Arc}; + + use arrow_array::{ArrayRef, Int64Array, RecordBatch, StringArray}; + use arrow_ipc::writer::StreamWriter; + + use super::*; + use crate::schema_registry::{ + protocol::{FieldResolution, Outcome}, + registry::StreamRegistry, + }; + + fn write_arrow(dir: &Path, name: &str, field: Field, array: ArrayRef) -> PathBuf { + let path = dir.join(name); + let schema = Arc::new(Schema::new(vec![field])); + let batch = RecordBatch::try_new(schema.clone(), vec![array]).unwrap(); + let mut w = StreamWriter::try_new(File::create(&path).unwrap(), &schema).unwrap(); + w.write(&batch).unwrap(); + w.finish().unwrap(); + path + } + + fn ctx<'a>( + dir: &'a Path, + resolver: &'a Resolver<'a>, + tenant: &'a Option, + ) -> ResolveContext<'a> { + ResolveContext { + stream: "s", + tenant_id: tenant, + node_id: "n".into(), + data_path: dir, + max_deferred_cycles: 3, + resolver, + } + } + + /// A resolver backed by an in-process registry, like the querier. + fn registry_resolver( + registry: &RefCell, + ) -> impl Fn(ResolveRequest) -> Result + '_ { + move |req| { + let mut reg = registry.borrow_mut(); + let res = reg.resolve(&req.fields); + reg.apply(&res.registered); + Ok(ResolveResponse { + protocol_version: PROTOCOL_VERSION, + request_id: req.request_id, + version: reg.version, + resolutions: res.resolutions, + delta: Vec::new(), + }) + } + } + + #[test] + fn two_ingestors_converge_on_one_type() { + let registry = RefCell::new(StreamRegistry::default()); + let resolver = registry_resolver(®istry); + let tenant = None; + + let d1 = tempfile::tempdir().unwrap(); + let d2 = tempfile::tempdir().unwrap(); + let f1 = write_arrow( + d1.path(), + "a.arrows", + Field::new("a", DataType::Int64, true), + Arc::new(Int64Array::from(vec![1])), + ); + let f2 = write_arrow( + d2.path(), + "a.arrows", + Field::new("a", DataType::Utf8, true), + Arc::new(StringArray::from(vec!["x"])), + ); + + let s1 = ResolutionState::default(); + let s2 = ResolutionState::default(); + let o1 = s1.ensure_resolved( + &ctx(d1.path(), &resolver, &tenant), + vec![f1.clone()], + |_| {}, + ); + let o2 = s2.ensure_resolved( + &ctx(d2.path(), &resolver, &tenant), + vec![f2.clone()], + |_| {}, + ); + assert_eq!(o1.ready, vec![f1]); + assert_eq!(o2.ready, vec![f2.clone()]); + + let v2 = s2.view(); + let renamed = v2 + .rename_schema(&read_file_schema(&f2).unwrap()) + .unwrap() + .unwrap(); + assert_eq!(renamed.field(0).name(), "a_utf8"); + assert_eq!(v2.fields["a"], DataType::Int64); + // Persisted: a fresh state on the same dir needs no RPC. + assert_eq!( + ResolutionState::load(d2.path()).view().as_ref(), + v2.as_ref() + ); + } + + /// `on_apply` moves the in-memory schema to the resolved types, so the view carrying + /// the matching renames must already be visible to other threads when it runs. + #[test] + fn resolved_view_is_visible_before_on_apply() { + let registry = RefCell::new(StreamRegistry::default()); + registry + .borrow_mut() + .apply(&[("a".to_string(), DataType::Int64)]); + let resolver = registry_resolver(®istry); + let tenant = None; + let dir = tempfile::tempdir().unwrap(); + let file = write_arrow( + dir.path(), + "a.arrows", + Field::new("a", DataType::Utf8, true), + Arc::new(StringArray::from(vec!["x"])), + ); + + let state = ResolutionState::default(); + let applied = RefCell::new(false); + state.ensure_resolved(&ctx(dir.path(), &resolver, &tenant), vec![file], |next| { + let visible = state.view(); + assert_eq!(visible.registry_version, next.registry_version); + assert!( + visible + .renames + .contains_key(&("a".to_string(), DataType::Utf8)) + ); + *applied.borrow_mut() = true; + }); + assert!(*applied.borrow()); + } + + #[test] + fn cached_rename_colliding_with_a_column_is_re_resolved() { + let registry = RefCell::new(StreamRegistry::default()); + let resolver = registry_resolver(®istry); + let tenant = None; + let dir = tempfile::tempdir().unwrap(); + let state = ResolutionState::default(); + + // Another node owns `a: Int64`; this node's first file maps `a: Utf8` to `a_utf8`. + registry + .borrow_mut() + .apply(&[("a".to_string(), DataType::Int64)]); + let first = write_arrow( + dir.path(), + "1.arrows", + Field::new("a", DataType::Utf8, true), + Arc::new(StringArray::from(vec!["x"])), + ); + let out = state.ensure_resolved(&ctx(dir.path(), &resolver, &tenant), vec![first], |_| {}); + assert_eq!(out.deferred.len(), 0); + assert_eq!( + state.view().renames[&("a".to_string(), DataType::Utf8)], + vec!["a_utf8".to_string()] + ); + + // A later file carries both `a: Utf8` and a literal `a_utf8`: both count as resolved, + // but the cached rename would duplicate `a_utf8`. + let path = dir.path().join("2.arrows"); + let schema = Arc::new(Schema::new(vec![ + Field::new("a", DataType::Utf8, true), + Field::new("a_utf8", DataType::Utf8, true), + ])); + let batch = RecordBatch::try_new( + schema.clone(), + vec![ + Arc::new(StringArray::from(vec!["x"])) as ArrayRef, + Arc::new(StringArray::from(vec!["y"])) as ArrayRef, + ], + ) + .unwrap(); + let mut w = StreamWriter::try_new(File::create(&path).unwrap(), &schema).unwrap(); + w.write(&batch).unwrap(); + w.finish().unwrap(); + + let out = state.ensure_resolved( + &ctx(dir.path(), &resolver, &tenant), + vec![path.clone()], + |_| {}, + ); + assert_eq!(out.ready, vec![path]); + assert!(!state.is_blocked()); + let renamed = state.view().rename_schema(&schema).unwrap().unwrap(); + let names: Vec<_> = renamed.fields().iter().map(|f| f.name().as_str()).collect(); + assert_eq!(names, ["a_utf8_utf8", "a_utf8"]); + + // The new target is appended: files carrying only `a: Utf8` keep `a_utf8`, so a file + // accepted before this answer still converts as it was accepted. + let only_a = Schema::new(vec![Field::new("a", DataType::Utf8, true)]); + let renamed = state.view().rename_schema(&only_a).unwrap().unwrap(); + assert_eq!(renamed.field(0).name(), "a_utf8"); + } + + #[test] + fn failures_defer_then_block_then_recover() { + let dir = tempfile::tempdir().unwrap(); + let tenant = None; + let file = write_arrow( + dir.path(), + "a.arrows", + Field::new("a", DataType::Int64, true), + Arc::new(Int64Array::from(vec![1])), + ); + let failing = |_req: ResolveRequest| Err(ResolveFailure::Transient("down".into())); + let state = ResolutionState::default(); + for cycle in 1..=3 { + let out = state.ensure_resolved( + &ctx(dir.path(), &failing, &tenant), + vec![file.clone()], + |_| {}, + ); + assert!(out.ready.is_empty()); + assert_eq!(out.deferred, vec![file.clone()]); + assert_eq!(state.is_blocked(), cycle >= 3); + } + let registry = RefCell::new(StreamRegistry::default()); + let ok = registry_resolver(®istry); + let out = state.ensure_resolved(&ctx(dir.path(), &ok, &tenant), vec![file.clone()], |_| {}); + assert_eq!(out.ready, vec![file]); + assert!(!state.is_blocked()); + assert_eq!(state.deferred_cycles(), 0); + } + + #[test] + fn retries_reuse_request_id() { + let dir = tempfile::tempdir().unwrap(); + let tenant = None; + let file = write_arrow( + dir.path(), + "a.arrows", + Field::new("a", DataType::Int64, true), + Arc::new(Int64Array::from(vec![1])), + ); + let ids = RefCell::new(Vec::new()); + let failing = |req: ResolveRequest| { + ids.borrow_mut().push(req.request_id); + Err(ResolveFailure::Transient("down".into())) + }; + let state = ResolutionState::default(); + state.ensure_resolved( + &ctx(dir.path(), &failing, &tenant), + vec![file.clone()], + |_| {}, + ); + state.ensure_resolved(&ctx(dir.path(), &failing, &tenant), vec![file], |_| {}); + let ids = ids.borrow(); + assert_eq!(ids.len(), 2); + assert_eq!(ids[0], ids[1]); + } + + #[test] + fn unresolvable_field_blocks_its_files() { + let dir = tempfile::tempdir().unwrap(); + let tenant = None; + let file = write_arrow( + dir.path(), + "a.arrows", + Field::new("ts", DataType::Utf8, true), + Arc::new(StringArray::from(vec!["x"])), + ); + let resolver = |req: ResolveRequest| { + Ok(ResolveResponse { + protocol_version: PROTOCOL_VERSION, + request_id: req.request_id, + version: 1, + resolutions: vec![FieldResolution { + original_name: "ts".into(), + proposed_type: DataType::Utf8, + resolved_name: "ts".into(), + data_type: DataType::Utf8, + registry_type: None, + outcome: Outcome::Unresolvable, + }], + delta: Vec::new(), + }) + }; + let state = ResolutionState::default(); + let out = state.ensure_resolved( + &ctx(dir.path(), &resolver, &tenant), + vec![file.clone()], + |_| {}, + ); + assert_eq!(out.deferred, vec![file]); + assert_eq!(state.deferred_cycles(), 1); + } + + #[test] + fn authority_without_resolution_defers() { + let dir = tempfile::tempdir().unwrap(); + let tenant = None; + let file = write_arrow( + dir.path(), + "a.arrows", + Field::new("a", DataType::Int64, true), + Arc::new(Int64Array::from(vec![1])), + ); + let old = |_req: ResolveRequest| Err(ResolveFailure::Rejected("upgrade".into())); + let state = ResolutionState::default(); + let out = + state.ensure_resolved(&ctx(dir.path(), &old, &tenant), vec![file.clone()], |_| {}); + assert_eq!(out.deferred, vec![file]); + assert_eq!(state.deferred_cycles(), 1); + } + + #[test] + fn overrides_flip_loser_type_and_add_rename_target() { + let view = ResolvedView::default().with_resolutions( + 1, + &[FieldResolution { + original_name: "a".into(), + proposed_type: DataType::Utf8, + resolved_name: "a_utf8".into(), + data_type: DataType::Utf8, + registry_type: Some(DataType::Int64), + outcome: Outcome::Renamed, + }], + &[], + ); + let current: HashMap> = HashMap::from([( + "a".to_string(), + Arc::new(Field::new("a", DataType::Utf8, true)), + )]); + let mut o = schema_overrides(&view, ¤t); + o.sort_by(|a, b| a.name().cmp(b.name())); + assert_eq!(o.len(), 2); + assert_eq!(o[0].data_type(), &DataType::Int64); + assert_eq!(o[1].name(), "a_utf8"); + } +} diff --git a/src/schema_registry/mod.rs b/src/schema_registry/mod.rs new file mode 100644 index 000000000..8abb12972 --- /dev/null +++ b/src/schema_registry/mod.rs @@ -0,0 +1,151 @@ +/* + * Parseable Server (C) 2022 - 2025 Parseable, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + */ + +//! Cluster-wide schema resolution for distributed ingestion. +//! +//! Ingestors infer types independently. Before converting arrow files to parquet they ask +//! a single authority (the querier, or Prism) which type each new field has. The authority +//! keeps a write-once name registry per stream; a later conflicting type is written under +//! `_`, the same rule the ingest-time conflict path uses. +//! See `docs/schema-resolution/` for the design. + +pub mod authority; +pub mod client; +pub mod ingestor; +pub mod protocol; +pub mod registry; +pub mod view; + +use std::{collections::HashMap, sync::Arc}; + +use arrow_schema::{Field, Schema}; +use relative_path::RelativePathBuf; +use tracing::warn; + +use crate::storage::{SCHEMA_FILE_NAME, STREAM_ROOT_DIRECTORY}; + +/// Node metadata of every node of `node_type`, tenant-scoped first, then cluster-wide. +pub async fn nodes_of_type( + node_type: crate::handlers::http::modal::NodeType, + tenant_id: &Option, +) -> anyhow::Result> { + use crate::handlers::http::cluster::get_node_info; + let mut nodes = get_node_info(node_type.clone(), tenant_id).await?; + if nodes.is_empty() && tenant_id.is_some() { + nodes = get_node_info(node_type, &None).await?; + } + Ok(nodes) +} + +/// Which node type is the schema authority: Prism when the cluster has one registered +/// (enterprise), otherwise the querier. Decided from registered metadata, not liveness, so +/// a Prism that is briefly down never causes ingestors to fall back to a second authority. +pub async fn authority_node_type( + tenant_id: &Option, +) -> anyhow::Result { + use crate::handlers::http::modal::NodeType; + if nodes_of_type(NodeType::Prism, tenant_id).await?.is_empty() { + Ok(NodeType::Querier) + } else { + Ok(NodeType::Prism) + } +} + +/// Path of the stream-level (authority owned) schema, independent of this node's mode. +pub fn authoritative_schema_path(stream: &str, tenant_id: &Option) -> RelativePathBuf { + let tenant = tenant_id.as_deref().unwrap_or(""); + RelativePathBuf::from_iter([tenant, stream, STREAM_ROOT_DIRECTORY, SCHEMA_FILE_NAME]) +} + +/// Unions schemas without ever failing: the first type seen for a field name wins and +/// later conflicting types are skipped with a warning. Metadata of the first schema is kept. +pub fn merge_schemas_tolerant(stream: &str, schemas: impl IntoIterator) -> Schema { + let mut fields: Vec = Vec::new(); + let mut index: HashMap = HashMap::new(); + let mut metadata = None; + + for schema in schemas { + if metadata.is_none() { + metadata = Some(schema.metadata().clone()); + } + for field in schema.fields() { + match index.get(field.name()) { + None => { + index.insert(field.name().clone(), fields.len()); + fields.push(field.as_ref().clone()); + } + Some(&i) => { + let mut merged = fields[i].clone(); + if merged.try_merge(field).is_ok() { + fields[i] = merged; + } else { + warn!( + stream, + "schema conflict on field {}: keeping {}, skipping {}", + field.name(), + fields[i].data_type(), + field.data_type() + ); + } + } + } + } + } + + Schema::new_with_metadata( + fields.into_iter().map(Arc::new).collect::>(), + metadata.unwrap_or_default(), + ) +} + +#[cfg(test)] +mod tests { + use super::*; + use arrow_schema::DataType; + + #[test] + fn tolerant_merge_keeps_first_type() { + let a = Schema::new(vec![ + Field::new("a", DataType::Int64, true), + Field::new("b", DataType::Utf8, true), + ]); + let b = Schema::new(vec![ + Field::new("a", DataType::Utf8, true), + Field::new("c", DataType::Boolean, true), + ]); + let merged = merge_schemas_tolerant("s", [a, b]); + assert_eq!(merged.fields().len(), 3); + assert_eq!( + merged.field_with_name("a").unwrap().data_type(), + &DataType::Int64 + ); + assert!(merged.field_with_name("c").is_ok()); + } + + #[test] + fn authoritative_path_ignores_node_mode() { + assert_eq!( + authoritative_schema_path("s", &None).as_str(), + "s/.stream/.schema" + ); + assert_eq!( + authoritative_schema_path("s", &Some("t".into())).as_str(), + "t/s/.stream/.schema" + ); + } +} diff --git a/src/schema_registry/protocol.rs b/src/schema_registry/protocol.rs new file mode 100644 index 000000000..90b074703 --- /dev/null +++ b/src/schema_registry/protocol.rs @@ -0,0 +1,143 @@ +/* + * Parseable Server (C) 2022 - 2025 Parseable, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + */ + +//! Wire types for the ingestor -> querier schema resolution RPC. + +use arrow_schema::{DataType, Field}; +use serde::{Deserialize, Serialize}; +use ulid::Ulid; + +pub const PROTOCOL_VERSION: u32 = 1; +pub const PROTOCOL_HEADER: &str = "x-p-schema-proto"; +/// Path (relative to the API base path) of the resolve endpoint on the authority. +pub const RESOLVE_PATH: &str = "internal/schema/resolve"; + +#[derive(Serialize, Deserialize, Debug, Clone)] +pub struct ResolveRequest { + pub protocol_version: u32, + /// Idempotency / correlation key. Reused verbatim on every retry of the same field set. + pub request_id: Ulid, + pub node_id: String, + pub tenant_id: Option, + pub stream: String, + /// Last `registry_version` this node observed for the stream (0 = never). + pub known_version: u64, + /// Fields the node has not resolved yet, with the type it wrote them as. + pub fields: Vec, +} + +#[derive(Serialize, Deserialize, Debug, Clone, Copy, PartialEq, Eq)] +#[serde(rename_all = "snake_case")] +pub enum Outcome { + /// The proposal was registered (first writer) or matches the registered type. + Accepted, + /// The name is registered with another type; write the column as `resolved_name`. + Renamed, + /// Like `Renamed`, and the suffixed name was already registered by an earlier request. + AlreadyRenamed, + /// No safe name exists: the field is a reserved column (`p_*` or a partition column) + /// registered with another type. + Unresolvable, +} + +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq)] +pub struct FieldResolution { + pub original_name: String, + /// Type the node proposed (as found in its arrow files). + pub proposed_type: DataType, + /// Name the node must write this column under. Equals `original_name` unless renamed. + pub resolved_name: String, + /// Authoritative type of `resolved_name`. + pub data_type: DataType, + /// Authoritative type of `original_name`, so a loser can flip its in-memory schema. + pub registry_type: Option, + pub outcome: Outcome, +} + +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq)] +pub struct RegistryEntry { + pub name: String, + pub data_type: DataType, +} + +#[derive(Serialize, Deserialize, Debug, Clone)] +pub struct ResolveResponse { + pub protocol_version: u32, + pub request_id: Ulid, + /// Registry version after this request (>= known_version). + pub version: u64, + /// One entry per proposed (non-null) field, in request order. + pub resolutions: Vec, + /// Full registry snapshot when the node is behind; lets it refresh its cache. + #[serde(default)] + pub delta: Vec, +} + +#[derive(Serialize, Deserialize, Debug, Clone, Copy, PartialEq, Eq)] +#[serde(rename_all = "SCREAMING_SNAKE_CASE")] +pub enum ErrorCode { + BadRequest, + StreamNotFound, + StreamDeleting, + Unavailable, + UnsupportedProtocol, +} + +#[derive(Serialize, Deserialize, Debug, Clone)] +pub struct ResolveErrorBody { + pub protocol_version: u32, + pub code: ErrorCode, + pub message: String, +} + +#[cfg(test)] +mod tests { + use super::*; + use arrow_schema::TimeUnit; + + #[test] + fn request_round_trips_arrow_types() { + let req = ResolveRequest { + protocol_version: PROTOCOL_VERSION, + request_id: Ulid::new(), + node_id: "n1".into(), + tenant_id: None, + stream: "s".into(), + known_version: 3, + fields: vec![ + Field::new("a", DataType::Int64, true), + Field::new("t", DataType::Timestamp(TimeUnit::Millisecond, None), true), + ], + }; + let json = serde_json::to_string(&req).unwrap(); + let back: ResolveRequest = serde_json::from_str(&json).unwrap(); + assert_eq!(back.fields, req.fields); + assert_eq!(back.request_id, req.request_id); + } + + #[test] + fn error_codes_are_screaming_snake_case() { + let body = ResolveErrorBody { + protocol_version: 1, + code: ErrorCode::StreamNotFound, + message: String::default(), + }; + let json = serde_json::to_value(&body).unwrap(); + assert_eq!(json["code"], "STREAM_NOT_FOUND"); + } +} diff --git a/src/schema_registry/registry.rs b/src/schema_registry/registry.rs new file mode 100644 index 000000000..d479a96e4 --- /dev/null +++ b/src/schema_registry/registry.rs @@ -0,0 +1,424 @@ +/* + * Parseable Server (C) 2022 - 2025 Parseable, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + */ + +//! Pure (I/O free) write-once field registry and the resolution rule. + +use std::collections::{HashMap, HashSet}; + +use arrow_schema::{DataType, Field}; + +use crate::event::format::get_datatype_suffix; + +use super::protocol::{FieldResolution, Outcome}; + +/// Types that are the same column type for registry purposes. +pub fn normalize(data_type: &DataType) -> DataType { + match data_type { + DataType::LargeUtf8 | DataType::Utf8View => DataType::Utf8, + DataType::LargeBinary | DataType::BinaryView => DataType::Binary, + other => other.clone(), + } +} + +pub fn compatible(registered: &DataType, proposed: &DataType) -> bool { + normalize(registered) == normalize(proposed) +} + +/// Authoritative name -> type map of one stream. +#[derive(Debug, Default, Clone)] +pub struct StreamRegistry { + pub fields: HashMap, + pub version: u64, + /// Columns that must never be renamed (time partition, custom partitions, p_* columns). + pub reserved: HashSet, +} + +/// Result of resolving one request against a registry snapshot. +#[derive(Debug, Default)] +pub struct Resolution { + pub resolutions: Vec, + /// Names first registered by this request, in registration order. + pub registered: Vec<(String, DataType)>, +} + +impl StreamRegistry { + /// Resolves `proposals` without mutating `self`. Deterministic: the answer is a pure + /// function of the registry contents and the proposals, so retries are idempotent. + pub fn resolve(&self, proposals: &[Field]) -> Resolution { + let mut overlay: HashMap = HashMap::new(); + let mut out = Resolution::default(); + // A rename target must not be a column the same files already carry, or renaming + // would leave two columns with one name in a file (e.g. `a` and `a_utf8` together). + let proposed_names: HashSet<&str> = proposals.iter().map(|f| f.name().as_str()).collect(); + // Nor may two proposals be renamed to the same column: a file carrying `a` and + // `a_utf8` (both Utf8) next to registered `a_utf8: Boolean` would otherwise send + // both to an existing `a_utf8_utf8`. + let mut granted: HashSet = HashSet::new(); + + for proposal in proposals { + let proposed = proposal.data_type(); + if proposed.is_null() { + continue; + } + let original = proposal.name().to_string(); + let lookup = |name: &str, overlay: &HashMap| { + overlay.get(name).or_else(|| self.fields.get(name)).cloned() + }; + let registry_type = lookup(&original, &overlay); + + let mut name = original.clone(); + let mut depth = 0; + let resolution = loop { + let taken_by_request = depth > 0 + && (proposed_names.contains(name.as_str()) || granted.contains(&name)); + match lookup(&name, &overlay) { + None if !taken_by_request => { + overlay.insert(name.clone(), proposed.clone()); + out.registered.push((name.clone(), proposed.clone())); + break FieldResolution { + original_name: original.clone(), + proposed_type: proposed.clone(), + resolved_name: name.clone(), + data_type: proposed.clone(), + registry_type: Some(if depth == 0 { + proposed.clone() + } else { + registry_type.clone().expect("conflict implies registered") + }), + outcome: if depth == 0 { + Outcome::Accepted + } else { + Outcome::Renamed + }, + }; + } + Some(existing) if !taken_by_request && compatible(&existing, proposed) => { + break FieldResolution { + original_name: original.clone(), + proposed_type: proposed.clone(), + resolved_name: name.clone(), + data_type: existing, + registry_type: registry_type.clone(), + outcome: if depth == 0 { + Outcome::Accepted + } else { + Outcome::AlreadyRenamed + }, + }; + } + // No depth limit: every step makes a longer name, and only finitely many + // names are registered or proposed, so the chain always reaches a free one. + // Types sharing a suffix (every list is `list`) chain one level per type. + _ => { + if self.reserved.contains(&original) { + break FieldResolution { + original_name: original.clone(), + proposed_type: proposed.clone(), + resolved_name: original.clone(), + data_type: proposed.clone(), + registry_type: registry_type.clone(), + outcome: Outcome::Unresolvable, + }; + } + depth += 1; + name = format!("{name}_{}", get_datatype_suffix(proposed)); + } + } + }; + if resolution.resolved_name != resolution.original_name { + granted.insert(resolution.resolved_name.clone()); + } + out.resolutions.push(resolution); + } + out + } + + pub fn apply(&mut self, registered: &[(String, DataType)]) { + if registered.is_empty() { + return; + } + for (name, data_type) in registered { + self.fields + .entry(name.clone()) + .or_insert_with(|| data_type.clone()); + } + self.version += 1; + } + + /// Seeds from an existing schema. The first type seen for a name wins; later conflicting + /// types are returned so the caller can log them. + pub fn seed<'a>( + &mut self, + fields: impl IntoIterator, + ) -> Vec<(String, DataType, DataType)> { + let mut conflicts = Vec::new(); + for field in fields { + if field.data_type().is_null() { + continue; + } + match self.fields.get(field.name()) { + None => { + self.fields + .insert(field.name().clone(), field.data_type().clone()); + } + Some(existing) if compatible(existing, field.data_type()) => {} + Some(existing) => conflicts.push(( + field.name().clone(), + existing.clone(), + field.data_type().clone(), + )), + } + } + conflicts + } +} + +#[cfg(test)] +mod tests { + use std::sync::Arc; + + use super::*; + use arrow_schema::TimeUnit; + + fn f(name: &str, t: DataType) -> Field { + Field::new(name, t, true) + } + + fn reg(fields: &[(&str, DataType)]) -> StreamRegistry { + StreamRegistry { + fields: fields + .iter() + .map(|(n, t)| (n.to_string(), t.clone())) + .collect(), + version: 1, + reserved: HashSet::new(), + } + } + + #[test] + fn unknown_field_is_registered() { + let r = reg(&[]); + let res = r.resolve(&[f("a", DataType::Int64)]); + assert_eq!(res.resolutions[0].outcome, Outcome::Accepted); + assert_eq!(res.resolutions[0].resolved_name, "a"); + assert_eq!(res.registered, vec![("a".to_string(), DataType::Int64)]); + } + + #[test] + fn same_type_is_accepted_without_registration() { + let r = reg(&[("a", DataType::Int64)]); + let res = r.resolve(&[f("a", DataType::Int64)]); + assert_eq!(res.resolutions[0].outcome, Outcome::Accepted); + assert!(res.registered.is_empty()); + } + + #[test] + fn conflicting_type_is_renamed_with_local_suffix_rule() { + let r = reg(&[("a", DataType::Int64)]); + let res = r.resolve(&[f("a", DataType::Utf8)]); + let fr = &res.resolutions[0]; + assert_eq!(fr.outcome, Outcome::Renamed); + assert_eq!(fr.resolved_name, "a_utf8"); + assert_eq!(fr.registry_type, Some(DataType::Int64)); + assert_eq!(res.registered, vec![("a_utf8".to_string(), DataType::Utf8)]); + } + + #[test] + fn existing_rename_target_is_reused() { + let r = reg(&[("a", DataType::Int64), ("a_utf8", DataType::Utf8)]); + let res = r.resolve(&[f("a", DataType::Utf8)]); + assert_eq!(res.resolutions[0].outcome, Outcome::AlreadyRenamed); + assert_eq!(res.resolutions[0].resolved_name, "a_utf8"); + assert!(res.registered.is_empty()); + } + + #[test] + fn rename_target_taken_by_other_type_chains_suffix() { + let r = reg(&[("a", DataType::Int64), ("a_utf8", DataType::Boolean)]); + let res = r.resolve(&[f("a", DataType::Utf8)]); + assert_eq!(res.resolutions[0].resolved_name, "a_utf8_utf8"); + assert_eq!(res.resolutions[0].outcome, Outcome::Renamed); + } + + #[test] + fn suffix_chain_continues_until_a_free_name() { + // The typed name is taken by another type twice over: the chain goes on until free. + let list = |t: DataType| DataType::List(Arc::new(Field::new_list_field(t, true))); + let r = reg(&[ + ("a", list(DataType::Int64)), + ("a_list_date", DataType::Boolean), + ("a_list_date_list_date", DataType::Utf8), + ]); + let res = r.resolve(&[f("a", list(DataType::Date32))]); + assert_eq!( + res.resolutions[0].resolved_name, + "a_list_date_list_date_list_date" + ); + assert_eq!(res.resolutions[0].outcome, Outcome::Renamed); + assert_eq!( + res.registered, + vec![( + "a_list_date_list_date_list_date".to_string(), + list(DataType::Date32) + )] + ); + } + + #[test] + fn each_list_type_gets_its_own_column() { + // The suffix names the element type, so list types of one field never share a + // column, however many there are, and an ingestor renaming locally lands on the + // same name. An old `a_list` column (named before suffixes carried the element + // type) is left as it is: new data of that type goes to the typed name. + let list = |t: DataType| DataType::List(Arc::new(Field::new_list_field(t, true))); + let r = reg(&[("a", DataType::Float64), ("a_list", list(DataType::Utf8))]); + let res = r.resolve(&[ + f("a", list(DataType::Utf8)), + f("a", list(list(DataType::Int64))), + ]); + let names: Vec<&str> = res + .resolutions + .iter() + .map(|x| x.resolved_name.as_str()) + .collect(); + assert_eq!(names, ["a_list_utf8", "a_list_list_int64"]); + assert!( + res.resolutions + .iter() + .all(|x| x.outcome == Outcome::Renamed) + ); + // Asked again, the same type resolves to its column. + let mut r = r; + r.apply(&res.registered); + let res = r.resolve(&[f("a", list(list(DataType::Int64)))]); + assert_eq!(res.resolutions[0].resolved_name, "a_list_list_int64"); + assert_eq!(res.resolutions[0].outcome, Outcome::AlreadyRenamed); + } + + #[test] + fn reserved_columns_are_never_renamed() { + let mut r = reg(&[("ts", DataType::Timestamp(TimeUnit::Millisecond, None))]); + r.reserved.insert("ts".into()); + let res = r.resolve(&[f("ts", DataType::Utf8)]); + assert_eq!(res.resolutions[0].outcome, Outcome::Unresolvable); + } + + #[test] + fn null_proposals_are_ignored() { + let r = reg(&[]); + let res = r.resolve(&[f("a", DataType::Null)]); + assert!(res.resolutions.is_empty()); + assert!(res.registered.is_empty()); + } + + #[test] + fn large_utf8_is_compatible_with_utf8() { + let r = reg(&[("a", DataType::Utf8)]); + let res = r.resolve(&[f("a", DataType::LargeUtf8)]); + assert_eq!(res.resolutions[0].outcome, Outcome::Accepted); + } + + #[test] + fn same_request_conflict_uses_overlay() { + let r = reg(&[]); + let res = r.resolve(&[f("a", DataType::Int64), f("a", DataType::Utf8)]); + assert_eq!(res.resolutions[0].resolved_name, "a"); + assert_eq!(res.resolutions[1].resolved_name, "a_utf8"); + assert_eq!(res.registered.len(), 2); + } + + #[test] + fn retry_after_apply_is_idempotent() { + // Ingestor 1 registers a:Int64, ingestor 2 gets a_utf8, ingestor 1 retries. + let mut r = reg(&[]); + let first = r.resolve(&[f("a", DataType::Int64)]); + r.apply(&first.registered); + let second = r.resolve(&[f("a", DataType::Utf8)]); + r.apply(&second.registered); + let retry = r.resolve(&[f("a", DataType::Int64)]); + assert_eq!(retry.resolutions[0].resolved_name, "a"); + assert_eq!(retry.resolutions[0].outcome, Outcome::Accepted); + let retry2 = r.resolve(&[f("a", DataType::Utf8)]); + assert_eq!(retry2.resolutions[0].resolved_name, "a_utf8"); + assert_eq!(r.version, 3); + } + + #[test] + fn seed_keeps_first_type_and_reports_conflicts() { + let mut r = StreamRegistry::default(); + let a = f("a", DataType::Int64); + let b = f("a", DataType::Utf8); + let conflicts = r.seed([&a, &b]); + assert_eq!(r.fields["a"], DataType::Int64); + assert_eq!(conflicts.len(), 1); + } + + #[test] + fn rename_skips_names_carried_by_the_same_request() { + // A file carrying both `a: Utf8` and `a_utf8: Utf8` must not map both to `a_utf8`. + for order in [ + [f("a", DataType::Utf8), f("a_utf8", DataType::Utf8)], + [f("a_utf8", DataType::Utf8), f("a", DataType::Utf8)], + ] { + let r = reg(&[("a", DataType::Int64)]); + let res = r.resolve(&order); + let by_name = |n: &str| { + res.resolutions + .iter() + .find(|x| x.original_name == n) + .unwrap() + .resolved_name + .clone() + }; + assert_eq!(by_name("a"), "a_utf8_utf8"); + assert_eq!(by_name("a_utf8"), "a_utf8"); + } + } + + #[test] + fn two_proposals_never_share_an_existing_rename_target() { + // Registered: `a` and `a_utf8` hold other types, `a_utf8_utf8` already holds Utf8 + // (an earlier rename). A file carrying `a: Utf8` and `a_utf8: Utf8` must get two + // different targets, whatever the order of the proposals. + for order in [ + [f("a", DataType::Utf8), f("a_utf8", DataType::Utf8)], + [f("a_utf8", DataType::Utf8), f("a", DataType::Utf8)], + ] { + let r = reg(&[ + ("a", DataType::Float64), + ("a_utf8", DataType::Boolean), + ("a_utf8_utf8", DataType::Utf8), + ]); + let res = r.resolve(&order); + let targets: Vec<&str> = res + .resolutions + .iter() + .map(|x| x.resolved_name.as_str()) + .collect(); + assert_eq!(targets.len(), 2); + assert_ne!(targets[0], targets[1], "{targets:?}"); + assert!( + res.resolutions + .iter() + .all(|x| x.data_type == DataType::Utf8 && x.resolved_name != x.original_name), + "{:?}", + res.resolutions + ); + } + } +} diff --git a/src/schema_registry/view.rs b/src/schema_registry/view.rs new file mode 100644 index 000000000..f04d3c99d --- /dev/null +++ b/src/schema_registry/view.rs @@ -0,0 +1,460 @@ +/* + * Parseable Server (C) 2022 - 2025 Parseable, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + */ + +//! Ingestor-side cache of what the authority decided for a stream. + +use std::{ + collections::{HashMap, HashSet}, + path::Path, + sync::Arc, +}; + +use arrow_array::RecordBatch; +use arrow_schema::{ArrowError, DataType, Field, Schema}; +use serde::{Deserialize, Serialize}; + +use super::{ + protocol::{FieldResolution, Outcome, RegistryEntry}, + registry::compatible, +}; + +pub const VIEW_FILE_NAME: &str = ".field_resolution.json"; + +/// Resolved fields and renames for one stream on one ingestor. +/// +/// Append-only by construction (it mirrors a write-once registry), so an older snapshot is +/// always a prefix-preserving subset of a newer one: a file converts the same way under any +/// later view. +#[derive(Debug, Default, Clone, PartialEq)] +pub struct ResolvedView { + pub registry_version: u64, + /// Authoritative type per column name, including rename targets. + pub fields: HashMap, + /// (name, type written in the arrow file) -> registered columns it may be written to, in + /// the order the authority granted them. Append-only: a file whose earlier target is + /// taken by another of its columns uses the next one, and existing entries never change. + pub renames: HashMap<(String, DataType), Vec>, + /// (name, type) pairs the authority could not place. Files carrying them are not converted. + pub unresolvable: HashSet<(String, DataType)>, +} + +#[derive(Serialize, Deserialize)] +struct RenameEntry { + from: String, + from_type: DataType, + to: String, +} + +#[derive(Serialize, Deserialize, Default)] +struct ViewFile { + registry_version: u64, + fields: Vec, + renames: Vec, +} + +/// Outcome of checking one `(name, type)` against the view. +#[derive(Debug, PartialEq, Eq)] +pub enum FieldState { + Resolved, + Unresolved, + Unresolvable, +} + +impl ResolvedView { + pub fn is_empty(&self) -> bool { + self.fields.is_empty() && self.renames.is_empty() + } + + pub fn field_state(&self, name: &str, data_type: &DataType) -> FieldState { + if data_type.is_null() { + return FieldState::Resolved; + } + if self + .fields + .get(name) + .is_some_and(|t| compatible(t, data_type)) + || self + .renames + .contains_key(&(name.to_string(), data_type.clone())) + { + FieldState::Resolved + } else if self + .unresolvable + .contains(&(name.to_string(), data_type.clone())) + { + FieldState::Unresolvable + } else { + FieldState::Unresolved + } + } + + /// Whether a batch field typed `incoming` predates a type the authority set for `name`: + /// the stream's type `current` is the authority's, and `incoming` is another one. + /// + /// Such a batch was built against the schema from before a resolution changed the + /// type (a stream switches to the authority's type whatever told it so: a rename, a + /// registry snapshot or a restart). It must still be accepted: its arrow file carries + /// `incoming`, which conversion resolves like any other unresolved type, renaming it or + /// holding the file back. + pub fn predates_resolved_type( + &self, + name: &str, + current: &DataType, + incoming: &DataType, + ) -> bool { + current != incoming + && self + .fields + .get(name) + .is_some_and(|resolved| compatible(resolved, current)) + } + + /// Folds an authority answer into a new view. + pub fn with_resolutions( + &self, + version: u64, + resolutions: &[FieldResolution], + delta: &[RegistryEntry], + ) -> Self { + let mut next = self.clone(); + for entry in delta { + next.fields + .insert(entry.name.clone(), entry.data_type.clone()); + } + for r in resolutions { + let key = (r.original_name.clone(), r.proposed_type.clone()); + match r.outcome { + Outcome::Unresolvable => { + next.unresolvable.insert(key); + if let Some(t) = &r.registry_type { + next.fields.insert(r.original_name.clone(), t.clone()); + } + } + Outcome::Accepted | Outcome::Renamed | Outcome::AlreadyRenamed => { + next.unresolvable.remove(&key); + next.fields + .insert(r.resolved_name.clone(), r.data_type.clone()); + if let Some(t) = &r.registry_type { + next.fields.insert(r.original_name.clone(), t.clone()); + } + if r.resolved_name != r.original_name { + let targets = next.renames.entry(key).or_default(); + if !targets.contains(&r.resolved_name) { + targets.push(r.resolved_name.clone()); + } + } + } + } + } + next.registry_version = next.registry_version.max(version); + next + } + + /// Returns the renamed schema of an arrow file, or `None` when no column is renamed. + /// + /// Each renamed column takes its first target not used by another column of the file. + /// Fails when every target is taken; the file then needs a new target from the authority. + pub fn rename_schema(&self, schema: &Schema) -> Result>, ArrowError> { + if self.renames.is_empty() { + return Ok(None); + } + let targets = |f: &Field| { + self.renames + .get(&(f.name().clone(), f.data_type().clone())) + .filter(|t| !t.is_empty()) + }; + if !schema.fields().iter().any(|f| targets(f).is_some()) { + return Ok(None); + } + let mut taken: HashSet<&str> = schema + .fields() + .iter() + .filter(|f| targets(f).is_none()) + .map(|f| f.name().as_str()) + .collect(); + let mut fields = Vec::with_capacity(schema.fields().len()); + for f in schema.fields() { + let Some(candidates) = targets(f) else { + fields.push(f.clone()); + continue; + }; + let Some(to) = candidates.iter().find(|t| !taken.contains(t.as_str())) else { + return Err(ArrowError::SchemaError(format!( + "renaming {} produced a duplicate column: {} already taken", + f.name(), + candidates.join(", ") + ))); + }; + taken.insert(to); + fields.push(Arc::new(f.as_ref().clone().with_name(to))); + } + Ok(Some(Arc::new(Schema::new_with_metadata( + fields, + schema.metadata().clone(), + )))) + } + + pub fn load(dir: &Path) -> Self { + let path = dir.join(VIEW_FILE_NAME); + let Ok(bytes) = std::fs::read(&path) else { + return Self::default(); + }; + match serde_json::from_slice::(&bytes) { + Ok(file) => Self { + registry_version: file.registry_version, + fields: file + .fields + .into_iter() + .map(|e| (e.name, e.data_type)) + .collect(), + renames: file.renames.into_iter().fold( + HashMap::<_, Vec>::new(), + |mut acc, r| { + acc.entry((r.from, r.from_type)).or_default().push(r.to); + acc + }, + ), + unresolvable: HashSet::new(), + }, + Err(err) => { + tracing::warn!("ignoring unreadable {}: {err}", path.display()); + Self::default() + } + } + } + + /// Atomically persists the view (tmp file + rename). + pub fn persist(&self, dir: &Path) -> std::io::Result<()> { + std::fs::create_dir_all(dir)?; + let mut fields: Vec = self + .fields + .iter() + .map(|(name, data_type)| RegistryEntry { + name: name.clone(), + data_type: data_type.clone(), + }) + .collect(); + fields.sort_by(|a, b| a.name.cmp(&b.name)); + // One entry per target, in grant order: `load` rebuilds each list in file order. + let mut keys: Vec<&(String, DataType)> = self.renames.keys().collect(); + keys.sort_by_cached_key(|(from, from_type)| (from.clone(), format!("{from_type:?}"))); + let renames: Vec = keys + .into_iter() + .flat_map(|key| { + self.renames[key].iter().map(|to| RenameEntry { + from: key.0.clone(), + from_type: key.1.clone(), + to: to.clone(), + }) + }) + .collect(); + let file = ViewFile { + registry_version: self.registry_version, + fields, + renames, + }; + let bytes = serde_json::to_vec(&file).map_err(std::io::Error::other)?; + let tmp = dir.join(format!("{VIEW_FILE_NAME}.tmp")); + std::fs::write(&tmp, bytes)?; + std::fs::rename(&tmp, dir.join(VIEW_FILE_NAME)) + } +} + +/// Zero-copy rename of a batch to `schema` (same column order, new names). +pub fn rename_batch( + schema: &Option>, + batch: RecordBatch, +) -> Result { + match schema { + Some(schema) => RecordBatch::try_new(schema.clone(), batch.columns().to_vec()), + None => Ok(batch), + } +} + +#[cfg(test)] +mod tests { + use super::*; + use arrow_array::{ArrayRef, Int64Array, StringArray}; + + fn renamed_view() -> ResolvedView { + ResolvedView::default().with_resolutions( + 2, + &[FieldResolution { + original_name: "a".into(), + proposed_type: DataType::Utf8, + resolved_name: "a_utf8".into(), + data_type: DataType::Utf8, + registry_type: Some(DataType::Int64), + outcome: Outcome::Renamed, + }], + &[], + ) + } + + #[test] + fn renamed_field_is_resolved_and_winner_type_recorded() { + let v = renamed_view(); + assert_eq!(v.field_state("a", &DataType::Utf8), FieldState::Resolved); + assert_eq!(v.field_state("a", &DataType::Int64), FieldState::Resolved); + assert_eq!(v.field_state("b", &DataType::Int64), FieldState::Unresolved); + assert_eq!(v.fields["a"], DataType::Int64); + assert_eq!(v.fields["a_utf8"], DataType::Utf8); + assert_eq!(v.registry_version, 2); + } + + #[test] + fn rename_is_zero_copy() { + let v = renamed_view(); + let schema = Schema::new(vec![ + Field::new("a", DataType::Utf8, true), + Field::new("b", DataType::Int64, true), + ]); + let renamed = v.rename_schema(&schema).unwrap().expect("renamed"); + assert_eq!(renamed.field(0).name(), "a_utf8"); + assert_eq!(renamed.field(1).name(), "b"); + + let a: ArrayRef = Arc::new(StringArray::from(vec!["x"])); + let b: ArrayRef = Arc::new(Int64Array::from(vec![1])); + let batch = RecordBatch::try_new(Arc::new(schema), vec![a.clone(), b.clone()]).unwrap(); + let out = rename_batch(&Some(renamed), batch).unwrap(); + assert!(Arc::ptr_eq(out.column(0), &a)); + assert_eq!(out.schema().field(0).name(), "a_utf8"); + } + + #[test] + fn winner_typed_column_is_not_renamed() { + let v = renamed_view(); + let schema = Schema::new(vec![Field::new("a", DataType::Int64, true)]); + assert!(v.rename_schema(&schema).unwrap().is_none()); + } + + #[test] + fn duplicate_after_rename_is_an_error() { + let v = renamed_view(); + let schema = Schema::new(vec![ + Field::new("a", DataType::Utf8, true), + Field::new("a_utf8", DataType::Utf8, true), + ]); + assert!(v.rename_schema(&schema).is_err()); + } + + /// `a: Utf8` granted `a_utf8`, then `a_utf8_utf8` for a file that also carries `a_utf8`. + fn two_target_view() -> ResolvedView { + renamed_view().with_resolutions( + 3, + &[FieldResolution { + original_name: "a".into(), + proposed_type: DataType::Utf8, + resolved_name: "a_utf8_utf8".into(), + data_type: DataType::Utf8, + registry_type: Some(DataType::Int64), + outcome: Outcome::Renamed, + }], + &[], + ) + } + + fn renamed_names(v: &ResolvedView, fields: &[&str]) -> Vec { + let schema = Schema::new( + fields + .iter() + .map(|n| Field::new(*n, DataType::Utf8, true)) + .collect::>(), + ); + v.rename_schema(&schema) + .unwrap() + .expect("renamed") + .fields() + .iter() + .map(|f| f.name().clone()) + .collect() + } + + #[test] + fn later_targets_never_change_earlier_renames() { + let v = two_target_view(); + assert_eq!( + v.renames[&("a".to_string(), DataType::Utf8)], + vec!["a_utf8".to_string(), "a_utf8_utf8".to_string()] + ); + // Files that converted under the first answer still convert the same way. + assert_eq!(renamed_names(&v, &["a"]), vec!["a_utf8"]); + assert_eq!( + renamed_names(&v, &["a", "a_utf8_utf8"]), + vec!["a_utf8", "a_utf8_utf8"] + ); + // Only a file whose first target is taken moves to the next one. + assert_eq!( + renamed_names(&v, &["a", "a_utf8"]), + vec!["a_utf8_utf8", "a_utf8"] + ); + // Every target taken: needs a new one from the authority. + let all = Schema::new( + ["a", "a_utf8", "a_utf8_utf8"] + .iter() + .map(|n| Field::new(*n, DataType::Utf8, true)) + .collect::>(), + ); + assert!(v.rename_schema(&all).is_err()); + } + + #[test] + fn persist_and_load_round_trip() { + let dir = tempfile::tempdir().unwrap(); + let v = renamed_view(); + v.persist(dir.path()).unwrap(); + assert_eq!(ResolvedView::load(dir.path()), v); + let v = two_target_view(); + v.persist(dir.path()).unwrap(); + assert_eq!(ResolvedView::load(dir.path()), v); + } + + /// A type learned only from a registry snapshot carries no rename, yet the stream + /// switches to it; a batch still carrying the local type must be accepted. + #[test] + fn snapshot_type_tolerates_batches_typed_before_it() { + let v = ResolvedView::default().with_resolutions( + 4, + &[], + &[RegistryEntry { + name: "b".into(), + data_type: DataType::Float64, + }], + ); + assert!(v.renames.is_empty()); + // The stream switched to the authority's type: an older Utf8 batch predates it. + assert!(v.predates_resolved_type("b", &DataType::Float64, &DataType::Utf8)); + // Same type: nothing to tolerate. + assert!(!v.predates_resolved_type("b", &DataType::Float64, &DataType::Float64)); + // The stream's type is not the authority's: a real conflict, not a stale batch. + assert!(!v.predates_resolved_type("b", &DataType::Utf8, &DataType::Float64)); + // Nothing resolved for the name. + assert!(!v.predates_resolved_type("c", &DataType::Float64, &DataType::Utf8)); + // A renamed type is covered the same way. + let v = renamed_view(); + assert!(v.predates_resolved_type("a", &DataType::Int64, &DataType::Utf8)); + } + + #[test] + fn missing_or_corrupt_file_is_empty_view() { + let dir = tempfile::tempdir().unwrap(); + assert!(ResolvedView::load(dir.path()).is_empty()); + std::fs::write(dir.path().join(VIEW_FILE_NAME), b"{not json").unwrap(); + assert!(ResolvedView::load(dir.path()).is_empty()); + } +} diff --git a/src/storage/object_storage.rs b/src/storage/object_storage.rs index 1a6f7320a..9f3b7edc4 100644 --- a/src/storage/object_storage.rs +++ b/src/storage/object_storage.rs @@ -861,6 +861,17 @@ pub trait ObjectStorage: Debug + Send + Sync + 'static { stream_name: &str, tenant_id: &Option, ) -> Result { + // On the schema authority, serialise this writer of the stream-level `.schema` + // with resolutions, and make the authority re-read it afterwards. + let _authority_guard = if matches!(PARSEABLE.options.mode, Mode::Query | Mode::Prism) { + Some( + crate::schema_registry::authority::SCHEMA_AUTHORITY + .lock_and_invalidate(tenant_id, stream_name) + .await, + ) + } else { + None + }; let schema = fetch_schema(stream_name, tenant_id).await?; let schema_bytes = Bytes::from(serde_json::to_vec(&schema)?); // convert to bytes @@ -1543,11 +1554,21 @@ async fn run_stream_deletion( return; } } + // Wait for schema resolutions of this stream in flight, and keep new ones out until its + // files are gone: one that already passed its deletion check would otherwise write + // `.schema` back after the prefix is removed. Held to the end of this function. + let _resolutions = crate::schema_registry::authority::SCHEMA_AUTHORITY + .lock_and_invalidate(tenant_id, stream_name) + .await; match storage.delete_stream(stream_name, tenant_id).await { Ok(()) => { // Evict and reset stats before clearing the tombstone: once the // marker is gone, a replacement can be created under this name. streams.delete_if_still_deleting(stream_name, tenant_id); + // The authority's cached registry belongs to the deleted stream; a stream + // recreated under this name must start from an empty registry. + crate::schema_registry::authority::SCHEMA_AUTHORITY + .forget_stream(tenant_id, stream_name); if let Err(e) = stats::delete_stats_for_mode(options.mode, stream_name, "json", tenant_id) { @@ -1813,11 +1834,16 @@ pub async fn commit_schema_to_storage( .await .map_err(|e| ObjectStorageError::MetastoreError(Box::new(e.to_detail())))?; - let new_schema = Schema::try_merge(vec![ - schema, - serde_json::from_slice::(&stream_schema)?, - ]) - .map_err(|e| ObjectStorageError::Custom(e.to_string()))?; + let stored = serde_json::from_slice::(&stream_schema)?; + let new_schema = + if PARSEABLE.options.mode == Mode::Ingest && PARSEABLE.options.schema_resolution { + // This node's own file: a type resolved by the schema authority replaces the type + // this node uploaded before the resolution. + crate::schema_registry::merge_schemas_tolerant(stream_name, [schema, stored]) + } else { + Schema::try_merge(vec![schema, stored]) + .map_err(|e| ObjectStorageError::Custom(e.to_string()))? + }; PARSEABLE .metastore diff --git a/src/utils/actix.rs b/src/utils/actix.rs index ac76075a8..a28189d44 100644 --- a/src/utils/actix.rs +++ b/src/utils/actix.rs @@ -1,21 +1,21 @@ /* -* Parseable Server (C) 2022 - 2025 Parseable, Inc. -* -* This program is free software: you can redistribute it and/or modify -* it under the terms of the GNU Affero General Public License as -* published by the Free Software Foundation, either version 3 of the -* License, or (at your option) any later version. -* -* This program is distributed in the hope that it will be useful, -* but WITHOUT ANY WARRANTY; without even the implied warranty of -* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -* GNU Affero General Public License for more details. -* -* You should have received a copy of the GNU Affero General Public License -* along with this program. If not, see . -* -* -*/ + * Parseable Server (C) 2022 - 2025 Parseable, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * + */ use actix_web::{ Error, FromRequest, HttpMessage, HttpRequest, diff --git a/src/utils/mod.rs b/src/utils/mod.rs index bbf2de6e2..300858eb6 100644 --- a/src/utils/mod.rs +++ b/src/utils/mod.rs @@ -211,6 +211,7 @@ pub async fn user_auth_for_datasets( Action::CreateUserGroup, Action::DeleteUserGroup, Action::DeleteNode, + Action::ResolveSchema, ] .contains(action) => {