diff --git a/nodedb-cluster-tests/tests/common_suite/cases/gateway_execute.rs b/nodedb-cluster-tests/tests/common_suite/cases/gateway_execute.rs index a2b02d8e6..c9ee4a8ab 100644 --- a/nodedb-cluster-tests/tests/common_suite/cases/gateway_execute.rs +++ b/nodedb-cluster-tests/tests/common_suite/cases/gateway_execute.rs @@ -20,7 +20,7 @@ use nodedb::control::gateway::core::QueryContext; use nodedb::control::gateway::plan_cache::PlanCacheKey; use nodedb::control::gateway::plan_cache::{hash_placeholder_types, hash_sql}; use nodedb::control::gateway::version_set::GatewayVersionSet; -use nodedb::control::gateway::{Gateway, PlanCache}; +use nodedb::control::gateway::{Gateway, LoweredPlan, PlanCache}; use nodedb::types::TenantId; use nodedb_physical::physical_plan::{KvOp, PhysicalPlan}; use nodedb_types::QualifiedCollection; @@ -135,7 +135,7 @@ async fn gateway_execute_sql_plan_cache_populated() { let sql = "GET gw_cache_smoke smoke-key"; let make_plan = || { - Ok(PhysicalPlan::Kv(KvOp::Get { + Ok(LoweredPlan::cacheable(PhysicalPlan::Kv(KvOp::Get { collection: QualifiedCollection::new( nodedb_types::id::DatabaseId::DEFAULT, "gw_cache_smoke", @@ -143,7 +143,7 @@ async fn gateway_execute_sql_plan_cache_populated() { key: b"smoke-key".to_vec(), rls_filters: vec![], surrogate_ceiling: None, - })) + }))) }; // Cache starts empty. diff --git a/nodedb-cluster-tests/tests/common_suite/cases/http_gateway_migration.rs b/nodedb-cluster-tests/tests/common_suite/cases/http_gateway_migration.rs index 59007d2ca..ef889768d 100644 --- a/nodedb-cluster-tests/tests/common_suite/cases/http_gateway_migration.rs +++ b/nodedb-cluster-tests/tests/common_suite/cases/http_gateway_migration.rs @@ -17,9 +17,9 @@ use std::sync::Arc; use std::time::Duration; use nodedb::Error; -use nodedb::control::gateway::Gateway; use nodedb::control::gateway::GatewayErrorMap; use nodedb::control::gateway::core::QueryContext; +use nodedb::control::gateway::{Gateway, LoweredPlan}; use nodedb::types::TenantId; use nodedb_physical::physical_plan::{KvOp, PhysicalPlan}; use nodedb_types::QualifiedCollection; @@ -184,7 +184,7 @@ async fn http_gateway_migration_cross_node_query() { get_sql, &[], || { - Ok(PhysicalPlan::Kv(KvOp::Get { + Ok(LoweredPlan::cacheable(PhysicalPlan::Kv(KvOp::Get { collection: QualifiedCollection::new( nodedb_types::id::DatabaseId::DEFAULT, "http_gw_cross_node", @@ -192,7 +192,7 @@ async fn http_gateway_migration_cross_node_query() { key: b"cross-key".to_vec(), rls_filters: vec![], surrogate_ceiling: None, - })) + }))) }, |plan| async { Ok(common::authorize_gateway_plan(&follower.shared, &ctx, plan).await) diff --git a/nodedb-cluster-tests/tests/common_suite/cases/pgwire_gateway_migration.rs b/nodedb-cluster-tests/tests/common_suite/cases/pgwire_gateway_migration.rs index c9867ae3f..d02c841b5 100644 --- a/nodedb-cluster-tests/tests/common_suite/cases/pgwire_gateway_migration.rs +++ b/nodedb-cluster-tests/tests/common_suite/cases/pgwire_gateway_migration.rs @@ -20,9 +20,9 @@ use crate::common; use std::sync::Arc; use std::time::Duration; -use nodedb::control::gateway::Gateway; use nodedb::control::gateway::core::QueryContext; use nodedb::control::gateway::version_set::GatewayVersionSet; +use nodedb::control::gateway::{Gateway, LoweredPlan}; use nodedb::types::TenantId; use nodedb_physical::physical_plan::{KvOp, PhysicalPlan}; use nodedb_types::QualifiedCollection; @@ -200,7 +200,7 @@ async fn pgwire_gateway_migration_plan_cache_hits() { let sql = "GET pgwire_gw_cache cache-key"; let make_plan = || { - Ok(PhysicalPlan::Kv(KvOp::Get { + Ok(LoweredPlan::cacheable(PhysicalPlan::Kv(KvOp::Get { collection: QualifiedCollection::new( nodedb_types::id::DatabaseId::DEFAULT, "pgwire_gw_cache", @@ -208,7 +208,7 @@ async fn pgwire_gateway_migration_plan_cache_hits() { key: b"cache-key".to_vec(), rls_filters: vec![], surrogate_ceiling: None, - })) + }))) }; let authorize_plan = |plan: PhysicalPlan| async { Ok(common::authorize_gateway_plan(&node.shared, &ctx, plan).await) diff --git a/nodedb-sql/src/catalog.rs b/nodedb-sql/src/catalog.rs index 41aa91772..4638193ea 100644 --- a/nodedb-sql/src/catalog.rs +++ b/nodedb-sql/src/catalog.rs @@ -192,6 +192,54 @@ pub trait SqlCatalog { fn resolve_regtype(&self, _name: &str) -> Option { None } + + /// Advance a sequence and return the allocated value, recording it as this + /// session's `currval`. Scoping mirrors `resolve_regclass`. + /// + /// An unknown name returns [`SqlError::UndefinedObject`] (SQLSTATE + /// `42704`). The default returns "sequence access unavailable" so + /// implementors with no sequence state compile without change. + fn sequence_nextval( + &self, + _database_id: nodedb_types::DatabaseId, + _tenant_id: u64, + _name: &str, + ) -> Result { + Err(sequence_access_unavailable()) + } + + /// Return the last value this session obtained from `sequence_nextval`. + /// + /// A sequence this session never advanced returns + /// [`SqlError::ObjectNotInPrerequisiteState`] (SQLSTATE `55000`). + fn sequence_currval( + &self, + _database_id: nodedb_types::DatabaseId, + _tenant_id: u64, + _name: &str, + ) -> Result { + Err(sequence_access_unavailable()) + } + + /// Position a sequence so the next `sequence_nextval` returns + /// `value + increment`. Returns `value`. + fn sequence_setval( + &self, + _database_id: nodedb_types::DatabaseId, + _tenant_id: u64, + _name: &str, + _value: i64, + ) -> Result { + Err(sequence_access_unavailable()) + } +} + +/// The error a catalog with no sequence state returns for every accessor. +fn sequence_access_unavailable() -> crate::SqlError { + crate::SqlError::ObjectNotInPrerequisiteState { + object: "sequence".into(), + detail: "sequence access unavailable: this catalog carries no sequence registry".into(), + } } /// View of a registered array, surfaced to the SQL planner. Decoded by diff --git a/nodedb-sql/src/engine_rules/columnar.rs b/nodedb-sql/src/engine_rules/columnar.rs index 366b86840..7decfbd69 100644 --- a/nodedb-sql/src/engine_rules/columnar.rs +++ b/nodedb-sql/src/engine_rules/columnar.rs @@ -13,6 +13,7 @@ impl EngineRules for ColumnarRules { Ok(vec![SqlPlan::Insert { collection: p.collection, engine: EngineType::Columnar, + route: WriteRoute::ColumnarFamily, rows: p.rows, column_defaults: p.column_defaults, if_absent: p.if_absent, @@ -29,6 +30,7 @@ impl EngineRules for ColumnarRules { Ok(vec![SqlPlan::Upsert { collection: p.collection, engine: EngineType::Columnar, + route: WriteRoute::ColumnarFamily, rows: p.rows, column_defaults: p.column_defaults, on_conflict_updates: p.on_conflict_updates, diff --git a/nodedb-sql/src/engine_rules/document_schemaless.rs b/nodedb-sql/src/engine_rules/document_schemaless.rs index 7b1af4059..17937962f 100644 --- a/nodedb-sql/src/engine_rules/document_schemaless.rs +++ b/nodedb-sql/src/engine_rules/document_schemaless.rs @@ -13,6 +13,7 @@ impl EngineRules for SchemalessRules { Ok(vec![SqlPlan::Insert { collection: p.collection, engine: EngineType::DocumentSchemaless, + route: WriteRoute::Document, rows: p.rows, column_defaults: p.column_defaults, if_absent: p.if_absent, @@ -25,6 +26,7 @@ impl EngineRules for SchemalessRules { Ok(vec![SqlPlan::Upsert { collection: p.collection, engine: EngineType::DocumentSchemaless, + route: WriteRoute::Document, rows: p.rows, column_defaults: p.column_defaults, on_conflict_updates: p.on_conflict_updates, diff --git a/nodedb-sql/src/engine_rules/document_strict.rs b/nodedb-sql/src/engine_rules/document_strict.rs index 1bc621cbc..0aaca86ac 100644 --- a/nodedb-sql/src/engine_rules/document_strict.rs +++ b/nodedb-sql/src/engine_rules/document_strict.rs @@ -13,6 +13,7 @@ impl EngineRules for StrictRules { Ok(vec![SqlPlan::Insert { collection: p.collection, engine: EngineType::DocumentStrict, + route: WriteRoute::Document, rows: p.rows, column_defaults: p.column_defaults, if_absent: p.if_absent, @@ -25,6 +26,7 @@ impl EngineRules for StrictRules { Ok(vec![SqlPlan::Upsert { collection: p.collection, engine: EngineType::DocumentStrict, + route: WriteRoute::Document, rows: p.rows, column_defaults: p.column_defaults, on_conflict_updates: p.on_conflict_updates, diff --git a/nodedb-sql/src/engine_rules/spatial.rs b/nodedb-sql/src/engine_rules/spatial.rs index 23396d985..afe708e52 100644 --- a/nodedb-sql/src/engine_rules/spatial.rs +++ b/nodedb-sql/src/engine_rules/spatial.rs @@ -13,6 +13,7 @@ impl EngineRules for SpatialRules { Ok(vec![SqlPlan::Insert { collection: p.collection, engine: EngineType::Spatial, + route: WriteRoute::ColumnarFamily, rows: p.rows, column_defaults: p.column_defaults, if_absent: p.if_absent, @@ -28,6 +29,7 @@ impl EngineRules for SpatialRules { Ok(vec![SqlPlan::Upsert { collection: p.collection, engine: EngineType::Spatial, + route: WriteRoute::ColumnarFamily, rows: p.rows, column_defaults: p.column_defaults, on_conflict_updates: p.on_conflict_updates, diff --git a/nodedb-sql/src/error.rs b/nodedb-sql/src/error.rs index b31a34c5b..ae2763b89 100644 --- a/nodedb-sql/src/error.rs +++ b/nodedb-sql/src/error.rs @@ -19,6 +19,35 @@ pub enum SqlError { #[error("function {name}(...) does not exist")] UndefinedFunction { name: String }, + /// A sequence accessor appeared where every output row needs its own + /// allocation, such as a SELECT list over a FROM clause. + /// + /// Rendered as SQLSTATE `0A000` (feature_not_supported). Constant + /// contexts evaluate the call for real: a FROM-less `SELECT`, a column + /// `DEFAULT`, a `VALUES` list. The refusal keeps a per-row call from + /// reaching the row evaluator, which has no sequence state and would + /// return `NULL` for every row. + #[error( + "{name}(...) is not supported in a per-row context; \ + a SELECT list, WHERE clause, or SET clause over a FROM relation \ + evaluates once per row. Call it in a FROM-less SELECT or a column \ + DEFAULT instead" + )] + SequencePerRowUnsupported { name: String }, + + /// A statement names a database object that does not exist — a sequence, + /// most commonly. Distinct from [`SqlError::UndefinedFunction`]: the + /// function exists, the object it names does not. PostgreSQL rejects the + /// same input with SQLSTATE `42704` (`undefined_object`). + #[error("{kind} \"{name}\" does not exist")] + UndefinedObject { kind: &'static str, name: String }, + + /// An object exists but a prerequisite step has not run, such as `currval` + /// before this session called `nextval`. PostgreSQL rejects the same input + /// with SQLSTATE `55000` (`object_not_in_prerequisite_state`). + #[error("{detail}")] + ObjectNotInPrerequisiteState { object: String, detail: String }, + #[error("unknown column '{column}' in table '{table}'")] UnknownColumn { table: String, column: String }, @@ -85,6 +114,25 @@ pub enum SqlError { #[error("unsupported: {detail}")] Unsupported { detail: String }, + /// A declared column DEFAULT the server cannot evaluate to a value. + /// + /// The column is never omitted instead. A DEFAULT that disappears stores + /// NULL where the declaration promised a value, and nothing reports it. + #[error("DEFAULT for column '{column}' cannot be evaluated: {expr}")] + UnevaluableDefault { column: String, expr: String }, + + /// `setval` appeared inside a column DEFAULT. + /// + /// A DEFAULT runs once per row, so evaluating `setval` there will move the + /// sequence's position on every inserted row. PostgreSQL reports a + /// function used in a context that forbids it as SQLSTATE `42601` + /// (`syntax_error`), and this refusal carries the same code. + #[error( + "setval() is not allowed in the DEFAULT for column '{column}'; \ + a DEFAULT must not move a sequence's position" + )] + SetvalInColumnDefault { column: String }, + #[error("invalid function call: {detail}")] InvalidFunction { detail: String }, diff --git a/nodedb-sql/src/functions/arg_types.rs b/nodedb-sql/src/functions/arg_types.rs index eac91261d..652aa8f17 100644 --- a/nodedb-sql/src/functions/arg_types.rs +++ b/nodedb-sql/src/functions/arg_types.rs @@ -391,3 +391,11 @@ pub static JSON_CONTAINS_ARGS: &[ArgTypeSpec] = &[any("container"), any("needle" /// `json_merge(base, overlay)` / `json_patch(base, overlay)`. pub static JSON_MERGE_ARGS: &[ArgTypeSpec] = &[any("base"), any("overlay")]; + +// ── Sequence accessors ─────────────────────────────────────────────────────── + +/// `nextval(name)` / `currval(name)` — one sequence name. +pub static SEQUENCE_NAME_ARGS: &[ArgTypeSpec] = &[typed("sequence", TEXT)]; + +/// `setval(name, value)` — a sequence name and the value it must take. +pub static SETVAL_ARGS: &[ArgTypeSpec] = &[typed("sequence", TEXT), typed("value", INT64_ONLY)]; diff --git a/nodedb-sql/src/functions/builtins/helpers.rs b/nodedb-sql/src/functions/builtins/helpers.rs index 6757194da..1eba3822a 100644 --- a/nodedb-sql/src/functions/builtins/helpers.rs +++ b/nodedb-sql/src/functions/builtins/helpers.rs @@ -30,5 +30,6 @@ pub(super) fn m( return_type, arg_types, since: V0_1_0, + volatility: nodedb_types::Volatility::Immutable, } } diff --git a/nodedb-sql/src/functions/builtins/scalars.rs b/nodedb-sql/src/functions/builtins/scalars.rs index f73556d45..bd23528e6 100644 --- a/nodedb-sql/src/functions/builtins/scalars.rs +++ b/nodedb-sql/src/functions/builtins/scalars.rs @@ -12,6 +12,7 @@ mod math; mod misc; mod pg_fts; mod pg_json; +mod sequence_fn; mod spatial; mod string; mod vector; @@ -32,6 +33,7 @@ pub(super) fn scalar_functions() -> Vec { fns.extend(array_fn::array_fn_functions()); fns.extend(array_elem::array_elem_functions()); fns.extend(id_fn::id_fn_functions()); + fns.extend(sequence_fn::sequence_fn_functions()); fns.extend(misc::misc_functions()); fns } diff --git a/nodedb-sql/src/functions/builtins/scalars/datetime.rs b/nodedb-sql/src/functions/builtins/scalars/datetime.rs index ec0dfcd08..18e8da509 100644 --- a/nodedb-sql/src/functions/builtins/scalars/datetime.rs +++ b/nodedb-sql/src/functions/builtins/scalars/datetime.rs @@ -44,7 +44,8 @@ pub(super) fn datetime_functions() -> Vec { no_trigger(), Some(ColumnType::Timestamptz), arg_types::NO_ARGS, - ), + ) + .volatile(), m( "current_timestamp", Scalar, @@ -53,7 +54,8 @@ pub(super) fn datetime_functions() -> Vec { no_trigger(), Some(ColumnType::Timestamptz), arg_types::NO_ARGS, - ), + ) + .volatile(), m( "datetime", Scalar, diff --git a/nodedb-sql/src/functions/builtins/scalars/id_fn.rs b/nodedb-sql/src/functions/builtins/scalars/id_fn.rs index 6b22aa059..bd5b2b737 100644 --- a/nodedb-sql/src/functions/builtins/scalars/id_fn.rs +++ b/nodedb-sql/src/functions/builtins/scalars/id_fn.rs @@ -31,7 +31,8 @@ pub(super) fn id_fn_functions() -> Vec { no_trigger(), Some(ColumnType::Uuid), arg_types::NO_ARGS, - ), + ) + .volatile(), // `uuid_v4` and `gen_random_uuid` are aliases for `uuid` — same // `nodedb_query::functions::id::try_eval` match arm // (`"uuid" | "uuid_v4" | "gen_random_uuid"`). @@ -43,7 +44,8 @@ pub(super) fn id_fn_functions() -> Vec { no_trigger(), Some(ColumnType::Uuid), arg_types::NO_ARGS, - ), + ) + .volatile(), m( "gen_random_uuid", Scalar, @@ -52,7 +54,8 @@ pub(super) fn id_fn_functions() -> Vec { no_trigger(), Some(ColumnType::Uuid), arg_types::NO_ARGS, - ), + ) + .volatile(), m( "uuid_v7", Scalar, @@ -61,7 +64,8 @@ pub(super) fn id_fn_functions() -> Vec { no_trigger(), Some(ColumnType::Uuid), arg_types::NO_ARGS, - ), + ) + .volatile(), m( "ulid", Scalar, @@ -70,7 +74,8 @@ pub(super) fn id_fn_functions() -> Vec { no_trigger(), Some(ColumnType::Ulid), arg_types::NO_ARGS, - ), + ) + .volatile(), m( "cuid2", Scalar, @@ -79,7 +84,8 @@ pub(super) fn id_fn_functions() -> Vec { no_trigger(), Some(ColumnType::String), arg_types::NO_ARGS, - ), + ) + .volatile(), // `nanoid(length?)` — the optional length argument means max_args is // 1 even though the common call form `nanoid()` takes none. m( @@ -90,7 +96,8 @@ pub(super) fn id_fn_functions() -> Vec { no_trigger(), Some(ColumnType::String), arg_types::NANOID_ARGS, - ), + ) + .volatile(), m( "is_uuid", Scalar, diff --git a/nodedb-sql/src/functions/builtins/scalars/sequence_fn.rs b/nodedb-sql/src/functions/builtins/scalars/sequence_fn.rs new file mode 100644 index 000000000..2a72468d9 --- /dev/null +++ b/nodedb-sql/src/functions/builtins/scalars/sequence_fn.rs @@ -0,0 +1,51 @@ +// SPDX-License-Identifier: Apache-2.0 + +//! Sequence accessor registrations: `nextval`, `currval`, `setval`. +//! +//! These are `Scalar` and `Volatile` at once. `Volatile` keeps the constant +//! folder from freezing a call into a literal and keeps a plan holding one +//! out of the plan cache, so each execution allocates a fresh value. +//! `nodedb_sql::planner::catalog_expr_fold` routes each call to the +//! `SqlCatalog` sequence methods at plan time. + +use nodedb_types::columnar::ColumnType; + +use crate::functions::arg_types; +use crate::functions::registry::{FunctionCategory::Scalar, FunctionMeta}; + +use super::super::helpers::{m, no_trigger}; + +pub(super) fn sequence_fn_functions() -> Vec { + vec![ + m( + "nextval", + Scalar, + 1, + 1, + no_trigger(), + Some(ColumnType::Int64), + arg_types::SEQUENCE_NAME_ARGS, + ) + .volatile(), + m( + "currval", + Scalar, + 1, + 1, + no_trigger(), + Some(ColumnType::Int64), + arg_types::SEQUENCE_NAME_ARGS, + ) + .volatile(), + m( + "setval", + Scalar, + 2, + 2, + no_trigger(), + Some(ColumnType::Int64), + arg_types::SETVAL_ARGS, + ) + .volatile(), + ] +} diff --git a/nodedb-sql/src/functions/builtins/scalars/spatial/registrations.rs b/nodedb-sql/src/functions/builtins/scalars/spatial/registrations.rs index c8b69027f..7ba32e813 100644 --- a/nodedb-sql/src/functions/builtins/scalars/spatial/registrations.rs +++ b/nodedb-sql/src/functions/builtins/scalars/spatial/registrations.rs @@ -31,6 +31,7 @@ pub(in crate::functions::builtins::scalars) fn spatial_functions() -> Vec Self { + self.volatility = Volatility::Volatile; + self + } } /// The function registry. @@ -135,6 +148,13 @@ impl FunctionRegistry { .is_some_and(|f| f.category == FunctionCategory::Aggregate) } + /// Whether a function must re-evaluate on every execution. + /// An unknown name is not volatile — the existence gate rejects it first. + pub fn is_volatile(&self, name: &str) -> bool { + self.lookup(name) + .is_some_and(|f| f.volatility.is_volatile()) + } + /// Check if a function is a window function. pub fn is_window(&self, name: &str) -> bool { self.lookup(name) diff --git a/nodedb-sql/src/functions/sequence_accessor.rs b/nodedb-sql/src/functions/sequence_accessor.rs new file mode 100644 index 000000000..74249dd61 --- /dev/null +++ b/nodedb-sql/src/functions/sequence_accessor.rs @@ -0,0 +1,14 @@ +// SPDX-License-Identifier: Apache-2.0 + +//! The sequence-accessor function names, named once for every gate that +//! treats them apart from other scalars. + +/// The accessors the planner routes to `SqlCatalog` sequence state. +pub const SEQUENCE_ACCESSORS: [&str; 3] = ["nextval", "currval", "setval"]; + +/// Whether `name` calls a sequence accessor. Comparison ignores ASCII case. +pub fn is_sequence_accessor(name: &str) -> bool { + SEQUENCE_ACCESSORS + .iter() + .any(|accessor| name.eq_ignore_ascii_case(accessor)) +} diff --git a/nodedb-sql/src/parser/type_expr.rs b/nodedb-sql/src/parser/type_expr.rs index d372658c5..7bcba19dc 100644 --- a/nodedb-sql/src/parser/type_expr.rs +++ b/nodedb-sql/src/parser/type_expr.rs @@ -7,6 +7,7 @@ //! at write time. use nodedb_types::Value; +use nodedb_types::columnar::ColumnType; use crate::error::SqlError; @@ -26,6 +27,10 @@ pub enum TypeExpr { } /// Leaf type variants that map to a single Value discriminant. +/// +/// Every variant except [`SimpleType::Object`] names one +/// [`ColumnType`](nodedb_types::columnar::ColumnType), so a spelling resolves +/// to the same type here and in the CONVERT path. #[derive(Debug, Clone, PartialEq)] pub enum SimpleType { Int, @@ -35,13 +40,23 @@ pub enum SimpleType { Bytes, Timestamp, Timestamptz, - Decimal, + /// Engine-assigned instant. Matching mirrors + /// `ColumnType::SystemTimestamp`: a client writes an instant, never text. + SystemTimestamp, + /// Declared precision and scale carry the spelling the author wrote. + /// Matching is variant-level, as it is for [`SimpleType::Vector`]. + Decimal { + precision: u8, + scale: u8, + }, Uuid, Ulid, Geometry, Duration, /// Untyped array (any element type). Array, + /// Typeguard-only leaf: a field holding a nested map. No declared DDL + /// spelling names it, and CONVERT maps such a field to `ColumnType::Json`. Object, Json, /// Untyped set (any element type). @@ -60,6 +75,12 @@ pub enum SimpleType { /// Parse a type expression string into a [`TypeExpr`]. /// +/// Each leaf spelling resolves through +/// [`ColumnType::from_str`](std::str::FromStr), the parser the CONVERT path +/// resolves the same typeguard text through. A leaf the shared parser does not +/// know is an error, so a trailing word such as `TIMESTAMP GARBAGE` is +/// rejected rather than read as `TIMESTAMP`. +/// /// # Examples /// /// ``` @@ -80,7 +101,15 @@ pub fn parse_type_expr(s: &str) -> Result { } let mut pos = 0usize; let chars: Vec = s.chars().collect(); - parse_union(&chars, &mut pos, false) + let expr = parse_union(&chars, &mut pos, false)?; + skip_ws(&chars, &mut pos); + if pos < chars.len() { + let rest: String = chars[pos..].iter().collect(); + return Err(SqlError::Parse { + detail: format!("unexpected trailing input in type expression: '{rest}'"), + }); + } + Ok(expr) } /// Parse `single_type ('|' single_type)*`. @@ -89,7 +118,7 @@ pub fn parse_type_expr(s: &str) -> Result { /// when parsing inside `ARRAY<...>` / `SET<...>`). fn parse_union(chars: &[char], pos: &mut usize, stop_at_gt: bool) -> Result { let mut variants: Vec = Vec::new(); - variants.push(parse_single(chars, pos, stop_at_gt)?); + variants.push(parse_single(chars, pos)?); loop { skip_ws(chars, pos); @@ -104,7 +133,7 @@ fn parse_union(chars: &[char], pos: &mut usize, stop_at_gt: bool) -> Result Result, SET<...>, VECTOR(N)). -fn parse_single(chars: &[char], pos: &mut usize, stop_at_gt: bool) -> Result { +/// Parse a single type token: a keyword, `ARRAY<...>`, or `SET<...>`. +fn parse_single(chars: &[char], pos: &mut usize) -> Result { skip_ws(chars, pos); let keyword = read_keyword(chars, pos); if keyword.is_empty() { @@ -128,122 +157,155 @@ fn parse_single(chars: &[char], pos: &mut usize, stop_at_gt: bool) -> Result Ok(TypeExpr::Null), - "INT" | "INTEGER" | "BIGINT" | "INT64" => Ok(TypeExpr::Simple(SimpleType::Int)), - "FLOAT" | "DOUBLE" | "REAL" | "FLOAT64" => Ok(TypeExpr::Simple(SimpleType::Float)), - "STRING" | "TEXT" | "VARCHAR" => Ok(TypeExpr::Simple(SimpleType::String)), - "BOOL" | "BOOLEAN" => Ok(TypeExpr::Simple(SimpleType::Bool)), - "BYTES" | "BYTEA" | "BLOB" => Ok(TypeExpr::Simple(SimpleType::Bytes)), - "TIMESTAMP" => Ok(TypeExpr::Simple(SimpleType::Timestamp)), - "TIMESTAMPTZ" => Ok(TypeExpr::Simple(SimpleType::Timestamptz)), - "DECIMAL" | "NUMERIC" => Ok(TypeExpr::Simple(SimpleType::Decimal)), - "UUID" => Ok(TypeExpr::Simple(SimpleType::Uuid)), - "ULID" => Ok(TypeExpr::Simple(SimpleType::Ulid)), - "GEOMETRY" => Ok(TypeExpr::Simple(SimpleType::Geometry)), - "DURATION" => Ok(TypeExpr::Simple(SimpleType::Duration)), + // Typeguard-only keyword: a field holding a nested map. The shared + // declared-type parser has no `OBJECT` spelling, and the CONVERT path + // maps such a field to `ColumnType::Json`. "OBJECT" => Ok(TypeExpr::Simple(SimpleType::Object)), - "JSON" => Ok(TypeExpr::Simple(SimpleType::Json)), - "REGEX" => Ok(TypeExpr::Simple(SimpleType::Regex)), - "RANGE" => Ok(TypeExpr::Simple(SimpleType::Range)), - "RECORD" => Ok(TypeExpr::Simple(SimpleType::Record)), - - // Dimensionless — matched before the parameterized "VECTOR" block. - // `read_keyword` reads the whole "SPARSEVECTOR" token, so this exact - // arm claims it and the "VECTOR" `(N)` parser never sees it. - "SPARSEVECTOR" => Ok(TypeExpr::Simple(SimpleType::SparseVector)), - - "VECTOR" => { - // Expect '(' digits ')'. - skip_ws(chars, pos); - if *pos >= chars.len() || chars[*pos] != '(' { - return Err(SqlError::Parse { - detail: format!("expected '(' after VECTOR at position {pos}"), - }); - } - *pos += 1; // consume '(' - skip_ws(chars, pos); - let digits = read_digits(chars, pos); - if digits.is_empty() { - return Err(SqlError::Parse { - detail: "expected dimension digits inside VECTOR(...)".to_string(), - }); - } - let dim: u32 = digits.parse().map_err(|_| SqlError::Parse { - detail: format!("invalid VECTOR dimension: '{digits}'"), - })?; - if dim == 0 { - return Err(SqlError::Parse { - detail: "VECTOR dimension must be > 0".to_string(), - }); - } - skip_ws(chars, pos); - if *pos >= chars.len() || chars[*pos] != ')' { - return Err(SqlError::Parse { - detail: format!("expected ')' to close VECTOR({dim} at position {pos}"), - }); - } - *pos += 1; // consume ')' - Ok(TypeExpr::Simple(SimpleType::Vector(dim))) - } - "ARRAY" => { - // Optional typed variant: ARRAY + // Typed generics: the typeguard grammar types the element, which no + // declared DDL spelling does. Bare `ARRAY` / `SET` resolve as leaves. + "ARRAY" | "SET" => { skip_ws(chars, pos); if *pos < chars.len() && chars[*pos] == '<' { *pos += 1; // consume '<' skip_ws(chars, pos); - let inner = parse_union(chars, pos, true)?; + let inner = Box::new(parse_union(chars, pos, true)?); skip_ws(chars, pos); if *pos >= chars.len() || chars[*pos] != '>' { return Err(SqlError::Parse { - detail: format!("expected '>' to close ARRAY<...> at position {pos}"), + detail: format!("expected '>' to close {keyword}<...> at position {pos}"), }); } *pos += 1; // consume '>' - Ok(TypeExpr::TypedArray(Box::new(inner))) - } else { - Ok(TypeExpr::Simple(SimpleType::Array)) + return Ok(if keyword == "ARRAY" { + TypeExpr::TypedArray(inner) + } else { + TypeExpr::TypedSet(inner) + }); } + parse_leaf(chars, pos, &keyword) } - "SET" => { - // Optional typed variant: SET - skip_ws(chars, pos); - if *pos < chars.len() && chars[*pos] == '<' && !stop_at_gt { - *pos += 1; // consume '<' - skip_ws(chars, pos); - let inner = parse_union(chars, pos, true)?; - skip_ws(chars, pos); - if *pos >= chars.len() || chars[*pos] != '>' { - return Err(SqlError::Parse { - detail: format!("expected '>' to close SET<...> at position {pos}"), - }); - } - *pos += 1; // consume '>' - Ok(TypeExpr::TypedSet(Box::new(inner))) - } else if *pos < chars.len() && chars[*pos] == '<' { - // Inside a nested context — consume the '<' and inner normally. - *pos += 1; - skip_ws(chars, pos); - let inner = parse_union(chars, pos, true)?; - skip_ws(chars, pos); - if *pos >= chars.len() || chars[*pos] != '>' { - return Err(SqlError::Parse { - detail: format!("expected '>' to close SET<...> at position {pos}"), - }); + _ => parse_leaf(chars, pos, &keyword), + } +} + +/// Resolve one leaf spelling through the shared declared-type parser. +/// +/// `ColumnType` answers which type a declared spelling names for both planes, +/// so routing here is what keeps typeguard enforcement and typeguard CONVERT +/// from reading one spelling two ways. +fn parse_leaf(chars: &[char], pos: &mut usize, keyword: &str) -> Result { + let spelling = read_spelling(chars, pos, keyword)?; + let column_type: ColumnType = spelling.parse().map_err(|e| SqlError::Parse { + detail: format!("type '{spelling}': {e}"), + })?; + Ok(TypeExpr::Simple(simple_from_column_type( + column_type, + &spelling, + )?)) +} + +/// Read one declared type spelling: the leading keyword, the words that +/// continue it, and an attached parameter list. +/// +/// `TIMESTAMP WITH TIME ZONE` is one type name, so continuation words join +/// with single spaces and the shared parser resolves the whole spelling. A +/// word that continues no known spelling makes the spelling unknown, which is +/// how a trailing garbage word is rejected instead of ignored. +/// +/// A parameter list attaches to its keyword with no space, as +/// `DECIMAL(10, 2)` does. A `(` behind whitespace stays unread and the caller +/// reports it as trailing input. +fn read_spelling(chars: &[char], pos: &mut usize, keyword: &str) -> Result { + let mut spelling = keyword.to_string(); + loop { + if *pos < chars.len() && chars[*pos] == '(' { + spelling.push_str(&read_paren_group(chars, pos)?); + continue; + } + let mut probe = *pos; + skip_ws(chars, &mut probe); + if probe == *pos { + break; + } + let word = read_keyword(chars, &mut probe); + if word.is_empty() { + break; + } + spelling.push(' '); + spelling.push_str(&word); + *pos = probe; + } + Ok(spelling) +} + +/// Read a balanced parenthesized group, both parentheses included. +fn read_paren_group(chars: &[char], pos: &mut usize) -> Result { + let start = *pos; + let mut depth = 0usize; + let mut group = String::new(); + while *pos < chars.len() { + let ch = chars[*pos]; + group.push(ch); + *pos += 1; + match ch { + '(' => depth += 1, + ')' => { + depth = depth.saturating_sub(1); + if depth == 0 { + return Ok(group); } - *pos += 1; - Ok(TypeExpr::TypedSet(Box::new(inner))) - } else { - Ok(TypeExpr::Simple(SimpleType::Set)) } + _ => {} } - - other => Err(SqlError::Parse { - detail: format!("unknown type keyword: '{other}'"), - }), } + Err(SqlError::Parse { + detail: format!("unclosed '(' at position {start}"), + }) +} + +/// Map a resolved column type onto the typeguard leaf that validates it. +/// +/// `ColumnType` is `#[non_exhaustive]`: a variant added later reaches the +/// typed error arm until a leaf is written for it. +fn simple_from_column_type( + column_type: ColumnType, + spelling: &str, +) -> Result { + Ok(match column_type { + ColumnType::Int64 => SimpleType::Int, + ColumnType::Float64 => SimpleType::Float, + ColumnType::String => SimpleType::String, + ColumnType::Bool => SimpleType::Bool, + ColumnType::Bytes => SimpleType::Bytes, + ColumnType::Timestamp => SimpleType::Timestamp, + ColumnType::Timestamptz => SimpleType::Timestamptz, + ColumnType::SystemTimestamp => SimpleType::SystemTimestamp, + ColumnType::Decimal { precision, scale } => SimpleType::Decimal { precision, scale }, + ColumnType::Geometry => SimpleType::Geometry, + ColumnType::Vector(dim) => SimpleType::Vector(dim), + ColumnType::SparseVector => SimpleType::SparseVector, + ColumnType::Uuid => SimpleType::Uuid, + ColumnType::Json => SimpleType::Json, + ColumnType::Ulid => SimpleType::Ulid, + ColumnType::Duration => SimpleType::Duration, + ColumnType::Array => SimpleType::Array, + ColumnType::Set => SimpleType::Set, + ColumnType::Regex => SimpleType::Regex, + ColumnType::Range => SimpleType::Range, + ColumnType::Record => SimpleType::Record, + other => { + return Err(SqlError::Parse { + detail: format!( + "type '{spelling}' resolves to {other}, which no typeguard validates" + ), + }); + } + }) } fn skip_ws(chars: &[char], pos: &mut usize) { @@ -267,16 +329,6 @@ fn read_keyword(chars: &[char], pos: &mut usize) -> String { s } -/// Read contiguous ASCII digits. -fn read_digits(chars: &[char], pos: &mut usize) -> String { - let mut s = String::new(); - while *pos < chars.len() && chars[*pos].is_ascii_digit() { - s.push(chars[*pos]); - *pos += 1; - } - s -} - // ── Validator ──────────────────────────────────────────────────────────────── /// Check if a [`Value`] matches a [`TypeExpr`]. @@ -286,7 +338,8 @@ fn read_digits(chars: &[char], pos: &mut usize) -> String { /// - `Simple(Timestamp)` accepts `Value::DateTime`, `Value::Integer`, and /// `Value::String` (same rules as `ColumnType::Timestamp.accepts()`). /// - `Simple(Decimal)` accepts `Value::Decimal`, `Value::Float`, `Value::Integer`, -/// and `Value::String`. +/// and `Value::String`. Declared precision and scale do not narrow it. +/// - `Simple(SystemTimestamp)` accepts `Value::DateTime` and `Value::Integer`. /// - `Simple(Uuid)` accepts `Value::Uuid` and `Value::String`. /// - `Simple(Geometry)` accepts `Value::Geometry` and `Value::String`. /// - `TypedArray(inner)` matches `Value::Array` where every element matches `inner`. @@ -327,7 +380,10 @@ fn value_matches_simple(value: &Value, simple: &SimpleType) -> bool { value, Value::DateTime(_) | Value::Integer(_) | Value::String(_) ), - SimpleType::Decimal => matches!( + // Mirrors `ColumnType::SystemTimestamp.accepts`: an engine-assigned + // instant takes no text form. + SimpleType::SystemTimestamp => matches!(value, Value::DateTime(_) | Value::Integer(_)), + SimpleType::Decimal { .. } => matches!( value, Value::Decimal(_) | Value::Float(_) | Value::Integer(_) | Value::String(_) ), @@ -485,6 +541,105 @@ mod tests { ); } + /// Both zone spellings are one type name each, so the parser reads the + /// zone the author wrote instead of the leading word alone. + #[test] + fn parse_timestamp_zone_spellings() { + assert_eq!( + parse_type_expr("TIMESTAMP WITH TIME ZONE").unwrap(), + TypeExpr::Simple(SimpleType::Timestamptz) + ); + assert_eq!( + parse_type_expr("timestamp without time zone").unwrap(), + TypeExpr::Simple(SimpleType::Timestamp) + ); + } + + /// A word that continues no known spelling is an error. Reading it as the + /// leading keyword is what let one declaration mean two types. + #[test] + fn parse_error_trailing_words() { + assert!(parse_type_expr("TIMESTAMP GARBAGE WORDS").is_err()); + assert!(parse_type_expr("STRING FOO").is_err()); + assert!(parse_type_expr("TIMESTAMP WITH").is_err()); + assert!(parse_type_expr("INT )").is_err()); + assert!(parse_type_expr("VECTOR(3) EXTRA").is_err()); + } + + /// Every integer and float spelling the shared parser knows parses here. + /// A keyword added to either list extends this test, so a declaration the + /// rest of the database resolves cannot fail typeguard enforcement. + #[test] + fn parse_every_declared_numeric_keyword() { + use nodedb_types::columnar::{DECLARED_FLOAT_KEYWORDS, DECLARED_INT_KEYWORDS}; + for keyword in DECLARED_INT_KEYWORDS { + assert_eq!( + parse_type_expr(keyword).unwrap(), + TypeExpr::Simple(SimpleType::Int), + "{keyword} must parse as an integer leaf" + ); + } + for keyword in DECLARED_FLOAT_KEYWORDS { + assert_eq!( + parse_type_expr(keyword).unwrap(), + TypeExpr::Simple(SimpleType::Float), + "{keyword} must parse as a float leaf" + ); + } + } + + #[test] + fn parse_jsonb() { + assert_eq!( + parse_type_expr("JSONB").unwrap(), + TypeExpr::Simple(SimpleType::Json) + ); + } + + #[test] + fn parse_decimal_carries_declared_params() { + assert_eq!( + parse_type_expr("DECIMAL(10, 2)").unwrap(), + TypeExpr::Simple(SimpleType::Decimal { + precision: 10, + scale: 2 + }) + ); + assert_eq!( + parse_type_expr("NUMERIC").unwrap(), + TypeExpr::Simple(SimpleType::Decimal { + precision: 38, + scale: 10 + }) + ); + assert!(parse_type_expr("DECIMAL(0)").is_err()); + } + + /// A declared character length resolves to the same leaf bare `VARCHAR` + /// gives, and matching stays a string check. + #[test] + fn parse_varchar_length() { + let expr = parse_type_expr("VARCHAR(255)").unwrap(); + assert_eq!(expr, TypeExpr::Simple(SimpleType::String)); + assert!(value_matches_type(&Value::String("x".into()), &expr)); + assert!(parse_type_expr("VARCHAR(0)").is_err()); + } + + /// `SYSTEM_TIMESTAMP` is engine-assigned, so it takes an instant and no + /// text — the rule `ColumnType::SystemTimestamp` enforces. + #[test] + fn parse_and_match_system_timestamp() { + let expr = parse_type_expr("SYSTEM_TIMESTAMP").unwrap(); + assert_eq!(expr, TypeExpr::Simple(SimpleType::SystemTimestamp)); + let dt = nodedb_types::NdbDateTime::from_micros(1_700_000_000_000_000); + assert!(value_matches_type(&Value::DateTime(dt), &expr)); + assert!(value_matches_type(&Value::Integer(1_700_000_000), &expr)); + assert!(!value_matches_type( + &Value::String("2024-01-01".into()), + &expr + )); + } + #[test] fn parse_error_unknown_keyword() { assert!(parse_type_expr("FOOBAR").is_err()); diff --git a/nodedb-sql/src/planner/catalog_expr_fold.rs b/nodedb-sql/src/planner/catalog_expr_fold.rs index 4a7339cbe..cf2d75ed8 100644 --- a/nodedb-sql/src/planner/catalog_expr_fold.rs +++ b/nodedb-sql/src/planner/catalog_expr_fold.rs @@ -6,6 +6,7 @@ use nodedb_types::DatabaseId; use crate::catalog::SqlCatalog; use crate::functions::registry::FunctionRegistry; +use crate::planner::const_fold::FoldScope; use crate::types::{SqlExpr, SqlValue}; pub(super) fn eval_catalog_constant( @@ -29,7 +30,55 @@ pub(super) fn eval_catalog_constant( })?; return Ok(SqlValue::String(normalized)); } - super::select::helpers::eval_constant_expr(expr, functions) + if let Some(value) = eval_sequence_accessor(expr, catalog)? { + return Ok(value); + } + // A from-less SELECT's plan is marked volatile when it holds a volatile + // call, so it is never cached. Evaluating the call here therefore serves + // this execution only, and the next execution re-plans and re-evaluates. + Ok( + crate::planner::const_fold::fold_constant_scoped(expr, functions, FoldScope::Once)? + .unwrap_or(SqlValue::Null), + ) +} + +/// Route `nextval` / `currval` / `setval` to the catalog's sequence state. +/// +/// Returns `Ok(None)` for every other expression. These calls are `Volatile`, +/// so the constant folder never reaches them and the plan holding the result +/// is never cached — each execution re-plans and allocates again. +pub(super) fn eval_sequence_accessor( + expr: &SqlExpr, + catalog: &dyn SqlCatalog, +) -> crate::Result> { + let SqlExpr::Function { name, args, .. } = expr else { + return Ok(None); + }; + let lowered = name.to_ascii_lowercase(); + if !crate::functions::sequence_accessor::is_sequence_accessor(&lowered) { + return Ok(None); + } + let sequence = match args.first() { + Some(SqlExpr::Literal(SqlValue::String(name))) => name.as_str(), + _ => { + return Err(crate::SqlError::Arity { + detail: format!("{lowered} requires a literal sequence name"), + }); + } + }; + let value = match lowered.as_str() { + "nextval" => catalog.sequence_nextval(DatabaseId::DEFAULT, 0, sequence)?, + "currval" => catalog.sequence_currval(DatabaseId::DEFAULT, 0, sequence)?, + _ => { + let Some(SqlExpr::Literal(SqlValue::Int(target))) = args.get(1) else { + return Err(crate::SqlError::Arity { + detail: "setval requires a literal bigint second argument".into(), + }); + }; + catalog.sequence_setval(DatabaseId::DEFAULT, 0, sequence, *target)? + } + }; + Ok(Some(SqlValue::Int(value))) } pub(super) fn validate_expr( diff --git a/nodedb-sql/src/planner/const_fold.rs b/nodedb-sql/src/planner/const_fold.rs index 8f94a81ea..c06a9fbbb 100644 --- a/nodedb-sql/src/planner/const_fold.rs +++ b/nodedb-sql/src/planner/const_fold.rs @@ -51,15 +51,38 @@ pub fn fold_constant_default(expr: &SqlExpr) -> FoldResult { fold_constant(expr, default_registry()) } +/// Whether a fold is allowed to evaluate `Volatile` functions. +/// +/// `Reuse` is plan-time folding whose result outlives this execution, so a +/// volatile call is never evaluated. `Once` is folding for a plan that is +/// itself marked volatile and never cached, so the call is evaluated here and +/// re-evaluated on the next execution's re-plan. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum FoldScope { + /// The folded value can be reused by a later execution. + Reuse, + /// The folded value serves this execution only. + Once, +} + /// Fold a `SqlExpr` to a literal `SqlValue` at plan time. See [`FoldResult`] -/// for what each outcome means. +/// for what each outcome means. Volatile calls are never folded. pub fn fold_constant(expr: &SqlExpr, registry: &FunctionRegistry) -> FoldResult { + fold_constant_scoped(expr, registry, FoldScope::Reuse) +} + +/// Fold a `SqlExpr` under an explicit [`FoldScope`]. +pub fn fold_constant_scoped( + expr: &SqlExpr, + registry: &FunctionRegistry, + scope: FoldScope, +) -> FoldResult { match expr { SqlExpr::Literal(v) => Ok(Some(v.clone())), SqlExpr::ArrayLiteral(items) => { let mut folded = Vec::with_capacity(items.len()); for item in items { - match fold_constant(item, registry)? { + match fold_constant_scoped(item, registry, scope)? { Some(v) => folded.push(v), None => return Ok(None), } @@ -69,7 +92,7 @@ pub fn fold_constant(expr: &SqlExpr, registry: &FunctionRegistry) -> FoldResult SqlExpr::UnaryOp { op: UnaryOp::Neg, expr, - } => Ok(match fold_constant(expr, registry)? { + } => Ok(match fold_constant_scoped(expr, registry, scope)? { // `checked_neg` so negating `i64::MIN` declines to fold rather // than wrapping (release) or panicking (debug). Some(SqlValue::Int(i)) => i.checked_neg().map(SqlValue::Int), @@ -79,17 +102,18 @@ pub fn fold_constant(expr: &SqlExpr, registry: &FunctionRegistry) -> FoldResult }), SqlExpr::BinaryOp { left, op, right } => { let (Some(l), Some(r)) = ( - fold_constant(left, registry)?, - fold_constant(right, registry)?, + fold_constant_scoped(left, registry, scope)?, + fold_constant_scoped(right, registry, scope)?, ) else { return Ok(None); }; fold_binary(l, *op, r) } - SqlExpr::Function { name, args, .. } => fold_function_call(name, args, registry), - SqlExpr::Cast { expr, to_type } => { - Ok(fold_constant(expr, registry)?.and_then(|inner| fold_cast(inner, to_type))) + SqlExpr::Function { name, args, .. } => { + fold_function_call_scoped(name, args, registry, scope) } + SqlExpr::Cast { expr, to_type } => Ok(fold_constant_scoped(expr, registry, scope)? + .and_then(|inner| fold_cast(inner, to_type))), _ => Ok(None), } } @@ -310,12 +334,23 @@ fn fold_binary(l: SqlValue, op: BinaryOp, r: SqlValue) -> FoldResult { /// through the shared scalar evaluator, and converting the result back to /// `SqlValue`. Only folds functions that are present in `registry`, so /// callers can distinguish "unknown function" from "known function, all -/// args folded". +/// args folded". A `Volatile` function is never folded. pub fn fold_function_call(name: &str, args: &[SqlExpr], registry: &FunctionRegistry) -> FoldResult { + fold_function_call_scoped(name, args, registry, FoldScope::Reuse) +} + +/// Fold a function call under an explicit [`FoldScope`]. +pub fn fold_function_call_scoped( + name: &str, + args: &[SqlExpr], + registry: &FunctionRegistry, + scope: FoldScope, +) -> FoldResult { // Gate on registry so unknown-function paths keep their existing // fallbacks instead of collapsing to SqlValue::Null. Aggregates and // window functions aren't foldable — they need a row stream. - let Some(meta) = registry.lookup(name) else { + let name_lower = name.to_lowercase(); + let Some(meta) = registry.lookup(&name_lower) else { return Ok(None); }; if matches!( @@ -324,10 +359,22 @@ pub fn fold_function_call(name: &str, args: &[SqlExpr], registry: &FunctionRegis ) { return Ok(None); } + // A volatile call must run per execution, so folding it into a reusable + // literal would freeze the first result into every later execution of the + // same plan. Same "not folded" outcome the category guard above produces. + if meta.volatility.is_volatile() && scope == FoldScope::Reuse { + return Ok(None); + } + // Sequence accessors read catalog state this folder has no handle on. + // `planner::catalog_expr_fold::eval_catalog_constant` resolves them + // through `SqlCatalog` before any fold runs. + if matches!(name_lower.as_str(), "nextval" | "currval" | "setval") { + return Ok(None); + } let mut folded_args = Vec::with_capacity(args.len()); for arg in args { - match fold_constant(arg, registry)? { + match fold_constant_scoped(arg, registry, scope)? { Some(v) => folded_args.push(sql_to_ndb_value(v)), None => return Ok(None), } @@ -339,7 +386,7 @@ pub fn fold_function_call(name: &str, args: &[SqlExpr], registry: &FunctionRegis // clause; `SELECT mod(5, 0)` has no row scope and became NULL. // Exhaustive on purpose: a new `EvalError` variant must be classified // here rather than defaulting to "defer to a runtime that may not exist". - match nodedb_query::functions::eval_function(&name.to_lowercase(), &folded_args) { + match nodedb_query::functions::eval_function(&name_lower, &folded_args) { Ok(result) => Ok(Some(ndb_to_sql_value(result))), Err(nodedb_query::EvalError::DivisionByZero) => Err(SqlError::DivisionByZero), } @@ -402,16 +449,21 @@ mod tests { use super::*; #[test] - fn fold_now_produces_timestamptz() { + fn fold_now_produces_timestamptz_once_only() { let registry = FunctionRegistry::new(); let expr = SqlExpr::Function { name: "now".into(), args: vec![], distinct: false, }; - let val = fold_constant(&expr, ®istry) + // `now` is volatile, so a reusable plan must never carry its value. + assert!( + matches!(fold_constant(&expr, ®istry), Ok(None)), + "now() must not fold into a reusable plan" + ); + let val = fold_constant_scoped(&expr, ®istry, FoldScope::Once) .expect("fold must not error") - .expect("now() should fold"); + .expect("now() must fold for a single execution"); match val { SqlValue::Timestamptz(dt) => { // Sanity: must not be epoch (year 1970). @@ -422,15 +474,19 @@ mod tests { } #[test] - fn fold_current_timestamp_produces_timestamptz() { + fn fold_current_timestamp_produces_timestamptz_once_only() { let registry = FunctionRegistry::new(); let expr = SqlExpr::Function { name: "current_timestamp".into(), args: vec![], distinct: false, }; + assert!( + matches!(fold_constant(&expr, ®istry), Ok(None)), + "current_timestamp must not fold into a reusable plan" + ); assert!(matches!( - fold_constant(&expr, ®istry), + fold_constant_scoped(&expr, ®istry, FoldScope::Once), Ok(Some(SqlValue::Timestamptz(_))) )); } diff --git a/nodedb-sql/src/planner/defaults.rs b/nodedb-sql/src/planner/defaults.rs deleted file mode 100644 index 5fb914978..000000000 --- a/nodedb-sql/src/planner/defaults.rs +++ /dev/null @@ -1,122 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 - -//! Column DEFAULT expression evaluation at insert time. -//! -//! Supports ID generation functions (UUIDv4/v7, ULID, CUID2, NANOID), `NOW()`, -//! and literal values. More complex defaults (arbitrary expressions) fall -//! through to the plan-time const-folder. -//! -//! Lives in the SQL crate rather than beside one engine's converter because -//! every engine that materializes a DEFAULT has to produce the SAME value for -//! the same expression — a `DEFAULT now()` that means one thing on a document -//! collection and another on a key-value one would be a difference nobody -//! declared. The key-value planner also needs it BEFORE its declared-type -//! coercion and range checks run, so a materialized default is validated -//! exactly like a supplied one. - -use nodedb_types::NodeDbError; - -pub fn evaluate_default_expr(expr: &str) -> Result, NodeDbError> { - let upper = expr.trim().to_uppercase(); - match upper.as_str() { - "UUID_V7" | "UUIDV7" | "GEN_UUID_V7()" | "UUID_V7()" => Ok(Some( - nodedb_types::Value::String(nodedb_types::id_gen::uuid_v7()), - )), - "UUID_V4" | "UUIDV4" | "UUID" | "GEN_UUID_V4()" | "UUID_V4()" => Ok(Some( - nodedb_types::Value::String(nodedb_types::id_gen::uuid_v4()), - )), - "ULID" | "GEN_ULID()" | "ULID()" => Ok(Some(nodedb_types::Value::String( - nodedb_types::id_gen::ulid(), - ))), - "CUID2" | "CUID2()" => Ok(Some(nodedb_types::Value::String( - nodedb_types::id_gen::cuid2(), - ))), - "NANOID" | "NANOID()" => Ok(Some(nodedb_types::Value::String( - nodedb_types::id_gen::nanoid(), - ))), - "NOW()" => { - let now = std::time::SystemTime::now() - .duration_since(std::time::UNIX_EPOCH) - .unwrap_or_default(); - Ok(Some(nodedb_types::Value::String( - chrono::DateTime::from_timestamp_millis(now.as_millis() as i64) - .map(|dt| dt.to_rfc3339()) - .unwrap_or_else(|| now.as_millis().to_string()), - ))) - } - _ => parse_parametric_or_literal(expr, &upper), - } -} - -fn parse_parametric_or_literal( - expr: &str, - upper: &str, -) -> Result, NodeDbError> { - // NANOID(N) — custom length. - if upper.starts_with("NANOID(") && upper.ends_with(')') { - let len_str = &upper[7..upper.len() - 1]; - if let Ok(len) = len_str.parse::() { - return Ok(Some(nodedb_types::Value::String( - nodedb_types::id_gen::nanoid_with_length(len), - ))); - } - } - // CUID2(N) — custom length; validates length range and surfaces planning errors. - if upper.starts_with("CUID2(") && upper.ends_with(')') { - let len_str = &upper[6..upper.len() - 1]; - if let Ok(len) = len_str.parse::() { - let id = nodedb_types::id_gen::cuid2_with_length(len).map_err(|e| { - NodeDbError::plan_error_at( - "defaults", - format!("CUID2({len}) default expression is invalid: {e}"), - ) - })?; - return Ok(Some(nodedb_types::Value::String(id))); - } - } - // Numeric literal. - if let Ok(i) = expr.trim().parse::() { - return Ok(Some(nodedb_types::Value::Integer(i))); - } - if let Ok(f) = expr.trim().parse::() { - return Ok(Some(nodedb_types::Value::Float(f))); - } - // Quoted string literal. - let trimmed = expr.trim(); - if (trimmed.starts_with('\'') && trimmed.ends_with('\'')) - || (trimmed.starts_with('"') && trimmed.ends_with('"')) - { - return Ok(Some(nodedb_types::Value::String( - trimmed[1..trimmed.len() - 1].to_string(), - ))); - } - - // Fallback: try the plan-time const-folder for arbitrary expressions - // (e.g. `upper('x')`, `1 + 2`, `concat('a', 'b')`). - Ok(try_const_fold_default(expr)) -} - -/// Attempt to parse the DEFAULT expression as SQL, then const-fold it. -fn try_const_fold_default(expr: &str) -> Option { - let sql_expr = crate::parse_expr_string(expr).ok()?; - let folded = crate::planner::const_fold::fold_constant_default(&sql_expr).ok()??; - Some(sql_value_to_ndb(folded)) -} - -fn sql_value_to_ndb(v: crate::types::SqlValue) -> nodedb_types::Value { - use crate::types::SqlValue; - match v { - SqlValue::Null => nodedb_types::Value::Null, - SqlValue::Bool(b) => nodedb_types::Value::Bool(b), - SqlValue::Int(i) => nodedb_types::Value::Integer(i), - SqlValue::Float(f) => nodedb_types::Value::Float(f), - SqlValue::Decimal(d) => nodedb_types::Value::Decimal(d), - SqlValue::String(s) => nodedb_types::Value::String(s), - SqlValue::Bytes(b) => nodedb_types::Value::Bytes(b), - SqlValue::Array(a) => { - nodedb_types::Value::Array(a.into_iter().map(sql_value_to_ndb).collect()) - } - SqlValue::Timestamp(dt) => nodedb_types::Value::NaiveDateTime(dt), - SqlValue::Timestamptz(dt) => nodedb_types::Value::DateTime(dt), - } -} diff --git a/nodedb-sql/src/planner/defaults/compiled.rs b/nodedb-sql/src/planner/defaults/compiled.rs new file mode 100644 index 000000000..9905e3e23 --- /dev/null +++ b/nodedb-sql/src/planner/defaults/compiled.rs @@ -0,0 +1,357 @@ +// SPDX-License-Identifier: Apache-2.0 + +//! Column DEFAULTs compiled once per statement and evaluated per row. + +use super::convert::{default_value_to_sql, sql_value_to_ndb}; +use super::kind::{DefaultKind, keyword_generator, parametric_or_literal}; +use crate::catalog::SqlCatalog; +use crate::error::SqlError; +use crate::types::{ColumnInfo, SqlExpr, SqlValue}; + +/// One column DEFAULT, classified and parsed. +/// +/// Compilation is the only step that reads the declaration text, so a caller +/// holding a `CompiledDefault` cannot parse the expression a second time. +#[derive(Debug, Clone)] +pub struct CompiledDefault { + column: String, + /// The declaration text, kept for the `UnevaluableDefault` message only. + text: String, + kind: DefaultKind, + volatile: bool, +} + +impl CompiledDefault { + /// Compile the DEFAULT declared on `column` without evaluating it. + /// + /// A parse error propagates verbatim, so the DDL gate can map an + /// unregistered function name onto SQLSTATE `42883`. + /// + /// A sequence accessor is parsed, never called, so declaring a column must + /// never advance a sequence. + pub fn declare(column: &str, expr: &str) -> crate::Result { + let kind = classify(column, expr)?; + let volatile = match &kind { + DefaultKind::Generator(_) => true, + DefaultKind::Literal(_) => false, + DefaultKind::Expr(parsed) => crate::types::plan::expr_is_volatile(parsed), + }; + Ok(Self { + column: column.to_string(), + text: expr.to_string(), + kind, + volatile, + }) + } + + /// Compile the DEFAULT declared on `column` for row materialization. + /// + /// An expression the parser refuses raises [`SqlError::UnevaluableDefault`] + /// rather than a parse error: DDL already refused those at declaration, so + /// a statement reaching here found one the catalog cannot resolve. + pub fn compile(column: &str, expr: &str) -> crate::Result { + Self::declare(column, expr).map_err(|error| match error { + SqlError::Parse { .. } | SqlError::UndefinedFunction { .. } => { + unevaluable(column, expr) + } + other => other, + }) + } + + /// The column this DEFAULT fills. + pub fn column(&self) -> &str { + &self.column + } + + /// Whether this DEFAULT produces a fresh value on every evaluation. + /// + /// A plan carrying one is never admitted to the plan cache, or the cache + /// replays one execution's value into every later one. + pub fn is_volatile(&self) -> bool { + self.volatile + } + + /// Evaluate to one value. `catalog` resolves `nextval` and `currval`. + /// + /// Takes no expression text, so the per-row path cannot re-parse. + pub fn evaluate(&self, catalog: &dyn SqlCatalog) -> crate::Result { + match &self.kind { + DefaultKind::Generator(generator) => generator.generate(), + DefaultKind::Literal(value) => Ok(value.clone()), + DefaultKind::Expr(parsed) => self.evaluate_expr(parsed, catalog), + } + } + + /// Resolve a parsed DEFAULT through the catalog, then the const-folder. + fn evaluate_expr( + &self, + parsed: &SqlExpr, + catalog: &dyn SqlCatalog, + ) -> crate::Result { + if let Some(value) = + crate::planner::catalog_expr_fold::eval_sequence_accessor(parsed, catalog)? + { + return Ok(sql_value_to_ndb(value)); + } + // `Once`: a materialized DEFAULT serves this insert only, and an INSERT + // plan carrying a volatile DEFAULT is never admitted to the plan cache. + let folded = crate::planner::const_fold::fold_constant_scoped( + parsed, + crate::planner::const_fold::default_registry(), + crate::planner::const_fold::FoldScope::Once, + ) + .map_err(|_| unevaluable(&self.column, &self.text))? + .ok_or_else(|| unevaluable(&self.column, &self.text))?; + Ok(sql_value_to_ndb(folded)) + } +} + +/// Every declared DEFAULT of one collection, compiled once for a statement. +/// +/// Build this OUTSIDE the row loop. `materialize_row` then fills each row from +/// the parsed forms, so a multi-row `VALUES` clause parses each declaration +/// exactly once however many rows it carries. +#[derive(Debug, Clone, Default)] +pub struct ColumnDefaults { + defaults: Vec, +} + +impl ColumnDefaults { + /// Compile the catalog's `(column_name, default_expr)` list. + pub fn compile_pairs(pairs: &[(String, String)]) -> crate::Result { + let defaults = pairs + .iter() + .map(|(column, expr)| CompiledDefault::compile(column, expr)) + .collect::>>()?; + Ok(Self { defaults }) + } + + /// Compile every declared column that carries a DEFAULT. + pub fn compile_columns(columns: &[ColumnInfo]) -> crate::Result { + let defaults = columns + .iter() + .filter_map(|column| { + column + .default + .as_deref() + .map(|expr| CompiledDefault::compile(&column.name, expr)) + }) + .collect::>>()?; + Ok(Self { defaults }) + } + + /// Whether no column declares a DEFAULT. + pub fn is_empty(&self) -> bool { + self.defaults.is_empty() + } + + /// Fill in every column of `row` that declares a DEFAULT and the statement + /// omitted. + /// + /// Each entry evaluates at most once per row, so a `nextval` DEFAULT + /// allocates exactly one value per row. + /// + /// A column the statement supplied stays untouched, an explicit `NULL` + /// included: `NULL` is a value the author chose, and overwriting it with + /// the default makes storing one impossible. + /// + /// Returns whether any default it materialized was volatile, so the caller + /// can keep the plan out of the plan cache. + pub fn materialize_row( + &self, + row: &mut Vec<(String, SqlValue)>, + catalog: &dyn SqlCatalog, + ) -> crate::Result { + let mut volatile = false; + for default in &self.defaults { + if row.iter().any(|(name, _)| name == default.column()) { + continue; + } + let evaluated = default.evaluate(catalog)?; + let value = default_value_to_sql(default.column(), evaluated)?; + volatile |= default.is_volatile(); + row.push((default.column().to_string(), value)); + } + Ok(volatile) + } +} + +/// Check that `expr`, the DEFAULT declared on `column`, can be evaluated. +/// +/// DDL calls this to refuse an unevaluable DEFAULT at declaration time. It +/// classifies and parses the expression, and evaluates nothing. Parsing runs +/// the resolver's `FunctionRegistry` gate, so an unregistered function name +/// raises [`SqlError::UndefinedFunction`]. +pub fn validate_default_expr(expr: &str, column: &str) -> crate::Result<()> { + CompiledDefault::declare(column, expr).map(|_| ()) +} + +/// Classify a DEFAULT into its compiled form. +fn classify(column: &str, expr: &str) -> crate::Result { + let upper = expr.trim().to_uppercase(); + if let Some(generator) = keyword_generator(&upper) { + return Ok(DefaultKind::Generator(generator)); + } + if let Some(kind) = parametric_or_literal(expr, &upper)? { + return Ok(kind); + } + let parsed = crate::parse_expr_string(expr)?; + reject_setval_default(&parsed, column)?; + Ok(DefaultKind::Expr(parsed)) +} + +/// Refuse `setval` as a column DEFAULT. +/// +/// `setval` moves a sequence rather than reading one, so a column cannot take +/// its result as a value. +fn reject_setval_default(expr: &SqlExpr, column: &str) -> crate::Result<()> { + if let SqlExpr::Function { name, .. } = expr + && name.eq_ignore_ascii_case("setval") + { + return Err(SqlError::SetvalInColumnDefault { + column: column.to_string(), + }); + } + Ok(()) +} + +fn unevaluable(column: &str, expr: &str) -> SqlError { + SqlError::UnevaluableDefault { + column: column.to_string(), + expr: expr.to_string(), + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::catalog::SqlCatalogError; + use std::cell::Cell; + + /// A catalog whose only state is a sequence counter, so a test can count + /// exactly how many times a DEFAULT reached `nextval`. + #[derive(Default)] + struct CountingCatalog { + nextval_calls: Cell, + } + + impl SqlCatalog for CountingCatalog { + fn get_collection( + &self, + _database_id: nodedb_types::DatabaseId, + _name: &str, + ) -> std::result::Result, SqlCatalogError> { + Ok(None) + } + + fn sequence_nextval( + &self, + _database_id: nodedb_types::DatabaseId, + _tenant_id: u64, + _name: &str, + ) -> crate::Result { + let next = self.nextval_calls.get() + 1; + self.nextval_calls.set(next); + Ok(next) + } + } + + fn pairs(column: &str, expr: &str) -> Vec<(String, String)> { + vec![(column.to_string(), expr.to_string())] + } + + #[test] + fn one_compilation_still_generates_a_fresh_value_per_row() { + let catalog = CountingCatalog::default(); + let compiled = ColumnDefaults::compile_pairs(&pairs("u", "UUID_V7()")).expect("compiles"); + let mut first = Vec::new(); + let mut second = Vec::new(); + compiled.materialize_row(&mut first, &catalog).expect("row"); + compiled + .materialize_row(&mut second, &catalog) + .expect("row"); + assert_ne!(first[0].1, second[0].1, "each row needs its own UUID"); + } + + #[test] + fn nextval_allocates_exactly_one_value_per_row() { + let catalog = CountingCatalog::default(); + let compiled = + ColumnDefaults::compile_pairs(&pairs("id", "nextval('s')")).expect("compiles"); + let mut first = Vec::new(); + let mut second = Vec::new(); + compiled.materialize_row(&mut first, &catalog).expect("row"); + compiled + .materialize_row(&mut second, &catalog) + .expect("row"); + assert_eq!(first[0].1, SqlValue::Int(1)); + assert_eq!(second[0].1, SqlValue::Int(2)); + assert_eq!(catalog.nextval_calls.get(), 2); + } + + #[test] + fn declaring_a_sequence_default_never_advances_it() { + let catalog = CountingCatalog::default(); + validate_default_expr("nextval('s')", "id").expect("declaration is valid"); + assert_eq!( + catalog.nextval_calls.get(), + 0, + "the DDL gate must parse, never call" + ); + } + + #[test] + fn a_supplied_column_keeps_its_value() { + let catalog = CountingCatalog::default(); + let compiled = + ColumnDefaults::compile_pairs(&pairs("id", "nextval('s')")).expect("compiles"); + let mut row = vec![("id".to_string(), SqlValue::Int(7))]; + compiled.materialize_row(&mut row, &catalog).expect("row"); + assert_eq!(row, vec![("id".to_string(), SqlValue::Int(7))]); + assert_eq!(catalog.nextval_calls.get(), 0); + } + + #[test] + fn volatility_is_decided_once_at_compile_time() { + assert!( + CompiledDefault::declare("u", "UUID_V7()") + .expect("compiles") + .is_volatile() + ); + assert!( + CompiledDefault::declare("id", "nextval('s')") + .expect("compiles") + .is_volatile() + ); + assert!( + !CompiledDefault::declare("s", "'active'") + .expect("compiles") + .is_volatile() + ); + assert!( + !CompiledDefault::declare("n", "1 + 2") + .expect("compiles") + .is_volatile() + ); + } + + #[test] + fn setval_is_refused_as_a_default() { + let error = CompiledDefault::declare("id", "setval('s', 10)").expect_err("setval refused"); + assert!(matches!(error, SqlError::SetvalInColumnDefault { .. })); + } + + #[test] + fn an_unregistered_function_is_refused_at_declaration() { + let error = CompiledDefault::declare("a", "no_such_function_here('x')") + .expect_err("unknown function refused"); + assert!(matches!(error, SqlError::UndefinedFunction { .. })); + } + + #[test] + fn an_unregistered_function_is_unevaluable_at_insert() { + let error = CompiledDefault::compile("a", "no_such_function_here('x')") + .expect_err("unknown function refused"); + assert!(matches!(error, SqlError::UnevaluableDefault { .. })); + } +} diff --git a/nodedb-sql/src/planner/defaults/convert.rs b/nodedb-sql/src/planner/defaults/convert.rs new file mode 100644 index 000000000..6817a5cca --- /dev/null +++ b/nodedb-sql/src/planner/defaults/convert.rs @@ -0,0 +1,60 @@ +// SPDX-License-Identifier: Apache-2.0 + +//! Conversion between the evaluator's value type and the planner's literal type. + +use crate::error::SqlError; +use crate::types::SqlValue; + +/// Convert an evaluated SQL literal into the engine-facing value type. +pub(super) fn sql_value_to_ndb(v: SqlValue) -> nodedb_types::Value { + match v { + SqlValue::Null => nodedb_types::Value::Null, + SqlValue::Bool(b) => nodedb_types::Value::Bool(b), + SqlValue::Int(i) => nodedb_types::Value::Integer(i), + SqlValue::Float(f) => nodedb_types::Value::Float(f), + SqlValue::Decimal(d) => nodedb_types::Value::Decimal(d), + SqlValue::String(s) => nodedb_types::Value::String(s), + SqlValue::Bytes(b) => nodedb_types::Value::Bytes(b), + SqlValue::Array(a) => { + nodedb_types::Value::Array(a.into_iter().map(sql_value_to_ndb).collect()) + } + SqlValue::Timestamp(dt) => nodedb_types::Value::NaiveDateTime(dt), + SqlValue::Timestamptz(dt) => nodedb_types::Value::DateTime(dt), + } +} + +/// Convert an evaluated default back into the planner's literal type. +/// +/// The inverse of `sql_value_to_ndb` above, which is the only producer of +/// these values — so every shape the evaluator can emit has an exact +/// counterpart here. Anything else raises rather than rendering through +/// `Debug`: a `DEFAULT` that stored `Uuid("…")` as its own debug text is the +/// same class of defect as dropping it, and harder to notice because the +/// column looks populated. +pub fn default_value_to_sql(column: &str, value: nodedb_types::Value) -> crate::Result { + Ok(match value { + nodedb_types::Value::Null => SqlValue::Null, + nodedb_types::Value::Bool(b) => SqlValue::Bool(b), + nodedb_types::Value::Integer(i) => SqlValue::Int(i), + nodedb_types::Value::Float(f) => SqlValue::Float(f), + nodedb_types::Value::Decimal(d) => SqlValue::Decimal(d), + nodedb_types::Value::String(s) => SqlValue::String(s), + nodedb_types::Value::Bytes(b) => SqlValue::Bytes(b), + nodedb_types::Value::NaiveDateTime(dt) => SqlValue::Timestamp(dt), + nodedb_types::Value::DateTime(dt) => SqlValue::Timestamptz(dt), + nodedb_types::Value::Array(items) => SqlValue::Array( + items + .into_iter() + .map(|item| default_value_to_sql(column, item)) + .collect::>>()?, + ), + other => { + return Err(SqlError::Unsupported { + detail: format!( + "default for column '{column}' evaluates to a value with no SQL literal \ + form: {other:?}" + ), + }); + } + }) +} diff --git a/nodedb-sql/src/planner/defaults/kind.rs b/nodedb-sql/src/planner/defaults/kind.rs new file mode 100644 index 000000000..2e1b2deab --- /dev/null +++ b/nodedb-sql/src/planner/defaults/kind.rs @@ -0,0 +1,138 @@ +// SPDX-License-Identifier: Apache-2.0 + +//! The compiled forms a column DEFAULT classifies into. + +use crate::error::SqlError; +use crate::types::SqlExpr; + +/// What a DEFAULT expression compiles to. +/// +/// Classification happens once per statement. Nothing here holds the original +/// text, so evaluation cannot fall back to parsing it again. +#[derive(Debug, Clone)] +pub(super) enum DefaultKind { + /// A generator that produces a fresh value on every call. + Generator(Generator), + /// A constant the declaration spells out. + Literal(nodedb_types::Value), + /// A parsed expression: a sequence accessor, or a const-folder target. + Expr(SqlExpr), +} + +/// A DEFAULT spelled as a value generator rather than a registered call. +/// +/// The catalog accepts both the bare form (`UUID_V7`) and the call form +/// (`UUID_V7()`), so both spellings classify to the same variant. +#[derive(Debug, Clone, Copy)] +pub(super) enum Generator { + UuidV7, + UuidV4, + Ulid, + Cuid2, + Cuid2Len(usize), + Nanoid, + NanoidLen(usize), + Now, +} + +impl Generator { + /// Produce one fresh value. + pub(super) fn generate(self) -> crate::Result { + let value = match self { + Self::UuidV7 => nodedb_types::Value::String(nodedb_types::id_gen::uuid_v7()), + Self::UuidV4 => nodedb_types::Value::String(nodedb_types::id_gen::uuid_v4()), + Self::Ulid => nodedb_types::Value::String(nodedb_types::id_gen::ulid()), + Self::Cuid2 => nodedb_types::Value::String(nodedb_types::id_gen::cuid2()), + Self::Cuid2Len(len) => nodedb_types::Value::String(cuid2_with_length(len)?), + Self::Nanoid => nodedb_types::Value::String(nodedb_types::id_gen::nanoid()), + Self::NanoidLen(len) => { + nodedb_types::Value::String(nodedb_types::id_gen::nanoid_with_length(len)) + } + Self::Now => nodedb_types::Value::String(now_rfc3339()), + }; + Ok(value) + } +} + +/// Generate a CUID2 of `len` characters, mapping a rejected length to a +/// planning error. +pub(super) fn cuid2_with_length(len: usize) -> crate::Result { + nodedb_types::id_gen::cuid2_with_length(len).map_err(|e| SqlError::Parse { + detail: format!("CUID2({len}) default expression is invalid: {e}"), + }) +} + +/// Render the current wall-clock instant the way `DEFAULT NOW()` stores it. +fn now_rfc3339() -> String { + let now = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap_or_default(); + chrono::DateTime::from_timestamp_millis(now.as_millis() as i64) + .map(|dt| dt.to_rfc3339()) + .unwrap_or_else(|| now.as_millis().to_string()) +} + +/// Classify the keyword-spelled defaults: the ID generators and `NOW()`. +/// +/// Returns `None` for every other expression. This is the one list of keyword +/// forms; the DDL gate classifies through it rather than repeating it. +pub(super) fn keyword_generator(upper: &str) -> Option { + let generator = match upper { + "UUID_V7" | "UUIDV7" | "GEN_UUID_V7()" | "UUID_V7()" => Generator::UuidV7, + "UUID_V4" | "UUIDV4" | "UUID" | "GEN_UUID_V4()" | "UUID_V4()" => Generator::UuidV4, + "ULID" | "GEN_ULID()" | "ULID()" => Generator::Ulid, + "CUID2" | "CUID2()" => Generator::Cuid2, + "NANOID" | "NANOID()" => Generator::Nanoid, + "NOW()" => Generator::Now, + _ => return None, + }; + Some(generator) +} + +/// Classify the parametric ID generators and the bare literals. +/// +/// Returns `Ok(None)` when `expr` is none of them, leaving it to the parser. +/// This is the one list of literal forms; the DDL gate reuses it. +/// +/// `CUID2(N)` validates its length here, so a rejected length raises at +/// declaration rather than at the first insert. +pub(super) fn parametric_or_literal(expr: &str, upper: &str) -> crate::Result> { + // NANOID(N) — custom length. + if upper.starts_with("NANOID(") && upper.ends_with(')') { + let len_str = &upper[7..upper.len() - 1]; + if let Ok(len) = len_str.parse::() { + return Ok(Some(DefaultKind::Generator(Generator::NanoidLen(len)))); + } + } + // CUID2(N) — custom length; validates length range and surfaces planning errors. + if upper.starts_with("CUID2(") && upper.ends_with(')') { + let len_str = &upper[6..upper.len() - 1]; + if let Ok(len) = len_str.parse::() { + // One generation checks the length range against the one authority + // for it. The value is discarded; each row generates its own. + cuid2_with_length(len)?; + return Ok(Some(DefaultKind::Generator(Generator::Cuid2Len(len)))); + } + } + // Numeric literal. + if let Ok(i) = expr.trim().parse::() { + return Ok(Some(DefaultKind::Literal(nodedb_types::Value::Integer(i)))); + } + if let Ok(f) = expr.trim().parse::() { + return Ok(Some(DefaultKind::Literal(nodedb_types::Value::Float(f)))); + } + // Quoted string literal. + // A length of two is the shortest quoted literal, the empty string. One + // lone quote character opens a literal nothing closes, so it is not one. + let trimmed = expr.trim(); + if trimmed.len() >= 2 + && ((trimmed.starts_with('\'') && trimmed.ends_with('\'')) + || (trimmed.starts_with('"') && trimmed.ends_with('"'))) + { + return Ok(Some(DefaultKind::Literal(nodedb_types::Value::String( + trimmed[1..trimmed.len() - 1].to_string(), + )))); + } + + Ok(None) +} diff --git a/nodedb-sql/src/planner/defaults/mod.rs b/nodedb-sql/src/planner/defaults/mod.rs new file mode 100644 index 000000000..98b05a51b --- /dev/null +++ b/nodedb-sql/src/planner/defaults/mod.rs @@ -0,0 +1,30 @@ +// SPDX-License-Identifier: Apache-2.0 + +//! Column DEFAULT expression compilation and evaluation at insert time. +//! +//! A declared DEFAULT is stored in the catalog as text. [`ColumnDefaults`] +//! compiles that text ONCE per statement; the per-row path evaluates the +//! compiled form and never sees a string, so it cannot re-parse. +//! +//! Supported forms: ID generation functions (UUIDv4/v7, ULID, CUID2, NANOID), +//! `NOW()`, sequence accessors (`nextval`, `currval`), literals, and any other +//! expression the plan-time const-folder can resolve. +//! +//! A DEFAULT that cannot be evaluated raises [`crate::SqlError::UnevaluableDefault`]. +//! The column is never omitted: an omitted column stores NULL where the +//! declaration promised a value, and nothing reports it. +//! +//! Lives in the SQL crate rather than beside one engine's converter because +//! every engine that materializes a DEFAULT has to produce the SAME value for +//! the same expression — a `DEFAULT now()` that means one thing on a document +//! collection and another on a key-value one would be a difference nobody +//! declared. The key-value planner also needs it BEFORE its declared-type +//! coercion and range checks run, so a materialized default is validated +//! exactly like a supplied one. + +mod compiled; +mod convert; +mod kind; + +pub use compiled::{ColumnDefaults, CompiledDefault, validate_default_expr}; +pub use convert::default_value_to_sql; diff --git a/nodedb-sql/src/planner/dml.rs b/nodedb-sql/src/planner/dml.rs index 5ee92a914..350e078fd 100644 --- a/nodedb-sql/src/planner/dml.rs +++ b/nodedb-sql/src/planner/dml.rs @@ -6,9 +6,10 @@ use nodedb_types::DatabaseId; use sqlparser::ast::{self}; use super::dml_helpers::{ - bind_insert_select_columns, build_kv_insert_plan, build_vector_primary_insert_plan, - check_declared_float_ranges_in_assignments, check_declared_int_ranges_in_assignments, - coerce_and_check_rows, convert_value_rows, resolve_insert_columns, + KvInsertParams, bind_insert_select_columns, build_kv_insert_plan, + build_vector_primary_insert_plan, check_declared_float_ranges_in_assignments, + check_declared_int_ranges_in_assignments, coerce_and_check_rows, convert_value_rows, + materialize_defaults_in_rows, resolve_insert_columns, }; use crate::engine_rules::{self, InsertParams}; use crate::error::{Result, SqlError}; @@ -177,15 +178,16 @@ pub fn plan_insert(ins: &ast::Insert, catalog: &dyn SqlCatalog) -> Result Result Result], + catalog: &dyn SqlCatalog, +) -> Result { + let compiled = ColumnDefaults::compile_columns(declared_columns)?; + if compiled.is_empty() { + return Ok(false); + } + let mut volatile = false; + for row in rows.iter_mut() { + volatile |= compiled.materialize_row(row, catalog)?; + } + Ok(volatile) +} diff --git a/nodedb-sql/src/planner/dml_helpers/kv_insert.rs b/nodedb-sql/src/planner/dml_helpers/kv_insert.rs index e3a7d0a3a..68525971e 100644 --- a/nodedb-sql/src/planner/dml_helpers/kv_insert.rs +++ b/nodedb-sql/src/planner/dml_helpers/kv_insert.rs @@ -3,8 +3,7 @@ //! Plan construction for the KV engine's `VALUES`-clause insert paths //! (plain `INSERT`, `UPSERT`, and `INSERT ... ON CONFLICT DO UPDATE`). -use sqlparser::ast; - +use super::params::KvInsertParams; use super::range_check::{ check_declared_float_ranges, check_declared_float_ranges_in_assignments, check_declared_int_ranges, check_declared_int_ranges_in_assignments, @@ -14,6 +13,7 @@ use crate::error::{Result, SqlError}; use crate::planner::declared_type_coerce::{ coerce_assignments_to_declared_types, coerce_row_to_declared_types, }; +use crate::planner::defaults::ColumnDefaults; use crate::types::*; /// Build a `SqlPlan::KvInsert` from a VALUES clause. Shared by plain INSERT, @@ -21,20 +21,22 @@ use crate::types::*; /// differ only in `intent` and `on_conflict_updates`, never in how entries /// are extracted from the row exprs. /// -/// `pk_col` is the schema-defined primary-key column name from +/// `params.pk_col` is the schema-defined primary-key column name from /// `CollectionInfo::primary_key`. When supplied, that column is used as /// the KV key regardless of whether it is named `"key"`. Falls back to /// the literal name `"key"` when `pk_col` is `None` (legacy / generic /// KV collections that use the built-in key/value column convention). -pub(crate) fn build_kv_insert_plan( - table_name: String, - columns: &[String], - rows_ast: &[ast::Parens>], - intent: KvInsertIntent, - mut on_conflict_updates: Vec<(String, SqlExpr)>, - pk_col: Option<&str>, - declared_columns: &[ColumnInfo], -) -> Result> { +pub(crate) fn build_kv_insert_plan(params: KvInsertParams<'_>) -> Result> { + let KvInsertParams { + collection: table_name, + columns, + rows_ast, + intent, + mut on_conflict_updates, + pk_col, + declared_columns, + catalog, + } = params; // Positional KV insert (no column list): the key/value split below is // driven entirely by matching column *names* against `key_col_name`/ // `"ttl"`. With an empty `columns` list there is no key to bind to, so @@ -70,14 +72,18 @@ pub(crate) fn build_kv_insert_plan( // string — while `RowDescription` advertises the declared numeric type, and // the read path can only encode SQL NULL for it. See // `declared_type_coerce` for the full rationale. + // Compiled once, outside the row loop: a DEFAULT expression's parse is a + // property of the declaration, not of the row. + let compiled_defaults = ColumnDefaults::compile_columns(declared_columns)?; let mut coerced_rows: Vec> = Vec::with_capacity(rows_ast.len()); + let mut volatile_defaults = false; for row_exprs in rows_ast { let mut row: Vec<(String, SqlValue)> = Vec::with_capacity(columns.len()); for (i, col) in columns.iter().enumerate() { let Some(expr) = row_exprs.get(i) else { break }; row.push((col.clone(), expr_to_sql_value(expr)?)); } - materialize_declared_defaults(declared_columns, &mut row)?; + volatile_defaults |= compiled_defaults.materialize_row(&mut row, catalog)?; // The key column is exempt — see `coerce_rows_to_declared_types`. coerce_row_to_declared_types(declared_columns, &mut row, Some(key_col_name))?; coerced_rows.push(row); @@ -130,95 +136,34 @@ pub(crate) fn build_kv_insert_plan( ttl_secs, intent, on_conflict_updates, + volatile_defaults, }]) } -/// Fill in every declared column the statement omitted that carries a DEFAULT. -/// -/// The key-value engine stores the bytes it is handed and has no typed write -/// path, so a DEFAULT that is not materialized HERE is materialized nowhere: -/// the catalog would keep the declaration and every read return nothing for it. -/// Documents and columnar rows expand theirs through the same -/// `evaluate_default_expr`, so one expression yields one value on every engine. -/// -/// Two rules the ordering encodes: -/// -/// - A column the statement SUPPLIED is never touched, and that includes an -/// explicit `NULL`. `NULL` is a value the author chose; overwriting it with -/// the default would make it impossible to store one. -/// - Materialized values are appended BEFORE the caller's declared-type -/// coercion and range checks, so a default is validated exactly like a -/// supplied literal. Filling them in afterwards would make `DEFAULT 999999` -/// on a `SMALLINT` column a way to store a value the same literal is -/// rejected for. -fn materialize_declared_defaults( - declared_columns: &[ColumnInfo], - row: &mut Vec<(String, SqlValue)>, -) -> Result<()> { - for column in declared_columns { - let Some(default_expr) = column.default.as_deref() else { - continue; - }; - if row.iter().any(|(name, _)| name == &column.name) { - continue; - } - let evaluated = - crate::planner::defaults::evaluate_default_expr(default_expr).map_err(|e| { - SqlError::Parse { - detail: format!("default for column '{}' is invalid: {e}", column.name), - } - })?; - let Some(evaluated) = evaluated else { continue }; - let value = nodedb_value_to_sql_value(&column.name, evaluated)?; - row.push((column.name.clone(), value)); - } - Ok(()) -} - -/// Convert an evaluated default back into the planner's literal type. -/// -/// The inverse of `sql_value_to_ndb` in `planner::defaults`, which is the only -/// producer of these values — so every shape the evaluator can emit has an -/// exact counterpart here. Anything else is rejected rather than rendered -/// through `Debug`: a `DEFAULT` that silently stored `Uuid("…")` as its own -/// debug text would be the same class of defect as dropping it entirely, but -/// harder to notice because the column would look populated. -fn nodedb_value_to_sql_value(column: &str, value: nodedb_types::Value) -> Result { - Ok(match value { - nodedb_types::Value::Null => SqlValue::Null, - nodedb_types::Value::Bool(b) => SqlValue::Bool(b), - nodedb_types::Value::Integer(i) => SqlValue::Int(i), - nodedb_types::Value::Float(f) => SqlValue::Float(f), - nodedb_types::Value::Decimal(d) => SqlValue::Decimal(d), - nodedb_types::Value::String(s) => SqlValue::String(s), - nodedb_types::Value::Bytes(b) => SqlValue::Bytes(b), - nodedb_types::Value::NaiveDateTime(dt) => SqlValue::Timestamp(dt), - nodedb_types::Value::DateTime(dt) => SqlValue::Timestamptz(dt), - nodedb_types::Value::Array(items) => SqlValue::Array( - items - .into_iter() - .map(|item| nodedb_value_to_sql_value(column, item)) - .collect::>>()?, - ), - other => { - return Err(SqlError::Unsupported { - detail: format!( - "default for column '{column}' evaluates to a value with no SQL literal \ - form: {other:?}" - ), - }); - } - }) -} - #[cfg(test)] mod kv_on_conflict_range_tests { + use sqlparser::ast; use sqlparser::ast::{Expr, Value, ValueWithSpan}; use sqlparser::tokenizer::Span; use super::*; + use crate::catalog::SqlCatalog; use nodedb_types::columnar::{FloatWidth, IntWidth}; + /// A catalog with no collections and no sequence state. These cases + /// declare no DEFAULT, so no accessor is ever reached. + struct NoCatalog; + + impl SqlCatalog for NoCatalog { + fn get_collection( + &self, + _database_id: nodedb_types::DatabaseId, + _name: &str, + ) -> std::result::Result, crate::catalog::SqlCatalogError> { + Ok(None) + } + } + fn string_column(name: &str) -> ColumnInfo { ColumnInfo { name: name.to_string(), @@ -274,15 +219,16 @@ mod kv_on_conflict_range_tests { int_column("n", Some(IntWidth::I32)), float_column("r", Some(FloatWidth::F32)), ]; - build_kv_insert_plan( - "t".to_string(), - &["key".to_string()], - &[ast::Parens::with_empty_span(vec![key_value_expr("a")])], - KvInsertIntent::Put, - updates, - Some("key"), - &declared, - ) + build_kv_insert_plan(KvInsertParams { + collection: "t".to_string(), + columns: &["key".to_string()], + rows_ast: &[ast::Parens::with_empty_span(vec![key_value_expr("a")])], + intent: KvInsertIntent::Put, + on_conflict_updates: updates, + pk_col: Some("key"), + declared_columns: &declared, + catalog: &NoCatalog, + }) } #[test] diff --git a/nodedb-sql/src/planner/dml_helpers/mod.rs b/nodedb-sql/src/planner/dml_helpers/mod.rs index 40fdeb125..242266de8 100644 --- a/nodedb-sql/src/planner/dml_helpers/mod.rs +++ b/nodedb-sql/src/planner/dml_helpers/mod.rs @@ -5,23 +5,29 @@ //! - [`range_check`] — declared-width coercion + range validation //! - [`insert_columns`] — positional-insert column resolution //! - [`ast_extract`] — table-name / primary-key point-lookup extraction +//! - [`declared_defaults`] — declared column DEFAULT materialization //! - [`vector_primary_insert`] — vector-primary collection insert plans //! - [`kv_insert`] — KV engine insert plans //! - [`insert_select_bind`] — `INSERT ... SELECT` target-column binding +//! - [`params`] — parameter structs for the helpers above mod ast_extract; +mod declared_defaults; mod insert_columns; mod insert_select_bind; mod kv_insert; +mod params; mod range_check; mod value_convert; mod vector_primary_insert; pub use ast_extract::extract_point_keys; pub(super) use ast_extract::extract_table_name_from_table_with_joins; +pub(super) use declared_defaults::materialize_defaults_in_rows; pub(super) use insert_columns::resolve_insert_columns; pub(super) use insert_select_bind::bind_insert_select_columns; pub(super) use kv_insert::build_kv_insert_plan; +pub(super) use params::KvInsertParams; pub(super) use range_check::{ check_declared_float_ranges_in_assignments, check_declared_int_ranges_in_assignments, coerce_and_check_rows, diff --git a/nodedb-sql/src/planner/dml_helpers/params.rs b/nodedb-sql/src/planner/dml_helpers/params.rs new file mode 100644 index 000000000..db7a678e9 --- /dev/null +++ b/nodedb-sql/src/planner/dml_helpers/params.rs @@ -0,0 +1,27 @@ +// SPDX-License-Identifier: Apache-2.0 + +//! Parameter structs passed into DML planning helpers. + +use sqlparser::ast; + +use crate::catalog::SqlCatalog; +use crate::types::*; + +/// Parameters for [`super::build_kv_insert_plan`], shared by plain `INSERT`, +/// `UPSERT`, and `INSERT ... ON CONFLICT (key) DO UPDATE` against the KV +/// engine — the three paths differ only in `intent` and +/// `on_conflict_updates`. +pub(crate) struct KvInsertParams<'a> { + pub collection: String, + pub columns: &'a [String], + pub rows_ast: &'a [ast::Parens>], + pub intent: KvInsertIntent, + pub on_conflict_updates: Vec<(String, SqlExpr)>, + /// Schema-defined primary-key column name from `CollectionInfo::primary_key`. + /// When supplied, that column is used as the KV key regardless of + /// whether it is named `"key"`. Falls back to the literal name `"key"` + /// when `None` (legacy / generic KV collections). + pub pk_col: Option<&'a str>, + pub declared_columns: &'a [ColumnInfo], + pub catalog: &'a dyn SqlCatalog, +} diff --git a/nodedb-sql/src/planner/dml_helpers/vector_primary_insert.rs b/nodedb-sql/src/planner/dml_helpers/vector_primary_insert.rs index 934888e84..1b48ba1ad 100644 --- a/nodedb-sql/src/planner/dml_helpers/vector_primary_insert.rs +++ b/nodedb-sql/src/planner/dml_helpers/vector_primary_insert.rs @@ -10,11 +10,17 @@ use crate::types::*; /// Extracts the vector-field column into `vector: Vec` and collects /// all remaining columns into `payload_fields`. Rows missing the vector /// column are rejected. +/// +/// `rows` arrive with every declared DEFAULT already materialized, so a +/// defaulted key or payload column reaches `payload_fields` like a supplied +/// one. `volatile_defaults` reports whether any of those defaults was volatile, +/// which keeps the plan out of the physical-plan cache. pub(crate) fn build_vector_primary_insert_plan( collection: &str, vpc: &nodedb_types::VectorPrimaryConfig, _columns: &[String], rows: Vec>, + volatile_defaults: bool, ) -> Result> { let mut result_rows = Vec::with_capacity(rows.len()); for row in rows { @@ -82,5 +88,6 @@ pub(crate) fn build_vector_primary_insert_plan( storage_dtype: vpc.storage_dtype, payload_indexes: vpc.payload_indexes.clone(), rows: result_rows, + volatile_defaults, }]) } diff --git a/nodedb-sql/src/planner/select/helpers.rs b/nodedb-sql/src/planner/select/helpers.rs index 08631f839..d8bcc2f2c 100644 --- a/nodedb-sql/src/planner/select/helpers.rs +++ b/nodedb-sql/src/planner/select/helpers.rs @@ -6,7 +6,6 @@ use sqlparser::ast; use crate::error::{Result, SqlError}; -use crate::functions::registry::FunctionRegistry; use crate::parser::normalize::{SCHEMA_QUALIFIED_MSG, normalize_ident}; use crate::resolver::ColumnScope; use crate::resolver::columns::TableScope; @@ -184,22 +183,6 @@ pub fn extract_func_args(func: &ast::Function) -> Result> { } } -/// Evaluate a constant SqlExpr to a SqlValue. Delegates to the shared -/// `const_fold::fold_constant` helper so that zero-arg scalar functions -/// like `now()` and `current_timestamp` go through the same evaluator -/// as the runtime expression path. -/// -/// An expression that is not constant yields NULL — this is the from-less -/// SELECT path, which has no row scope to defer to. An expression that *is* -/// constant and failed to evaluate raises instead: it can never succeed, so -/// NULL would be a wrong answer rather than an unknown one. -pub(crate) fn eval_constant_expr( - expr: &SqlExpr, - functions: &FunctionRegistry, -) -> crate::Result { - Ok(crate::planner::const_fold::fold_constant(expr, functions)?.unwrap_or(SqlValue::Null)) -} - pub(super) fn extract_column_name(expr: &ast::Expr) -> Result { match expr { ast::Expr::Identifier(ident) => Ok(normalize_ident(ident)), diff --git a/nodedb-sql/src/planner/select/select_stmt.rs b/nodedb-sql/src/planner/select/select_stmt.rs index e6b6fa18e..3ab107f9e 100644 --- a/nodedb-sql/src/planner/select/select_stmt.rs +++ b/nodedb-sql/src/planner/select/select_stmt.rs @@ -75,10 +75,12 @@ pub(super) fn plan_select( let projection = convert_projection(&select.projection, &scope)?; let mut columns = Vec::new(); let mut values = Vec::new(); + let mut volatile = false; for (i, proj) in projection.iter().enumerate() { match proj { Projection::Computed { expr, alias } => { columns.push(alias.clone()); + volatile |= crate::types::plan::expr_is_volatile(expr); values.push(crate::planner::catalog_expr_fold::eval_catalog_constant( expr, catalog, functions, )?); @@ -94,7 +96,11 @@ pub(super) fn plan_select( } } return Ok(PlannedSelect { - plan: SqlPlan::ConstantResult { columns, values }, + plan: SqlPlan::ConstantResult { + columns, + values, + volatile, + }, scope, }); } @@ -188,6 +194,7 @@ pub(super) fn plan_select( Box::new(SqlPlan::ConstantResult { columns: Vec::new(), values: Vec::new(), + volatile: false, }), ); for sq in subquery_joins diff --git a/nodedb-sql/src/resolver/columns.rs b/nodedb-sql/src/resolver/columns.rs index 463859824..cc5d7f18b 100644 --- a/nodedb-sql/src/resolver/columns.rs +++ b/nodedb-sql/src/resolver/columns.rs @@ -123,6 +123,15 @@ impl TableScope { } } + /// Whether an expression here is evaluated once per row of some relation. + /// + /// A scope with no relation of its own and no enclosing query stands + /// behind a FROM-less `SELECT`, which produces exactly one row and + /// evaluates its projection at plan time. + pub fn is_row_scope(&self) -> bool { + !self.tables.is_empty() || self.outer.is_some() + } + /// A copy of this scope nested inside `outer`, for planning a correlated /// subquery body. pub fn nested_in(mut self, outer: TableScope) -> Self { diff --git a/nodedb-sql/src/resolver/expr/functions.rs b/nodedb-sql/src/resolver/expr/functions.rs index f7bdf1980..10f0f6b58 100644 --- a/nodedb-sql/src/resolver/expr/functions.rs +++ b/nodedb-sql/src/resolver/expr/functions.rs @@ -86,6 +86,16 @@ pub(super) fn convert_function_depth( return Err(SqlError::UndefinedFunction { name }); } + // Per-row gate: a sequence accessor is evaluated at plan time, by + // `planner::catalog_expr_fold` for a FROM-less SELECT and by + // `planner::defaults` for a column DEFAULT. A call over a FROM relation + // reaches the row evaluator instead, which holds no sequence state and + // answers `NULL` for every row. Refuse it here so the statement fails + // loudly at plan time. + if scope.is_row_scope() && crate::functions::sequence_accessor::is_sequence_accessor(&name) { + return Err(SqlError::SequencePerRowUnsupported { name }); + } + let args = collect_function_args(func, depth, scope)?; let distinct = match &func.args { @@ -297,4 +307,41 @@ mod tests { "quoted 'UPPER' must still resolve via case-insensitive registry lookup" ); } + + /// A FROM-less SELECT resolves against an empty relation set, where the + /// planner evaluates the accessor through the catalog. + #[test] + fn a_sequence_accessor_resolves_without_a_relation() { + let func = function_ast("SELECT nextval('s')"); + let mut depth = 0; + let scope = crate::resolver::columns::TableScope::new(); + let expr = convert_function_depth(&func, &mut depth, &ColumnScope::Relations(&scope)) + .expect("a FROM-less sequence accessor must resolve"); + match expr { + SqlExpr::Function { name, .. } => assert_eq!(name, "nextval"), + other => panic!("expected SqlExpr::Function, got {other:?}"), + } + } + + /// A relation in scope makes the call per-row, which the row evaluator + /// cannot serve, so the resolver refuses it. + #[test] + fn a_sequence_accessor_over_a_relation_is_refused() { + let mut depth = 0; + let outer = crate::resolver::columns::TableScope::new(); + let scope = crate::resolver::columns::TableScope::new().nested_in(outer); + for call in [ + "SELECT nextval('s')", + "SELECT currval('s')", + "SELECT setval('s', 1)", + ] { + let func = function_ast(call); + let err = convert_function_depth(&func, &mut depth, &ColumnScope::Relations(&scope)) + .unwrap_err(); + assert!( + matches!(err, SqlError::SequencePerRowUnsupported { .. }), + "expected SqlError::SequencePerRowUnsupported for {call}, got {err:?}" + ); + } + } } diff --git a/nodedb-sql/src/resolver/scope.rs b/nodedb-sql/src/resolver/scope.rs index 378bd6bc5..e829dd79d 100644 --- a/nodedb-sql/src/resolver/scope.rs +++ b/nodedb-sql/src/resolver/scope.rs @@ -31,4 +31,15 @@ impl ColumnScope<'_> { Self::Relations(scope) => scope.check_name(table_ref, column), } } + + /// Whether an expression here is evaluated once per row of some relation. + /// + /// `Unchecked` stands behind stored DEFAULTs, index predicates, and the + /// constant folders, none of which iterate rows. + pub fn is_row_scope(&self) -> bool { + match self { + Self::Unchecked => false, + Self::Relations(scope) => scope.is_row_scope(), + } + } } diff --git a/nodedb-sql/src/types/mod.rs b/nodedb-sql/src/types/mod.rs index 167995ebe..064e722b9 100644 --- a/nodedb-sql/src/types/mod.rs +++ b/nodedb-sql/src/types/mod.rs @@ -15,7 +15,7 @@ pub use filter::{CompareOp, Filter, FilterExpr}; pub use plan::{ ArrayPrefilter, DistanceMetric, KvInsertIntent, MergeClauseKind, MergePlanAction, MergePlanClause, PlanCacheEligibility, SqlPlan, VectorAnnOptions, VectorPrimaryRow, - VectorQuantization, + VectorQuantization, WriteRoute, }; pub use query::{ AggOutputSlot, AggregateExpr, EngineType, JoinType, Projection, SortKey, SpatialPredicate, diff --git a/nodedb-sql/src/types/plan/cacheability.rs b/nodedb-sql/src/types/plan/cacheability.rs index 5f5e1f24c..c03d0b509 100644 --- a/nodedb-sql/src/types/plan/cacheability.rs +++ b/nodedb-sql/src/types/plan/cacheability.rs @@ -9,7 +9,8 @@ use super::SqlPlan; pub enum PlanCacheEligibility { /// Lowering depends only on schema/catalog descriptors tracked by the cache. Cacheable, - /// Lowering consults mutable row identity and must run for every execution. + /// Lowering consults mutable row identity, or the plan holds a volatile + /// expression, and must run for every execution. DataDependent, } @@ -34,10 +35,29 @@ impl SqlPlan { /// Document point operations resolve primary-key bytes to a surrogate while /// lowering. That binding can appear after an earlier miss without any /// schema-version change, so those physical tasks cannot be cached. + /// + /// A plan holding a volatile call is `DataDependent` for the same reason: + /// the call was evaluated while the plan was built, so caching it would + /// replay one execution's value into every later one. pub fn cache_eligibility(&self) -> PlanCacheEligibility { use PlanCacheEligibility::{Cacheable, DataDependent}; match self { + Self::ConstantResult { volatile: true, .. } => DataDependent, + Self::Insert { + column_defaults, .. + } + | Self::Upsert { + column_defaults, .. + } if super::volatility_scan::defaults_are_volatile(column_defaults) => DataDependent, + Self::KvInsert { + volatile_defaults: true, + .. + } + | Self::VectorPrimaryInsert { + volatile_defaults: true, + .. + } => DataDependent, Self::PointGet { engine: EngineType::DocumentSchemaless | EngineType::DocumentStrict, .. diff --git a/nodedb-sql/src/types/plan/mod.rs b/nodedb-sql/src/types/plan/mod.rs index 2f3388f7f..ad6020b61 100644 --- a/nodedb-sql/src/types/plan/mod.rs +++ b/nodedb-sql/src/types/plan/mod.rs @@ -8,9 +8,11 @@ mod row_types; mod variant_name; mod variants; mod vector_opts; +mod volatility_scan; pub use cacheability::PlanCacheEligibility; pub use merge_types::{MergeClauseKind, MergePlanAction, MergePlanClause}; -pub use row_types::{KvInsertIntent, VectorPrimaryRow}; +pub use row_types::{KvInsertIntent, VectorPrimaryRow, WriteRoute}; pub use variants::{DistanceMetric, SqlPlan}; pub use vector_opts::{ArrayPrefilter, VectorAnnOptions, VectorQuantization}; +pub use volatility_scan::{default_expr_is_volatile, defaults_are_volatile, expr_is_volatile}; diff --git a/nodedb-sql/src/types/plan/row_types.rs b/nodedb-sql/src/types/plan/row_types.rs index 3e24c9e92..b002fc657 100644 --- a/nodedb-sql/src/types/plan/row_types.rs +++ b/nodedb-sql/src/types/plan/row_types.rs @@ -35,3 +35,18 @@ pub enum KvInsertIntent { /// duplicate key overwrites. Also the shape used by the RESP SET path. Put, } + +/// The lowering a row-shaped write takes, carried on `SqlPlan::Insert` and +/// `SqlPlan::Upsert`. +/// +/// The engine's `EngineRules` picks it while it builds the variant, so the +/// conversion layer reads the route instead of re-deciding from `EngineType`. +/// An engine with no row-shaped lowering builds another variant entirely and +/// never names a route. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum WriteRoute { + /// One task per row, carrying a `DocumentOp` or a `CrdtOp`. + Document, + /// One batched columnar task for the whole statement. + ColumnarFamily, +} diff --git a/nodedb-sql/src/types/plan/variants.rs b/nodedb-sql/src/types/plan/variants.rs index 5e539e7b8..f5ad2b736 100644 --- a/nodedb-sql/src/types/plan/variants.rs +++ b/nodedb-sql/src/types/plan/variants.rs @@ -15,7 +15,7 @@ use crate::types::query::{ }; use super::merge_types::MergePlanClause; -use super::row_types::{KvInsertIntent, VectorPrimaryRow}; +use super::row_types::{KvInsertIntent, VectorPrimaryRow, WriteRoute}; use super::vector_opts::{ArrayPrefilter, VectorAnnOptions}; /// The top-level plan produced by the SQL planner. @@ -27,6 +27,10 @@ pub enum SqlPlan { ConstantResult { columns: Vec, values: Vec, + /// Whether any projected expression called a `Volatile` function. + /// The values were evaluated while this plan was built, so a cached + /// plan would replay them; a volatile plan is never cached. + volatile: bool, }, // ── Reads ── @@ -100,6 +104,9 @@ pub enum SqlPlan { Insert { collection: String, engine: EngineType, + /// The lowering these rows take, chosen by the engine's `EngineRules`. + /// The conversion layer reads it instead of re-deciding from `engine`. + route: WriteRoute, rows: Vec>, /// Column defaults from schema: `(column_name, default_expr)`. /// Used to auto-generate values for missing columns (e.g. `id` with `UUID_V7`). @@ -138,11 +145,18 @@ pub enum SqlPlan { /// Empty for plain UPSERT (whole-value overwrite) and for INSERT /// variants. on_conflict_updates: Vec<(String, SqlExpr)>, + /// Whether any DEFAULT materialized into `entries` came from a + /// `Volatile` expression. The key-value planner evaluates declared + /// defaults while building this plan, so a cached plan would replay + /// one execution's value; a volatile plan is never cached. + volatile_defaults: bool, }, /// UPSERT: insert or merge if document exists. Upsert { collection: String, engine: EngineType, + /// The lowering these rows take. Mirrors `Insert::route`. + route: WriteRoute, rows: Vec>, column_defaults: Vec<(String, String)>, /// `ON CONFLICT (...) DO UPDATE SET field = expr` assignments. @@ -717,6 +731,11 @@ pub enum SqlPlan { /// via `payload.add_index` on the first DirectUpsert. payload_indexes: Vec<(String, nodedb_types::PayloadIndexKind)>, rows: Vec, + /// Whether any row carries a value materialized from a volatile + /// DEFAULT such as `nextval`. The value was evaluated while the plan + /// was built, so caching the lowered tasks would replay one + /// execution's value into every later one. + volatile_defaults: bool, }, // ── Index DDL ─────────────────────────────────────────────────────── diff --git a/nodedb-sql/src/types/plan/volatility_scan.rs b/nodedb-sql/src/types/plan/volatility_scan.rs new file mode 100644 index 000000000..b72965c36 --- /dev/null +++ b/nodedb-sql/src/types/plan/volatility_scan.rs @@ -0,0 +1,80 @@ +// SPDX-License-Identifier: Apache-2.0 + +//! Volatility scanning over plan expressions and column DEFAULT strings. +//! +//! A plan holding a volatile call must re-plan per execution, so the plan +//! cache never replays a frozen value. + +use crate::planner::const_fold::default_registry; +use crate::types_expr::SqlExpr; + +/// Whether an expression tree contains a call to a `Volatile` function. +pub fn expr_is_volatile(expr: &SqlExpr) -> bool { + match expr { + SqlExpr::Function { name, args, .. } => { + default_registry().is_volatile(name) || args.iter().any(expr_is_volatile) + } + SqlExpr::BinaryOp { left, right, .. } => expr_is_volatile(left) || expr_is_volatile(right), + SqlExpr::UnaryOp { expr, .. } + | SqlExpr::Cast { expr, .. } + | SqlExpr::IsNull { expr, .. } => expr_is_volatile(expr), + SqlExpr::Case { + operand, + when_then, + else_expr, + } => { + operand.as_deref().is_some_and(expr_is_volatile) + || when_then + .iter() + .any(|(when, then)| expr_is_volatile(when) || expr_is_volatile(then)) + || else_expr.as_deref().is_some_and(expr_is_volatile) + } + SqlExpr::InList { expr, list, .. } => { + expr_is_volatile(expr) || list.iter().any(expr_is_volatile) + } + SqlExpr::Between { + expr, low, high, .. + } => expr_is_volatile(expr) || expr_is_volatile(low) || expr_is_volatile(high), + SqlExpr::Like { expr, pattern, .. } => expr_is_volatile(expr) || expr_is_volatile(pattern), + SqlExpr::ArrayLiteral(items) => items.iter().any(expr_is_volatile), + SqlExpr::Column { .. } | SqlExpr::Literal(_) | SqlExpr::Subquery(_) | SqlExpr::Wildcard => { + false + } + } +} + +/// Column DEFAULT spellings that generate a fresh value per row but name no +/// registered function. Kept in step with the generator arms of +/// `crate::planner::defaults`. +const VOLATILE_DEFAULT_ALIASES: &[&str] = + &["uuidv7", "uuidv4", "gen_uuid_v7", "gen_uuid_v4", "gen_ulid"]; + +/// Whether a stored column DEFAULT expression re-evaluates per execution. +/// +/// The catalog stores a DEFAULT as text, in either a bare form (`UUID_V7`) or +/// a call form (`uuid_v7()`, `nextval('s')`), so the leading identifier is +/// checked before the string is parsed as an expression. +pub fn default_expr_is_volatile(expr: &str) -> bool { + let trimmed = expr.trim(); + let head: String = trimmed + .chars() + .take_while(|c| c.is_alphanumeric() || *c == '_') + .collect::() + .to_ascii_lowercase(); + if !head.is_empty() + && (default_registry().is_volatile(&head) + || VOLATILE_DEFAULT_ALIASES.contains(&head.as_str())) + { + return true; + } + crate::parse_expr_string(trimmed) + .ok() + .is_some_and(|parsed| expr_is_volatile(&parsed)) +} + +/// Whether any `(column, default_expr)` pair re-evaluates per execution. +pub fn defaults_are_volatile(defaults: &[(String, String)]) -> bool { + defaults + .iter() + .any(|(_, expr)| default_expr_is_volatile(expr)) +} diff --git a/nodedb-sql/src/visitor/plan_visitor/args.rs b/nodedb-sql/src/visitor/plan_visitor/args.rs index 206457f44..81207a39a 100644 --- a/nodedb-sql/src/visitor/plan_visitor/args.rs +++ b/nodedb-sql/src/visitor/plan_visitor/args.rs @@ -9,7 +9,7 @@ use crate::temporal::TemporalScope; use crate::types::SqlPlan; use crate::types::filter::Filter; -use crate::types::plan::{ArrayPrefilter, MergePlanClause, VectorAnnOptions}; +use crate::types::plan::{ArrayPrefilter, MergePlanClause, VectorAnnOptions, WriteRoute}; use crate::types::query::{ AggregateExpr, EngineType, JoinType, Projection, SortKey, SpatialPredicate, WindowSpec, }; @@ -65,6 +65,8 @@ pub struct DocumentIndexLookupVisitArgs<'a> { pub struct InsertVisitArgs<'a> { pub collection: &'a str, pub engine: EngineType, + /// The lowering these rows take, decided by the engine's `EngineRules`. + pub route: WriteRoute, pub rows: &'a [Vec<(String, SqlValue)>], pub column_defaults: &'a [(String, String)], pub if_absent: bool, @@ -76,6 +78,8 @@ pub struct InsertVisitArgs<'a> { pub struct UpsertVisitArgs<'a> { pub collection: &'a str, pub engine: EngineType, + /// The lowering these rows take, decided by the engine's `EngineRules`. + pub route: WriteRoute, pub rows: &'a [Vec<(String, SqlValue)>], pub column_defaults: &'a [(String, String)], pub on_conflict_updates: &'a [(String, SqlExpr)], diff --git a/nodedb-sql/src/visitor/plan_visitor/dispatch.rs b/nodedb-sql/src/visitor/plan_visitor/dispatch.rs index 1357c1b1d..dc8c98930 100644 --- a/nodedb-sql/src/visitor/plan_visitor/dispatch.rs +++ b/nodedb-sql/src/visitor/plan_visitor/dispatch.rs @@ -17,7 +17,9 @@ use crate::types::SqlPlan; pub fn dispatch(visitor: &mut V, plan: &SqlPlan) -> Result { match plan { - SqlPlan::ConstantResult { columns, values } => visitor.constant_result(columns, values), + SqlPlan::ConstantResult { + columns, values, .. + } => visitor.constant_result(columns, values), SqlPlan::Scan { collection, alias, @@ -93,6 +95,7 @@ pub fn dispatch(visitor: &mut V, plan: &SqlPlan) -> Result(visitor: &mut V, plan: &SqlPlan) -> Result visitor.insert(InsertVisitArgs { collection, engine: *engine, + route: *route, rows, column_defaults, if_absent: *if_absent, @@ -113,10 +117,12 @@ pub fn dispatch(visitor: &mut V, plan: &SqlPlan) -> Result visitor.kv_insert(collection, entries, *ttl_secs, *intent, on_conflict_updates), SqlPlan::Upsert { collection, engine, + route, rows, column_defaults, on_conflict_updates, @@ -125,6 +131,7 @@ pub fn dispatch(visitor: &mut V, plan: &SqlPlan) -> Result visitor.upsert(UpsertVisitArgs { collection, engine: *engine, + route: *route, rows, column_defaults, on_conflict_updates, diff --git a/nodedb-sql/src/visitor/plan_visitor/dispatch_rest.rs b/nodedb-sql/src/visitor/plan_visitor/dispatch_rest.rs index 2572ae135..0f5f61452 100644 --- a/nodedb-sql/src/visitor/plan_visitor/dispatch_rest.rs +++ b/nodedb-sql/src/visitor/plan_visitor/dispatch_rest.rs @@ -162,6 +162,8 @@ pub(super) fn dispatch_rest( storage_dtype, payload_indexes, rows, + // Cache eligibility only; lowering does not read it. + volatile_defaults: _, } => visitor.vector_primary_insert( collection, field, diff --git a/nodedb-types/src/columnar/column_parse.rs b/nodedb-types/src/columnar/column_parse.rs index 880ac3d8c..b184352da 100644 --- a/nodedb-types/src/columnar/column_parse.rs +++ b/nodedb-types/src/columnar/column_parse.rs @@ -8,6 +8,36 @@ use std::str::FromStr; use super::column_type::ColumnType; +/// Every declared spelling that resolves to [`ColumnType::Int64`]. +/// +/// nodedb stores every integer as a full `i64`, so all of these collapse to +/// one storage variant. The declared width travels separately as an +/// [`IntWidth`](super::IntWidth), which narrows the advertised wire OID and +/// bounds writes. [`IntWidth::from_declared_type`](super::IntWidth::from_declared_type) +/// must recognize every spelling listed here — a spelling it does not know +/// advertises OID 20 (`bigint`) whatever the author declared. +pub const DECLARED_INT_KEYWORDS: [&str; 8] = [ + "BIGINT", "INT64", "INTEGER", "INT", "INT4", "INT8", "SMALLINT", "INT2", +]; + +/// Every declared spelling that resolves to [`ColumnType::Float64`]. +/// +/// The float counterpart of [`DECLARED_INT_KEYWORDS`]: nodedb stores every +/// float as a full `f64`, and +/// [`FloatWidth::from_declared_type`](super::FloatWidth::from_declared_type) +/// carries the declared width that narrows the advertised wire OID. It must +/// recognize every spelling listed here. +pub const DECLARED_FLOAT_KEYWORDS: [&str; 8] = [ + "FLOAT64", + "DOUBLE", + "DOUBLE PRECISION", + "FLOAT8", + "REAL", + "FLOAT4", + "FLOAT32", + "FLOAT", +]; + /// Error from parsing a column type string. #[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)] #[non_exhaustive] @@ -18,6 +48,8 @@ pub enum ColumnTypeParseError { UseTimestamp, #[error("invalid VECTOR dimension: '{0}' (must be a positive integer)")] InvalidVectorDim(String), + #[error("invalid VARCHAR length: '{0}' (must be a positive integer)")] + InvalidCharLength(String), #[error( "invalid DECIMAL/NUMERIC params: '{0}' (expected DECIMAL(precision, scale) with precision 1-38 and scale <= precision)" )] @@ -120,33 +152,35 @@ impl FromStr for ColumnType { return Ok(Self::Vector(dim)); } - match upper.as_str() { - // `INT4`/`INT8`/`SMALLINT`/`INT2` are PostgreSQL wire-width integer - // keywords: strict/kv `CREATE COLLECTION` must accept - // them as valid aliases, not reject them as unknown types. They - // all collapse to the same `Int64` storage variant as - // `BIGINT`/`INTEGER`/`INT` — nodedb always stores integers as a - // full i64. The declared width is carried separately as an - // [`super::IntWidth`], which is what bounds writes and narrows the - // advertised wire OID; it is deliberately not a storage variant. - "BIGINT" | "INT64" | "INTEGER" | "INT" | "INT4" | "INT8" | "SMALLINT" | "INT2" => { - Ok(Self::Int64) + // VARCHAR(n) special case. A declared character length is a wire + // contract, not a storage property — nodedb stores every string + // unbounded — so `VARCHAR(255)` resolves to the answer bare `VARCHAR` + // gives. The length still parses, so a malformed one errors here + // instead of resolving to a type the author did not write. + if let Some(rest) = upper.strip_prefix("VARCHAR") + && let Some(inner) = rest.strip_prefix('(').and_then(|r| r.strip_suffix(')')) + { + let length: u32 = inner + .trim() + .parse() + .map_err(|_| ColumnTypeParseError::InvalidCharLength(inner.trim().to_string()))?; + if length == 0 { + return Err(ColumnTypeParseError::InvalidCharLength("0".into())); } - // `FLOAT4`/`FLOAT8`/`FLOAT32`/`DOUBLE PRECISION` are PostgreSQL - // wire-width float keywords, rejected as unknown types here for - // the same reason `INT4` was. They all collapse to - // the same `Float64` storage variant as `DOUBLE`/`REAL`/`FLOAT` — - // nodedb always stores floats as a full f64. The declared width is - // carried separately as a [`super::FloatWidth`], which narrows the - // advertised wire OID; it is deliberately not a storage variant. - // Unlike integers it bounds no writes: narrowing a float rounds - // rather than wraps, and PostgreSQL accepts-and-rounds too. - "FLOAT64" | "DOUBLE" | "DOUBLE PRECISION" | "FLOAT8" | "REAL" | "FLOAT4" - | "FLOAT32" | "FLOAT" => Ok(Self::Float64), + return Ok(Self::String); + } + + match upper.as_str() { + // Every PostgreSQL wire-width integer keyword collapses to the + // one `Int64` storage variant; `DECLARED_INT_KEYWORDS` lists them + // and carries the width contract. + keyword if DECLARED_INT_KEYWORDS.contains(&keyword) => Ok(Self::Int64), + // The float counterpart, listed by `DECLARED_FLOAT_KEYWORDS`. + keyword if DECLARED_FLOAT_KEYWORDS.contains(&keyword) => Ok(Self::Float64), "TEXT" | "STRING" | "VARCHAR" => Ok(Self::String), "BOOL" | "BOOLEAN" => Ok(Self::Bool), "BYTES" | "BYTEA" | "BLOB" => Ok(Self::Bytes), - "TIMESTAMP" => Ok(Self::Timestamp), + "TIMESTAMP" | "TIMESTAMP WITHOUT TIME ZONE" => Ok(Self::Timestamp), "TIMESTAMPTZ" | "TIMESTAMP WITH TIME ZONE" => Ok(Self::Timestamptz), "SYSTEM_TIMESTAMP" | "SYSTEMTIMESTAMP" => Ok(Self::SystemTimestamp), "GEOMETRY" => Ok(Self::Geometry), @@ -169,3 +203,180 @@ impl FromStr for ColumnType { } } } + +impl ColumnType { + /// Resolve a declared DDL type string to a column type. + /// + /// This is the single answer to "which [`ColumnType`] does this declared + /// string denote", shared by both planes. The catalog records the raw DDL + /// text that followed the column name, so an entry reads `INT DEFAULT 5` + /// or `DECIMAL(10, 2) NOT NULL`, not `INT`. This resolves the leading type + /// token, so a trailing modifier never changes the answer. + /// + /// `None` means the token names no known type. A caller that needs a + /// fallback picks its own — this reports the absence rather than guessing. + /// + /// A multi-word spelling resolves from its first word: `DOUBLE PRECISION` + /// is [`ColumnType::Float64`], and `TIMESTAMP WITH TIME ZONE` reaching + /// here as catalog text resolves to [`ColumnType::Timestamp`]. Pass such a + /// spelling to [`str::parse`] instead to resolve it whole. + pub fn from_declared_type(declared: &str) -> Option { + bare_declared_token(declared).parse().ok() + } +} + +/// The leading type token of a declared DDL type string. +/// +/// The cut is the first whitespace outside parentheses, so a parameter list +/// keeps its internal spaces and `DECIMAL(10, 2) NOT NULL` yields +/// `DECIMAL(10, 2)`. A trailing comma left by a column-list split is dropped. +fn bare_declared_token(declared: &str) -> &str { + let trimmed = declared.trim_start(); + let mut depth = 0usize; + let mut end = trimmed.len(); + for (index, ch) in trimmed.char_indices() { + match ch { + '(' => depth += 1, + ')' => depth = depth.saturating_sub(1), + _ if depth == 0 && ch.is_whitespace() => { + end = index; + break; + } + _ => {} + } + } + trimmed.get(..end).unwrap_or(trimmed).trim_end_matches(',') +} + +#[cfg(test)] +mod tests { + use super::super::{FloatWidth, IntWidth}; + use super::*; + + /// Every spelling the parser resolves to `Int64` must also resolve to a + /// declared [`IntWidth`]. A spelling one side knows and the other does not + /// advertises the wrong `RowDescription` OID for the column. + /// + /// The list is the one the parser itself matches on, so adding a spelling + /// there extends this test rather than leaving it behind. + #[test] + fn every_declared_int_keyword_resolves_to_int64_and_a_width() { + for keyword in DECLARED_INT_KEYWORDS { + assert_eq!( + keyword.parse::(), + Ok(ColumnType::Int64), + "{keyword} must resolve to Int64" + ); + assert!( + IntWidth::from_declared_type(keyword).is_some(), + "{keyword} must resolve to a declared IntWidth" + ); + } + } + + /// The float counterpart of + /// [`every_declared_int_keyword_resolves_to_int64_and_a_width`]. + #[test] + fn every_declared_float_keyword_resolves_to_float64_and_a_width() { + for keyword in DECLARED_FLOAT_KEYWORDS { + assert_eq!( + keyword.parse::(), + Ok(ColumnType::Float64), + "{keyword} must resolve to Float64" + ); + assert!( + FloatWidth::from_declared_type(keyword).is_some(), + "{keyword} must resolve to a declared FloatWidth" + ); + } + } + + #[test] + fn declared_type_ignores_trailing_modifiers() { + assert_eq!( + ColumnType::from_declared_type("INT DEFAULT 5"), + Some(ColumnType::Int64) + ); + assert_eq!( + ColumnType::from_declared_type("TEXT NOT NULL PRIMARY KEY"), + Some(ColumnType::String) + ); + assert_eq!( + ColumnType::from_declared_type("timestamp time_key"), + Some(ColumnType::Timestamp) + ); + assert_eq!( + ColumnType::from_declared_type("BIGINT,"), + Some(ColumnType::Int64) + ); + } + + /// A parameter list keeps its internal spaces: cutting at the first + /// whitespace would leave `DECIMAL(10,` and lose the type. + #[test] + fn declared_type_keeps_a_spaced_parameter_list() { + assert_eq!( + ColumnType::from_declared_type("DECIMAL(10, 2) NOT NULL"), + Some(ColumnType::Decimal { + precision: 10, + scale: 2 + }) + ); + assert_eq!( + ColumnType::from_declared_type("VECTOR(768)"), + Some(ColumnType::Vector(768)) + ); + } + + /// The two zone spellings resolve whole, so a parser that reads the full + /// string reads the zone the author wrote. + #[test] + fn both_timestamp_zone_spellings_resolve_whole() { + assert_eq!( + "TIMESTAMP WITH TIME ZONE".parse::(), + Ok(ColumnType::Timestamptz) + ); + assert_eq!( + "timestamp without time zone".parse::(), + Ok(ColumnType::Timestamp) + ); + } + + /// A declared character length resolves to `String` and a malformed one + /// errors rather than resolving to an unwritten type. + #[test] + fn varchar_length_resolves_to_string() { + assert_eq!("VARCHAR(255)".parse::(), Ok(ColumnType::String)); + assert_eq!("varchar(1)".parse::(), Ok(ColumnType::String)); + assert_eq!( + ColumnType::from_declared_type("VARCHAR(8) NOT NULL"), + Some(ColumnType::String) + ); + assert_eq!( + "VARCHAR(0)".parse::(), + Err(ColumnTypeParseError::InvalidCharLength("0".into())) + ); + assert_eq!( + "VARCHAR(abc)".parse::(), + Err(ColumnTypeParseError::InvalidCharLength("ABC".into())) + ); + } + + #[test] + fn declared_type_reports_an_unknown_token_as_none() { + assert_eq!(ColumnType::from_declared_type("SOMETHING_ELSE"), None); + assert_eq!(ColumnType::from_declared_type(""), None); + assert_eq!(ColumnType::from_declared_type(" "), None); + } + + /// `Timestamp` and `Timestamptz` are instants; `SystemTimestamp` is + /// engine-assigned and is not. + #[test] + fn only_timestamp_types_are_instants() { + assert!(ColumnType::Timestamp.is_instant()); + assert!(ColumnType::Timestamptz.is_instant()); + assert!(!ColumnType::SystemTimestamp.is_instant()); + assert!(!ColumnType::Int64.is_instant()); + assert!(!ColumnType::String.is_instant()); + } +} diff --git a/nodedb-types/src/columnar/column_type.rs b/nodedb-types/src/columnar/column_type.rs index ec369fc6f..262ef622a 100644 --- a/nodedb-types/src/columnar/column_type.rs +++ b/nodedb-types/src/columnar/column_type.rs @@ -108,6 +108,19 @@ impl ColumnType { self.fixed_size().is_none() } + /// Whether a column of this type carries an instant. + /// + /// This is the single answer to that question for both planes: the + /// Control Plane types such a column as a timestamp on the wire, and the + /// Data Plane scales exactly these columns from the epoch milliseconds + /// storage holds to the epoch microseconds a client reads. + /// + /// `SystemTimestamp` is not an instant. It is engine-assigned from HLC at + /// commit and the planner types it as text. + pub const fn is_instant(&self) -> bool { + matches!(self, Self::Timestamp | Self::Timestamptz) + } + /// Return the canonical PostgreSQL type OID for this column type. /// /// This is the single authoritative mapping between NodeDB `ColumnType` diff --git a/nodedb-types/src/columnar/mod.rs b/nodedb-types/src/columnar/mod.rs index 2d55ef73c..b7741b8c4 100644 --- a/nodedb-types/src/columnar/mod.rs +++ b/nodedb-types/src/columnar/mod.rs @@ -13,7 +13,7 @@ pub mod schema; pub mod wal_record; pub use column_def::{ColumnDef, ColumnModifier}; -pub use column_parse::ColumnTypeParseError; +pub use column_parse::{ColumnTypeParseError, DECLARED_FLOAT_KEYWORDS, DECLARED_INT_KEYWORDS}; pub use column_type::ColumnType; pub use declared_type_keyword::declared_type_matches; pub use dml_wal_record::ColumnarDmlWalRecord; diff --git a/nodedb-types/src/error/ctors/read_query_auth.rs b/nodedb-types/src/error/ctors/read_query_auth.rs index 446d5fd97..9ac2d70a5 100644 --- a/nodedb-types/src/error/ctors/read_query_auth.rs +++ b/nodedb-types/src/error/ctors/read_query_auth.rs @@ -134,6 +134,32 @@ impl NodeDbError { } } + /// A statement names a database object that does not exist. Distinct from + /// `undefined_function`: the function exists, the object it names does + /// not. Renders as SQLSTATE `42704` (`undefined_object`). + pub fn undefined_object(object: impl Into) -> Self { + let object = object.into(); + Self { + code: ErrorCode::UNDEFINED_OBJECT, + message: format!("{object} does not exist"), + details: ErrorDetails::UndefinedObject { object }, + cause: None, + } + } + + /// An object exists but a prerequisite step has not run, such as `currval` + /// before this session called `nextval`. Renders as SQLSTATE `55000` + /// (`object_not_in_prerequisite_state`). + pub fn object_not_ready(object: impl Into, detail: impl Into) -> Self { + let object = object.into(); + Self { + code: ErrorCode::OBJECT_NOT_READY, + message: detail.into(), + details: ErrorDetails::ObjectNotReady { object }, + cause: None, + } + } + /// A column reference names no column of any relation in scope. Distinct /// from `plan_error` so clients match on the code rather than parsing the /// message. diff --git a/nodedb-types/src/error/sqlstate.rs b/nodedb-types/src/error/sqlstate.rs index ffea17029..021e979ac 100644 --- a/nodedb-types/src/error/sqlstate.rs +++ b/nodedb-types/src/error/sqlstate.rs @@ -175,6 +175,10 @@ pub const STATEMENT_TOO_COMPLEX: &str = "54001"; // ── Class 55 — Object Not In Prerequisite State ────────────────────────────── +/// `55000` — `object_not_in_prerequisite_state` (the object exists but the +/// step this statement needs has not run, e.g. `currval` before `nextval`) +pub const OBJECT_NOT_IN_PREREQUISITE_STATE: &str = "55000"; + /// `55P03` — `lock_not_available` (no cluster leader) pub const LOCK_NOT_AVAILABLE: &str = "55P03"; @@ -362,6 +366,7 @@ mod tests { CONFIGURATION_LIMIT_EXCEEDED, PROGRAM_LIMIT_EXCEEDED, STATEMENT_TOO_COMPLEX, + OBJECT_NOT_IN_PREREQUISITE_STATE, LOCK_NOT_AVAILABLE, QUERY_CANCELED, CANNOT_CONNECT_NOW, diff --git a/nodedb-types/src/lib.rs b/nodedb-types/src/lib.rs index 7d34bd951..afcad8014 100644 --- a/nodedb-types/src/lib.rs +++ b/nodedb-types/src/lib.rs @@ -69,6 +69,7 @@ pub mod vector_dtype; pub mod vector_index_params; pub mod vector_index_stats; pub mod vector_model; +pub mod volatility; pub mod wire_version; pub use approx::{CountMinSketch, HyperLogLog, SpaceSaving, TDigest}; @@ -139,3 +140,4 @@ pub use vector_dtype::VectorStorageDtype; pub use vector_index_params::StoredVectorIndexParams; pub use vector_index_stats::{VectorIndexQuantization, VectorIndexStats, VectorIndexType}; pub use vector_model::{VectorModelEntry, VectorModelMetadata}; +pub use volatility::Volatility; diff --git a/nodedb-types/src/volatility.rs b/nodedb-types/src/volatility.rs new file mode 100644 index 000000000..b1fb7b497 --- /dev/null +++ b/nodedb-types/src/volatility.rs @@ -0,0 +1,33 @@ +// SPDX-License-Identifier: Apache-2.0 + +//! Function volatility: how far a call's result can be reused. +//! +//! Volatility is orthogonal to a function's category. A function is +//! `Scalar` and `Volatile` at the same time. + +/// How far a function call's result can be reused. +/// +/// Two states, because the engine has two reuse boundaries: a value folded +/// into a cached plan, and a value produced fresh for one execution. A +/// per-statement memoization boundary would need a third state; no such +/// boundary exists, so no third state does either. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] +pub enum Volatility { + /// Same arguments always give the same result. Foldable at plan time. + #[default] + Immutable, + /// Can change per call, or has side effects. + Volatile, +} + +impl Volatility { + /// Whether a call can be folded to a literal at plan time. + pub fn is_foldable(self) -> bool { + matches!(self, Self::Immutable) + } + + /// Whether a call must re-evaluate on every execution. + pub fn is_volatile(self) -> bool { + matches!(self, Self::Volatile) + } +} diff --git a/nodedb/src/control/event_trigger.rs b/nodedb/src/control/event_trigger.rs index 35f387348..217e58a09 100644 --- a/nodedb/src/control/event_trigger.rs +++ b/nodedb/src/control/event_trigger.rs @@ -340,7 +340,7 @@ pub async fn run_event_action_sql( tenant_id, database_id, &security.context(&shared), - false, + None, ) .await .map_err(|source| TriggerActionError::Plan { source })?; diff --git a/nodedb/src/control/gateway/lowered_plan.rs b/nodedb/src/control/gateway/lowered_plan.rs new file mode 100644 index 000000000..e0a5d3b1c --- /dev/null +++ b/nodedb/src/control/gateway/lowered_plan.rs @@ -0,0 +1,48 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! The lowered plan the gateway executes, paired with its plan-cache verdict. + +use nodedb_physical::physical_plan::PhysicalPlan; +use nodedb_sql::types::PlanCacheEligibility; + +/// A `SqlPlan` lowered to a `PhysicalPlan`, carrying the verdict that decides +/// admission to the gateway plan cache. +/// +/// The verdict rides beside the plan instead of inside it. `PhysicalPlan` is +/// the Data Plane wire enum; cacheability is a Control Plane planning property +/// the Data Plane never reads. A field on the enum would widen the wire shape +/// and every match on it for no execution benefit. +#[derive(Debug, Clone)] +pub struct LoweredPlan { + /// The physical operation to execute. + pub plan: PhysicalPlan, + /// Whether the gateway can admit this plan to its cache. + pub cache_eligibility: PlanCacheEligibility, +} + +impl LoweredPlan { + /// Pair a lowered plan with the verdict its `SqlPlan` batch produced. + /// + /// Pass the value returned by + /// `crate::control::planner::sql_plan_convert::batch_cache_eligibility`. + /// Never derive a second volatility verdict here. + pub fn new(plan: PhysicalPlan, cache_eligibility: PlanCacheEligibility) -> Self { + Self { + plan, + cache_eligibility, + } + } + + /// Pair a plan built directly, with no `SqlPlan` behind it. + /// + /// A directly constructed operation carries no expression tree, so it + /// carries no volatile call. + pub fn cacheable(plan: PhysicalPlan) -> Self { + Self::new(plan, PlanCacheEligibility::Cacheable) + } + + /// Whether the gateway can admit this plan to its cache. + pub fn is_cacheable(&self) -> bool { + self.cache_eligibility.is_cacheable() + } +} diff --git a/nodedb/src/control/gateway/mod.rs b/nodedb/src/control/gateway/mod.rs index fc73a4c54..50058b703 100644 --- a/nodedb/src/control/gateway/mod.rs +++ b/nodedb/src/control/gateway/mod.rs @@ -9,6 +9,7 @@ pub mod error_map; pub mod fuser; pub mod invalidation; pub mod key_extractor; +pub mod lowered_plan; pub mod plan_cache; pub mod retry; pub mod route; @@ -22,6 +23,7 @@ pub use core::Gateway; pub use error_map::GatewayErrorMap; pub use invalidation::PlanCacheInvalidator; pub use key_extractor::{KeyExtractor, UnwiredKeyExtractor}; +pub use lowered_plan::LoweredPlan; pub use plan_cache::PlanCache; pub use route::{RouteDecision, TaskRoute}; pub use version_check::{DescriptorCheckError, check_descriptor_versions}; diff --git a/nodedb/src/control/gateway/sql_execute.rs b/nodedb/src/control/gateway/sql_execute.rs index 7fbe6f4f3..fc063cf2b 100644 --- a/nodedb/src/control/gateway/sql_execute.rs +++ b/nodedb/src/control/gateway/sql_execute.rs @@ -11,6 +11,7 @@ use crate::control::server::shared::clone_write::CloneCheckedTask; use nodedb_physical::physical_plan::PhysicalPlan; use super::core::{Gateway, QueryContext, authorized_plan_for_context}; +use super::lowered_plan::LoweredPlan; use super::plan_cache::{PlanCacheKey, SqlKey, hash_placeholder_types, hash_sql}; use super::version_set::{permission_tree_version_key, rls_version_key}; @@ -20,12 +21,17 @@ impl Gateway { /// `plan_fn` runs at most once on a cache miss. `authorize_fn` clone-checks /// and authorizes the exact plan — the cached plan or the newly planned /// value — the same way every other dispatch entry point does. + /// + /// `plan_fn` returns the plan together with its + /// [`nodedb_sql::types::PlanCacheEligibility`] verdict. A `DataDependent` + /// plan enters neither the plan cache nor the version-set side cache, so + /// the next execution re-plans and re-evaluates its volatile calls. pub async fn execute_sql( &self, ctx: &QueryContext, sql: &str, placeholder_types: &[&str], - plan_fn: impl FnOnce() -> Result, + plan_fn: impl FnOnce() -> Result, authorize_fn: Authorize, ) -> Result>, Error> where @@ -88,19 +94,29 @@ impl Gateway { } } - let plan = plan_fn()?; + let lowered = plan_fn()?; + let cacheable = lowered.is_cacheable(); + let plan = lowered.plan; let actual_vs = self .collect_version_set(&plan, ctx.tenant_id.as_u64(), ctx.database_id) .await?; - let actual_key = PlanCacheKey { - sql_text_hash: sql_hash, - placeholder_types_hash: ph_hash, - version_set: actual_vs.clone(), - }; - self.plan_cache - .insert_version_set(sql_key, actual_vs.clone()); - self.plan_cache.insert(actual_key, Arc::new(plan.clone())); + // A volatile plan froze its `nextval` / `now()` / UUID values while it + // was built. Admitting it would replay this execution's values into + // every later one, so it skips both caches and re-plans next time. + // The side cache is skipped too: its entry is pruned only alongside the + // plan entry it maps to, so storing one without a plan leaves an orphan + // that survives every DDL invalidation and buys no hit. + if cacheable { + let actual_key = PlanCacheKey { + sql_text_hash: sql_hash, + placeholder_types_hash: ph_hash, + version_set: actual_vs.clone(), + }; + self.plan_cache + .insert_version_set(sql_key, actual_vs.clone()); + self.plan_cache.insert(actual_key, Arc::new(plan.clone())); + } let checked = authorize_fn(plan.clone()).await?; let plan = authorized_plan_for_context(ctx, checked)?; diff --git a/nodedb/src/control/planner/catalog_adapter/adapter.rs b/nodedb/src/control/planner/catalog_adapter/adapter.rs index 8b6842f5b..73f120e4b 100644 --- a/nodedb/src/control/planner/catalog_adapter/adapter.rs +++ b/nodedb/src/control/planner/catalog_adapter/adapter.rs @@ -54,6 +54,14 @@ pub struct OriginCatalog { /// negligible — `get_collection` is called only a handful /// of times per plan. pub(super) recorded_versions: Mutex, + /// Node-wide sequence counters. `None` for adapters built without a + /// `SharedState`; every sequence accessor then reports that sequence + /// access is unavailable. + pub(super) sequence_registry: Option>, + /// The calling session's `currval` map. `nextval` records into it and + /// `currval` reads only from it, so one session never sees another + /// session's allocation. `None` for planning with no session behind it. + pub(super) session_sequences: Option>, } impl OriginCatalog { @@ -83,6 +91,8 @@ impl OriginCatalog { drain_tracker: None, recorded_versions: Mutex::new(DescriptorVersionSet::new()), array_catalog: None, + sequence_registry: None, + session_sequences: None, } } @@ -107,9 +117,20 @@ impl OriginCatalog { drain_tracker: Some(Arc::clone(&shared.lease_drain)), recorded_versions: Mutex::new(DescriptorVersionSet::new()), array_catalog: Some(shared.array_catalog.clone()), + sequence_registry: Some(Arc::clone(&shared.sequence_registry)), + session_sequences: None, } } + /// Bind the calling session's `currval` map to this adapter. + pub fn with_session_sequences( + mut self, + values: Option>, + ) -> Self { + self.session_sequences = values; + self + } + /// Drain the recorded descriptor-version set and return it. /// Callers capture this after planning finishes and use it /// as the plan cache key + freshness witness. diff --git a/nodedb/src/control/planner/catalog_adapter/mod.rs b/nodedb/src/control/planner/catalog_adapter/mod.rs index 84514f2db..fe5c8beca 100644 --- a/nodedb/src/control/planner/catalog_adapter/mod.rs +++ b/nodedb/src/control/planner/catalog_adapter/mod.rs @@ -31,6 +31,7 @@ //! queries. mod adapter; +mod sequence_access; mod sql_catalog_impl; mod type_convert; diff --git a/nodedb/src/control/planner/catalog_adapter/sequence_access.rs b/nodedb/src/control/planner/catalog_adapter/sequence_access.rs new file mode 100644 index 000000000..020fdd543 --- /dev/null +++ b/nodedb/src/control/planner/catalog_adapter/sequence_access.rs @@ -0,0 +1,84 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! Sequence accessor bodies behind `OriginCatalog`'s `SqlCatalog` methods. +//! +//! `nextval` advances the node counter AND records the value in the calling +//! session's map. `currval` reads only that session map, so one session never +//! reports another session's allocation. + +use nodedb_sql::SqlError; + +use crate::control::sequence::{SequenceError, SequenceRegistry}; + +use super::adapter::OriginCatalog; + +impl OriginCatalog { + /// Advance `name` and record the value as this session's `currval`. + pub(super) fn advance_sequence(&self, name: &str) -> Result { + let registry = self.require_sequence_registry()?; + let database_id = self.database_id.as_u64(); + let value = registry + .nextval(database_id, self.tenant_id, name) + .map_err(|e| map_sequence_error(name, e))?; + if let Some(session) = &self.session_sequences { + session.record(database_id, self.tenant_id, name, value); + } + Ok(value) + } + + /// The last value THIS SESSION obtained from `nextval` on `name`. + pub(super) fn session_sequence_value(&self, name: &str) -> Result { + // The sequence must exist before its absence from the session map can + // mean "not yet called in this session" rather than "no such object". + let registry = self.require_sequence_registry()?; + if !registry.exists(self.database_id.as_u64(), self.tenant_id, name) { + return Err(undefined_sequence(name)); + } + self.session_sequences + .as_ref() + .and_then(|session| session.last(self.database_id.as_u64(), self.tenant_id, name)) + .ok_or_else(|| SqlError::ObjectNotInPrerequisiteState { + object: name.to_string(), + detail: format!( + "currval of sequence \"{name}\" is not yet defined in this session" + ), + }) + } + + /// Position `name` so the next `nextval` returns `value + increment`. + pub(super) fn position_sequence(&self, name: &str, value: i64) -> Result { + let registry = self.require_sequence_registry()?; + registry + .setval(self.database_id.as_u64(), self.tenant_id, name, value) + .map_err(|e| map_sequence_error(name, e)) + } + + fn require_sequence_registry(&self) -> Result<&SequenceRegistry, SqlError> { + self.sequence_registry + .as_deref() + .ok_or_else(|| SqlError::ObjectNotInPrerequisiteState { + object: "sequence".into(), + detail: "sequence access unavailable: this planner holds no sequence registry" + .into(), + }) + } +} + +/// The function exists, the object does not — SQLSTATE `42704`, never `42883`. +fn undefined_sequence(name: &str) -> SqlError { + SqlError::UndefinedObject { + kind: "sequence", + name: name.to_string(), + } +} + +/// Map a registry error onto its planner equivalent. +fn map_sequence_error(name: &str, error: SequenceError) -> SqlError { + match error { + SequenceError::NotFound { .. } => undefined_sequence(name), + other => SqlError::ObjectNotInPrerequisiteState { + object: name.to_string(), + detail: other.to_string(), + }, + } +} diff --git a/nodedb/src/control/planner/catalog_adapter/sql_catalog_impl.rs b/nodedb/src/control/planner/catalog_adapter/sql_catalog_impl.rs index e9c7db299..4bac4b2e6 100644 --- a/nodedb/src/control/planner/catalog_adapter/sql_catalog_impl.rs +++ b/nodedb/src/control/planner/catalog_adapter/sql_catalog_impl.rs @@ -220,6 +220,37 @@ impl SqlCatalog for OriginCatalog { ) } + /// Database and tenant come from the adapter's own scope, mirroring + /// `resolve_regclass`; the trait's arguments are the caller's plan-time + /// defaults and carry no session scope. + fn sequence_nextval( + &self, + _database_id: nodedb_types::DatabaseId, + _tenant_id: u64, + name: &str, + ) -> Result { + self.advance_sequence(name) + } + + fn sequence_currval( + &self, + _database_id: nodedb_types::DatabaseId, + _tenant_id: u64, + name: &str, + ) -> Result { + self.session_sequence_value(name) + } + + fn sequence_setval( + &self, + _database_id: nodedb_types::DatabaseId, + _tenant_id: u64, + name: &str, + value: i64, + ) -> Result { + self.position_sequence(name, value) + } + fn resolve_regtype(&self, name: &str) -> Option { crate::control::server::pgwire::catalog::tables::pg_type::type_oid_map() .get(name) diff --git a/nodedb/src/control/planner/catalog_adapter/type_convert.rs b/nodedb/src/control/planner/catalog_adapter/type_convert.rs index 02c8f8af6..8aa2a1dc8 100644 --- a/nodedb/src/control/planner/catalog_adapter/type_convert.rs +++ b/nodedb/src/control/planner/catalog_adapter/type_convert.rs @@ -61,12 +61,20 @@ pub(super) fn convert_collection_type( .declared_primary_key .clone() .unwrap_or_else(|| "id".to_string()); + // `stored.fields` carries every declared column by name, the pk + // included, so its DEFAULT clause is recovered the same way the + // columnar arm recovers one for its synthetic-pk field. + let pk_default = stored + .fields + .iter() + .find(|(name, _)| name.eq_ignore_ascii_case(&pk_name)) + .and_then(|(_, type_str)| declared_default(type_str)); let mut columns = vec![ColumnInfo { name: pk_name.clone(), data_type: SqlDataType::String, nullable: false, is_primary_key: true, - default: None, + default: pk_default, raw_type: None, int_width: None, float_width: None, @@ -81,7 +89,7 @@ pub(super) fn convert_collection_type( data_type: parse_type_str(type_str), nullable: true, is_primary_key: false, - default: None, + default: declared_default(type_str), raw_type: None, int_width: IntWidth::from_declared_type(type_str), float_width: FloatWidth::from_declared_type(type_str), @@ -261,31 +269,50 @@ fn convert_column_type(ct: &nodedb_types::columnar::ColumnType) -> SqlDataType { } } +/// Resolve the declared SQL type of a catalog `fields` entry. +/// +/// The catalog records the raw DDL text that followed the column name, so an +/// entry reads `INT DEFAULT 5` or `INT NOT NULL`, not `INT`. +/// [`nodedb_types::columnar::ColumnType::from_declared_type`] is the single +/// classifier both planes resolve that text through, so a trailing modifier +/// never changes the resolved type and the Data Plane cannot disagree about +/// which columns carry an instant. +/// +/// A token that names no known type resolves to `SqlDataType::String`, the +/// widest rendering a catalog column can fall back to. fn parse_type_str(s: &str) -> SqlDataType { - let upper = s.to_uppercase(); - // Handle DECIMAL/NUMERIC with optional (p,s) params. - if upper.starts_with("DECIMAL") || upper.starts_with("NUMERIC") { - return SqlDataType::Decimal; + match nodedb_types::columnar::ColumnType::from_declared_type(s) { + Some(declared) => declared_column_type_to_sql(declared), + None => SqlDataType::String, } - match upper.as_str() { - // Every spelling `IntWidth::from_declared_type` recognizes must appear - // here too, or the column resolves to the `_ => String` default and - // advertises OID 25 (text) — the exact failure that made `SMALLINT` - // columns unreadable. `parse_type_str` decides *whether* - // the column is an integer; `IntWidth` decides *how wide*. - "INT" | "INTEGER" | "INT4" | "INT8" | "INT64" | "BIGINT" | "SMALLINT" | "INT2" => { - SqlDataType::Int64 - } - // Same contract as the integer arm above, for the float family: every - // spelling `FloatWidth::from_declared_type` recognizes must appear - // here, or the column falls through to `_ => String` and advertises - // OID 25 (text) no matter what width was declared. - "FLOAT" | "FLOAT4" | "FLOAT8" | "FLOAT32" | "FLOAT64" | "DOUBLE" | "DOUBLE PRECISION" - | "REAL" => SqlDataType::Float64, - "BOOL" | "BOOLEAN" => SqlDataType::Bool, - "BYTES" | "BYTEA" | "BLOB" => SqlDataType::Bytes, - "TIMESTAMP" | "TIMESTAMPTZ" => SqlDataType::Timestamp, - _ => SqlDataType::String, +} + +/// Map a declared column type onto the SQL type a `fields` column advertises. +/// +/// Every type whose declared spelling and whose resolved strict/kv schema +/// type advertise the same SQL type defers to [`convert_column_type`], so the +/// two mappings cannot drift. The arms above that tail are the exceptions: a +/// schemaless or columnar-family column stores these as the text the client +/// wrote, not in the strict engine's binary encoding, so it renders as text. +fn declared_column_type_to_sql(declared: nodedb_types::columnar::ColumnType) -> SqlDataType { + use nodedb_types::columnar::ColumnType; + match declared { + // A declared TIMESTAMPTZ column reads back in the naive timestamp + // shape these engines store, so it advertises OID 1114. + ColumnType::Timestamptz => SqlDataType::Timestamp, + // Bitemporal system time is engine-assigned and renders as text. + ColumnType::SystemTimestamp => SqlDataType::String, + // Stored as the client's own text: WKT geometry, JSON text, a vector + // or duration literal, and the collection literals. + ColumnType::Geometry + | ColumnType::Json + | ColumnType::Vector(_) + | ColumnType::Duration + | ColumnType::Array + | ColumnType::Set + | ColumnType::Range + | ColumnType::Record => SqlDataType::String, + other => convert_column_type(&other), } } @@ -296,6 +323,41 @@ mod tests { use super::{SqlDataType, convert_collection_type, parse_type_str}; use crate::control::security::catalog::StoredCollection; + /// The planner reads a cell as an instant for exactly the declared types + /// `ColumnType::is_instant` names, which is the same predicate the Data + /// Plane scales emission by. Pinning the equivalence here is what a + /// comment could not do: a spelling added to one side and not the other + /// fails this test instead of shipping a millisecond value labelled as + /// microseconds. + #[test] + fn parse_type_str_reads_exactly_the_instant_declared_types_as_timestamps() { + use nodedb_types::columnar::ColumnType; + for declared in [ + "TIMESTAMP", + "TIMESTAMPTZ", + "timestamp", + "TIMESTAMP TIME_KEY", + "TIMESTAMPTZ NOT NULL", + "SYSTEM_TIMESTAMP", + "BIGINT TIME_KEY", + "INT", + "TEXT", + "GEOMETRY", + "DECIMAL(10, 2)", + "VECTOR(768)", + "SOMETHING_ELSE", + "", + ] { + let is_instant = + ColumnType::from_declared_type(declared).is_some_and(|ty| ty.is_instant()); + assert_eq!( + is_instant, + parse_type_str(declared) == SqlDataType::Timestamp, + "{declared}: the instant predicate and the planner type must agree" + ); + } + } + /// `SMALLINT`/`INT2` are valid PostgreSQL wire-width integer keywords /// that must resolve to the same `SqlDataType::Int64` arm as /// `INT`/`INTEGER`/`INT4`/`INT8`/`BIGINT` — previously they were unlisted diff --git a/nodedb/src/control/planner/context/query/context.rs b/nodedb/src/control/planner/context/query/context.rs index a7cee59a2..c9f96e67f 100644 --- a/nodedb/src/control/planner/context/query/context.rs +++ b/nodedb/src/control/planner/context/query/context.rs @@ -98,6 +98,13 @@ pub struct QueryContext { /// mirrors `broadcast_threshold_bytes` so `&self` plan calls read it without /// an exclusive borrow. pub(super) shuffle_agg_threshold: std::sync::atomic::AtomicUsize, + /// The calling connection's `currval` map, forwarded into the catalog + /// adapter each plan call. Written per request by + /// `apply_planning_session_overrides`, exactly like the knobs above. + /// `None` for planning with no session behind it, which then reports + /// that sequence access is unavailable rather than crossing sessions. + pub(super) session_sequences: + std::sync::Mutex>>, } impl QueryContext { @@ -122,6 +129,7 @@ impl QueryContext { shuffle_agg_threshold: std::sync::atomic::AtomicUsize::new( super::tuning::DEFAULT_SHUFFLE_AGG_THRESHOLD, ), + session_sequences: std::sync::Mutex::new(None), } } @@ -191,6 +199,7 @@ impl QueryContext { shuffle_agg_threshold: std::sync::atomic::AtomicUsize::new( super::tuning::DEFAULT_SHUFFLE_AGG_THRESHOLD, ), + session_sequences: std::sync::Mutex::new(None), } } @@ -228,10 +237,35 @@ impl QueryContext { shuffle_agg_threshold: std::sync::atomic::AtomicUsize::new( super::tuning::DEFAULT_SHUFFLE_AGG_THRESHOLD, ), + session_sequences: std::sync::Mutex::new(None), } } } +impl QueryContext { + /// Bind the calling connection's `currval` map for the next plan call. + pub fn set_session_sequences( + &self, + values: Option>, + ) { + let mut slot = self + .session_sequences + .lock() + .unwrap_or_else(|p| p.into_inner()); + *slot = values; + } + + /// The `currval` map bound for the current plan call. + pub(super) fn session_sequences( + &self, + ) -> Option> { + self.session_sequences + .lock() + .unwrap_or_else(|p| p.into_inner()) + .clone() + } +} + /// The node-default broadcast threshold (bytes) for fixtures that have no /// `SharedState` tuning to read. Sourced from `ClusterTransportTuning::default()` /// so the planner default and the config default never drift. diff --git a/nodedb/src/control/planner/context/query/planning.rs b/nodedb/src/control/planner/context/query/planning.rs index 8a845dcd7..6d46558ef 100644 --- a/nodedb/src/control/planner/context/query/planning.rs +++ b/nodedb/src/control/planner/context/query/planning.rs @@ -11,57 +11,10 @@ use std::sync::Arc; use super::QueryContext; use crate::control::planner::context::security::PlanSecurityContext; +use crate::control::planner::plan_error_map::map_plan_error; use crate::control::planner::sql_plan_convert::PlanningPurpose; use crate::control::server::response_shape::schema::OutputSchema; - -/// Map a planner error onto its Control-Plane equivalent. -/// -/// One mapping for every `plan_sql*` call site. Four copies of this match -/// existed and had already drifted — only one of them mapped -/// `RetryableSchemaChanged`, so the same condition was retryable on one path -/// and a flat plan error on the others. A new variant added to `SqlError` -/// reaches every site through here or none. -fn map_plan_error(error: nodedb_sql::SqlError, tenant_id: crate::types::TenantId) -> crate::Error { - match error { - nodedb_sql::SqlError::RetryableSchemaChanged { descriptor } => { - crate::Error::RetryableSchemaChanged { descriptor } - } - nodedb_sql::SqlError::CollectionDeactivated { - name, - retention_expires_at_ns, - .. - } => crate::Error::CollectionDeactivated { - tenant_id, - collection: name, - retention_expires_at_ns, - }, - nodedb_sql::SqlError::UnknownTable { name } => crate::Error::CollectionNotFound { - tenant_id, - collection: name, - }, - nodedb_sql::SqlError::UndefinedFunction { name } => { - crate::Error::UndefinedFunction { name } - } - // A constant expression that divides by zero is the same condition the - // row-scope evaluator raises, so it carries the same code. - nodedb_sql::SqlError::DivisionByZero => crate::Error::DivisionByZero, - nodedb_sql::SqlError::InvalidLimitValue { clause, value } => { - crate::Error::InvalidLimitValue { clause, value } - } - nodedb_sql::SqlError::UnknownColumn { column, .. } => { - crate::Error::UndefinedColumn { column } - } - nodedb_sql::SqlError::AmbiguousColumn { column } => { - crate::Error::AmbiguousColumn { column } - } - // A target/expression count mismatch is a syntax error in PostgreSQL, - // so it renders 42601 through `BadRequest`. - nodedb_sql::SqlError::Arity { detail } => crate::Error::BadRequest { detail }, - other => crate::Error::PlanError { - detail: other.to_string(), - }, - } -} +use nodedb_physical::physical_plan::ReturningSpec; /// Bundled arguments for [`QueryContext::plan_sql_with_rls`]. pub struct PlanSqlWithRlsParams<'a> { @@ -97,6 +50,7 @@ impl QueryContext { tenant_id: crate::types::TenantId, database_id: crate::types::DatabaseId, purpose: PlanningPurpose, + returning: Option<&ReturningSpec>, ) -> crate::Result<( Vec, OutputSchema, @@ -128,8 +82,13 @@ impl QueryContext { } else { inputs.build_adapter(tenant_id.as_u64(), database_id) }; - let plans = - nodedb_sql::plan_sql(sql, &catalog).map_err(|e| map_plan_error(e, tenant_id))?; + // `nextval` records into the calling session's map and `currval` reads + // only from it, so the adapter must know which session is planning. + // `Arc` because the converters evaluate column DEFAULTs that read the + // catalog — `nextval` and `currval` — after planning returns. + let catalog = Arc::new(catalog.with_session_sequences(self.session_sequences())); + let plans = nodedb_sql::plan_sql(sql, catalog.as_ref()) + .map_err(|e| map_plan_error(e, tenant_id))?; // Fold catalog-dependent cast expressions (::regclass, ::regtype) to // constant OID literals at plan time, before crossing the bridge. // The data-plane evaluator is pure and has no catalog access. @@ -138,7 +97,7 @@ impl QueryContext { .map(|p| { nodedb_sql::planner::catalog_fold::fold_catalog_exprs_in_plan( p, - &catalog, + catalog.as_ref(), database_id, tenant_id.as_u64(), ) @@ -182,21 +141,17 @@ impl QueryContext { .load(std::sync::atomic::Ordering::Relaxed), database_id, tenant_id, + sql_catalog: Some(Arc::clone(&catalog) as _), }; let output_schema = crate::control::planner::sql_plan_convert::output_schema::build_output_schema( &plans, - &catalog, + catalog.as_ref(), database_id, + returning, ); - let cache_eligibility = if plans - .iter() - .all(|plan| plan.cache_eligibility().is_cacheable()) - { - nodedb_sql::types::PlanCacheEligibility::Cacheable - } else { - nodedb_sql::types::PlanCacheEligibility::DataDependent - }; + let cache_eligibility = + crate::control::planner::sql_plan_convert::batch_cache_eligibility(&plans); let tasks = crate::control::planner::sql_plan_convert::convert(&plans, tenant_id, &ctx)?; Ok((tasks, output_schema, version_set, cache_eligibility)) } @@ -217,18 +172,22 @@ impl QueryContext { database_id, sec, } = params; - self.plan_sql_with_rls_returning(sql, tenant_id, database_id, sec, false) + self.plan_sql_with_rls_returning(sql, tenant_id, database_id, sec, None) .await } - /// Plan SQL with RLS injection, optionally propagating a RETURNING flag. + /// Plan SQL with RLS injection, announcing a DML `RETURNING` clause. + /// + /// `returning` is the spec `strip_returning` parsed off the statement, so + /// the write announces the columns it projects. `None` for a statement + /// that carries no clause. pub async fn plan_sql_with_rls_returning( &self, sql: &str, tenant_id: crate::types::TenantId, database_id: crate::types::DatabaseId, sec: &PlanSecurityContext<'_>, - returning: bool, + returning: Option<&ReturningSpec>, ) -> crate::Result<( Vec, OutputSchema, @@ -250,7 +209,7 @@ impl QueryContext { tenant_id: crate::types::TenantId, database_id: crate::types::DatabaseId, sec: &PlanSecurityContext<'_>, - returning: bool, + returning: Option<&ReturningSpec>, ) -> crate::Result<( Vec, OutputSchema, @@ -289,7 +248,7 @@ impl QueryContext { tenant_id, database_id, sec, - false, + None, PlanningPurpose::Metadata, ) .await @@ -302,7 +261,7 @@ impl QueryContext { tenant_id: crate::types::TenantId, database_id: crate::types::DatabaseId, sec: &PlanSecurityContext<'_>, - _returning: bool, + returning: Option<&ReturningSpec>, purpose: PlanningPurpose, ) -> crate::Result<( Vec, @@ -311,7 +270,7 @@ impl QueryContext { nodedb_sql::types::PlanCacheEligibility, )> { let (mut tasks, output_schema, mut version_set, cache_eligibility) = - self.plan_with_nodedb_sql_for_purpose(sql, tenant_id, database_id, purpose)?; + self.plan_with_nodedb_sql_for_purpose(sql, tenant_id, database_id, purpose, returning)?; // Versions read BEFORE injection, never after: injection reads live // policy/grant state under its own lock, and a mutation racing in @@ -361,13 +320,21 @@ impl QueryContext { tenant_id: crate::types::TenantId, database_id: crate::types::DatabaseId, sec: &PlanSecurityContext<'_>, + returning: Option<&ReturningSpec>, ) -> crate::Result<( Vec, OutputSchema, )> { - self.plan_sql_with_params_and_rls_and_versions(sql, params, tenant_id, database_id, sec) - .await - .map(|(tasks, schema, _)| (tasks, schema)) + self.plan_sql_with_params_and_rls_and_versions( + sql, + params, + tenant_id, + database_id, + sec, + returning, + ) + .await + .map(|(tasks, schema, _)| (tasks, schema)) } /// Parameterized RLS planning plus the descriptor versions observed by its @@ -380,6 +347,7 @@ impl QueryContext { tenant_id: crate::types::TenantId, database_id: crate::types::DatabaseId, sec: &PlanSecurityContext<'_>, + returning: Option<&ReturningSpec>, ) -> crate::Result<( Vec, OutputSchema, @@ -396,15 +364,23 @@ impl QueryContext { // Fresh adapter per plan call: same rationale as // `plan_with_nodedb_sql_for_purpose`. Its recorded version set is returned to the // caller so parameterized plans participate in descriptor admission. - let catalog = inputs.build_adapter(tenant_id.as_u64(), database_id); - let raw_plans = nodedb_sql::plan_sql_with_params(sql, params, &catalog) + // `nextval` records into the calling session's map and `currval` reads + // only from it, so the adapter must know which session is planning. + // `Arc` because the converters evaluate column DEFAULTs that read the + // catalog after planning returns. + let catalog = Arc::new( + inputs + .build_adapter(tenant_id.as_u64(), database_id) + .with_session_sequences(self.session_sequences()), + ); + let raw_plans = nodedb_sql::plan_sql_with_params(sql, params, catalog.as_ref()) .map_err(|error| map_plan_error(error, tenant_id))?; let plans: Vec<_> = raw_plans .into_iter() .map(|p| { nodedb_sql::planner::catalog_fold::fold_catalog_exprs_in_plan( p, - &catalog, + catalog.as_ref(), database_id, tenant_id.as_u64(), ) @@ -447,12 +423,14 @@ impl QueryContext { .load(std::sync::atomic::Ordering::Relaxed), database_id, tenant_id, + sql_catalog: Some(Arc::clone(&catalog) as _), }; let output_schema = crate::control::planner::sql_plan_convert::output_schema::build_output_schema( &plans, - &catalog, + catalog.as_ref(), database_id, + returning, ); let mut tasks = crate::control::planner::sql_plan_convert::convert(&plans, tenant_id, &ctx)?; diff --git a/nodedb/src/control/planner/mod.rs b/nodedb/src/control/planner/mod.rs index bb5951700..ac07b12b7 100644 --- a/nodedb/src/control/planner/mod.rs +++ b/nodedb/src/control/planner/mod.rs @@ -7,6 +7,7 @@ pub mod context; pub mod descriptor_set; pub mod implicit_edges; pub mod materialized_sum; +pub(crate) mod plan_error_map; pub mod procedural; pub mod redaction_refusal; pub mod rls_injection; diff --git a/nodedb/src/control/planner/plan_error_map.rs b/nodedb/src/control/planner/plan_error_map.rs new file mode 100644 index 000000000..1e9dd8dbc --- /dev/null +++ b/nodedb/src/control/planner/plan_error_map.rs @@ -0,0 +1,70 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! `nodedb_sql::SqlError` to Control-Plane `crate::Error` mapping. + +/// Map a planner error onto its Control-Plane equivalent. +/// +/// One mapping for every site that surfaces a `nodedb_sql` error: the +/// `plan_sql*` calls and the `SqlPlan` -> `PhysicalPlan` converters that +/// evaluate column DEFAULTs. Four copies of this match existed and had +/// already drifted — only one of them mapped `RetryableSchemaChanged`, so the +/// same condition was retryable on one path and a flat plan error on the +/// others. A new variant added to `SqlError` reaches every site through here +/// or none. +pub(crate) fn map_plan_error( + error: nodedb_sql::SqlError, + tenant_id: crate::types::TenantId, +) -> crate::Error { + match error { + nodedb_sql::SqlError::RetryableSchemaChanged { descriptor } => { + crate::Error::RetryableSchemaChanged { descriptor } + } + nodedb_sql::SqlError::CollectionDeactivated { + name, + retention_expires_at_ns, + .. + } => crate::Error::CollectionDeactivated { + tenant_id, + collection: name, + retention_expires_at_ns, + }, + nodedb_sql::SqlError::UnknownTable { name } => crate::Error::CollectionNotFound { + tenant_id, + collection: name, + }, + nodedb_sql::SqlError::UndefinedFunction { name } => { + crate::Error::UndefinedFunction { name } + } + // A per-row sequence accessor is a refusal, not a syntax error, so it + // keeps SQLSTATE `0A000` rather than the `42601` the fallback gives. + nodedb_sql::SqlError::SequencePerRowUnsupported { .. } => { + crate::Error::FeatureNotSupported { + detail: error.to_string(), + } + } + nodedb_sql::SqlError::UndefinedObject { kind, name } => { + crate::Error::UndefinedObject { kind, name } + } + nodedb_sql::SqlError::ObjectNotInPrerequisiteState { object, detail } => { + crate::Error::ObjectNotInPrerequisiteState { object, detail } + } + // A constant expression that divides by zero is the same condition the + // row-scope evaluator raises, so it carries the same code. + nodedb_sql::SqlError::DivisionByZero => crate::Error::DivisionByZero, + nodedb_sql::SqlError::InvalidLimitValue { clause, value } => { + crate::Error::InvalidLimitValue { clause, value } + } + nodedb_sql::SqlError::UnknownColumn { column, .. } => { + crate::Error::UndefinedColumn { column } + } + nodedb_sql::SqlError::AmbiguousColumn { column } => { + crate::Error::AmbiguousColumn { column } + } + // A target/expression count mismatch is a syntax error in PostgreSQL, + // so it renders 42601 through `BadRequest`. + nodedb_sql::SqlError::Arity { detail } => crate::Error::BadRequest { detail }, + other => crate::Error::PlanError { + detail: other.to_string(), + }, + } +} diff --git a/nodedb/src/control/planner/procedural/executor/core/dispatch.rs b/nodedb/src/control/planner/procedural/executor/core/dispatch.rs index 44bcf3bed..0606e485e 100644 --- a/nodedb/src/control/planner/procedural/executor/core/dispatch.rs +++ b/nodedb/src/control/planner/procedural/executor/core/dispatch.rs @@ -94,7 +94,7 @@ impl<'a> StatementExecutor<'a> { self.tenant_id, self.database_id, &security.context(self.state), - false, + None, ) .await?; let lease_scope = self.state.acquire_plan_lease_scope(&versions)?; diff --git a/nodedb/src/control/planner/sql_plan_convert/array_fn_convert/aggregate.rs b/nodedb/src/control/planner/sql_plan_convert/array_fn_convert/aggregate.rs index d52fc535b..c61cf62bc 100644 --- a/nodedb/src/control/planner/sql_plan_convert/array_fn_convert/aggregate.rs +++ b/nodedb/src/control/planner/sql_plan_convert/array_fn_convert/aggregate.rs @@ -153,6 +153,7 @@ mod tests { shuffle_agg_num_parts: 0, broadcast_threshold_bytes: 8 * 1024 * 1024, shuffle_agg_threshold: 10_000, + sql_catalog: None, database_id: crate::types::DatabaseId::DEFAULT, tenant_id: crate::types::TenantId::new(0), } diff --git a/nodedb/src/control/planner/sql_plan_convert/array_fn_convert/slice.rs b/nodedb/src/control/planner/sql_plan_convert/array_fn_convert/slice.rs index ed997b9f4..845605a93 100644 --- a/nodedb/src/control/planner/sql_plan_convert/array_fn_convert/slice.rs +++ b/nodedb/src/control/planner/sql_plan_convert/array_fn_convert/slice.rs @@ -255,6 +255,7 @@ mod tests { shuffle_agg_num_parts: 0, broadcast_threshold_bytes: 8 * 1024 * 1024, shuffle_agg_threshold: 10_000, + sql_catalog: None, database_id: crate::types::DatabaseId::DEFAULT, tenant_id: crate::types::TenantId::new(0), }; diff --git a/nodedb/src/control/planner/sql_plan_convert/cache_verdict.rs b/nodedb/src/control/planner/sql_plan_convert/cache_verdict.rs new file mode 100644 index 000000000..92e94a8c9 --- /dev/null +++ b/nodedb/src/control/planner/sql_plan_convert/cache_verdict.rs @@ -0,0 +1,21 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! Plan-cache verdict for a batch of logical plans. + +use nodedb_sql::types::{PlanCacheEligibility, SqlPlan}; + +/// Fold one statement batch into a single plan-cache verdict. +/// +/// The batch is cacheable only when every plan in it is cacheable. +/// `SqlPlan::cache_eligibility` is the one source of truth for volatile calls +/// and for row identity bound while lowering. Never derive a second verdict. +pub fn batch_cache_eligibility(plans: &[SqlPlan]) -> PlanCacheEligibility { + if plans + .iter() + .all(|plan| plan.cache_eligibility().is_cacheable()) + { + PlanCacheEligibility::Cacheable + } else { + PlanCacheEligibility::DataDependent + } +} diff --git a/nodedb/src/control/planner/sql_plan_convert/convert.rs b/nodedb/src/control/planner/sql_plan_convert/convert.rs index 7f2a67670..7887e2056 100644 --- a/nodedb/src/control/planner/sql_plan_convert/convert.rs +++ b/nodedb/src/control/planner/sql_plan_convert/convert.rs @@ -129,6 +129,12 @@ pub struct ConvertContext { /// `DEFAULT_SHUFFLE_AGG_THRESHOLD`; overridable per-session via /// `nodedb.shuffle_agg_threshold` for operator control and test determinism. pub shuffle_agg_threshold: usize, + /// The catalog the plan was built against. INSERT/UPSERT conversion + /// evaluates each declared column DEFAULT here, and a sequence-backed + /// DEFAULT (`nextval`, `currval`) reads its value through this handle. + /// `None` for converters built by sub-planners that hold no catalog; a + /// catalog-reading DEFAULT then raises instead of dropping the column. + pub sql_catalog: Option>, } impl ConvertContext { @@ -136,6 +142,22 @@ impl ConvertContext { self.purpose == PlanningPurpose::Metadata } + /// The catalog a column DEFAULT is evaluated against. + /// + /// Raises when the converter holds none. A DEFAULT that reads the catalog + /// must fail loudly: omitting the column stores NULL where the + /// declaration promised a value. + pub fn sql_catalog( + &self, + ) -> crate::Result<&(dyn nodedb_sql::catalog::SqlCatalog + Send + Sync)> { + self.sql_catalog + .as_deref() + .ok_or_else(|| crate::Error::PlanError { + detail: "plan conversion holds no catalog, so a column DEFAULT that reads one cannot be evaluated" + .into(), + }) + } + /// Resolve an existing surrogate without creating a mapping while planning /// metadata. Execute planning retains the allocating assignment behavior. pub fn surrogate_for_pk( @@ -292,6 +314,7 @@ mod tests { shuffle_agg_num_parts: 0, broadcast_threshold_bytes: 0, shuffle_agg_threshold: 0, + sql_catalog: None, } } diff --git a/nodedb/src/control/planner/sql_plan_convert/dml/insert.rs b/nodedb/src/control/planner/sql_plan_convert/dml/insert.rs index 57bcb697e..8bd48d4fa 100644 --- a/nodedb/src/control/planner/sql_plan_convert/dml/insert.rs +++ b/nodedb/src/control/planner/sql_plan_convert/dml/insert.rs @@ -1,6 +1,6 @@ // SPDX-License-Identifier: BUSL-1.1 -use nodedb_sql::types::{EngineType, SqlValue}; +use nodedb_sql::types::{SqlValue, WriteRoute}; use nodedb_types::Surrogate; use nodedb_types::columnar::{ColumnDef, ColumnType, ColumnarSchema}; @@ -10,7 +10,9 @@ use nodedb_physical::physical_plan::ColumnarInsertIntent; use nodedb_physical::physical_plan::*; use super::super::convert::ConvertContext; -use super::super::value::{row_to_msgpack, rows_to_msgpack_array, sql_value_to_string}; +use super::super::value::{ + expand_row_defaults, row_to_msgpack, rows_to_msgpack_array, sql_value_to_string, +}; use nodedb_physical::physical_task::{PhysicalTask, PostSetOp}; /// Build a `ColumnarSchema` from raw catalog column-type strings. @@ -213,21 +215,11 @@ pub(super) fn columnar_row_surrogates( Ok(out) } -pub(in super::super) fn nodedb_value_to_sql(val: nodedb_types::Value) -> SqlValue { - match val { - nodedb_types::Value::Integer(n) => SqlValue::Int(n), - nodedb_types::Value::Float(f) => SqlValue::Float(f), - nodedb_types::Value::String(s) => SqlValue::String(s), - nodedb_types::Value::Bool(b) => SqlValue::Bool(b), - nodedb_types::Value::Null => SqlValue::Null, - _ => SqlValue::String(format!("{val:?}")), - } -} - /// Bundled arguments for [`convert_insert`]. pub(in super::super) struct ConvertInsertArgs<'a> { pub collection: &'a str, - pub engine: &'a EngineType, + /// The lowering these rows take, decided by `nodedb-sql`. + pub route: WriteRoute, pub rows: &'a [Vec<(String, SqlValue)>], pub column_defaults: &'a [(String, String)], pub column_schema: &'a [(String, String)], @@ -242,7 +234,7 @@ pub(in super::super) fn convert_insert( ) -> crate::Result> { let ConvertInsertArgs { collection, - engine, + route, rows, column_defaults, column_schema, @@ -295,44 +287,17 @@ pub(in super::super) fn convert_insert( let mut balanced_documents: Vec<(String, Vec)> = Vec::new(); let mut balanced_surrogates: Vec = Vec::new(); - let mut expanded_rows: Vec> = Vec::with_capacity(rows.len()); - for row in rows { - if column_defaults.is_empty() { - expanded_rows.push(row.clone()); - continue; - } - let mut expanded = row.clone(); - for (col_name, default_expr) in column_defaults { - if !expanded.iter().any(|(k, _)| k == col_name) - && let Some(val) = super::super::value::evaluate_default_expr(default_expr) - .map_err(|e| crate::Error::PlanError { - detail: format!("default for column '{col_name}': {e}"), - })? - { - expanded.push((col_name.clone(), nodedb_value_to_sql(val))); - } - } - expanded_rows.push(expanded); - } + // Every engine's rows expand their DEFAULTs here, ahead of identity + // derivation. A DEFAULT materialized after the primary-key NOT NULL gate + // refuses a key the declaration supplies. + let expanded_rows = expand_row_defaults(rows, column_defaults, tenant_id, ctx)?; - for (i, row) in expanded_rows.iter().enumerate() { - match engine { - EngineType::KeyValue => { - return Err(crate::Error::PlanError { - detail: "KV INSERT must use SqlPlan::KvInsert path".into(), - }); - } - EngineType::Timeseries => { - return Err(crate::Error::PlanError { - detail: format!( - "INSERT into '{collection}': timeseries collections use TimeseriesIngest, not Insert" - ), - }); + for row in &expanded_rows { + match route { + WriteRoute::ColumnarFamily => { + columnar_rows.push(row); } - EngineType::Columnar | EngineType::Spatial => { - columnar_rows.push(&rows[i]); - } - EngineType::DocumentSchemaless | EngineType::DocumentStrict => { + WriteRoute::Document => { let value_bytes = row_to_msgpack(row)?; let (doc_id, surrogate) = resolve_doc_identity(ctx, collection, primary_key, row)?; // One page for the whole statement: the rows of a balanced @@ -381,13 +346,6 @@ pub(in super::super) fn convert_insert( txn_id: None, }); } - EngineType::Array => { - return Err(crate::Error::PlanError { - detail: format!( - "INSERT into '{collection}': array engine uses INSERT INTO ARRAY syntax" - ), - }); - } } } @@ -405,7 +363,7 @@ pub(in super::super) fn convert_insert( } if !columnar_rows.is_empty() { - let payload = rows_to_msgpack_array(&columnar_rows, column_defaults)?; + let payload = rows_to_msgpack_array(&columnar_rows)?; let intent = if if_absent { ColumnarInsertIntent::InsertIfAbsent } else { @@ -495,6 +453,7 @@ mod tests { shuffle_agg_num_parts: 0, broadcast_threshold_bytes: 8 * 1024 * 1024, shuffle_agg_threshold: 10_000, + sql_catalog: None, database_id: crate::types::DatabaseId::DEFAULT, tenant_id: crate::types::TenantId::new(0), }; @@ -514,7 +473,7 @@ mod tests { let rows = vec![crdt_row("k1")]; let tasks = convert_insert(ConvertInsertArgs { collection: "crdt_coll", - engine: &EngineType::DocumentSchemaless, + route: WriteRoute::Document, rows: &rows, column_defaults: &[], column_schema: &[], @@ -546,7 +505,7 @@ mod tests { let rows = vec![crdt_row("k1")]; let tasks = convert_insert(ConvertInsertArgs { collection: "plain", - engine: &EngineType::DocumentSchemaless, + route: WriteRoute::Document, rows: &rows, column_defaults: &[], column_schema: &[], diff --git a/nodedb/src/control/planner/sql_plan_convert/dml/kv_and_vector.rs b/nodedb/src/control/planner/sql_plan_convert/dml/kv_and_vector.rs index 43c2b03f5..5b3130117 100644 --- a/nodedb/src/control/planner/sql_plan_convert/dml/kv_and_vector.rs +++ b/nodedb/src/control/planner/sql_plan_convert/dml/kv_and_vector.rs @@ -171,7 +171,10 @@ pub(in super::super) fn convert_vector_primary_insert( .iter() .map(|(k, v)| (k.clone(), sql_value_to_nodedb_value(v))) .collect(); - zerompk::to_msgpack_vec(&value_map).unwrap_or_default() + zerompk::to_msgpack_vec(&value_map).map_err(|e| crate::Error::Serialization { + format: "msgpack".into(), + detail: format!("vector-primary payload: {e}"), + })? }; tasks.push(PhysicalTask { @@ -223,6 +226,7 @@ mod tests { shuffle_agg_num_parts: 0, broadcast_threshold_bytes: 8 * 1024 * 1024, shuffle_agg_threshold: 10_000, + sql_catalog: None, database_id: crate::types::DatabaseId::DEFAULT, tenant_id: crate::types::TenantId::new(0), } diff --git a/nodedb/src/control/planner/sql_plan_convert/dml/update_delete/delete.rs b/nodedb/src/control/planner/sql_plan_convert/dml/update_delete/delete.rs index 29246ede5..073e58865 100644 --- a/nodedb/src/control/planner/sql_plan_convert/dml/update_delete/delete.rs +++ b/nodedb/src/control/planner/sql_plan_convert/dml/update_delete/delete.rs @@ -259,6 +259,7 @@ mod tests { shuffle_agg_num_parts: 0, broadcast_threshold_bytes: 8 * 1024 * 1024, shuffle_agg_threshold: 10_000, + sql_catalog: None, database_id: crate::types::DatabaseId::DEFAULT, tenant_id: crate::types::TenantId::new(0), }; diff --git a/nodedb/src/control/planner/sql_plan_convert/dml/update_delete/update.rs b/nodedb/src/control/planner/sql_plan_convert/dml/update_delete/update.rs index f9c75d93f..bfd2894fb 100644 --- a/nodedb/src/control/planner/sql_plan_convert/dml/update_delete/update.rs +++ b/nodedb/src/control/planner/sql_plan_convert/dml/update_delete/update.rs @@ -386,6 +386,7 @@ mod tests { shuffle_agg_num_parts: 0, broadcast_threshold_bytes: 8 * 1024 * 1024, shuffle_agg_threshold: 10_000, + sql_catalog: None, database_id: crate::types::DatabaseId::DEFAULT, tenant_id: crate::types::TenantId::new(0), }; diff --git a/nodedb/src/control/planner/sql_plan_convert/dml/upsert.rs b/nodedb/src/control/planner/sql_plan_convert/dml/upsert.rs index a9ec7435b..9813c9305 100644 --- a/nodedb/src/control/planner/sql_plan_convert/dml/upsert.rs +++ b/nodedb/src/control/planner/sql_plan_convert/dml/upsert.rs @@ -6,7 +6,7 @@ //! identity helper there (`resolve_doc_identity`) so a row's surrogate is //! derived identically whichever statement wrote it. -use nodedb_sql::types::{EngineType, SqlExpr, SqlValue}; +use nodedb_sql::types::{SqlExpr, SqlValue, WriteRoute}; use crate::bridge::envelope::PhysicalPlan; use crate::types::{TenantId, VShardId}; @@ -14,14 +14,17 @@ use nodedb_physical::physical_plan::ColumnarInsertIntent; use nodedb_physical::physical_plan::*; use super::super::convert::ConvertContext; -use super::super::value::{assignments_to_update_values, row_to_msgpack, rows_to_msgpack_array}; +use super::super::value::{ + assignments_to_update_values, expand_row_defaults, row_to_msgpack, rows_to_msgpack_array, +}; use super::insert::{build_schema_bytes, columnar_row_surrogates, resolve_doc_identity}; use nodedb_physical::physical_task::{PhysicalTask, PostSetOp}; /// Bundled arguments for [`convert_upsert`]. pub(in super::super) struct ConvertUpsertArgs<'a> { pub collection: &'a str, - pub engine: &'a EngineType, + /// The lowering these rows take, decided by `nodedb-sql`. + pub route: WriteRoute, pub rows: &'a [Vec<(String, SqlValue)>], pub column_defaults: &'a [(String, String)], pub column_schema: &'a [(String, String)], @@ -36,7 +39,7 @@ pub(in super::super) fn convert_upsert( ) -> crate::Result> { let ConvertUpsertArgs { collection, - engine, + route, rows, column_defaults, column_schema, @@ -73,9 +76,14 @@ pub(in super::super) fn convert_upsert( let mut columnar_rows: Vec<&Vec<(String, SqlValue)>> = Vec::new(); - for row in rows { - match engine { - EngineType::DocumentSchemaless | EngineType::DocumentStrict => { + // Every engine's rows expand their DEFAULTs here, ahead of identity + // derivation, so the primary-key NOT NULL gate reads the row the + // declaration promises — see `expand_row_defaults`. + let expanded_rows = expand_row_defaults(rows, column_defaults, tenant_id, ctx)?; + + for row in &expanded_rows { + match route { + WriteRoute::Document => { let value_bytes = row_to_msgpack(row)?; let (doc_id, surrogate) = resolve_doc_identity(ctx, collection, primary_key, row)?; let plan = if is_crdt { @@ -114,21 +122,14 @@ pub(in super::super) fn convert_upsert( txn_id: None, }); } - EngineType::Columnar | EngineType::Spatial => { + WriteRoute::ColumnarFamily => { columnar_rows.push(row); } - EngineType::Timeseries | EngineType::KeyValue | EngineType::Array => { - return Err(crate::Error::PlanError { - detail: format!( - "UPSERT into '{collection}': engine type {engine:?} does not support upsert" - ), - }); - } } } if !columnar_rows.is_empty() { - let payload = rows_to_msgpack_array(&columnar_rows, column_defaults)?; + let payload = rows_to_msgpack_array(&columnar_rows)?; let surrogates = columnar_row_surrogates(ctx, collection, &columnar_rows, primary_key)?; let schema_bytes = build_schema_bytes(column_schema); tasks.push(PhysicalTask { diff --git a/nodedb/src/control/planner/sql_plan_convert/mod.rs b/nodedb/src/control/planner/sql_plan_convert/mod.rs index bdfb309b6..ba341a353 100644 --- a/nodedb/src/control/planner/sql_plan_convert/mod.rs +++ b/nodedb/src/control/planner/sql_plan_convert/mod.rs @@ -4,6 +4,7 @@ pub mod aggregate; pub mod array_alter_convert; pub mod array_convert; pub mod array_fn_convert; +pub mod cache_verdict; pub mod convert; pub mod dml; pub mod expr; @@ -19,4 +20,5 @@ pub mod set_ops; pub mod value; pub mod visitor; +pub use cache_verdict::batch_cache_eligibility; pub use convert::{ConvertContext, PlanningPurpose, convert}; diff --git a/nodedb/src/control/planner/sql_plan_convert/output_schema.rs b/nodedb/src/control/planner/sql_plan_convert/output_schema/build.rs similarity index 64% rename from nodedb/src/control/planner/sql_plan_convert/output_schema.rs rename to nodedb/src/control/planner/sql_plan_convert/output_schema/build.rs index 4e5df4633..f7b9fa374 100644 --- a/nodedb/src/control/planner/sql_plan_convert/output_schema.rs +++ b/nodedb/src/control/planner/sql_plan_convert/output_schema/build.rs @@ -4,234 +4,41 @@ //! `SqlPlan` list, threaded into response shaping so the pgwire encoder can //! advertise correct RowDescription type OIDs. //! -//! Bare columns carry their real catalog type; computed SELECT expressions, -//! GROUP BY keys, and aggregate results are typed conservatively via -//! [`output_schema_types`](super::output_schema_types). A wrong non-TEXT OID -//! makes clients fail to parse the text value, so every uncertain case falls -//! back to `DdlColType::Text`, the safe default. +//! A read plan announces its projection. A write plan announces the columns +//! its `RETURNING` clause projects, and nothing when it carries none — see +//! [`build_returning_schema`](super::returning::build_returning_schema). use std::collections::HashMap; +use nodedb_physical::physical_plan::ReturningSpec; +use nodedb_query::agg_key::canonical_agg_key; use nodedb_sql::catalog::SqlCatalog; use nodedb_sql::types::SqlPlan; -use nodedb_sql::types::query::{AggOutputSlot, Projection}; -use nodedb_sql::types_expr::SqlExpr; +use nodedb_sql::types::query::AggOutputSlot; -use super::lateral::collection_name_from_plan; -use super::output_schema_types::{infer_aggregate_type, infer_computed_expr_type}; -use crate::control::server::response_shape::schema::{ - OutputColumn, OutputSchema, sql_data_type_to_ddl_col_type_with_width, -}; +use crate::control::planner::sql_plan_convert::aggregate::agg_expr_to_pair; +use crate::control::planner::sql_plan_convert::lateral::collection_name_from_plan; +use crate::control::planner::sql_plan_convert::output_schema_types::infer_aggregate_type; +use crate::control::server::response_shape::schema::{OutputColumn, OutputSchema}; use crate::control::server::response_shape::types::DdlColType; -/// Maps one `Projection` entry to an `OutputColumn`, given a map of bare -/// column name -> resolved wire type for the collection in scope. -/// -/// This is the authoritative derivation rule (see also `schema_from_projection` -/// in this module): for a qualified `table.column` reference, `lookup_key` keeps the full -/// dot-joined form (the join executor prefixes every key with its source -/// collection name) while `display_name` is the last segment. For a bare -/// column both are identical. -/// -/// `Projection::Star` / `Projection::QualifiedStar` have no single concrete -/// column and return `None`; the caller sets `is_star` instead. -fn projection_to_column( - p: &Projection, - types: &HashMap, -) -> Option { - match p { - Projection::Column(qname) => { - let display_name = qname - .rsplit('.') - .next() - .map(str::to_string) - .unwrap_or_else(|| qname.clone()); - let ty = types - .get(&display_name) - .copied() - .unwrap_or(DdlColType::Text); - Some(OutputColumn { - display_name, - lookup_key: qname.clone(), - ty, - }) - } - Projection::Computed { expr, alias } => { - // For an aliased column reference (`o.id AS oid`) the Data Plane - // keys the value by the underlying column, not the alias — so the - // lookup_key must be the qualified expression (matching the join - // executor's prefixed keys) while the alias is only the display - // name. A genuine computed expression (`price * qty AS total`) is - // emitted by the executor under its alias, so that stays the key. - let lookup_key = match expr { - SqlExpr::Column { - table: Some(t), - name, - } => format!("{t}.{name}"), - SqlExpr::Column { table: None, name } => name.clone(), - _ => alias.clone(), - }; - Some(OutputColumn { - display_name: alias.clone(), - lookup_key, - ty: infer_computed_expr_type(expr, types), - }) - } - Projection::Star | Projection::QualifiedStar(_) => None, - } -} - -/// Builds a `HashMap` of bare column name -> resolved wire type for -/// `collection`, via a best-effort catalog lookup. Returns an empty map -/// (never an error) when the lookup fails or the collection is unknown — -/// callers fall back to `DdlColType::Text` for every column in that case. -fn column_types_for( - catalog: &C, - database_id: nodedb_types::DatabaseId, - collection: &str, -) -> HashMap { - match catalog.get_collection(database_id, collection) { - Ok(Some(info)) => info - .columns - .iter() - .map(|c| { - ( - c.name.clone(), - sql_data_type_to_ddl_col_type_with_width( - &c.data_type, - c.int_width, - c.float_width, - ), - ) - }) - .collect(), - _ => HashMap::new(), - } -} - -/// Derives an `OutputColumn` for one GROUP BY key expression. -/// -/// The `display_name` is the SELECT-list output name: the explicit alias when -/// the projection aliased the key (`SELECT k AS label ... GROUP BY k` yields -/// output column `label`, matching Postgres), otherwise the key's own column -/// name. The `lookup_key` always stays the raw grouped column name (the key -/// the aggregate executor emits the value under), so `project_row` still finds -/// the value. -/// -/// The output type is the grouped column's catalog type when the key is a bare -/// column (resolved from `types`, default `Text`); a computed-expression key is -/// typed conservatively via [`infer_computed_expr_type`], defaulting to `Text`. -/// -/// A non-`Column` GROUP BY key (a computed expression) derives its `lookup_key` -/// from the shared index-based `computed_group_key_name` rule — the exact name -/// the aggregate spec emits the evaluated value under, so the two can never -/// diverge. Its `display_name` is the SELECT-list alias when present -/// (`UPPER(label) AS u` shows column `u`), else the same placeholder. -fn group_by_key_column( - expr: &SqlExpr, - index: usize, - alias: Option<&str>, - types: &HashMap, -) -> OutputColumn { - match expr { - SqlExpr::Column { table, name } => { - let lookup_key = match table { - Some(t) => format!("{t}.{name}"), - None => name.clone(), - }; - let display_name = alias.map(str::to_string).unwrap_or_else(|| name.clone()); - let ty = types.get(name).copied().unwrap_or(DdlColType::Text); - OutputColumn { - display_name, - lookup_key, - ty, - } - } - _ => { - // The executor emits the evaluated value under the shared - // index-based name (see `group_by_to_specs`), so `lookup_key` MUST - // equal it. `display_name` is the SELECT-list alias when present - // (`UPPER(label) AS u` shows column `u`), else the same placeholder. - let lookup_key = super::group_key_name::computed_group_key_name(index); - let display_name = alias - .map(str::to_string) - .unwrap_or_else(|| lookup_key.clone()); - OutputColumn { - display_name, - lookup_key, - ty: infer_computed_expr_type(expr, types), - } - } - } -} - -/// Returns the collection's columns in declared catalog order, mapped to -/// `OutputColumn`s (`display_name` = `lookup_key` = column name). Returns an -/// empty `Vec` when the catalog/collection lookup fails or the collection has -/// no declared columns (e.g. a schemaless collection) — so a schemaless -/// `SELECT *` still yields empty columns, deriving its shape from the rows. -fn ordered_columns_for( - catalog: &C, - database_id: nodedb_types::DatabaseId, - collection: &str, -) -> Vec { - match catalog.get_collection(database_id, collection) { - Ok(Some(info)) => info - .columns - .iter() - .map(|c| OutputColumn { - display_name: c.name.clone(), - lookup_key: c.name.clone(), - ty: sql_data_type_to_ddl_col_type_with_width( - &c.data_type, - c.int_width, - c.float_width, - ), - }) - .collect(), - _ => Vec::new(), - } -} - -/// Maps a projection list to an `OutputSchema` fragment using `types`. -/// -/// A `Star` / `QualifiedStar` in the projection sets `is_star` and expands -/// into `ordered_cols` (the collection's catalog columns in declared order), -/// appending only entries not already produced by a named projection. When -/// the projection has no star, `ordered_cols` is ignored and behavior is the -/// named-columns-only, `is_star=false` case. -fn schema_from_projection( - projection: &[Projection], - types: &HashMap, - ordered_cols: &[OutputColumn], -) -> OutputSchema { - let mut columns = Vec::with_capacity(projection.len()); - let mut is_star = false; - for p in projection { - match projection_to_column(p, types) { - Some(col) => columns.push(col), - None => { - is_star = true; - for oc in ordered_cols { - if !columns.iter().any(|c| c.lookup_key == oc.lookup_key) { - columns.push(oc.clone()); - } - } - } - } - } - OutputSchema { columns, is_star } -} +use super::columns::{ + column_types_for, group_by_key_column, ordered_columns_for, schema_from_projection, +}; +use super::returning::build_returning_schema; /// Derives the planner-authoritative output schema of a compiled plan list. /// -/// Only the plan variants that carry a resolvable projection against a -/// single named collection are handled directly; other plan variants are -/// handled by later units in this effort and fall back to an empty schema. -pub fn build_output_schema( +/// A read plan announces the columns its projection names. A write plan +/// announces the columns `returning` projects: the clause is stripped from the +/// statement text before planning, so the plan itself carries no column list +/// and the caller supplies the parsed spec. `None` means the statement carries +/// no `RETURNING` clause, and a write then announces nothing. +pub fn build_output_schema( plans: &[SqlPlan], catalog: &C, database_id: nodedb_types::DatabaseId, + returning: Option<&ReturningSpec>, ) -> OutputSchema { let Some(plan) = plans.first() else { return OutputSchema { @@ -241,6 +48,47 @@ pub fn build_output_schema( }; match plan { + // A grouped timeseries scan announces the columns its aggregate + // encoder emits, in that encoder's order: each GROUP BY key, then + // each aggregate. A GROUP BY key carries its own catalog type, so one + // stored instant renders the same grouped as it does through a plain + // `SELECT`. An aggregate result stays `Text`: the timeseries plan + // carries no SELECT-list alias for it, so its type cannot be resolved + // with certainty. + // + // A `time_bucket` query is excluded: its encoder prepends a `bucket` + // boundary column that the plan's GROUP BY list does not name, so the + // announced shape would not be the emitted one. + SqlPlan::TimeseriesScan { + collection, + group_by, + aggregates, + bucket_interval_ms, + .. + } if !group_by.is_empty() && *bucket_interval_ms == 0 => { + let types = column_types_for(catalog, database_id, collection); + let mut columns = Vec::with_capacity(group_by.len() + aggregates.len()); + for key in group_by { + columns.push(OutputColumn { + display_name: key.clone(), + lookup_key: key.clone(), + ty: types.get(key).copied().unwrap_or(DdlColType::Text), + }); + } + for agg in aggregates { + let (function, field) = agg_expr_to_pair(agg); + let key = canonical_agg_key(&function, &field); + columns.push(OutputColumn { + display_name: key.clone(), + lookup_key: key, + ty: DdlColType::Text, + }); + } + OutputSchema { + columns, + is_star: false, + } + } SqlPlan::Scan { collection, projection, @@ -310,12 +158,19 @@ pub fn build_output_schema( let ordered_cols = ordered_columns_for(catalog, database_id, collection); schema_from_projection(projection, &types, &ordered_cols) } - SqlPlan::Join { projection, .. } => { - // A join has no single source collection; column types default - // to `Text` for every projected field rather than picking one - // side arbitrarily. A star here has no single catalog to expand - // against, so no ordered columns are supplied. - let types = HashMap::new(); + SqlPlan::Join { + left, + right, + projection, + .. + } => { + // Each projected column resolves against the catalog of the side + // it came from, keyed on the qualified name the join executor + // emits (`orders.ts`). A bare name two sides declare differently + // cannot be attributed and stays `Text`. A star here has no + // single catalog to expand against, so no ordered columns are + // supplied. + let types = super::join_types::join_column_types(left, right, catalog, database_id); schema_from_projection(projection, &types, &[]) } SqlPlan::ConstantResult { columns, .. } => OutputSchema { @@ -397,12 +252,17 @@ pub fn build_output_schema( // Set operations take their column names/types from the first // (left) branch, matching standard SQL set-op semantics. SqlPlan::Union { inputs, .. } => match inputs.first() { - Some(first) => build_output_schema(std::slice::from_ref(first), catalog, database_id), + Some(first) => { + build_output_schema(std::slice::from_ref(first), catalog, database_id, returning) + } None => OutputSchema::default(), }, - SqlPlan::Intersect { left, .. } | SqlPlan::Except { left, .. } => { - build_output_schema(std::slice::from_ref(left.as_ref()), catalog, database_id) - } + SqlPlan::Intersect { left, .. } | SqlPlan::Except { left, .. } => build_output_schema( + std::slice::from_ref(left.as_ref()), + catalog, + database_id, + returning, + ), SqlPlan::RecursiveValue { columns, .. } => OutputSchema { columns: columns .iter() @@ -416,9 +276,12 @@ pub fn build_output_schema( }, // The outer query determines the final projected shape; the CTE // definitions themselves are only inputs to it. - SqlPlan::Cte { outer, .. } => { - build_output_schema(std::slice::from_ref(outer.as_ref()), catalog, database_id) - } + SqlPlan::Cte { outer, .. } => build_output_schema( + std::slice::from_ref(outer.as_ref()), + catalog, + database_id, + returning, + ), // A post-processor's projected shape is its outer projection; an empty // projection (SELECT *) inherits the body's columns. (Synthesized during // conversion, so this is normally unreached — the schema is derived from @@ -428,7 +291,12 @@ pub fn build_output_schema( input, projection, .. } => { if projection.is_empty() { - build_output_schema(std::slice::from_ref(input.as_ref()), catalog, database_id) + build_output_schema( + std::slice::from_ref(input.as_ref()), + catalog, + database_id, + returning, + ) } else { let types = HashMap::new(); schema_from_projection(projection, &types, &[]) @@ -458,38 +326,40 @@ pub fn build_output_schema( .collect(), is_star: false, }, - // Writes / DDL: no output rows, nothing to shape. - SqlPlan::Insert { .. } - | SqlPlan::KvInsert { .. } - | SqlPlan::Upsert { .. } - | SqlPlan::Update { .. } - | SqlPlan::UpdateFrom { .. } - | SqlPlan::Delete { .. } - | SqlPlan::Truncate { .. } - | SqlPlan::TimeseriesIngest { .. } - | SqlPlan::InsertSelect { .. } + // A write announces exactly what its `RETURNING` clause projects, from + // the target collection's declared columns. `RETURNING` is a + // projection, so it is typed like one: a `SELECT ts, host, v` and an + // `INSERT ... RETURNING ts, host, v` announce the same three types and + // render the same stored row identically. + SqlPlan::Insert { collection, .. } + | SqlPlan::KvInsert { collection, .. } + | SqlPlan::Upsert { collection, .. } + | SqlPlan::Update { collection, .. } + | SqlPlan::UpdateFrom { collection, .. } + | SqlPlan::Delete { collection, .. } + | SqlPlan::TimeseriesIngest { collection, .. } + | SqlPlan::VectorPrimaryInsert { collection, .. } => { + build_returning_schema(returning, collection, catalog, database_id) + } + // Same rule, for the two writes that name their target `target`. + SqlPlan::Merge { target, .. } | SqlPlan::InsertSelect { target, .. } => { + build_returning_schema(returning, target, catalog, database_id) + } + // No rows to shape. `TRUNCATE`, index DDL, and the whole `CREATE ARRAY` + // family answer with a command tag; the array DML ops answer with an + // affected count, and `inject_returning_spec` attaches no spec to them, + // so announcing columns for one would hold a count payload to a row + // shape it does not have. + SqlPlan::Truncate { .. } | SqlPlan::CreateArray { .. } | SqlPlan::DropArray { .. } | SqlPlan::AlterArray { .. } | SqlPlan::InsertArray { .. } | SqlPlan::DeleteArray { .. } - | SqlPlan::VectorPrimaryInsert { .. } | SqlPlan::CreateIndex { .. } | SqlPlan::DropIndex { .. } | SqlPlan::ArrayFlush { .. } | SqlPlan::ArrayCompact { .. } => OutputSchema::default(), - // `Merge` (with or without RETURNING) and `Update`/`UpdateFrom` with - // `returning: true` are shaped downstream via - // `PlanKind::ReturningRows` -> `shape_returning_rows`, which reads - // column names/values directly out of the response payload - // (`RowsPayload` msgpack) when no columns were announced to the - // client. An empty schema here is therefore correct, not a - // placeholder: it says "nothing announced", which is exactly true of - // the simple-query path, where the RowDescription is built from those - // same payload-derived rows. The extended-query path announces its - // columns at Describe time and supplies them as the projection - // instead, and the shaper then holds the rows to them. - SqlPlan::Merge { .. } => OutputSchema::default(), // `ArrayAgg` / `ArrayElementwise` compile to `ArrayOp::Aggregate` / // `ArrayOp::Elementwise`, which `describe_plan` classifies as // `PlanKind::MultiRow`. `MultiRow` responses are shaped by @@ -503,49 +373,8 @@ pub fn build_output_schema( #[cfg(test)] mod tests { use super::*; - - #[test] - fn bare_column_uses_matching_type_from_map() { - let mut types = HashMap::new(); - types.insert("foo".to_string(), DdlColType::Int8); - let p = Projection::Column("foo".to_string()); - let col = projection_to_column(&p, &types).expect("Some for Column"); - assert_eq!(col.lookup_key, "foo"); - assert_eq!(col.display_name, "foo"); - assert_eq!(col.ty, DdlColType::Int8); - } - - #[test] - fn qualified_column_display_is_last_segment() { - let types = HashMap::new(); - let p = Projection::Column("t.bar".to_string()); - let col = projection_to_column(&p, &types).expect("Some for Column"); - assert_eq!(col.lookup_key, "t.bar"); - assert_eq!(col.display_name, "bar"); - assert_eq!(col.ty, DdlColType::Text); - } - - #[test] - fn computed_uses_alias_for_both_and_defaults_to_text() { - let types = HashMap::new(); - let p = Projection::Computed { - expr: nodedb_sql::types_expr::SqlExpr::Wildcard, - alias: "total".to_string(), - }; - let col = projection_to_column(&p, &types).expect("Some for Computed"); - assert_eq!(col.lookup_key, "total"); - assert_eq!(col.display_name, "total"); - assert_eq!(col.ty, DdlColType::Text); - } - - #[test] - fn star_returns_none() { - let types = HashMap::new(); - assert!(projection_to_column(&Projection::Star, &types).is_none()); - assert!( - projection_to_column(&Projection::QualifiedStar("t".to_string()), &types).is_none() - ); - } + use nodedb_sql::types::query::Projection; + use nodedb_sql::types_expr::SqlExpr; /// Catalog stub whose `get_collection` is never called by the /// `ConstantResult` branch under test; only required to satisfy the @@ -568,8 +397,10 @@ mod tests { let plans = vec![SqlPlan::ConstantResult { columns: vec!["a".to_string(), "b".to_string()], values: vec![], + volatile: false, }]; - let schema = build_output_schema(&plans, &NoCatalog, nodedb_types::DatabaseId::DEFAULT); + let schema = + build_output_schema(&plans, &NoCatalog, nodedb_types::DatabaseId::DEFAULT, None); assert_eq!(schema.columns.len(), 2); assert_eq!(schema.columns[0].display_name, "a"); assert_eq!(schema.columns[0].lookup_key, "a"); @@ -637,7 +468,8 @@ mod tests { sort_keys: Vec::new(), }]; - let schema = build_output_schema(&plans, &NoCatalog, nodedb_types::DatabaseId::DEFAULT); + let schema = + build_output_schema(&plans, &NoCatalog, nodedb_types::DatabaseId::DEFAULT, None); assert_eq!(schema.columns.len(), 3); // Group-key display_name is the SELECT-list alias; lookup_key stays // the raw grouped column name (the executor's emitted value key). @@ -664,7 +496,8 @@ mod tests { ], distinct: false, }]; - let schema = build_output_schema(&plans, &NoCatalog, nodedb_types::DatabaseId::DEFAULT); + let schema = + build_output_schema(&plans, &NoCatalog, nodedb_types::DatabaseId::DEFAULT, None); assert_eq!(schema.columns.len(), 1); assert_eq!(schema.columns[0].display_name, "id"); } @@ -680,7 +513,8 @@ mod tests { max_depth: 100, distinct: false, }]; - let schema = build_output_schema(&plans, &NoCatalog, nodedb_types::DatabaseId::DEFAULT); + let schema = + build_output_schema(&plans, &NoCatalog, nodedb_types::DatabaseId::DEFAULT, None); assert_eq!(schema.columns.len(), 1); assert_eq!(schema.columns[0].display_name, "n"); assert_eq!(schema.columns[0].lookup_key, "n"); @@ -721,7 +555,8 @@ mod tests { key_value: nodedb_sql::types_expr::SqlValue::Null, projection: id_and_dist_projection(), }]; - let schema = build_output_schema(&plans, &NoCatalog, nodedb_types::DatabaseId::DEFAULT); + let schema = + build_output_schema(&plans, &NoCatalog, nodedb_types::DatabaseId::DEFAULT, None); assert_id_and_dist_schema(&schema); } @@ -741,7 +576,8 @@ mod tests { payload_filters: Vec::new(), projection: id_and_dist_projection(), }]; - let schema = build_output_schema(&plans, &NoCatalog, nodedb_types::DatabaseId::DEFAULT); + let schema = + build_output_schema(&plans, &NoCatalog, nodedb_types::DatabaseId::DEFAULT, None); assert_id_and_dist_schema(&schema); } @@ -758,7 +594,8 @@ mod tests { score_alias: None, projection: id_and_dist_projection(), }]; - let schema = build_output_schema(&plans, &NoCatalog, nodedb_types::DatabaseId::DEFAULT); + let schema = + build_output_schema(&plans, &NoCatalog, nodedb_types::DatabaseId::DEFAULT, None); assert_id_and_dist_schema(&schema); } @@ -775,7 +612,8 @@ mod tests { score_alias: None, projection: id_and_dist_projection(), }]; - let schema = build_output_schema(&plans, &NoCatalog, nodedb_types::DatabaseId::DEFAULT); + let schema = + build_output_schema(&plans, &NoCatalog, nodedb_types::DatabaseId::DEFAULT, None); assert_id_and_dist_schema(&schema); } @@ -873,7 +711,12 @@ mod tests { grouping_sets: None, sort_keys: Vec::new(), }]; - let schema = build_output_schema(&plans, &TypedCatalog, nodedb_types::DatabaseId::DEFAULT); + let schema = build_output_schema( + &plans, + &TypedCatalog, + nodedb_types::DatabaseId::DEFAULT, + None, + ); // group-keys-first fallback (empty output_order): region, then aggs. assert_eq!(schema.columns.len(), 5); // GROUP BY text column -> the text column's catalog type. @@ -913,7 +756,12 @@ mod tests { grouping_sets: None, sort_keys: Vec::new(), }]; - let schema = build_output_schema(&plans, &TypedCatalog, nodedb_types::DatabaseId::DEFAULT); + let schema = build_output_schema( + &plans, + &TypedCatalog, + nodedb_types::DatabaseId::DEFAULT, + None, + ); assert_eq!(schema.columns.len(), 1); assert_eq!(schema.columns[0].display_name, "u"); assert_eq!(schema.columns[0].ty, DdlColType::Text); @@ -938,7 +786,12 @@ mod tests { }, ]; let plans = vec![scan_plan("metrics", projection)]; - let schema = build_output_schema(&plans, &TypedCatalog, nodedb_types::DatabaseId::DEFAULT); + let schema = build_output_schema( + &plans, + &TypedCatalog, + nodedb_types::DatabaseId::DEFAULT, + None, + ); assert_eq!(schema.columns.len(), 2); // Bare-column-passthrough computed expr -> the column's catalog type. assert_eq!(schema.columns[0].display_name, "aliased_n"); diff --git a/nodedb/src/control/planner/sql_plan_convert/output_schema/columns.rs b/nodedb/src/control/planner/sql_plan_convert/output_schema/columns.rs new file mode 100644 index 000000000..78cea8a45 --- /dev/null +++ b/nodedb/src/control/planner/sql_plan_convert/output_schema/columns.rs @@ -0,0 +1,275 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! Column-level derivation shared by every output-schema rule. +//! +//! One projection entry, one GROUP BY key, or one collection's declared column +//! list maps to [`OutputColumn`]s here. Bare columns carry their real catalog +//! type; computed expressions are typed conservatively via +//! [`output_schema_types`](crate::control::planner::sql_plan_convert::output_schema_types). +//! A wrong non-TEXT OID makes clients fail to parse the text value, so every +//! uncertain case falls back to `DdlColType::Text`, the safe default. + +use std::collections::HashMap; + +use nodedb_sql::catalog::SqlCatalog; +use nodedb_sql::types::query::Projection; +use nodedb_sql::types_expr::SqlExpr; + +use crate::control::planner::sql_plan_convert::group_key_name::computed_group_key_name; +use crate::control::planner::sql_plan_convert::output_schema_types::infer_computed_expr_type; +use crate::control::server::response_shape::schema::{ + OutputColumn, OutputSchema, sql_data_type_to_ddl_col_type_with_width, +}; +use crate::control::server::response_shape::types::DdlColType; + +/// Maps one `Projection` entry to an `OutputColumn`, given a map of bare +/// column name -> resolved wire type for the collection in scope. +/// +/// This is the authoritative derivation rule (see also [`schema_from_projection`]): +/// for a qualified `table.column` reference, `lookup_key` keeps the full +/// dot-joined form (the join executor prefixes every key with its source +/// collection name) while `display_name` is the last segment. For a bare +/// column both are identical. +/// +/// `Projection::Star` / `Projection::QualifiedStar` have no single concrete +/// column and return `None`; the caller sets `is_star` instead. +pub(super) fn projection_to_column( + p: &Projection, + types: &HashMap, +) -> Option { + match p { + Projection::Column(qname) => { + let display_name = qname + .rsplit('.') + .next() + .map(str::to_string) + .unwrap_or_else(|| qname.clone()); + // A join's map is keyed on the qualified name, because two + // sides can carry the same bare column with different types. A + // single-collection map holds bare names only, so the qualified + // lookup misses there and the bare name answers. + let ty = types + .get(qname) + .or_else(|| types.get(&display_name)) + .copied() + .unwrap_or(DdlColType::Text); + Some(OutputColumn { + display_name, + lookup_key: qname.clone(), + ty, + }) + } + Projection::Computed { expr, alias } => { + // For an aliased column reference (`o.id AS oid`) the Data Plane + // keys the value by the underlying column, not the alias — so the + // lookup_key must be the qualified expression (matching the join + // executor's prefixed keys) while the alias is only the display + // name. A genuine computed expression (`price * qty AS total`) is + // emitted by the executor under its alias, so that stays the key. + let lookup_key = match expr { + SqlExpr::Column { + table: Some(t), + name, + } => format!("{t}.{name}"), + SqlExpr::Column { table: None, name } => name.clone(), + _ => alias.clone(), + }; + Some(OutputColumn { + display_name: alias.clone(), + lookup_key, + ty: infer_computed_expr_type(expr, types), + }) + } + Projection::Star | Projection::QualifiedStar(_) => None, + } +} + +/// Builds a `HashMap` of bare column name -> resolved wire type for +/// `collection`, via a best-effort catalog lookup. Returns an empty map +/// (never an error) when the lookup fails or the collection is unknown — +/// callers fall back to `DdlColType::Text` for every column in that case. +pub(super) fn column_types_for( + catalog: &C, + database_id: nodedb_types::DatabaseId, + collection: &str, +) -> HashMap { + match catalog.get_collection(database_id, collection) { + Ok(Some(info)) => info + .columns + .iter() + .map(|c| { + ( + c.name.clone(), + sql_data_type_to_ddl_col_type_with_width( + &c.data_type, + c.int_width, + c.float_width, + ), + ) + }) + .collect(), + _ => HashMap::new(), + } +} + +/// Derives an `OutputColumn` for one GROUP BY key expression. +/// +/// The `display_name` is the SELECT-list output name: the explicit alias when +/// the projection aliased the key (`SELECT k AS label ... GROUP BY k` yields +/// output column `label`, matching Postgres), otherwise the key's own column +/// name. The `lookup_key` always stays the raw grouped column name (the key +/// the aggregate executor emits the value under), so `project_row` still finds +/// the value. +/// +/// The output type is the grouped column's catalog type when the key is a bare +/// column (resolved from `types`, default `Text`); a computed-expression key is +/// typed conservatively via [`infer_computed_expr_type`], defaulting to `Text`. +/// +/// A non-`Column` GROUP BY key (a computed expression) derives its `lookup_key` +/// from the shared index-based `computed_group_key_name` rule — the exact name +/// the aggregate spec emits the evaluated value under, so the two can never +/// diverge. Its `display_name` is the SELECT-list alias when present +/// (`UPPER(label) AS u` shows column `u`), else the same placeholder. +pub(super) fn group_by_key_column( + expr: &SqlExpr, + index: usize, + alias: Option<&str>, + types: &HashMap, +) -> OutputColumn { + match expr { + SqlExpr::Column { table, name } => { + let lookup_key = match table { + Some(t) => format!("{t}.{name}"), + None => name.clone(), + }; + let display_name = alias.map(str::to_string).unwrap_or_else(|| name.clone()); + let ty = types.get(name).copied().unwrap_or(DdlColType::Text); + OutputColumn { + display_name, + lookup_key, + ty, + } + } + _ => { + // The executor emits the evaluated value under the shared + // index-based name (see `group_by_to_specs`), so `lookup_key` MUST + // equal it. `display_name` is the SELECT-list alias when present + // (`UPPER(label) AS u` shows column `u`), else the same placeholder. + let lookup_key = computed_group_key_name(index); + let display_name = alias + .map(str::to_string) + .unwrap_or_else(|| lookup_key.clone()); + OutputColumn { + display_name, + lookup_key, + ty: infer_computed_expr_type(expr, types), + } + } + } +} + +/// Returns the collection's columns in declared catalog order, mapped to +/// `OutputColumn`s (`display_name` = `lookup_key` = column name). Returns an +/// empty `Vec` when the catalog/collection lookup fails or the collection has +/// no declared columns (e.g. a schemaless collection) — so a schemaless +/// `SELECT *` still yields empty columns, deriving its shape from the rows. +pub(super) fn ordered_columns_for( + catalog: &C, + database_id: nodedb_types::DatabaseId, + collection: &str, +) -> Vec { + match catalog.get_collection(database_id, collection) { + Ok(Some(info)) => info + .columns + .iter() + .map(|c| OutputColumn { + display_name: c.name.clone(), + lookup_key: c.name.clone(), + ty: sql_data_type_to_ddl_col_type_with_width( + &c.data_type, + c.int_width, + c.float_width, + ), + }) + .collect(), + _ => Vec::new(), + } +} + +/// Maps a projection list to an `OutputSchema` fragment using `types`. +/// +/// A `Star` / `QualifiedStar` in the projection sets `is_star` and expands +/// into `ordered_cols` (the collection's catalog columns in declared order), +/// appending only entries not already produced by a named projection. When +/// the projection has no star, `ordered_cols` is ignored and behavior is the +/// named-columns-only, `is_star=false` case. +pub(super) fn schema_from_projection( + projection: &[Projection], + types: &HashMap, + ordered_cols: &[OutputColumn], +) -> OutputSchema { + let mut columns = Vec::with_capacity(projection.len()); + let mut is_star = false; + for p in projection { + match projection_to_column(p, types) { + Some(col) => columns.push(col), + None => { + is_star = true; + for oc in ordered_cols { + if !columns.iter().any(|c| c.lookup_key == oc.lookup_key) { + columns.push(oc.clone()); + } + } + } + } + } + OutputSchema { columns, is_star } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn bare_column_uses_matching_type_from_map() { + let mut types = HashMap::new(); + types.insert("foo".to_string(), DdlColType::Int8); + let p = Projection::Column("foo".to_string()); + let col = projection_to_column(&p, &types).expect("Some for Column"); + assert_eq!(col.lookup_key, "foo"); + assert_eq!(col.display_name, "foo"); + assert_eq!(col.ty, DdlColType::Int8); + } + + #[test] + fn qualified_column_display_is_last_segment() { + let types = HashMap::new(); + let p = Projection::Column("t.bar".to_string()); + let col = projection_to_column(&p, &types).expect("Some for Column"); + assert_eq!(col.lookup_key, "t.bar"); + assert_eq!(col.display_name, "bar"); + assert_eq!(col.ty, DdlColType::Text); + } + + #[test] + fn computed_uses_alias_for_both_and_defaults_to_text() { + let types = HashMap::new(); + let p = Projection::Computed { + expr: nodedb_sql::types_expr::SqlExpr::Wildcard, + alias: "total".to_string(), + }; + let col = projection_to_column(&p, &types).expect("Some for Computed"); + assert_eq!(col.lookup_key, "total"); + assert_eq!(col.display_name, "total"); + assert_eq!(col.ty, DdlColType::Text); + } + + #[test] + fn star_returns_none() { + let types = HashMap::new(); + assert!(projection_to_column(&Projection::Star, &types).is_none()); + assert!( + projection_to_column(&Projection::QualifiedStar("t".to_string()), &types).is_none() + ); + } +} diff --git a/nodedb/src/control/planner/sql_plan_convert/output_schema/join_types.rs b/nodedb/src/control/planner/sql_plan_convert/output_schema/join_types.rs new file mode 100644 index 000000000..3fed56b73 --- /dev/null +++ b/nodedb/src/control/planner/sql_plan_convert/output_schema/join_types.rs @@ -0,0 +1,276 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! Per-side column types for a join's output schema. +//! +//! A join has no single source collection, so each projected column resolves +//! against the catalog of the side it came from. Two sides can carry the same +//! bare column name with different types, so the map is keyed on the +//! qualified name the executor emits (`orders.ts`). The bare name is added +//! too, and only while exactly one side claims it: an ambiguous bare name +//! stays `Text`, which every client parses. + +use std::collections::HashMap; + +use nodedb_sql::catalog::SqlCatalog; +use nodedb_sql::types::SqlPlan; + +use crate::control::server::response_shape::types::DdlColType; + +use super::columns::column_types_for; + +/// One join side: the collection it reads and the alias qualifying its +/// columns in the merged row. `None` means the columns are qualified by the +/// collection name. +struct JoinSide { + collection: String, + alias: Option, +} + +/// Column types of every side of a join, keyed by qualified name. +/// +/// Each side contributes `.` and `.` — a +/// projection can spell either — plus the bare `` when no other side +/// declares that name with a different type. +pub(super) fn join_column_types( + left: &SqlPlan, + right: &SqlPlan, + catalog: &C, + database_id: nodedb_types::DatabaseId, +) -> HashMap { + let mut sides = Vec::new(); + collect_sides(left, &mut sides); + collect_sides(right, &mut sides); + + let mut qualified: HashMap = HashMap::new(); + // Bare names seen so far, and whether one side still owns the name. A + // second side declaring the same name with a different type marks it + // ambiguous, and an ambiguous bare name resolves to `Text`. + let mut bare: HashMap = HashMap::new(); + let mut ambiguous: Vec = Vec::new(); + + for side in &sides { + let types = column_types_for(catalog, database_id, &side.collection); + for (column, ty) in &types { + qualified.insert(format!("{}.{column}", side.collection), *ty); + if let Some(alias) = &side.alias { + qualified.insert(format!("{alias}.{column}"), *ty); + } + // `copied` ends the borrow on `bare` before the arms write to it. + match bare.get(column).copied() { + Some(seen) if seen == *ty => {} + Some(_) => { + if !ambiguous.iter().any(|c| c == column) { + ambiguous.push(column.clone()); + } + } + None => { + bare.insert(column.clone(), *ty); + } + } + } + } + + for column in &ambiguous { + bare.insert(column.clone(), DdlColType::Text); + } + for (column, ty) in bare { + qualified.entry(column).or_insert(ty); + } + qualified +} + +/// Collect the scan-like leaves of one join side. +/// +/// A nested join contributes both of its own sides. A plan shape with no +/// single source collection contributes none, so every column it projects +/// stays `Text`. +fn collect_sides(plan: &SqlPlan, out: &mut Vec) { + match plan { + SqlPlan::Join { left, right, .. } => { + collect_sides(left, out); + collect_sides(right, out); + } + // The three read shapes that carry a FROM-clause alias. + SqlPlan::Scan { + collection, alias, .. + } + | SqlPlan::PointGet { + collection, alias, .. + } + | SqlPlan::DocumentIndexLookup { + collection, alias, .. + } => out.push(JoinSide { + collection: collection.clone(), + alias: alias.clone(), + }), + // Reads over one collection that carry no alias slot: the merged row + // qualifies their columns by the collection name. + SqlPlan::RangeScan { collection, .. } + | SqlPlan::TimeseriesScan { collection, .. } + | SqlPlan::SpatialScan { collection, .. } + | SqlPlan::VectorSearch { collection, .. } + | SqlPlan::MultiVectorSearch { collection, .. } + | SqlPlan::SparseSearch { collection, .. } + | SqlPlan::TextSearch { collection, .. } + | SqlPlan::HybridSearch { collection, .. } + | SqlPlan::HybridSearchTriple { collection, .. } + | SqlPlan::RecursiveScan { collection, .. } => out.push(JoinSide { + collection: collection.clone(), + alias: None, + }), + // No single source collection to attribute a column to: set + // operations, aggregates, CTE/subquery bodies, lateral shapes, + // constant and recursive-value rows, every write, and the whole + // array and index DDL family. + SqlPlan::ConstantResult { .. } + | SqlPlan::Insert { .. } + | SqlPlan::KvInsert { .. } + | SqlPlan::Upsert { .. } + | SqlPlan::InsertSelect { .. } + | SqlPlan::Update { .. } + | SqlPlan::UpdateFrom { .. } + | SqlPlan::Delete { .. } + | SqlPlan::Truncate { .. } + | SqlPlan::Aggregate { .. } + | SqlPlan::TimeseriesIngest { .. } + | SqlPlan::Union { .. } + | SqlPlan::Intersect { .. } + | SqlPlan::Except { .. } + | SqlPlan::RecursiveValue { .. } + | SqlPlan::Cte { .. } + | SqlPlan::Subquery { .. } + | SqlPlan::CreateArray { .. } + | SqlPlan::DropArray { .. } + | SqlPlan::AlterArray { .. } + | SqlPlan::InsertArray { .. } + | SqlPlan::DeleteArray { .. } + | SqlPlan::ArraySlice { .. } + | SqlPlan::ArrayProject { .. } + | SqlPlan::ArrayAgg { .. } + | SqlPlan::ArrayElementwise { .. } + | SqlPlan::ArrayFlush { .. } + | SqlPlan::ArrayCompact { .. } + | SqlPlan::Merge { .. } + | SqlPlan::LateralTopK { .. } + | SqlPlan::LateralLoop { .. } + | SqlPlan::VectorPrimaryInsert { .. } + | SqlPlan::CreateIndex { .. } + | SqlPlan::DropIndex { .. } => {} + } +} + +#[cfg(test)] +mod tests { + use super::*; + use nodedb_sql::types::collection::ColumnInfo; + use nodedb_sql::types::query::EngineType; + use nodedb_sql::types_expr::SqlDataType; + + /// Two collections whose `id` columns differ in type, plus a `ts` + /// declared only on `events`. + struct TwoSideCatalog; + + fn column(name: &str, ty: SqlDataType) -> ColumnInfo { + ColumnInfo { + name: name.to_string(), + data_type: ty, + nullable: true, + is_primary_key: false, + default: None, + raw_type: None, + int_width: None, + float_width: None, + } + } + + impl SqlCatalog for TwoSideCatalog { + fn get_collection( + &self, + _database_id: nodedb_types::DatabaseId, + name: &str, + ) -> Result, nodedb_sql::catalog::SqlCatalogError> + { + let columns = match name { + "events" => vec![ + column("ts", SqlDataType::Timestamp), + column("id", SqlDataType::Int64), + ], + "hosts" => vec![column("id", SqlDataType::String)], + _ => return Ok(None), + }; + Ok(Some(nodedb_sql::types::CollectionInfo { + name: name.to_string(), + engine: EngineType::DocumentStrict, + columns, + primary_key: None, + has_auto_tier: false, + indexes: Vec::new(), + bitemporal: false, + primary: nodedb_types::PrimaryEngine::Document, + vector_primary: None, + partition_strategy: nodedb_types::PartitionStrategy::CollectionHomed, + open_schema: nodedb_sql::types::CollectionInfo::open_schema_for( + EngineType::DocumentStrict, + ), + })) + } + } + + fn scan(collection: &str, alias: Option<&str>) -> SqlPlan { + SqlPlan::Scan { + collection: collection.to_string(), + alias: alias.map(str::to_string), + engine: EngineType::DocumentStrict, + filters: Vec::new(), + projection: Vec::new(), + sort_keys: Vec::new(), + limit: None, + offset: 0, + distinct: false, + window_functions: Vec::new(), + temporal: nodedb_sql::temporal::TemporalScope::default(), + } + } + + /// A column declared on one side only keeps that side's catalog type, + /// under both its qualified and its bare name. + #[test] + fn a_column_unique_to_one_side_keeps_its_type() { + let types = join_column_types( + &scan("events", None), + &scan("hosts", None), + &TwoSideCatalog, + nodedb_types::DatabaseId::DEFAULT, + ); + assert_eq!(types.get("events.ts"), Some(&DdlColType::Timestamp)); + assert_eq!(types.get("ts"), Some(&DdlColType::Timestamp)); + } + + /// An alias qualifies the same columns the collection name does. + #[test] + fn an_alias_names_the_same_side() { + let types = join_column_types( + &scan("events", Some("e")), + &scan("hosts", None), + &TwoSideCatalog, + nodedb_types::DatabaseId::DEFAULT, + ); + assert_eq!(types.get("e.ts"), Some(&DdlColType::Timestamp)); + assert_eq!(types.get("events.ts"), Some(&DdlColType::Timestamp)); + } + + /// A bare name two sides declare with different types cannot be + /// attributed, so it stays `Text` while the qualified names keep theirs. + #[test] + fn an_ambiguous_bare_name_stays_text() { + let types = join_column_types( + &scan("events", None), + &scan("hosts", None), + &TwoSideCatalog, + nodedb_types::DatabaseId::DEFAULT, + ); + assert_eq!(types.get("id"), Some(&DdlColType::Text)); + assert_eq!(types.get("events.id"), Some(&DdlColType::Int8)); + assert_eq!(types.get("hosts.id"), Some(&DdlColType::Text)); + } +} diff --git a/nodedb/src/control/planner/sql_plan_convert/output_schema/mod.rs b/nodedb/src/control/planner/sql_plan_convert/output_schema/mod.rs new file mode 100644 index 000000000..7c219e6e8 --- /dev/null +++ b/nodedb/src/control/planner/sql_plan_convert/output_schema/mod.rs @@ -0,0 +1,9 @@ +// SPDX-License-Identifier: BUSL-1.1 + +pub mod build; +pub mod columns; +pub mod join_types; +pub mod returning; + +pub use build::build_output_schema; +pub use returning::build_returning_schema; diff --git a/nodedb/src/control/planner/sql_plan_convert/output_schema/returning.rs b/nodedb/src/control/planner/sql_plan_convert/output_schema/returning.rs new file mode 100644 index 000000000..6f95a8fdb --- /dev/null +++ b/nodedb/src/control/planner/sql_plan_convert/output_schema/returning.rs @@ -0,0 +1,217 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! Derives the announced [`OutputSchema`] of a DML `RETURNING` clause. +//! +//! A `RETURNING` clause is a projection over the target collection, so it is +//! typed by the same rule a `SELECT` projection is: the clause's column list is +//! mapped to [`Projection`] entries and handed to +//! [`schema_from_projection`](super::columns::schema_from_projection), against +//! the same catalog column types and declared order. There is one derivation, +//! so a write and a read of the same column can never announce different types. +//! +//! `RETURNING *` sets `is_star`, exactly as `SELECT *` does. The concrete +//! column list of a star is only knowable from the returned rows — a +//! schemaless row carries fields no catalog column declares — so the shaper +//! keeps the row-derived list and renders those cells as text, the same answer +//! `SELECT *` gives for the same row. + +use nodedb_physical::physical_plan::{ReturningColumns, ReturningSpec}; +use nodedb_sql::catalog::SqlCatalog; +use nodedb_sql::types::query::Projection; +use nodedb_sql::types_expr::SqlExpr; + +use crate::control::server::response_shape::schema::OutputSchema; + +use super::columns::{column_types_for, ordered_columns_for, schema_from_projection}; + +/// The output schema a write announces for `returning` against `collection`. +/// +/// `None` — the statement carries no `RETURNING` clause — announces nothing, +/// which is what a write with no result set must say. +pub fn build_returning_schema( + returning: Option<&ReturningSpec>, + collection: &str, + catalog: &C, + database_id: nodedb_types::DatabaseId, +) -> OutputSchema { + let Some(spec) = returning else { + return OutputSchema::default(); + }; + let projection = returning_projection(spec); + let types = column_types_for(catalog, database_id, collection); + let ordered_cols = ordered_columns_for(catalog, database_id, collection); + schema_from_projection(&projection, &types, &ordered_cols) +} + +/// Maps a `RETURNING` column list to the projection entries the shared +/// derivation reads. +/// +/// The clause's grammar admits a bare column name and an optional alias, and +/// nothing else: `parse_returning_columns` rejects every expression form with a +/// typed error before a spec exists. So an aliased item maps to a +/// `Projection::Computed` wrapping the column reference — the form that keeps +/// the alias as the display name while the value is still looked up under the +/// source column — and a bare item maps to `Projection::Column`. +fn returning_projection(spec: &ReturningSpec) -> Vec { + match &spec.columns { + ReturningColumns::Star => vec![Projection::Star], + ReturningColumns::Named(items) => items + .iter() + .map(|item| match &item.alias { + Some(alias) => Projection::Computed { + expr: SqlExpr::Column { + table: None, + name: item.name.clone(), + }, + alias: alias.clone(), + }, + None => Projection::Column(item.name.clone()), + }) + .collect(), + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::control::server::response_shape::types::DdlColType; + use nodedb_physical::physical_plan::ReturningItem; + + /// Catalog exposing one `points` collection: a `TIMESTAMP` time key, a + /// `TEXT` tag, and a `FLOAT` measurement — the shape a timeseries + /// collection declares. + struct PointsCatalog; + + impl SqlCatalog for PointsCatalog { + fn get_collection( + &self, + _database_id: nodedb_types::DatabaseId, + name: &str, + ) -> Result, nodedb_sql::catalog::SqlCatalogError> + { + use nodedb_sql::types::collection::ColumnInfo; + use nodedb_sql::types::query::EngineType; + use nodedb_sql::types_expr::SqlDataType; + + if name != "points" { + return Ok(None); + } + let col = |n: &str, t: SqlDataType| ColumnInfo { + name: n.to_string(), + data_type: t, + nullable: true, + is_primary_key: false, + default: None, + raw_type: None, + int_width: None, + float_width: None, + }; + Ok(Some(nodedb_sql::types::CollectionInfo { + name: "points".to_string(), + engine: EngineType::Timeseries, + columns: vec![ + col("ts", SqlDataType::Timestamp), + col("host", SqlDataType::String), + col("v", SqlDataType::Float64), + ], + primary_key: None, + has_auto_tier: false, + indexes: Vec::new(), + bitemporal: false, + primary: nodedb_types::PrimaryEngine::Document, + vector_primary: None, + partition_strategy: nodedb_types::PartitionStrategy::CollectionHomed, + open_schema: nodedb_sql::types::CollectionInfo::open_schema_for( + EngineType::Timeseries, + ), + })) + } + } + + fn named(items: &[(&str, Option<&str>)]) -> ReturningSpec { + ReturningSpec { + columns: ReturningColumns::Named( + items + .iter() + .map(|(name, alias)| ReturningItem { + name: (*name).to_string(), + alias: alias.map(str::to_string), + }) + .collect(), + ), + } + } + + fn schema(spec: Option<&ReturningSpec>) -> OutputSchema { + let database_id = nodedb_types::DatabaseId::DEFAULT; + build_returning_schema(spec, "points", &PointsCatalog, database_id) + } + + /// Named columns carry the declared catalog type, in clause order — the + /// same types `SELECT ts, host, v` announces for the same row. + #[test] + fn named_columns_carry_their_declared_types() { + let spec = named(&[("ts", None), ("host", None), ("v", None)]); + let out = schema(Some(&spec)); + assert!(!out.is_star); + let got: Vec<(&str, DdlColType)> = out + .columns + .iter() + .map(|c| (c.display_name.as_str(), c.ty)) + .collect(); + assert_eq!( + got, + vec![ + ("ts", DdlColType::Timestamp), + ("host", DdlColType::Text), + ("v", DdlColType::Float8), + ] + ); + } + + /// An alias names the output column while the value is still looked up + /// under the source column, and it keeps the source column's type. + #[test] + fn an_alias_renames_the_column_and_keeps_its_type() { + let spec = named(&[("v", Some("reading"))]); + let out = schema(Some(&spec)); + assert_eq!(out.columns.len(), 1); + assert_eq!(out.columns[0].display_name, "reading"); + assert_eq!(out.columns[0].lookup_key, "v"); + assert_eq!(out.columns[0].ty, DdlColType::Float8); + } + + /// A column the catalog does not declare falls back to `Text`, the safe + /// default for a schemaless field. + #[test] + fn an_undeclared_column_falls_back_to_text() { + let spec = named(&[("undeclared", None)]); + let out = schema(Some(&spec)); + assert_eq!(out.columns[0].ty, DdlColType::Text); + } + + /// `RETURNING *` sets `is_star`, so the shaper keeps the row-derived + /// column list — the same answer `SELECT *` gives. + #[test] + fn a_star_is_marked_as_one() { + let spec = ReturningSpec { + columns: ReturningColumns::Star, + }; + let out = schema(Some(&spec)); + assert!(out.is_star); + let names: Vec<&str> = out + .columns + .iter() + .map(|c| c.display_name.as_str()) + .collect(); + assert_eq!(names, vec!["ts", "host", "v"]); + } + + /// A write with no `RETURNING` clause announces nothing. + #[test] + fn no_clause_announces_nothing() { + let out = schema(None); + assert!(out.columns.is_empty()); + assert!(!out.is_star); + } +} diff --git a/nodedb/src/control/planner/sql_plan_convert/output_schema_types.rs b/nodedb/src/control/planner/sql_plan_convert/output_schema_types.rs index 69d9cad86..f6c0f623f 100644 --- a/nodedb/src/control/planner/sql_plan_convert/output_schema_types.rs +++ b/nodedb/src/control/planner/sql_plan_convert/output_schema_types.rs @@ -21,14 +21,23 @@ use nodedb_sql::types_expr::{BinaryOp, SqlExpr, SqlValue, UnaryOp}; use crate::control::server::response_shape::types::DdlColType; -/// Resolves a bare (unqualified or table-qualified) column reference to its -/// catalog type from `types`. Returns `None` for any non-column expression, or -/// for a column absent from the map. `types` is keyed by bare column name (the -/// last dotted segment), matching how the catalog map is built for the -/// collection in scope. +/// Resolves a column reference to its catalog type from `types`. Returns +/// `None` for any non-column expression, or for a column absent from the map. +/// +/// A join's map is keyed on the qualified name, because two sides can carry +/// the same bare column with different types, so a table-qualified reference +/// tries `.` first. A single-collection map holds bare names +/// only, where that lookup misses and the bare name answers. fn bare_column_type(expr: &SqlExpr, types: &HashMap) -> Option { match expr { - SqlExpr::Column { name, .. } => types.get(name).copied(), + SqlExpr::Column { + table: Some(table), + name, + } => types + .get(&format!("{table}.{name}")) + .or_else(|| types.get(name)) + .copied(), + SqlExpr::Column { table: None, name } => types.get(name).copied(), _ => None, } } diff --git a/nodedb/src/control/planner/sql_plan_convert/set_ops.rs b/nodedb/src/control/planner/sql_plan_convert/set_ops.rs index 8b4d6dacb..4493c7f02 100644 --- a/nodedb/src/control/planner/sql_plan_convert/set_ops.rs +++ b/nodedb/src/control/planner/sql_plan_convert/set_ops.rs @@ -410,6 +410,7 @@ mod tests { shuffle_agg_num_parts: 0, broadcast_threshold_bytes: 8 * 1024 * 1024, shuffle_agg_threshold: 10_000, + sql_catalog: None, database_id: crate::types::DatabaseId::DEFAULT, tenant_id: crate::types::TenantId::new(0), }, @@ -469,6 +470,7 @@ mod tests { shuffle_agg_num_parts: 0, broadcast_threshold_bytes: 8 * 1024 * 1024, shuffle_agg_threshold: 10_000, + sql_catalog: None, database_id: crate::types::DatabaseId::DEFAULT, tenant_id: crate::types::TenantId::new(0), }, diff --git a/nodedb/src/control/planner/sql_plan_convert/value/defaults.rs b/nodedb/src/control/planner/sql_plan_convert/value/defaults.rs new file mode 100644 index 000000000..cb6e4aba2 --- /dev/null +++ b/nodedb/src/control/planner/sql_plan_convert/value/defaults.rs @@ -0,0 +1,45 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! The one DEFAULT materialization point for the row-shaped DML converters. + +use nodedb_sql::types::SqlValue; + +use super::super::convert::ConvertContext; +use crate::control::planner::plan_error_map::map_plan_error; +use crate::types::TenantId; + +/// Expand each row's omitted DEFAULT columns, before engine dispatch. +/// +/// INSERT and UPSERT call this once per statement, so identity derivation, +/// the primary-key NOT NULL gate, and the stored payload all read the same +/// row. A DEFAULT materialized per engine after that gate refuses a key the +/// declaration supplies. +/// +/// Each DEFAULT compiles once per statement and evaluates once per row, so a +/// `nextval` DEFAULT allocates exactly one value per row of a multi-row VALUES +/// clause and the expression is parsed once however many rows it fills. +/// +/// `column_defaults` empty means nothing to expand: the rows pass through and +/// the catalog is never read. +pub(in super::super) fn expand_row_defaults( + rows: &[Vec<(String, SqlValue)>], + column_defaults: &[(String, String)], + tenant_id: TenantId, + ctx: &ConvertContext, +) -> crate::Result>> { + if column_defaults.is_empty() { + return Ok(rows.to_vec()); + } + let catalog = ctx.sql_catalog()?; + let compiled = nodedb_sql::planner::defaults::ColumnDefaults::compile_pairs(column_defaults) + .map_err(|e| map_plan_error(e, tenant_id))?; + let mut expanded = Vec::with_capacity(rows.len()); + for row in rows { + let mut row = row.clone(); + compiled + .materialize_row(&mut row, catalog) + .map_err(|e| map_plan_error(e, tenant_id))?; + expanded.push(row); + } + Ok(expanded) +} diff --git a/nodedb/src/control/planner/sql_plan_convert/value/mod.rs b/nodedb/src/control/planner/sql_plan_convert/value/mod.rs index 778074ead..851ae2e54 100644 --- a/nodedb/src/control/planner/sql_plan_convert/value/mod.rs +++ b/nodedb/src/control/planner/sql_plan_convert/value/mod.rs @@ -1,10 +1,11 @@ // SPDX-License-Identifier: BUSL-1.1 -//! Value conversion utilities: SqlValue ↔ nodedb_types::Value, msgpack encoding, -//! and the re-export of the shared column-default evaluator. +//! Value conversion utilities: SqlValue ↔ nodedb_types::Value, msgpack +//! encoding, and the one DEFAULT materialization point. pub(super) mod assignments; pub(super) mod convert; +pub(super) mod defaults; pub(super) mod msgpack_write; pub(super) mod rows; @@ -14,12 +15,9 @@ pub(super) use assignments::{ pub(super) use convert::{ sql_value_to_bytes, sql_value_to_msgpack, sql_value_to_nodedb_value, sql_value_to_string, }; -// The evaluator lives in the SQL crate so every engine that materializes a -// DEFAULT produces the same value for the same expression; re-exported here so -// the document/columnar converters keep their existing import path. +pub(super) use defaults::expand_row_defaults; pub(super) use msgpack_write::{ row_to_msgpack, write_msgpack_array_header, write_msgpack_map_header, write_msgpack_str, write_msgpack_value, }; -pub(super) use nodedb_sql::planner::defaults::evaluate_default_expr; pub(super) use rows::rows_to_msgpack_array; diff --git a/nodedb/src/control/planner/sql_plan_convert/value/rows.rs b/nodedb/src/control/planner/sql_plan_convert/value/rows.rs index 2fcf789af..ab35416aa 100644 --- a/nodedb/src/control/planner/sql_plan_convert/value/rows.rs +++ b/nodedb/src/control/planner/sql_plan_convert/value/rows.rs @@ -5,28 +5,18 @@ use nodedb_sql::types::SqlValue; use super::convert::sql_value_to_nodedb_value; -use nodedb_sql::planner::defaults::evaluate_default_expr; -pub(crate) fn rows_to_msgpack_array( - rows: &[&Vec<(String, SqlValue)>], - column_defaults: &[(String, String)], -) -> crate::Result> { +/// Encode already-expanded rows as one msgpack array of maps. +/// +/// Callers materialize DEFAULTs through `expand_row_defaults` before routing, +/// so every column the declaration promises is already present in `rows`. +pub(crate) fn rows_to_msgpack_array(rows: &[&Vec<(String, SqlValue)>]) -> crate::Result> { let mut arr: Vec = Vec::with_capacity(rows.len()); for row in rows { let mut map = std::collections::HashMap::new(); for (key, val) in row.iter() { map.insert(key.clone(), sql_value_to_nodedb_value(val)); } - for (col_name, default_expr) in column_defaults { - if !map.contains_key(col_name) - && let Some(val) = - evaluate_default_expr(default_expr).map_err(|e| crate::Error::PlanError { - detail: format!("default for column '{col_name}': {e}"), - })? - { - map.insert(col_name.clone(), val); - } - } arr.push(nodedb_types::Value::Object(map)); } let val = nodedb_types::Value::Array(arr); diff --git a/nodedb/src/control/planner/sql_plan_convert/visitor/arms_dml.rs b/nodedb/src/control/planner/sql_plan_convert/visitor/arms_dml.rs index 93f937107..90dffec3d 100644 --- a/nodedb/src/control/planner/sql_plan_convert/visitor/arms_dml.rs +++ b/nodedb/src/control/planner/sql_plan_convert/visitor/arms_dml.rs @@ -10,7 +10,8 @@ macro_rules! impl_dml_arms_for_convert_visitor { ) -> crate::Result> { let nodedb_sql::InsertVisitArgs { collection, - engine, + engine: _engine, + route, rows, column_defaults, if_absent, @@ -19,7 +20,7 @@ macro_rules! impl_dml_arms_for_convert_visitor { } = args; super::super::dml::convert_insert(super::super::dml::ConvertInsertArgs { collection, - engine: &engine, + route, rows, column_defaults, column_schema, @@ -36,7 +37,8 @@ macro_rules! impl_dml_arms_for_convert_visitor { ) -> crate::Result> { let nodedb_sql::UpsertVisitArgs { collection, - engine, + engine: _engine, + route, rows, column_defaults, on_conflict_updates, @@ -45,7 +47,7 @@ macro_rules! impl_dml_arms_for_convert_visitor { } = args; super::super::dml::convert_upsert(super::super::dml::ConvertUpsertArgs { collection, - engine: &engine, + route, rows, column_defaults, column_schema, diff --git a/nodedb/src/control/scatter_gather/hop.rs b/nodedb/src/control/scatter_gather/hop.rs index 4cb4af739..cd436aa37 100644 --- a/nodedb/src/control/scatter_gather/hop.rs +++ b/nodedb/src/control/scatter_gather/hop.rs @@ -192,7 +192,7 @@ pub async fn coordinate_cross_shard_hop( crate::types::TenantId::new(tenant_id_u64), database_id, &security.context(shared), - false, + None, ) .await { diff --git a/nodedb/src/control/sequence/mod.rs b/nodedb/src/control/sequence/mod.rs index a4f7525a2..16ba6f427 100644 --- a/nodedb/src/control/sequence/mod.rs +++ b/nodedb/src/control/sequence/mod.rs @@ -6,10 +6,12 @@ pub mod gap_free; pub mod log; pub mod range_alloc; pub mod registry; +pub mod session_values; pub mod types; pub use self::format::{FormatToken, ResetScope}; pub use self::gap_free::GapFreeManager; pub use self::range_alloc::RangeAllocator; pub use self::registry::SequenceRegistry; +pub use self::session_values::SessionSequenceValues; pub use self::types::SequenceError; diff --git a/nodedb/src/control/sequence/range_alloc.rs b/nodedb/src/control/sequence/range_alloc.rs index abff0d486..f469eacc9 100644 --- a/nodedb/src/control/sequence/range_alloc.rs +++ b/nodedb/src/control/sequence/range_alloc.rs @@ -141,7 +141,7 @@ impl RangeAllocator { let current_val = state .sequence_registry - .currval(database_id, tenant_id, sequence_name) + .node_current_value(database_id, tenant_id, sequence_name) .unwrap_or(0); let range_start = current_val + increment; diff --git a/nodedb/src/control/sequence/registry.rs b/nodedb/src/control/sequence/registry.rs index 8edb090b3..3b4b7d4a2 100644 --- a/nodedb/src/control/sequence/registry.rs +++ b/nodedb/src/control/sequence/registry.rs @@ -239,8 +239,10 @@ impl SequenceRegistry { handle.nextval_batch(n) } - /// Get the current value (last nextval result on this node). - pub fn currval( + /// Read this NODE's counter — the last value any session on this node + /// allocated. SQL `currval` is session-scoped and does not read this; + /// range allocation and diagnostics do. + pub fn node_current_value( &self, database_id: u64, tenant_id: u64, @@ -419,7 +421,7 @@ pub enum SequenceValue { Formatted(String), } -fn registry_key(database_id: u64, tenant_id: u64, name: &str) -> String { +pub(crate) fn registry_key(database_id: u64, tenant_id: u64, name: &str) -> String { format!("{database_id}:{tenant_id}:{name}") } @@ -452,9 +454,9 @@ mod tests { ); assert_eq!(registry.nextval(4, 1, "orders_seq").unwrap(), 1); assert_eq!(registry.nextval(4, 1, "orders_seq").unwrap(), 2); - assert_eq!(registry.currval(4, 1, "orders_seq").unwrap(), 2); + assert_eq!(registry.node_current_value(4, 1, "orders_seq").unwrap(), 2); assert_eq!(registry.setval(4, 1, "orders_seq", 10).unwrap(), 10); - assert_eq!(registry.currval(4, 1, "orders_seq").unwrap(), 10); + assert_eq!(registry.node_current_value(4, 1, "orders_seq").unwrap(), 10); }) .await; } diff --git a/nodedb/src/control/sequence/session_values.rs b/nodedb/src/control/sequence/session_values.rs new file mode 100644 index 000000000..09c738a4b --- /dev/null +++ b/nodedb/src/control/sequence/session_values.rs @@ -0,0 +1,52 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! Per-session record of the last `nextval` each sequence handed this session. +//! +//! SQL `currval` returns the last value THIS SESSION obtained from `nextval`, +//! never the node-wide counter that `SequenceRegistry::node_current_value` +//! reads. Keys match `SequenceRegistry`'s: `"{database_id}:{tenant_id}:{name}"`. + +use std::collections::HashMap; +use std::sync::Mutex; + +use super::registry::registry_key; + +/// One connection's `currval` state, shared with the plan-time catalog adapter. +#[derive(Default)] +pub struct SessionSequenceValues { + values: Mutex>, +} + +impl SessionSequenceValues { + pub fn new() -> Self { + Self::default() + } + + /// Record the value `nextval` just handed this session. + pub fn record(&self, database_id: u64, tenant_id: u64, name: &str, value: i64) { + let key = registry_key(database_id, tenant_id, name); + self.values + .lock() + .unwrap_or_else(|p| p.into_inner()) + .insert(key, value); + } + + /// The last value this session obtained for `name`, or `None` when this + /// session never called `nextval` on it. + pub fn last(&self, database_id: u64, tenant_id: u64, name: &str) -> Option { + let key = registry_key(database_id, tenant_id, name); + self.values + .lock() + .unwrap_or_else(|p| p.into_inner()) + .get(&key) + .copied() + } + + /// Drop every recorded value. Called on a full session reset. + pub fn clear(&self) { + self.values + .lock() + .unwrap_or_else(|p| p.into_inner()) + .clear(); + } +} diff --git a/nodedb/src/control/server/pgwire/handler/cursor_query.rs b/nodedb/src/control/server/pgwire/handler/cursor_query.rs index 9cae1f414..7d0310141 100644 --- a/nodedb/src/control/server/pgwire/handler/cursor_query.rs +++ b/nodedb/src/control/server/pgwire/handler/cursor_query.rs @@ -68,7 +68,7 @@ impl NodeDbPgHandler { permission_cache: Some(&*perm_cache), }; let (tasks, _output_schema, versions, _) = query_ctx - .plan_sql_with_rls_and_versions(sql, tenant_id, database_id, &sec, false) + .plan_sql_with_rls_and_versions(sql, tenant_id, database_id, &sec, None) .await .map_err(StatementSetupError::from)?; drop(perm_cache); diff --git a/nodedb/src/control/server/pgwire/handler/prepared/parser.rs b/nodedb/src/control/server/pgwire/handler/prepared/parser.rs index 524fbb171..1ae47ff18 100644 --- a/nodedb/src/control/server/pgwire/handler/prepared/parser.rs +++ b/nodedb/src/control/server/pgwire/handler/prepared/parser.rs @@ -23,19 +23,19 @@ use crate::control::state::SharedState; use super::super::auth::{pgwire_authorization_error, resolve_session_identity}; use super::statement::ParsedStatement; -use parser_schema::{ - count_placeholders, is_dsl_statement, result_fields_for_returning, - substitute_placeholders_with_null, -}; +use parser_schema::{count_placeholders, is_dsl_statement, substitute_placeholders_with_null}; #[path = "parser_schema.rs"] mod parser_schema; -/// Maps the response shaper's protocol-neutral wire type to a pgwire -/// `Type` for RowDescription. Mirrors the (now-deleted) SQL-reparse path's -/// `sql_data_type_to_pg`; variants with no dedicated wire type still fall -/// back to `Type::TEXT` where that was the prior fallback, matching -/// today's behavior. +/// Maps the response shaper's protocol-neutral wire type to a pgwire `Type` +/// for RowDescription. +/// +/// This is the only mapping the extended-query path uses. Every result column +/// — a `SELECT` projection and a `RETURNING` clause alike — reaches it through +/// `build_output_schema`, so Describe and the simple-query path answer one +/// question with one rule. A `DdlColType` with no dedicated wire type resolves +/// to `Type::TEXT`, the safe default a client can always parse. fn ddl_col_type_to_pg(ty: &DdlColType) -> Type { match ty { DdlColType::Int8 => Type::INT8, @@ -302,25 +302,21 @@ impl NodeDbQueryParser { Err(_) => return (param_types, Vec::new()), }; - // When the original SQL had a RETURNING clause on a DML statement, - // build result fields from the collection schema and the RETURNING spec. - if let Some(spec) = returning_spec - && let Some(fields) = result_fields_for_returning(&spec, plans.first(), catalog) - { - return (param_types, fields); - } - // Infer result fields from the planner's authoritative output // schema — the same derivation used to shape response rows, so // Describe's RowDescription always matches what Execute returns. - // Empty `plans` (already handled above) or a plan variant with no - // resolvable projection yields an empty `OutputSchema`, matching - // today's `Vec::new()` fallback for DSL/non-SELECT statements. + // A write plan announces what its `RETURNING` spec projects, through + // that same derivation, so Describe and the simple-query path can + // never disagree on a column's type. Empty `plans` (already handled + // above) or a plan variant with no resolvable projection yields an + // empty `OutputSchema`, matching the `Vec::new()` fallback for + // DSL/non-SELECT statements. let output_schema = crate::control::planner::sql_plan_convert::output_schema::build_output_schema( &plans, catalog, database_id, + returning_spec.as_ref(), ); let result_fields: Vec = output_schema .columns diff --git a/nodedb/src/control/server/pgwire/handler/prepared/parser_schema.rs b/nodedb/src/control/server/pgwire/handler/prepared/parser_schema.rs index f812c04de..4457aa8d4 100644 --- a/nodedb/src/control/server/pgwire/handler/prepared/parser_schema.rs +++ b/nodedb/src/control/server/pgwire/handler/prepared/parser_schema.rs @@ -1,14 +1,10 @@ // SPDX-License-Identifier: BUSL-1.1 -//! Schema-inference utilities for `NodeDbQueryParser`. +//! SQL-text utilities for `NodeDbQueryParser`. //! -//! These free functions are called by `parser.rs` during Parse-message -//! handling to infer parameter and result-field types from SQL text and -//! catalog metadata. - -use nodedb_types::DatabaseId; -use pgwire::api::Type; -use pgwire::api::results::FieldInfo; +//! `parser.rs` calls these during Parse-message handling to classify a +//! statement and to count and neutralise its `$N` placeholders. Result-column +//! types come from the planner's `OutputSchema`, never from here. /// Return true if `sql` starts with a DSL or DDL keyword that `plan_sql` /// cannot parse and must be routed through `execute_sql` at Execute time. @@ -123,99 +119,6 @@ pub(super) fn count_placeholders(sql: &str) -> usize { max_idx } -/// Build result `FieldInfo`s for a DML statement with a RETURNING clause. -/// -/// Resolves the target collection from the DML plan, looks up its schema, and -/// projects the RETURNING spec onto it. Returns `None` if the plan isn't a -/// recognized DML type or the collection schema cannot be found. -pub(super) fn result_fields_for_returning( - spec: &nodedb_physical::physical_plan::ReturningSpec, - plan: Option<&nodedb_sql::SqlPlan>, - catalog: &dyn nodedb_sql::SqlCatalog, -) -> Option> { - use nodedb_physical::physical_plan::{ReturningColumns, ReturningItem}; - use nodedb_sql::types::SqlDataType; - use pgwire::api::results::FieldFormat; - - // Local, self-contained mapping from the planner's `SqlDataType` to a - // pgwire `Type`; kept private to this function since RETURNING is the - // only remaining caller after the DESCRIBE path moved to the planner's - // authoritative `OutputSchema` (see `ddl_col_type_to_pg` in parser.rs). - fn returning_col_type_to_pg(dt: &SqlDataType) -> Type { - match dt { - SqlDataType::Int64 => Type::INT8, - SqlDataType::Float64 => Type::FLOAT8, - SqlDataType::String => Type::TEXT, - SqlDataType::Bool => Type::BOOL, - SqlDataType::Bytes => Type::BYTEA, - SqlDataType::Timestamp => Type::TIMESTAMP, - SqlDataType::Timestamptz => Type::TIMESTAMPTZ, - SqlDataType::Decimal => Type::NUMERIC, - SqlDataType::Uuid => Type::TEXT, - SqlDataType::Vector(_) => Type::BYTEA, - SqlDataType::Geometry => Type::BYTEA, - SqlDataType::Unknown => Type::TEXT, - } - } - - // Every write that can carry a RETURNING clause resolves its target here. - // A write whose target is missed announces NO result columns while its - // response still ships one field per stored column, which the client - // cannot read against the RowDescription it was given — so the list is - // the write plans, not just the two that first needed it. - let collection = match plan? { - nodedb_sql::SqlPlan::Update { collection, .. } - | nodedb_sql::SqlPlan::UpdateFrom { collection, .. } - | nodedb_sql::SqlPlan::Delete { collection, .. } - | nodedb_sql::SqlPlan::Insert { collection, .. } - | nodedb_sql::SqlPlan::KvInsert { collection, .. } - | nodedb_sql::SqlPlan::Upsert { collection, .. } - | nodedb_sql::SqlPlan::TimeseriesIngest { collection, .. } - | nodedb_sql::SqlPlan::VectorPrimaryInsert { collection, .. } => collection.as_str(), - nodedb_sql::SqlPlan::Merge { target, .. } - | nodedb_sql::SqlPlan::InsertSelect { target, .. } => target.as_str(), - _ => return None, - }; - - let info = catalog - .get_collection(DatabaseId::DEFAULT, collection) - .ok() - .flatten()?; - - let columns_to_field_info = |columns: &[nodedb_sql::ColumnInfo]| -> Vec { - columns - .iter() - .map(|c| { - FieldInfo::new( - c.name.clone(), - None, - None, - returning_col_type_to_pg(&c.data_type), - FieldFormat::Text, - ) - }) - .collect() - }; - - let fields = match &spec.columns { - ReturningColumns::Star => columns_to_field_info(&info.columns), - ReturningColumns::Named(items) => items - .iter() - .map(|item: &ReturningItem| { - let display_name = item.alias.clone().unwrap_or_else(|| item.name.clone()); - let pg_type = info - .columns - .iter() - .find(|c| c.name == item.name) - .map(|c| returning_col_type_to_pg(&c.data_type)) - .unwrap_or(Type::TEXT); - FieldInfo::new(display_name, None, None, pg_type, FieldFormat::Text) - }) - .collect(), - }; - Some(fields) -} - #[cfg(test)] mod tests { use super::*; diff --git a/nodedb/src/control/server/pgwire/handler/routing/calvin_dispatch.rs b/nodedb/src/control/server/pgwire/handler/routing/calvin_dispatch.rs index 2a00170c7..a9208b00e 100644 --- a/nodedb/src/control/server/pgwire/handler/routing/calvin_dispatch.rs +++ b/nodedb/src/control/server/pgwire/handler/routing/calvin_dispatch.rs @@ -53,14 +53,16 @@ fn meter_calvin_task( /// Who issued the statement and how its rows must be encoded back. /// -/// Bundled because these four are the connection's identity, not parameters of -/// the dispatch: they are looked up together at the call site and travel -/// unchanged through every branch below. +/// Bundled because these are the connection's identity and result contract, +/// not parameters of the dispatch: they are looked up together at the call +/// site and travel unchanged through every branch below. pub(super) struct CalvinDispatchSession<'a> { pub identity: &'a AuthenticatedIdentity, pub session_id: SessionId, pub result_formats: &'a [pgwire::api::results::FieldFormat], pub auth: &'a crate::control::security::auth_context::AuthContext, + /// The statement's announced output columns, when it announced any. + pub projection: Option<&'a crate::control::server::response_shape::schema::OutputSchema>, } impl NodeDbPgHandler { @@ -81,6 +83,7 @@ impl NodeDbPgHandler { session_id, result_formats, auth, + projection, } = session; let cross_shard_mode = self.sessions.cross_shard_txn_mode(session_id); let tx_state = self.sessions.transaction_state(session_id); @@ -180,6 +183,7 @@ impl NodeDbPgHandler { task, apply_resp.as_ref(), CalvinResponseCtx { + projection, state: &self.state, tenant_id, database_id, @@ -259,6 +263,7 @@ impl NodeDbPgHandler { task, outcome.apply_result.as_ref(), CalvinResponseCtx { + projection, state: &self.state, tenant_id, database_id, diff --git a/nodedb/src/control/server/pgwire/handler/routing/calvin_response.rs b/nodedb/src/control/server/pgwire/handler/routing/calvin_response.rs index a73d059e2..dd305ec63 100644 --- a/nodedb/src/control/server/pgwire/handler/routing/calvin_response.rs +++ b/nodedb/src/control/server/pgwire/handler/routing/calvin_response.rs @@ -16,6 +16,11 @@ use nodedb_physical::physical_task::PhysicalTask; /// Shared inputs for shaping one task of a completed Calvin batch. pub(super) struct CalvinResponseCtx<'a> { + /// The statement's announced output columns, when it announced any. A + /// `RETURNING` write is held to them here exactly as the single-shard + /// dispatch loop holds it, so the same statement renders the same row + /// whichever route it took. + pub(super) projection: Option<&'a crate::control::server::response_shape::schema::OutputSchema>, pub(super) state: &'a crate::control::state::SharedState, pub(super) tenant_id: TenantId, pub(super) database_id: crate::types::DatabaseId, @@ -55,6 +60,7 @@ pub(super) fn calvin_execution_response( use crate::control::server::response_shape::types::{PlanKind, describe_plan}; let CalvinResponseCtx { + projection, state, tenant_id, database_id, @@ -70,7 +76,7 @@ pub(super) fn calvin_execution_response( payload: resp.payload.as_bytes(), plan: &task.plan, plan_kind: PlanKind::ReturningRows, - projection: None, + projection, state, database_id, tenant_id, diff --git a/nodedb/src/control/server/pgwire/handler/routing/execute.rs b/nodedb/src/control/server/pgwire/handler/routing/execute.rs index 81a737d1b..26538d414 100644 --- a/nodedb/src/control/server/pgwire/handler/routing/execute.rs +++ b/nodedb/src/control/server/pgwire/handler/routing/execute.rs @@ -144,7 +144,10 @@ impl NodeDbPgHandler { tenant_id, identity, session_id, - shaping.formats, + ResultShaping { + projection: effective_schema, + formats: shaping.formats, + }, &auth_ctx, ) .await? @@ -214,6 +217,7 @@ impl NodeDbPgHandler { session_id, result_formats: shaping.formats, auth: &auth_ctx, + projection: effective_schema, }, &sum_target_reads, ) diff --git a/nodedb/src/control/server/pgwire/handler/routing/planning.rs b/nodedb/src/control/server/pgwire/handler/routing/planning.rs index 41f600a6b..194f2d032 100644 --- a/nodedb/src/control/server/pgwire/handler/routing/planning.rs +++ b/nodedb/src/control/server/pgwire/handler/routing/planning.rs @@ -156,13 +156,12 @@ impl NodeDbPgHandler { // Request-admission already ran once in `execute_single_sql` — must not admit again. // Per-query ON DENY always wins over the session-level override in `scope`. - let (clean_sql, scope) = + let (statement_sql, scope) = crate::control::server::session_auth::apply_per_query_on_deny(sql, scope); // Strip RETURNING clause before DataFusion planning. let (clean_sql, returning_spec) = - returning::strip_returning(&clean_sql).map_err(StatementSetupError::from)?; - let has_returning = returning_spec.is_some(); + returning::strip_returning(&statement_sql).map_err(StatementSetupError::from)?; // Forwards per-session planning GUCs into the shared query context, protocol-neutral // so pgwire and native honor them identically; flags drive the cache bypass below. @@ -215,9 +214,13 @@ impl NodeDbPgHandler { let state = Arc::clone(&self.state); let tenant = tenant_id.as_u64(); let db = database_id; + // Keyed on the statement INCLUDING its `RETURNING` clause: the + // clause is stripped before planning, so two statements that + // differ only in what they project share the stripped text — and + // the cached entry carries the announced output columns. self.sessions.get_cached_plan( session_id, - &clean_sql, + &statement_sql, move |id| current_descriptor_version(&state, tenant, db, id), current_permission_tree_version, current_rls_version, @@ -243,6 +246,7 @@ impl NodeDbPgHandler { tenant_id, database_id, &sec, + returning_spec.as_ref(), ) .await .map_err(StatementSetupError::from)?; @@ -275,7 +279,7 @@ impl NodeDbPgHandler { tenant_id, database_id, &sec, - has_returning, + returning_spec.as_ref(), ) .await .map_err(StatementSetupError::from)? @@ -286,7 +290,7 @@ impl NodeDbPgHandler { if !bypass_cache && cache_eligibility.is_cacheable() { self.sessions.put_cached_plan( session_id, - &clean_sql, + &statement_sql, planned.clone(), versions.clone(), output_schema.clone(), diff --git a/nodedb/src/control/server/pgwire/handler/routing/pre_dispatch.rs b/nodedb/src/control/server/pgwire/handler/routing/pre_dispatch.rs index 90af0937a..a6978eabf 100644 --- a/nodedb/src/control/server/pgwire/handler/routing/pre_dispatch.rs +++ b/nodedb/src/control/server/pgwire/handler/routing/pre_dispatch.rs @@ -2,7 +2,7 @@ //! Pre-dispatch routing gates for pgwire planned task sets. -use pgwire::api::results::{FieldFormat, Response}; +use pgwire::api::results::Response; use pgwire::error::{ErrorInfo, PgWireError, PgWireResult}; use nodedb_physical::physical_task::PhysicalTask; @@ -90,9 +90,13 @@ impl NodeDbPgHandler { tenant_id: TenantId, identity: &AuthenticatedIdentity, session_id: SessionId, - result_formats: &[FieldFormat], + shaping: ResultShaping<'_>, auth: &crate::control::security::auth_context::AuthContext, ) -> PgWireResult>> { + let ResultShaping { + projection, + formats: result_formats, + } = shaping; let tx_state = self.sessions.transaction_state(session_id); if tx_state == crate::control::server::shared::session::TransactionState::InBlock || self.state.calvin_completion_registry.get().is_none() @@ -121,6 +125,7 @@ impl NodeDbPgHandler { session_id, result_formats, auth, + projection, }, // No settled image read to carry — this fires before materialized-sum settlement. &[], diff --git a/nodedb/src/control/server/pgwire/types/error_map.rs b/nodedb/src/control/server/pgwire/types/error_map.rs index d4299d39c..976634640 100644 --- a/nodedb/src/control/server/pgwire/types/error_map.rs +++ b/nodedb/src/control/server/pgwire/types/error_map.rs @@ -45,11 +45,22 @@ pub fn error_to_sqlstate(err: &crate::Error) -> (&'static str, &'static str, Str it is hard-deleted" ), ), + crate::Error::FeatureNotSupported { detail } => { + ("ERROR", sqlstate::FEATURE_NOT_SUPPORTED, detail.clone()) + } crate::Error::UndefinedFunction { name } => ( "ERROR", sqlstate::UNDEFINED_FUNCTION, format!("function {name}(...) does not exist"), ), + crate::Error::UndefinedObject { .. } => { + ("ERROR", sqlstate::UNDEFINED_OBJECT, err.to_string()) + } + crate::Error::ObjectNotInPrerequisiteState { detail, .. } => ( + "ERROR", + sqlstate::OBJECT_NOT_IN_PREREQUISITE_STATE, + detail.clone(), + ), crate::Error::UndefinedColumn { column } => ( "ERROR", sqlstate::UNDEFINED_COLUMN, diff --git a/nodedb/src/control/server/response_shape/returning.rs b/nodedb/src/control/server/response_shape/returning.rs index f2ef2c848..82af689ec 100644 --- a/nodedb/src/control/server/response_shape/returning.rs +++ b/nodedb/src/control/server/response_shape/returning.rs @@ -24,11 +24,15 @@ //! construction* rather than by coincidence, and every cell stays under the //! name it was stored with — no padding, no truncation, no re-alignment. //! -//! The simple-query protocol passes no projection (a DML plan's `OutputSchema` -//! is empty) and keeps the row-derived list. That divergence is correct: it -//! emits the RowDescription and the DataRows together out of this one -//! `ShapedRows`, so there is no earlier announcement to honour, and a -//! schemaless row's undeclared fields stay visible. +//! The simple-query protocol announces the same list: the planner derives a +//! DML plan's `OutputSchema` from its `RETURNING` column list, so a named +//! clause is held to those columns and their catalog types, and a returned +//! cell renders exactly as the same column renders under `SELECT`. +//! +//! `RETURNING *` announces `is_star` instead, and a star keeps the row-derived +//! list. The concrete columns of a star are only knowable once the rows exist — +//! a schemaless row carries fields no catalog column declares — which is the +//! same answer `SELECT *` gives for the same row. use serde_json::{Map, Value as JsonValue}; diff --git a/nodedb/src/control/server/shared/ddl/neutral/collection/create/build.rs b/nodedb/src/control/server/shared/ddl/neutral/collection/create/build.rs index 8d695562e..a3ed56209 100644 --- a/nodedb/src/control/server/shared/ddl/neutral/collection/create/build.rs +++ b/nodedb/src/control/server/shared/ddl/neutral/collection/create/build.rs @@ -37,6 +37,7 @@ use super::request::CreateCollectionRequest; use super::build_flags::{err, resolve_crdt_flag, validate_crdt_signing_storage, validate_name}; use super::build_post_create::{create_serial_sequences, log_vector_fields}; use super::build_primary_engine::resolve_primary_engine; +use crate::control::server::shared::ddl::neutral::column_default::validate_column_defaults; /// Per-surface configuration. The fields are the entire surface-level /// difference between `CREATE COLLECTION` and `CREATE TABLE`. @@ -86,6 +87,13 @@ pub async fn build_and_persist( )); } + // Refuse a DEFAULT the server cannot evaluate here, not at the first + // INSERT. It runs before any lifecycle guard or predecessor purge, so a + // rejected declaration leaves the existing state untouched. A SERIAL + // column carries no DEFAULT text yet; the `nextval` this build generates + // for it names a registered function and clears the same gate. + validate_column_defaults(columns)?; + let tenant_id = identity.tenant_id; // Metadata Raft serializes clustered DDL. Without it, hold an exclusive @@ -179,10 +187,22 @@ pub async fn build_and_persist( let canonical_engine = validate_engine_name(engine, options)?; let bitemporal_flag = flags.iter().any(|f| f == "BITEMPORAL"); + // Expand SERIAL / BIGSERIAL first, so every later reader of the column + // list sees the backing type plus the `nextval` DEFAULT naming the + // sequence `create_serial_sequences` materializes below. + let (expanded_columns, serial_fields) = + crate::control::server::shared::ddl::schema_validation::parse_fields_clause_from_pairs( + name, columns, + ); + // Resolve user-defined type names to TEXT for schema building. // Original names are preserved in `fields` for drop-protection. - let resolved_columns: Vec<(String, String)> = - resolve_custom_type_columns(columns, state, database_id.as_u64(), tenant_id.as_u64()); + let resolved_columns: Vec<(String, String)> = resolve_custom_type_columns( + &expanded_columns, + state, + database_id.as_u64(), + tenant_id.as_u64(), + ); let (collection_type, columnar_schema_columns) = nodedb_sql::ddl_ast::build_collection_type( canonical_engine, @@ -193,10 +213,7 @@ pub async fn build_and_persist( ) .map_err(|e| err("42601", e.to_string()))?; - let (mut fields, serial_fields) = - crate::control::server::shared::ddl::schema_validation::parse_fields_clause_from_pairs( - columns, - ); + let mut fields = expanded_columns.clone(); if fields.is_empty() && !columnar_schema_columns.is_empty() { fields = columnar_schema_columns; } @@ -210,7 +227,7 @@ pub async fn build_and_persist( }; let (primary, vector_primary) = - resolve_primary_engine(options, columns, &fields, &collection_type)?; + resolve_primary_engine(options, &expanded_columns, &fields, &collection_type)?; let append_only = flags.iter().any(|f| f == "APPEND_ONLY"); let hash_chain = flags.iter().any(|f| f == "HASH_CHAIN"); @@ -245,7 +262,7 @@ pub async fn build_and_persist( // column list. Recorded on every engine so schemaless collections can // key their document id off it instead of the hardcoded `id` field; // harmless for strict/KV, which already track the PK on their schema. - let declared_primary_key = columns.iter().find_map(|(col_name, type_str)| { + let declared_primary_key = expanded_columns.iter().find_map(|(col_name, type_str)| { let (_, is_pk, _, _) = nodedb_sql::ddl_ast::collection_type::parse_column_type_str_full(type_str); is_pk.then(|| col_name.clone()) diff --git a/nodedb/src/control/server/shared/ddl/neutral/collection/dml/parse/dispatch.rs b/nodedb/src/control/server/shared/ddl/neutral/collection/dml/parse/dispatch.rs index a4c49ab5d..8370b4aee 100644 --- a/nodedb/src/control/server/shared/ddl/neutral/collection/dml/parse/dispatch.rs +++ b/nodedb/src/control/server/shared/ddl/neutral/collection/dml/parse/dispatch.rs @@ -134,7 +134,7 @@ pub(in crate::control::server::shared::ddl::neutral::collection) async fn plan_a // Injection happens HERE, before the task set is consumed: implicit-edge // extraction, authorization, staging, and dispatch all read `tasks` after // this point, and injecting later would hand them un-injected copies. - let (mut tasks, versions) = { + let (mut tasks, output_schema, versions) = { let scope = RequestAuthScope::for_database(identity, state.auth_stores(), database_id); let permission_cache = state.permission_cache.read().await; let sec = PlanSecurityContext { @@ -147,14 +147,20 @@ pub(in crate::control::server::shared::ddl::neutral::collection) async fn plan_a permission_cache: Some(&*permission_cache), }; let query_ctx = crate::control::planner::context::QueryContext::for_state(state); - let (tasks, _output_schema, versions, _) = query_ctx - .plan_sql_with_rls_and_versions(sql, tenant_id, database_id, &sec, false) + let (tasks, output_schema, versions, _) = query_ctx + .plan_sql_with_rls_and_versions( + sql, + tenant_id, + database_id, + &sec, + returning_spec.as_ref(), + ) .await .map_err(|error| { let (_, sqlstate, message) = error_to_sqlstate(&error); ddl_err(sqlstate, message) })?; - (tasks, versions) + (tasks, output_schema, versions) }; // Attach the projection to every planned write, refusing any insert shape @@ -414,7 +420,9 @@ pub(in crate::control::server::shared::ddl::neutral::collection) async fn plan_a payload: response.payload.as_bytes(), plan: &task.plan, plan_kind: PlanKind::ReturningRows, - projection: None, + // The statement's announced `RETURNING` columns, so this + // transport renders a returned cell exactly as pgwire does. + projection: Some(&output_schema), state, database_id, tenant_id, diff --git a/nodedb/src/control/server/shared/ddl/neutral/collection/register.rs b/nodedb/src/control/server/shared/ddl/neutral/collection/register.rs index 6d3f8e789..f0674723a 100644 --- a/nodedb/src/control/server/shared/ddl/neutral/collection/register.rs +++ b/nodedb/src/control/server/shared/ddl/neutral/collection/register.rs @@ -62,7 +62,7 @@ pub async fn dispatch_register_if_needed( return Ok(()); }; let (fields, _serial_fields) = - crate::control::server::shared::ddl::schema_validation::parse_fields_clause(parts); + crate::control::server::shared::ddl::schema_validation::parse_fields_clause(&name, parts); let mut indexes = derive_auto_indexes(fields.iter().map(|(n, _)| n.as_str())); extend_with_catalog_indexes(&mut indexes, &coll); // `sql` is unused on this leader-side path: index derivation reads diff --git a/nodedb/src/control/server/shared/ddl/neutral/column_default.rs b/nodedb/src/control/server/shared/ddl/neutral/column_default.rs new file mode 100644 index 000000000..390409a66 --- /dev/null +++ b/nodedb/src/control/server/shared/ddl/neutral/column_default.rs @@ -0,0 +1,70 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! DDL-time gate for declared value-producing clauses. +//! +//! A column `DEFAULT`, a typeguard `DEFAULT` and a typeguard `VALUE` all pass +//! through here, so one registry decides which function names a declaration +//! can name. +//! +//! A `DEFAULT` clause is stored as raw text on the column and is evaluated +//! only when an INSERT omits the column. Without this gate a `CREATE` or a +//! `CONVERT` accepts a call to a function that does not exist, and the author +//! learns about it at the first insert instead of at the declaration. +//! +//! The check runs `nodedb_sql`'s DEFAULT classifier, which consults the same +//! `FunctionRegistry` the resolver's undefined-function gate consults. No +//! second registry and no second name list exist here. + +use nodedb_sql::SqlError; +use nodedb_sql::ddl_ast::collection_type::parse_column_type_str_full; +use nodedb_types::error::sqlstate; + +use super::super::result::DdlError; + +/// Refuse a declared column `DEFAULT` the server cannot evaluate. +/// +/// Each pair carries a column name and its declared type text, and the +/// `DEFAULT` clause is read out of that text. +pub(super) fn validate_column_defaults(columns: &[(String, String)]) -> Result<(), DdlError> { + for (column, type_str) in columns { + let (_, _, _, default_expr) = parse_column_type_str_full(type_str); + let Some(expr) = default_expr else { + continue; + }; + validate_column_default(column, &expr)?; + } + Ok(()) +} + +/// Refuse one declared column `DEFAULT` the server cannot evaluate. +pub(super) fn validate_column_default(column: &str, expr: &str) -> Result<(), DdlError> { + validate_clause_expr("DEFAULT", column, expr) +} + +/// Refuse one declared value-producing clause the server cannot evaluate. +/// +/// `clause` names the keyword the author wrote and `owner` the column or field +/// carrying it, so a typeguard `VALUE` reports itself rather than borrowing the +/// `DEFAULT` wording. +/// +/// The expression is classified and parsed, never evaluated, so a +/// `DEFAULT nextval('s')` column never advances its sequence at DDL time. +/// +/// An unregistered function name raises SQLSTATE `42883`; every other +/// rejection raises SQLSTATE `42601`. +pub(super) fn validate_clause_expr(clause: &str, owner: &str, expr: &str) -> Result<(), DdlError> { + nodedb_sql::planner::defaults::validate_default_expr(expr, owner) + .map_err(|error| clause_error(clause, owner, &error)) +} + +/// Map a clause validation error onto its SQLSTATE. +fn clause_error(clause: &str, owner: &str, error: &SqlError) -> DdlError { + let sqlstate = match error { + SqlError::UndefinedFunction { .. } => sqlstate::UNDEFINED_FUNCTION, + _ => sqlstate::SYNTAX_ERROR, + }; + DdlError::new( + sqlstate, + format!("{clause} for '{owner}' is invalid: {error}"), + ) +} diff --git a/nodedb/src/control/server/shared/ddl/neutral/convert.rs b/nodedb/src/control/server/shared/ddl/neutral/convert.rs deleted file mode 100644 index 46c218ba8..000000000 --- a/nodedb/src/control/server/shared/ddl/neutral/convert.rs +++ /dev/null @@ -1,515 +0,0 @@ -// SPDX-License-Identifier: BUSL-1.1 - -//! Protocol-neutral DDL handler for CONVERT COLLECTION. -//! -//! Syntax: -//! - `CONVERT COLLECTION TO document` -//! - `CONVERT COLLECTION TO strict (col1 TYPE, col2 TYPE, ...)` -//! - `CONVERT COLLECTION TO kv` -//! -//! Ported from the pgwire `ddl::convert` handler. The accepted-target -//! validation (document_schemaless / document_strict / kv — columnar / -//! timeseries / spatial rejected), the catalog read + write, the Data Plane -//! conversion dispatch, and the typeguard → CHECK-constraint carry-over are -//! preserved verbatim; only the result construction changed from pgwire -//! `Response` / `PgWireError` to the protocol-neutral [`DdlResult`] / -//! [`DdlError`]. - -use nodedb_sql::parser::preprocess::lex::{ - find_ascii_case_insensitive, find_ascii_case_insensitive_from, -}; -use nodedb_types::DatabaseId; -use std::time::Duration; - -use sonic_rs; - -use crate::bridge::envelope::PhysicalPlan; -use crate::control::catalog_entry::persist_collection_replicated; -use crate::control::security::identity::AuthenticatedIdentity; -use crate::control::server::shared::ddl::sql_parse::parse_ident_token; -use crate::control::server::shared::ddl::sync_dispatch::{ - SystemReason, SystemTask, dispatch_system, -}; -use crate::control::state::SharedState; -use nodedb_physical::physical_plan::MetaOp; - -use super::super::result::{DdlError, DdlResult}; - -fn err(sqlstate: &str, message: &str) -> DdlError { - DdlError::new(sqlstate, message) -} - -/// CONVERT COLLECTION TO [()] -pub async fn convert_collection( - state: &SharedState, - identity: &AuthenticatedIdentity, - database_id: DatabaseId, - sql: &str, -) -> Result, DdlError> { - let (collection, target_type, explicit_columns) = parse_convert_sql(sql)?; - let tenant_id = identity.tenant_id; - - // Validate collection exists. - let catalog = state.credentials.catalog(); - - let mut coll = catalog - .get_collection(database_id, tenant_id.as_u64(), &collection) - .map_err(|e| err("XX000", &e.to_string()))? - .ok_or_else(|| { - err( - "42P01", - &format!("collection '{collection}' does not exist"), - ) - })?; - - // Build columns before dispatch — needed for both Data Plane and catalog. - let columns: Option> = match target_type.as_str() { - "document_strict" | "kv" => { - let cols = if let Some(cols) = explicit_columns { - cols - } else if !coll.type_guards.is_empty() { - typeguards_to_column_defs(&coll.type_guards)? - } else { - return Err(err( - "42601", - "CONVERT TO strict requires column definitions or active typeguards", - )); - }; - Some(cols) - } - _ => None, - }; - - let schema_json_for_dp = if let Some(ref cols) = columns { - sonic_rs::to_string(cols) - .map_err(|e| err("XX000", &format!("schema serialization: {e}")))? - } else { - String::new() - }; - - // Dispatch to Data Plane: re-encode if needed (strict = Binary Tuple). - let plan = PhysicalPlan::Meta(MetaOp::ConvertCollection { - collection: nodedb_types::QualifiedCollection::new(database_id, &collection), - target_type: target_type.clone(), - schema_json: schema_json_for_dp, - }); - - dispatch_system( - state, - SystemTask::new( - SystemReason::DdlApply, - tenant_id, - database_id, - &collection, - plan, - ), - Duration::from_secs(60), - ) - .await - .map_err(|e| err("XX000", &format!("conversion failed: {e}")))?; - - // Update catalog collection type. - let new_type = match target_type.as_str() { - "document_schemaless" => nodedb_types::CollectionType::document(), - "document_strict" | "kv" => { - let columns = columns.expect( - "invariant: columns is Some for document_strict/kv targets, validated above", - ); - let schema = nodedb_types::columnar::StrictSchema { - columns, - version: 1, - dropped_columns: Vec::new(), - bitemporal: false, - }; - if target_type == "kv" { - nodedb_types::CollectionType::kv(schema) - } else { - nodedb_types::CollectionType::strict(schema) - } - } - _ => { - return Err(err( - "42601", - &format!("unsupported target type: {target_type}"), - )); - } - }; - - coll.collection_type = new_type; - - // CONVERT TO document_strict: if collection had typeguards, carry over CHECK constraints - // and drop typeguard definitions (strict schema subsumes type checking). - if target_type == "document_strict" && !coll.type_guards.is_empty() { - for guard in &coll.type_guards { - if let Some(ref check_expr) = guard.check_expr { - // Avoid duplicate names. - let name = format!("_guard_{}", guard.field); - if !coll.check_constraints.iter().any(|c| c.name == name) { - coll.check_constraints.push( - crate::control::security::catalog::types::CheckConstraintDef { - name, - check_sql: check_expr.clone(), - has_subquery: false, - }, - ); - } - } - } - coll.type_guards.clear(); - } - - persist_collection_replicated(state, database_id, &coll) - .map_err(|e| err("XX000", &e.to_string()))?; - - tracing::info!( - %collection, - target_type, - tenant = tenant_id.as_u64(), - "collection converted" - ); - - Ok(vec![DdlResult::Status { - command: "CONVERT COLLECTION".to_string(), - rows_affected: None, - }]) -} - -// ── SQL Parsing ────────────────────────────────────────────────────────── - -/// Parse CONVERT COLLECTION SQL. -/// -/// Returns `(collection_name, target_type, explicit_columns)`. -/// `explicit_columns` is `None` for `TO document` or `TO strict` without parens. -fn parse_convert_sql( - sql: &str, -) -> Result< - ( - String, - String, - Option>, - ), - DdlError, -> { - // Extract collection name: CONVERT COLLECTION TO ... - let coll_pos = find_ascii_case_insensitive(sql, "COLLECTION ") - .ok_or_else(|| err("42601", "expected COLLECTION keyword"))?; - let after_coll = sql[coll_pos + 11..].trim_start(); - let collection = parse_ident_token( - after_coll - .split_whitespace() - .next() - .ok_or_else(|| err("42601", "missing collection name"))?, - )?; - - // Extract target type: TO - let to_pos = find_ascii_case_insensitive_from(sql, " TO ", coll_pos + 11) - .ok_or_else(|| err("42601", "expected TO clause"))?; - let after_to = sql[to_pos + 4..].trim_start(); - let target_type = after_to - .split_whitespace() - .next() - .ok_or_else(|| err("42601", "missing target type after TO"))? - .to_lowercase() - .trim_matches('(') - .to_string(); - - match target_type.as_str() { - "document_schemaless" => Ok((collection, "document_schemaless".into(), None)), - "document_strict" | "kv" => { - if after_to.contains('(') { - let cols = parse_column_defs(after_to)?; - Ok((collection, target_type, Some(cols))) - } else { - Ok((collection, target_type, None)) - } - } - "document" | "doc" => Err(err( - "42601", - "deprecated target type 'document'; use 'document_schemaless'", - )), - "strict" => Err(err( - "42601", - "deprecated target type 'strict'; use 'document_strict'", - )), - "key_value" | "keyvalue" => Err(err("42601", "deprecated target type; use 'kv'")), - other => Err(err( - "42601", - &format!( - "unsupported target type: '{other}' \ - (expected document_schemaless, document_strict, kv)" - ), - )), - } -} - -/// Parse `(col1 TYPE, col2 TYPE, ...)` into `Vec`. -fn parse_column_defs(s: &str) -> Result, DdlError> { - use nodedb_types::columnar::ColumnDef; - - let open = s - .find('(') - .ok_or_else(|| err("42601", "expected (column definitions) after type"))?; - let close = s - .rfind(')') - .ok_or_else(|| err("42601", "missing closing parenthesis"))?; - if close <= open { - return Err(err("42601", "empty column definitions")); - } - - let inner = &s[open + 1..close]; - let mut columns: Vec = Vec::new(); - - for part in inner.split(',') { - let part = part.trim(); - if part.is_empty() { - continue; - } - let tokens: Vec<&str> = part.split_whitespace().collect(); - if tokens.len() < 2 { - return Err(err( - "42601", - &format!("expected 'name TYPE' in column def: {part}"), - )); - } - let col_name = parse_ident_token(tokens[0])?; - let col_type = tokens[1].to_uppercase(); - let nullable = !tokens - .windows(2) - .any(|w| w[0].eq_ignore_ascii_case("NOT") && w[1].eq_ignore_ascii_case("NULL")) - && !tokens.iter().any(|t| t.eq_ignore_ascii_case("NOTNULL")); - let primary_key = tokens - .windows(2) - .any(|w| w[0].eq_ignore_ascii_case("PRIMARY") && w[1].eq_ignore_ascii_case("KEY")); - - let ct = sql_type_to_column_type(&col_type); - let mut col = if nullable { - ColumnDef::nullable(col_name, ct) - } else { - ColumnDef::required(col_name, ct) - }; - if primary_key { - col = col.with_primary_key(); - } - columns.push(col); - } - - if columns.is_empty() { - return Err(err("42601", "at least one column required")); - } - - Ok(columns) -} - -/// Map SQL type names to ColumnType. -fn sql_type_to_column_type(sql_type: &str) -> nodedb_types::columnar::ColumnType { - use nodedb_types::columnar::ColumnType; - match sql_type { - "INT" | "INTEGER" | "INT8" | "BIGINT" | "INT4" | "INT2" | "SMALLINT" => ColumnType::Int64, - "FLOAT" | "FLOAT8" | "DOUBLE" | "REAL" => ColumnType::Float64, - "NUMERIC" | "DECIMAL" => ColumnType::Decimal { - precision: 38, - scale: 10, - }, - "BOOL" | "BOOLEAN" => ColumnType::Bool, - "TIMESTAMP" | "TIMESTAMP WITHOUT TIME ZONE" => ColumnType::Timestamp, - "TIMESTAMPTZ" | "TIMESTAMP WITH TIME ZONE" => ColumnType::Timestamptz, - "BLOB" | "BYTEA" | "BINARY" => ColumnType::Bytes, - "UUID" => ColumnType::Uuid, - "JSON" | "JSONB" => ColumnType::Json, - "SPARSEVECTOR" => ColumnType::SparseVector, - "GEOMETRY" => ColumnType::Geometry, - _ => ColumnType::String, // VARCHAR, TEXT, STRING, and anything else - } -} - -/// Convert typeguard field definitions to strict schema column definitions. -/// -/// Each typeguard field becomes a column. The type expression is mapped to ColumnType. -/// REQUIRED fields become NOT NULL. DEFAULT expressions carry over. -fn typeguards_to_column_defs( - guards: &[nodedb_types::TypeGuardFieldDef], -) -> Result, DdlError> { - use nodedb_types::columnar::{ColumnDef, ColumnType}; - - // Always include an `id` column as primary key. - let mut columns = vec![ColumnDef::required("id", ColumnType::String).with_primary_key()]; - - for guard in guards { - // Skip dot-path fields (nested) — strict schema is flat. - if guard.field.contains('.') { - continue; - } - // Skip if already added (e.g., "id"). - if guard.field == "id" || columns.iter().any(|c| c.name == guard.field) { - continue; - } - - let ct = typeguard_type_to_column_type(&guard.type_expr); - let mut col = if guard.required { - ColumnDef::required(guard.field.clone(), ct) - } else { - ColumnDef::nullable(guard.field.clone(), ct) - }; - col.default = guard.default_expr.clone().or(guard.value_expr.clone()); - columns.push(col); - } - - Ok(columns) -} - -/// Map a typeguard type expression string to a ColumnType. -fn typeguard_type_to_column_type(type_expr: &str) -> nodedb_types::columnar::ColumnType { - use nodedb_types::columnar::ColumnType; - - // Strip union types — take the first non-NULL type. - let base = type_expr - .split('|') - .map(|s| s.trim()) - .find(|s| !s.eq_ignore_ascii_case("NULL")) - .unwrap_or(type_expr) - .to_uppercase(); - - // Strip generic parameters: ARRAY → ARRAY, SET → SET. - let base = base.split('<').next().unwrap_or(&base); - - match base { - "INT" | "INTEGER" | "BIGINT" | "INT64" => ColumnType::Int64, - "FLOAT" | "DOUBLE" | "REAL" | "FLOAT64" => ColumnType::Float64, - "STRING" | "TEXT" | "VARCHAR" => ColumnType::String, - "BOOL" | "BOOLEAN" => ColumnType::Bool, - "BYTES" | "BYTEA" | "BLOB" => ColumnType::Bytes, - "TIMESTAMP" | "TIMESTAMP WITHOUT TIME ZONE" => ColumnType::Timestamp, - "TIMESTAMPTZ" | "TIMESTAMP WITH TIME ZONE" => ColumnType::Timestamptz, - "DECIMAL" | "NUMERIC" => ColumnType::Decimal { - precision: 38, - scale: 10, - }, - "UUID" => ColumnType::Uuid, - "GEOMETRY" => ColumnType::Geometry, - "JSON" | "JSONB" | "OBJECT" => ColumnType::Json, - "SPARSEVECTOR" => ColumnType::SparseVector, - _ => ColumnType::String, // fallback - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn parse_convert_to_document_schemaless() { - let (coll, target, cols) = - parse_convert_sql("CONVERT COLLECTION users TO document_schemaless").unwrap(); - assert_eq!(coll, "users"); - assert_eq!(target, "document_schemaless"); - assert!(cols.is_none()); - } - - #[test] - fn parse_convert_deprecated_document_rejected() { - assert!(parse_convert_sql("CONVERT COLLECTION users TO document").is_err()); - } - - #[test] - fn parse_convert_to_document_strict() { - let sql = - "CONVERT COLLECTION users TO document_strict (name VARCHAR, age INT, active BOOLEAN)"; - let (coll, target, cols) = parse_convert_sql(sql).unwrap(); - assert_eq!(coll, "users"); - assert_eq!(target, "document_strict"); - - let cols = cols.unwrap(); - assert_eq!(cols.len(), 3); - assert_eq!(cols[0].name, "name"); - assert!(matches!( - cols[0].column_type, - nodedb_types::columnar::ColumnType::String - )); - assert_eq!(cols[1].name, "age"); - assert!(matches!( - cols[1].column_type, - nodedb_types::columnar::ColumnType::Int64 - )); - assert_eq!(cols[2].name, "active"); - assert!(matches!( - cols[2].column_type, - nodedb_types::columnar::ColumnType::Bool - )); - } - - #[test] - fn parse_convert_deprecated_strict_rejected() { - let sql = "CONVERT COLLECTION users TO strict (name VARCHAR)"; - assert!(parse_convert_sql(sql).is_err()); - } - - #[test] - fn parse_convert_to_kv() { - let sql = "CONVERT COLLECTION cache TO kv (key VARCHAR, value BLOB)"; - let (coll, target, cols) = parse_convert_sql(sql).unwrap(); - assert_eq!(coll, "cache"); - assert_eq!(target, "kv"); - assert!(cols.is_some()); - } - - #[test] - fn parse_convert_not_null_constraint() { - let sql = "CONVERT COLLECTION users TO document_strict (id INT NOT NULL, name VARCHAR)"; - let (_, _, cols) = parse_convert_sql(sql).unwrap(); - let cols = cols.unwrap(); - assert!(!cols[0].nullable); - assert!(cols[1].nullable); - } - - #[test] - fn parse_convert_missing_to_errors() { - assert!(parse_convert_sql("CONVERT COLLECTION users").is_err()); - } - - #[test] - fn parse_convert_unknown_type_errors() { - assert!(parse_convert_sql("CONVERT COLLECTION users TO graph").is_err()); - } - - #[test] - fn sql_type_to_column_type_distinguishes_timestamp_from_timestamptz() { - use nodedb_types::columnar::ColumnType; - assert!(matches!( - sql_type_to_column_type("TIMESTAMP"), - ColumnType::Timestamp - )); - assert!(matches!( - sql_type_to_column_type("TIMESTAMP WITHOUT TIME ZONE"), - ColumnType::Timestamp - )); - assert!(matches!( - sql_type_to_column_type("TIMESTAMPTZ"), - ColumnType::Timestamptz - )); - assert!(matches!( - sql_type_to_column_type("TIMESTAMP WITH TIME ZONE"), - ColumnType::Timestamptz - )); - } - - #[test] - fn typeguard_type_to_column_type_distinguishes_timestamp_from_timestamptz() { - use nodedb_types::columnar::ColumnType; - assert!(matches!( - typeguard_type_to_column_type("TIMESTAMP"), - ColumnType::Timestamp - )); - assert!(matches!( - typeguard_type_to_column_type("TIMESTAMP WITHOUT TIME ZONE"), - ColumnType::Timestamp - )); - assert!(matches!( - typeguard_type_to_column_type("TIMESTAMPTZ"), - ColumnType::Timestamptz - )); - assert!(matches!( - typeguard_type_to_column_type("TIMESTAMP WITH TIME ZONE"), - ColumnType::Timestamptz - )); - } -} diff --git a/nodedb/src/control/server/shared/ddl/neutral/convert/column_defs.rs b/nodedb/src/control/server/shared/ddl/neutral/convert/column_defs.rs new file mode 100644 index 000000000..f72364ef9 --- /dev/null +++ b/nodedb/src/control/server/shared/ddl/neutral/convert/column_defs.rs @@ -0,0 +1,373 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! CONVERT COLLECTION statement text parsed into a target type and columns. +//! +//! A column definition is `name TYPE [NOT NULL] [PRIMARY KEY] [DEFAULT expr]`. +//! The `DEFAULT` clause is validated here and stored on the created column, so +//! a later insert that omits the column takes its value. + +use nodedb_sql::ddl_ast::collection_type::parse_column_type_str_full; +use nodedb_sql::parser::preprocess::lex::{ + find_ascii_case_insensitive, find_ascii_case_insensitive_from, +}; + +use crate::control::server::shared::ddl::sql_parse::{parse_ident_token, split_values}; + +use super::super::super::result::DdlError; +use super::super::column_default::validate_column_default; +use super::support::err; +use super::type_map::sql_type_to_column_type; + +/// Parse CONVERT COLLECTION SQL. +/// +/// Returns `(collection_name, target_type, explicit_columns)`. +/// `explicit_columns` is `None` for `TO document` or `TO strict` without parens. +pub(super) fn parse_convert_sql( + sql: &str, +) -> Result< + ( + String, + String, + Option>, + ), + DdlError, +> { + // Extract collection name: CONVERT COLLECTION TO ... + let coll_pos = find_ascii_case_insensitive(sql, "COLLECTION ") + .ok_or_else(|| err("42601", "expected COLLECTION keyword"))?; + let after_coll = sql[coll_pos + 11..].trim_start(); + let collection = parse_ident_token( + after_coll + .split_whitespace() + .next() + .ok_or_else(|| err("42601", "missing collection name"))?, + )?; + + // Extract target type: TO + let to_pos = find_ascii_case_insensitive_from(sql, " TO ", coll_pos + 11) + .ok_or_else(|| err("42601", "expected TO clause"))?; + let after_to = sql[to_pos + 4..].trim_start(); + let target_type = after_to + .split_whitespace() + .next() + .ok_or_else(|| err("42601", "missing target type after TO"))? + .to_lowercase() + .trim_matches('(') + .to_string(); + + match target_type.as_str() { + "document_schemaless" => Ok((collection, "document_schemaless".into(), None)), + "document_strict" | "kv" => { + if after_to.contains('(') { + let cols = parse_column_defs(after_to)?; + Ok((collection, target_type, Some(cols))) + } else { + Ok((collection, target_type, None)) + } + } + "document" | "doc" => Err(err( + "42601", + "deprecated target type 'document'; use 'document_schemaless'", + )), + "strict" => Err(err( + "42601", + "deprecated target type 'strict'; use 'document_strict'", + )), + "key_value" | "keyvalue" => Err(err("42601", "deprecated target type; use 'kv'")), + other => Err(err( + "42601", + format!( + "unsupported target type: '{other}' \ + (expected document_schemaless, document_strict, kv)" + ), + )), + } +} + +/// Words that end a type spelling in a CONVERT column definition. +const COLUMN_MODIFIER_KEYWORDS: [&str; 6] = ["NOT", "NULL", "NOTNULL", "PRIMARY", "KEY", "DEFAULT"]; + +/// Parse `(col1 TYPE, col2 TYPE, ...)` into `Vec`. +fn parse_column_defs(s: &str) -> Result, DdlError> { + use nodedb_types::columnar::ColumnDef; + + let open = s + .find('(') + .ok_or_else(|| err("42601", "expected (column definitions) after type"))?; + let close = s + .rfind(')') + .ok_or_else(|| err("42601", "missing closing parenthesis"))?; + if close <= open { + return Err(err("42601", "empty column definitions")); + } + + let inner = &s[open + 1..close]; + let mut columns: Vec = Vec::new(); + + // The list splits on top-level commas, so a parameter list keeps its own + // comma and `amount DECIMAL(10, 2)` stays one column definition. + for part in split_values(inner) { + let part = part.trim(); + if part.is_empty() { + continue; + } + let name_token = part.split_whitespace().next().ok_or_else(|| { + err( + "42601", + format!("expected 'name TYPE' in column def: {part}"), + ) + })?; + let (head, tail, default_expr) = split_column_default(part, name_token)?; + // The modifier scan reads the definition without its DEFAULT + // expression, so `NOT NULL` written after the clause still lands. + let declaration = format!("{head} {tail}"); + let tokens: Vec<&str> = declaration.split_whitespace().collect(); + if tokens.len() < 2 { + return Err(err( + "42601", + format!("expected 'name TYPE' in column def: {part}"), + )); + } + let col_name = parse_ident_token(tokens[0])?; + // The type spelling runs from the second token to the first modifier + // word, so `TIMESTAMP WITH TIME ZONE` resolves whole. + let type_end = tokens[1..] + .iter() + .position(|token| { + COLUMN_MODIFIER_KEYWORDS + .iter() + .any(|keyword| token.eq_ignore_ascii_case(keyword)) + }) + .map(|offset| offset + 1) + .unwrap_or(tokens.len()); + if type_end < 2 { + return Err(err( + "42601", + format!("expected 'name TYPE' in column def: {part}"), + )); + } + let col_type = tokens[1..type_end].join(" ").to_uppercase(); + let nullable = !tokens + .windows(2) + .any(|w| w[0].eq_ignore_ascii_case("NOT") && w[1].eq_ignore_ascii_case("NULL")) + && !tokens.iter().any(|t| t.eq_ignore_ascii_case("NOTNULL")); + let primary_key = tokens + .windows(2) + .any(|w| w[0].eq_ignore_ascii_case("PRIMARY") && w[1].eq_ignore_ascii_case("KEY")); + + let ct = sql_type_to_column_type(&col_type)?; + let mut col = if nullable { + ColumnDef::nullable(col_name, ct) + } else { + ColumnDef::required(col_name, ct) + }; + if primary_key { + col = col.with_primary_key(); + } + if let Some(expr) = default_expr { + validate_column_default(&col.name, &expr)?; + col = col.with_default(expr); + } + columns.push(col); + } + + if columns.is_empty() { + return Err(err("42601", "at least one column required")); + } + + Ok(columns) +} + +/// Split a column definition around its `DEFAULT` clause. +/// +/// Returns the text before the keyword, the text after the expression, and +/// the expression itself. The clause is read by +/// `parse_column_type_str_full`, the parser `CREATE COLLECTION` reads a +/// column `DEFAULT` with, so both statements accept one syntax. +fn split_column_default<'a>( + part: &'a str, + name_token: &str, +) -> Result<(&'a str, &'a str, Option), DdlError> { + let after_name = &part[name_token.len()..]; + let Some(offset) = find_ascii_case_insensitive(after_name, "DEFAULT") else { + return Ok((part, "", None)); + }; + let (_, _, _, default_expr) = parse_column_type_str_full(after_name); + let expr = default_expr.ok_or_else(|| { + err( + "42601", + format!("DEFAULT needs an expression in column def: {part}"), + ) + })?; + let after_keyword = after_name[offset + "DEFAULT".len()..].trim(); + let Some(tail) = after_keyword.strip_prefix(expr.as_str()) else { + return Err(err( + "42601", + format!("DEFAULT clause is malformed in column def: {part}"), + )); + }; + Ok((&part[..name_token.len() + offset], tail, Some(expr))) +} + +#[cfg(test)] +mod tests { + use super::*; + use nodedb_types::columnar::ColumnType; + + /// A CONVERT column definition resolves a multi-word type spelling whole, + /// so the declared zone survives. + #[test] + fn convert_column_defs_resolve_a_multi_word_type_spelling() { + let sql = "CONVERT COLLECTION events TO document_strict \ + (id TEXT PRIMARY KEY, at TIMESTAMP WITH TIME ZONE)"; + let (_, _, cols) = parse_convert_sql(sql).expect("multi-word column type must parse"); + let cols = cols.expect("explicit column defs must be present"); + assert_eq!(cols[1].name, "at"); + assert_eq!(cols[1].column_type, ColumnType::Timestamptz); + } + + /// A CONVERT column definition naming no known type stops the statement. + #[test] + fn convert_column_defs_refuse_an_unknown_type() { + let sql = "CONVERT COLLECTION events TO document_strict (id TEXT, payload WIDGET)"; + let error = parse_convert_sql(sql).expect_err("an unknown column type must refuse"); + assert_eq!(error.sqlstate, "42601"); + } + + /// A parameter list keeps its own comma, so the space after it changes + /// nothing about how the column list splits. + #[test] + fn convert_column_defs_split_around_a_parameter_list_comma() { + for sql in [ + "CONVERT COLLECTION sales TO document_strict (id TEXT, amount DECIMAL(10, 2))", + "CONVERT COLLECTION sales TO document_strict (id TEXT, amount DECIMAL(10,2))", + ] { + let (_, _, cols) = parse_convert_sql(sql).expect("a parameter list must parse"); + let cols = cols.expect("explicit column defs must be present"); + assert_eq!( + cols.iter().map(|c| c.name.as_str()).collect::>(), + ["id", "amount"], + "column list must split on the top-level comma only: {sql}" + ); + assert_eq!( + cols[1].column_type, + ColumnType::Decimal { + precision: 10, + scale: 2 + } + ); + } + } + + /// A `DEFAULT` clause reaches the created column instead of being dropped. + #[test] + fn convert_column_defs_carry_a_default_clause() { + let sql = "CONVERT COLLECTION orders TO document_strict \ + (id TEXT PRIMARY KEY, status TEXT NOT NULL DEFAULT 'pending')"; + let (_, _, cols) = parse_convert_sql(sql).expect("a DEFAULT clause must parse"); + let cols = cols.expect("explicit column defs must be present"); + assert_eq!(cols[1].name, "status"); + assert_eq!(cols[1].column_type, ColumnType::String); + assert!( + !cols[1].nullable, + "NOT NULL must survive the DEFAULT clause" + ); + assert_eq!(cols[1].default.as_deref(), Some("'pending'")); + } + + /// `NOT NULL` written after the `DEFAULT` clause still reaches the column. + #[test] + fn convert_column_defs_read_a_modifier_after_the_default_clause() { + let sql = "CONVERT COLLECTION orders TO document_strict \ + (id TEXT, status TEXT DEFAULT 'pending' NOT NULL)"; + let (_, _, cols) = parse_convert_sql(sql).expect("a trailing modifier must parse"); + let cols = cols.expect("explicit column defs must be present"); + assert_eq!(cols[1].column_type, ColumnType::String); + assert!(!cols[1].nullable, "NOT NULL after DEFAULT must land"); + assert_eq!(cols[1].default.as_deref(), Some("'pending'")); + } + + /// A `DEFAULT` naming a function the server cannot evaluate stops CONVERT + /// with the SQLSTATE `CREATE COLLECTION` raises for the same clause. + #[test] + fn convert_column_defs_refuse_an_unevaluable_default() { + let sql = "CONVERT COLLECTION orders TO document_strict \ + (id TEXT, status TEXT DEFAULT no_such_function())"; + let error = parse_convert_sql(sql).expect_err("an unevaluable DEFAULT must refuse"); + assert_eq!(error.sqlstate, "42883"); + } + + /// A `DEFAULT` with no expression is a syntax error, not a dropped clause. + #[test] + fn convert_column_defs_refuse_an_empty_default() { + let sql = "CONVERT COLLECTION orders TO document_strict (id TEXT, status TEXT DEFAULT)"; + let error = parse_convert_sql(sql).expect_err("an empty DEFAULT must refuse"); + assert_eq!(error.sqlstate, "42601"); + } + + #[test] + fn parse_convert_to_document_schemaless() { + let (coll, target, cols) = + parse_convert_sql("CONVERT COLLECTION users TO document_schemaless").unwrap(); + assert_eq!(coll, "users"); + assert_eq!(target, "document_schemaless"); + assert!(cols.is_none()); + } + + #[test] + fn parse_convert_deprecated_document_rejected() { + assert!(parse_convert_sql("CONVERT COLLECTION users TO document").is_err()); + } + + #[test] + fn parse_convert_to_document_strict() { + let sql = + "CONVERT COLLECTION users TO document_strict (name VARCHAR, age INT, active BOOLEAN)"; + let (coll, target, cols) = parse_convert_sql(sql).unwrap(); + assert_eq!(coll, "users"); + assert_eq!(target, "document_strict"); + + let cols = cols.unwrap(); + assert_eq!(cols.len(), 3); + assert_eq!(cols[0].name, "name"); + assert!(matches!(cols[0].column_type, ColumnType::String)); + assert_eq!(cols[1].name, "age"); + assert!(matches!(cols[1].column_type, ColumnType::Int64)); + assert_eq!(cols[2].name, "active"); + assert!(matches!(cols[2].column_type, ColumnType::Bool)); + } + + #[test] + fn parse_convert_deprecated_strict_rejected() { + let sql = "CONVERT COLLECTION users TO strict (name VARCHAR)"; + assert!(parse_convert_sql(sql).is_err()); + } + + #[test] + fn parse_convert_to_kv() { + let sql = "CONVERT COLLECTION cache TO kv (key VARCHAR, value BLOB)"; + let (coll, target, cols) = parse_convert_sql(sql).unwrap(); + assert_eq!(coll, "cache"); + assert_eq!(target, "kv"); + assert!(cols.is_some()); + } + + #[test] + fn parse_convert_not_null_constraint() { + let sql = "CONVERT COLLECTION users TO document_strict (id INT NOT NULL, name VARCHAR)"; + let (_, _, cols) = parse_convert_sql(sql).unwrap(); + let cols = cols.unwrap(); + assert!(!cols[0].nullable); + assert!(cols[1].nullable); + } + + #[test] + fn parse_convert_missing_to_errors() { + assert!(parse_convert_sql("CONVERT COLLECTION users").is_err()); + } + + #[test] + fn parse_convert_unknown_type_errors() { + assert!(parse_convert_sql("CONVERT COLLECTION users TO graph").is_err()); + } +} diff --git a/nodedb/src/control/server/shared/ddl/neutral/convert/driver.rs b/nodedb/src/control/server/shared/ddl/neutral/convert/driver.rs new file mode 100644 index 000000000..18ffe1ed8 --- /dev/null +++ b/nodedb/src/control/server/shared/ddl/neutral/convert/driver.rs @@ -0,0 +1,155 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! CONVERT COLLECTION execution: catalog read, Data Plane re-encode, catalog write. +//! +//! Accepted targets are `document_schemaless`, `document_strict` and `kv`. +//! The columnar, timeseries and spatial engines are creation-time choices and +//! are rejected here. + +use nodedb_types::DatabaseId; +use std::time::Duration; + +use sonic_rs; + +use crate::bridge::envelope::PhysicalPlan; +use crate::control::catalog_entry::persist_collection_replicated; +use crate::control::security::identity::AuthenticatedIdentity; +use crate::control::server::shared::ddl::sync_dispatch::{ + SystemReason, SystemTask, dispatch_system, +}; +use crate::control::state::SharedState; +use nodedb_physical::physical_plan::MetaOp; + +use super::super::super::result::{DdlError, DdlResult}; +use super::column_defs::parse_convert_sql; +use super::support::err; +use super::typeguard_columns::typeguards_to_column_defs; + +/// CONVERT COLLECTION TO [()] +pub async fn convert_collection( + state: &SharedState, + identity: &AuthenticatedIdentity, + database_id: DatabaseId, + sql: &str, +) -> Result, DdlError> { + let (collection, target_type, explicit_columns) = parse_convert_sql(sql)?; + let tenant_id = identity.tenant_id; + + // Validate collection exists. + let catalog = state.credentials.catalog(); + + let mut coll = catalog + .get_collection(database_id, tenant_id.as_u64(), &collection) + .map_err(|e| err("XX000", e.to_string()))? + .ok_or_else(|| err("42P01", format!("collection '{collection}' does not exist")))?; + + // Build columns before dispatch — needed for both Data Plane and catalog. + let columns: Option> = match target_type.as_str() { + "document_strict" | "kv" => { + let cols = if let Some(cols) = explicit_columns { + cols + } else if !coll.type_guards.is_empty() { + typeguards_to_column_defs(&coll.type_guards)? + } else { + return Err(err( + "42601", + "CONVERT TO strict requires column definitions or active typeguards", + )); + }; + Some(cols) + } + _ => None, + }; + + let schema_json_for_dp = if let Some(ref cols) = columns { + sonic_rs::to_string(cols).map_err(|e| err("XX000", format!("schema serialization: {e}")))? + } else { + String::new() + }; + + // Dispatch to Data Plane: re-encode if needed (strict = Binary Tuple). + let plan = PhysicalPlan::Meta(MetaOp::ConvertCollection { + collection: nodedb_types::QualifiedCollection::new(database_id, &collection), + target_type: target_type.clone(), + schema_json: schema_json_for_dp, + }); + + dispatch_system( + state, + SystemTask::new( + SystemReason::DdlApply, + tenant_id, + database_id, + &collection, + plan, + ), + Duration::from_secs(60), + ) + .await + .map_err(|e| err("XX000", format!("conversion failed: {e}")))?; + + // Update catalog collection type. + let new_type = match target_type.as_str() { + "document_schemaless" => nodedb_types::CollectionType::document(), + "document_strict" | "kv" => { + let columns = columns.expect( + "invariant: columns is Some for document_strict/kv targets, validated above", + ); + let schema = nodedb_types::columnar::StrictSchema { + columns, + version: 1, + dropped_columns: Vec::new(), + bitemporal: false, + }; + if target_type == "kv" { + nodedb_types::CollectionType::kv(schema) + } else { + nodedb_types::CollectionType::strict(schema) + } + } + _ => { + return Err(err( + "42601", + format!("unsupported target type: {target_type}"), + )); + } + }; + + coll.collection_type = new_type; + + // CONVERT TO document_strict: if collection had typeguards, carry over CHECK constraints + // and drop typeguard definitions (strict schema subsumes type checking). + if target_type == "document_strict" && !coll.type_guards.is_empty() { + for guard in &coll.type_guards { + if let Some(ref check_expr) = guard.check_expr { + // Avoid duplicate names. + let name = format!("_guard_{}", guard.field); + if !coll.check_constraints.iter().any(|c| c.name == name) { + coll.check_constraints.push( + crate::control::security::catalog::types::CheckConstraintDef { + name, + check_sql: check_expr.clone(), + has_subquery: false, + }, + ); + } + } + } + coll.type_guards.clear(); + } + + persist_collection_replicated(state, database_id, &coll) + .map_err(|e| err("XX000", e.to_string()))?; + + tracing::info!( + %collection, + target_type, + tenant = tenant_id.as_u64(), + "collection converted" + ); + + Ok(vec![DdlResult::Status { + command: "CONVERT COLLECTION".to_string(), + rows_affected: None, + }]) +} diff --git a/nodedb/src/control/server/shared/ddl/neutral/convert/mod.rs b/nodedb/src/control/server/shared/ddl/neutral/convert/mod.rs new file mode 100644 index 000000000..1a2704585 --- /dev/null +++ b/nodedb/src/control/server/shared/ddl/neutral/convert/mod.rs @@ -0,0 +1,16 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! Protocol-neutral DDL handler for CONVERT COLLECTION. +//! +//! Syntax: +//! - `CONVERT COLLECTION TO document_schemaless` +//! - `CONVERT COLLECTION TO document_strict (col1 TYPE, col2 TYPE, ...)` +//! - `CONVERT COLLECTION TO kv` + +pub mod column_defs; +pub mod driver; +mod support; +pub mod type_map; +pub mod typeguard_columns; + +pub use driver::convert_collection; diff --git a/nodedb/src/control/server/shared/ddl/neutral/convert/support.rs b/nodedb/src/control/server/shared/ddl/neutral/convert/support.rs new file mode 100644 index 000000000..816fe3b9e --- /dev/null +++ b/nodedb/src/control/server/shared/ddl/neutral/convert/support.rs @@ -0,0 +1,13 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! Shared helpers for the CONVERT COLLECTION family. + +use super::super::super::result::DdlError; + +/// Build a protocol-neutral [`DdlError`] with the given SQLSTATE and message. +/// +/// `message` takes anything convertible to `String`, so a `format!` result +/// moves straight in without a borrow and a second allocation. +pub(super) fn err(sqlstate: &str, message: impl Into) -> DdlError { + DdlError::new(sqlstate, message) +} diff --git a/nodedb/src/control/server/shared/ddl/neutral/convert/type_map.rs b/nodedb/src/control/server/shared/ddl/neutral/convert/type_map.rs new file mode 100644 index 000000000..f9c3eaf5d --- /dev/null +++ b/nodedb/src/control/server/shared/ddl/neutral/convert/type_map.rs @@ -0,0 +1,397 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! Declared type spellings resolved to a `ColumnType` for CONVERT COLLECTION. +//! +//! Two mappers share one resolver: the column-definition mapper reads the +//! spelling an author wrote in the statement, the typeguard mapper reads the +//! spelling a guard declared. Both answer through +//! `nodedb_sql::parser::type_expr::parse_type_expr`'s resolver, so one +//! spelling resolves to one type on every path. + +use super::super::super::result::DdlError; +use super::support::err; + +/// Resolve a declared type spelling to a `ColumnType`. +/// +/// The whole spelling resolves, so `TIMESTAMP WITH TIME ZONE` keeps the zone +/// the author wrote and `TIMESTAMP GARBAGE` names no type at all. +/// +/// This is the parser `nodedb_sql::parser::type_expr::parse_type_expr` +/// resolves the same text through, so a spelling added there reaches both +/// CONVERT mappers with no edit here. +fn resolve_declared_type( + declared: &str, +) -> Result { + declared.trim().parse() +} + +/// Map a declared SQL type spelling to a `ColumnType`. +/// +/// The spelling resolves through [`resolve_declared_type`]. +/// +/// `BINARY` resolves ahead of that call. It is a CONVERT-only spelling for +/// `ColumnType::Bytes` that the shared parser rejects. +/// +/// A spelling the shared parser rejects raises `42601`. The author names the +/// stored column type, so an unresolvable spelling must never become +/// `ColumnType::String`. +pub(super) fn sql_type_to_column_type( + sql_type: &str, +) -> Result { + use nodedb_types::columnar::ColumnType; + + let declared = sql_type.trim(); + if declared.eq_ignore_ascii_case("BINARY") { + return Ok(ColumnType::Bytes); + } + resolve_declared_type(declared) + .map_err(|e| err("42601", format!("column type '{declared}': {e}"))) +} + +/// Map a typeguard type expression string to a `ColumnType`. +/// +/// Union and generic wrappers reduce to their leaf spelling first. The leaf +/// then resolves through [`resolve_declared_type`], apart from the one +/// typeguard-only spelling named below. +/// +/// A leaf the shared parser rejects raises `42601`. The guard declares what +/// the column holds, so an unresolvable leaf must never become +/// `ColumnType::String`. +pub(super) fn typeguard_type_to_column_type( + type_expr: &str, +) -> Result { + use nodedb_types::columnar::ColumnType; + + // Strip union types — take the first non-NULL type. + let base = type_expr + .split('|') + .map(|s| s.trim()) + .find(|s| !s.eq_ignore_ascii_case("NULL")) + .unwrap_or(type_expr) + .to_uppercase(); + + // Strip generic parameters: ARRAY → ARRAY, SET → SET. + let base = base.split('<').next().unwrap_or(&base).trim(); + + // `OBJECT` is a typeguard-only keyword. It names a field holding a nested + // map, has no declared-DDL spelling, and stores as inline MessagePack. + if base.eq_ignore_ascii_case("OBJECT") { + return Ok(ColumnType::Json); + } + + // Every other spelling resolves through the one shared parser, so a + // spelling added there reaches typeguard conversion with no edit here. + resolve_declared_type(base).map_err(|e| err("42601", format!("type guard type '{base}': {e}"))) +} + +#[cfg(test)] +mod tests { + use super::*; + use nodedb_types::columnar::{ColumnType, DECLARED_FLOAT_KEYWORDS, DECLARED_INT_KEYWORDS}; + + /// Both CONVERT type mappers must answer with the shared parser for every + /// integer spelling it knows. A keyword added to `DECLARED_INT_KEYWORDS` + /// extends this test, so neither mapper can fall to its `String` default + /// for a spelling the rest of the database resolves to `Int64`. + #[test] + fn convert_mappers_resolve_every_declared_int_keyword() { + for keyword in DECLARED_INT_KEYWORDS { + assert_eq!( + sql_type_to_column_type(keyword) + .expect("the column def mapper resolves a declared int/float keyword"), + ColumnType::Int64, + "column def mapper must resolve {keyword}" + ); + assert_eq!( + typeguard_type_to_column_type(keyword) + .expect("the typeguard mapper resolves a declared int/float keyword"), + ColumnType::Int64, + "typeguard mapper must resolve {keyword}" + ); + } + } + + /// The float counterpart of + /// [`convert_mappers_resolve_every_declared_int_keyword`]. + #[test] + fn convert_mappers_resolve_every_declared_float_keyword() { + for keyword in DECLARED_FLOAT_KEYWORDS { + assert_eq!( + sql_type_to_column_type(keyword) + .expect("the column def mapper resolves a declared int/float keyword"), + ColumnType::Float64, + "column def mapper must resolve {keyword}" + ); + assert_eq!( + typeguard_type_to_column_type(keyword) + .expect("the typeguard mapper resolves a declared int/float keyword"), + ColumnType::Float64, + "typeguard mapper must resolve {keyword}" + ); + } + } + + /// Both mappers delegate to `ColumnType::from_declared_type` outside their + /// documented exceptions. This pins the delegation itself, so a mapper + /// cannot regrow a private keyword table that answers differently. + #[test] + fn convert_mappers_delegate_to_the_shared_parser() { + for declared in [ + "TEXT", + "VARCHAR", + "STRING", + "BOOL", + "BOOLEAN", + "BYTES", + "BYTEA", + "BLOB", + "TIMESTAMP", + "TIMESTAMPTZ", + "NUMERIC", + "DECIMAL", + "UUID", + "ULID", + "JSON", + "JSONB", + "GEOMETRY", + "SPARSEVECTOR", + "VECTOR(384)", + "DURATION", + "ARRAY", + "SET", + "REGEX", + "RANGE", + "RECORD", + "SYSTEM_TIMESTAMP", + ] { + let shared = ColumnType::from_declared_type(declared) + .expect("test input names a type the shared parser knows"); + assert_eq!( + sql_type_to_column_type(declared) + .expect("the column def mapper resolves a shared-parser spelling"), + shared, + "column def mapper must match the shared parser on {declared}" + ); + assert_eq!( + typeguard_type_to_column_type(declared) + .expect("the typeguard mapper resolves a shared-parser spelling"), + shared, + "typeguard mapper must match the shared parser on {declared}" + ); + } + } + + /// A spelling the shared parser rejects raises `42601` on both mappers. + /// + /// A silent `String` answer would store a column type the author never + /// declared, so the refusal is the whole contract here. `TIMESTAMP + /// GARBAGE` covers the trailing-word case, which resolving the leading + /// token alone would read as `TIMESTAMP`. + #[test] + fn convert_mappers_refuse_a_spelling_the_shared_parser_rejects() { + for declared in ["WIDGET", "TIMESTAMP GARBAGE"] { + let column_def = sql_type_to_column_type(declared) + .expect_err("the column def mapper must refuse an unresolvable spelling"); + assert_eq!( + column_def.sqlstate, "42601", + "column def mapper must refuse {declared} with 42601" + ); + let typeguard = typeguard_type_to_column_type(declared) + .expect_err("the typeguard mapper must refuse an unresolvable spelling"); + assert_eq!( + typeguard.sqlstate, "42601", + "typeguard mapper must refuse {declared} with 42601" + ); + } + } + + /// `BINARY` is a CONVERT-only spelling for `Bytes`; the shared parser + /// rejects it, so the column def mapper resolves it itself. + #[test] + fn binary_stays_a_convert_only_bytes_spelling() { + assert_eq!( + sql_type_to_column_type("BINARY").expect("BINARY resolves"), + ColumnType::Bytes + ); + assert_eq!(ColumnType::from_declared_type("BINARY"), None); + } + + /// `OBJECT` is a typeguard-only keyword the shared parser rejects, and a + /// multi-word timestamp spelling keeps the zone the author wrote. + #[test] + fn typeguard_only_spellings_keep_their_own_answer() { + assert_eq!( + typeguard_type_to_column_type("OBJECT").expect("OBJECT resolves"), + ColumnType::Json + ); + assert_eq!(ColumnType::from_declared_type("OBJECT"), None); + assert_eq!( + typeguard_type_to_column_type("TIMESTAMP WITH TIME ZONE") + .expect("the zone spelling resolves"), + ColumnType::Timestamptz + ); + assert_eq!( + typeguard_type_to_column_type("TIMESTAMP WITHOUT TIME ZONE") + .expect("the no-zone spelling resolves"), + ColumnType::Timestamp + ); + } + + /// Union and generic wrappers resolve from their inner leaf type. + #[test] + fn typeguard_unions_and_generics_resolve_to_the_leaf() { + assert_eq!( + typeguard_type_to_column_type("INT|NULL").expect("INT|NULL resolves"), + ColumnType::Int64 + ); + assert_eq!( + typeguard_type_to_column_type("NULL|STRING").expect("NULL|STRING resolves"), + ColumnType::String + ); + assert_eq!( + typeguard_type_to_column_type("ARRAY").expect("ARRAY resolves"), + ColumnType::Array + ); + assert_eq!( + typeguard_type_to_column_type("SET").expect("SET resolves"), + ColumnType::Set + ); + } + + /// One spelling, one answer. The typeguard ENFORCEMENT parser + /// (`nodedb_sql::parser::type_expr::parse_type_expr`, which decides + /// whether a written value is valid) and this CONVERT mapper (which + /// decides the column type the same guard becomes) must resolve every + /// declared spelling to the same type. + /// + /// A spelling one path reads as `Timestamp` and the other as + /// `Timestamptz` is a document validated under one rule and stored under + /// another. Adding a spelling to either parser extends this list, and a + /// leaf added to `SimpleType` fails the mapping below until it names its + /// column type. + #[test] + fn enforcement_and_convert_agree_on_every_typeguard_spelling() { + use nodedb_sql::parser::type_expr::{SimpleType, TypeExpr, parse_type_expr}; + + // The column type each typeguard leaf denotes. `Object` is the + // typeguard-only leaf; every other leaf names its own `ColumnType`. + fn leaf_column_type(leaf: &SimpleType) -> ColumnType { + match leaf { + SimpleType::Int => ColumnType::Int64, + SimpleType::Float => ColumnType::Float64, + SimpleType::String => ColumnType::String, + SimpleType::Bool => ColumnType::Bool, + SimpleType::Bytes => ColumnType::Bytes, + SimpleType::Timestamp => ColumnType::Timestamp, + SimpleType::Timestamptz => ColumnType::Timestamptz, + SimpleType::SystemTimestamp => ColumnType::SystemTimestamp, + SimpleType::Decimal { precision, scale } => ColumnType::Decimal { + precision: *precision, + scale: *scale, + }, + SimpleType::Uuid => ColumnType::Uuid, + SimpleType::Ulid => ColumnType::Ulid, + SimpleType::Geometry => ColumnType::Geometry, + SimpleType::Duration => ColumnType::Duration, + SimpleType::Array => ColumnType::Array, + SimpleType::Object => ColumnType::Json, + SimpleType::Json => ColumnType::Json, + SimpleType::Set => ColumnType::Set, + SimpleType::Regex => ColumnType::Regex, + SimpleType::Range => ColumnType::Range, + SimpleType::Record => ColumnType::Record, + SimpleType::Vector(dim) => ColumnType::Vector(*dim), + SimpleType::SparseVector => ColumnType::SparseVector, + } + } + + let mut spellings: Vec = [ + "TEXT", + "VARCHAR", + "VARCHAR(255)", + "STRING", + "BOOL", + "BOOLEAN", + "BYTES", + "BYTEA", + "BLOB", + "TIMESTAMP", + "TIMESTAMPTZ", + "TIMESTAMP WITH TIME ZONE", + "TIMESTAMP WITHOUT TIME ZONE", + "SYSTEM_TIMESTAMP", + "DECIMAL", + "NUMERIC", + "DECIMAL(10, 2)", + "UUID", + "ULID", + "GEOMETRY", + "DURATION", + "JSON", + "JSONB", + "OBJECT", + "ARRAY", + "SET", + "REGEX", + "RANGE", + "RECORD", + "SPARSEVECTOR", + "VECTOR(384)", + ] + .iter() + .map(|spelling| spelling.to_string()) + .collect(); + spellings.extend( + DECLARED_INT_KEYWORDS + .iter() + .chain(DECLARED_FLOAT_KEYWORDS.iter()) + .map(|keyword| keyword.to_string()), + ); + + for spelling in spellings { + let parsed = parse_type_expr(&spelling) + .unwrap_or_else(|e| panic!("enforcement must parse {spelling}: {e}")); + let TypeExpr::Simple(leaf) = parsed else { + panic!("{spelling} must parse to a leaf type, got {parsed:?}") + }; + assert_eq!( + leaf_column_type(&leaf), + typeguard_type_to_column_type(&spelling) + .unwrap_or_else(|e| panic!("CONVERT must resolve {spelling}: {}", e.message)), + "enforcement and CONVERT must agree on {spelling}" + ); + } + } + + #[test] + fn sql_type_to_column_type_distinguishes_timestamp_from_timestamptz() { + for (spelling, expected) in [ + ("TIMESTAMP", ColumnType::Timestamp), + ("TIMESTAMP WITHOUT TIME ZONE", ColumnType::Timestamp), + ("TIMESTAMPTZ", ColumnType::Timestamptz), + ("TIMESTAMP WITH TIME ZONE", ColumnType::Timestamptz), + ] { + assert_eq!( + sql_type_to_column_type(spelling).expect("the spelling resolves"), + expected, + "column def mapper on {spelling}" + ); + } + } + + #[test] + fn typeguard_type_to_column_type_distinguishes_timestamp_from_timestamptz() { + for (spelling, expected) in [ + ("TIMESTAMP", ColumnType::Timestamp), + ("TIMESTAMP WITHOUT TIME ZONE", ColumnType::Timestamp), + ("TIMESTAMPTZ", ColumnType::Timestamptz), + ("TIMESTAMP WITH TIME ZONE", ColumnType::Timestamptz), + ] { + assert_eq!( + typeguard_type_to_column_type(spelling).expect("the spelling resolves"), + expected, + "typeguard mapper on {spelling}" + ); + } + } +} diff --git a/nodedb/src/control/server/shared/ddl/neutral/convert/typeguard_columns.rs b/nodedb/src/control/server/shared/ddl/neutral/convert/typeguard_columns.rs new file mode 100644 index 000000000..0bc1749cc --- /dev/null +++ b/nodedb/src/control/server/shared/ddl/neutral/convert/typeguard_columns.rs @@ -0,0 +1,123 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! Typeguard field definitions turned into strict-schema columns. +//! +//! `CONVERT COLLECTION TO document_strict` with no explicit column list +//! reads the collection's active typeguards instead. + +use super::super::super::result::DdlError; +use super::super::column_default::validate_column_default; +use super::support::err; +use super::type_map::typeguard_type_to_column_type; + +/// Convert typeguard field definitions to strict schema column definitions. +/// +/// Each typeguard field becomes a column. The type expression is mapped to ColumnType. +/// REQUIRED fields become NOT NULL. DEFAULT expressions carry over. +/// +/// A carried-over expression becomes a column `DEFAULT`, so it passes the gate +/// every declared column `DEFAULT` passes. An unregistered function name +/// raises `42883`; every other rejection raises `42601`. +pub(super) fn typeguards_to_column_defs( + guards: &[nodedb_types::TypeGuardFieldDef], +) -> Result, DdlError> { + use nodedb_types::columnar::{ColumnDef, ColumnType}; + + // Always include an `id` column as primary key. + let mut columns = vec![ColumnDef::required("id", ColumnType::String).with_primary_key()]; + + for guard in guards { + // Skip dot-path fields (nested) — strict schema is flat. + if guard.field.contains('.') { + continue; + } + // Skip if already added (e.g., "id"). + if guard.field == "id" || columns.iter().any(|c| c.name == guard.field) { + continue; + } + + let ct = typeguard_type_to_column_type(&guard.type_expr).map_err(|e| { + err( + &e.sqlstate, + format!("field '{}': {}", guard.field, e.message), + ) + })?; + let mut col = if guard.required { + ColumnDef::required(guard.field.clone(), ct) + } else { + ColumnDef::nullable(guard.field.clone(), ct) + }; + // A guard carries either DEFAULT or VALUE, never both. Strict schema + // has one materialization slot, so both land on the column `DEFAULT`. + if let Some(expr) = guard.default_expr.clone().or(guard.value_expr.clone()) { + validate_column_default(&col.name, &expr)?; + col.default = Some(expr); + } + columns.push(col); + } + + Ok(columns) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn guard(field: &str, type_expr: &str) -> nodedb_types::TypeGuardFieldDef { + nodedb_types::TypeGuardFieldDef { + field: field.to_string(), + type_expr: type_expr.to_string(), + required: false, + check_expr: None, + default_expr: None, + value_expr: None, + } + } + + /// An unresolvable guard type stops CONVERT instead of storing a column + /// type the guard never declared. + #[test] + fn typeguards_to_column_defs_refuses_an_unresolvable_guard_type() { + let guards = vec![guard("gadget", "WIDGET")]; + let error = + typeguards_to_column_defs(&guards).expect_err("an unresolvable guard type must refuse"); + assert_eq!(error.sqlstate, "42601"); + assert!( + error.message.contains("gadget"), + "the error must name the field: {}", + error.message + ); + } + + /// A guard DEFAULT the column path can evaluate reaches the column. + #[test] + fn typeguards_to_column_defs_carry_a_literal_default() { + let mut status = guard("status", "STRING"); + status.default_expr = Some("'draft'".to_string()); + let columns = typeguards_to_column_defs(&[status]).expect("a literal DEFAULT must convert"); + assert_eq!(columns[1].name, "status"); + assert_eq!(columns[1].default.as_deref(), Some("'draft'")); + } + + /// A guard VALUE lands on the column `DEFAULT`, the one strict-schema slot + /// that materializes an omitted field. + #[test] + fn typeguards_to_column_defs_carry_a_value_expression() { + let mut computed = guard("computed", "STRING"); + computed.value_expr = Some("'server'".to_string()); + let columns = + typeguards_to_column_defs(&[computed]).expect("a VALUE expression must convert"); + assert_eq!(columns[1].default.as_deref(), Some("'server'")); + } + + /// A carried-over expression naming an unregistered function stops CONVERT + /// with the SQLSTATE every other DEFAULT source raises for it. + #[test] + fn typeguards_to_column_defs_refuse_an_unevaluable_default() { + let mut status = guard("status", "STRING"); + status.default_expr = Some("no_such_function()".to_string()); + let error = typeguards_to_column_defs(&[status]) + .expect_err("an unevaluable carried-over DEFAULT must refuse"); + assert_eq!(error.sqlstate, "42883"); + } +} diff --git a/nodedb/src/control/server/shared/ddl/neutral/mod.rs b/nodedb/src/control/server/shared/ddl/neutral/mod.rs index 5d0c9664e..7d6d82e97 100644 --- a/nodedb/src/control/server/shared/ddl/neutral/mod.rs +++ b/nodedb/src/control/server/shared/ddl/neutral/mod.rs @@ -18,6 +18,7 @@ pub mod change_stream; pub mod chunk_text; pub mod cluster; pub mod collection; +mod column_default; pub mod conflict_policy; pub mod constraint; pub mod consumer_group; diff --git a/nodedb/src/control/server/shared/ddl/neutral/planning.rs b/nodedb/src/control/server/shared/ddl/neutral/planning.rs index 7c29e27dc..acf185a58 100644 --- a/nodedb/src/control/server/shared/ddl/neutral/planning.rs +++ b/nodedb/src/control/server/shared/ddl/neutral/planning.rs @@ -48,7 +48,7 @@ pub async fn plan_authorized_sql( }; let query_ctx = QueryContext::for_state(state); let (tasks, output_schema, versions, _) = query_ctx - .plan_sql_with_rls_and_versions(sql, identity.tenant_id, database_id, &sec, false) + .plan_sql_with_rls_and_versions(sql, identity.tenant_id, database_id, &sec, None) .await .map_err(|error| DdlError::new("42601", format!("query planning failed: {error}")))?; diff --git a/nodedb/src/control/server/shared/ddl/neutral/typeguard/injected_expr.rs b/nodedb/src/control/server/shared/ddl/neutral/typeguard/injected_expr.rs new file mode 100644 index 000000000..1fb3afed4 --- /dev/null +++ b/nodedb/src/control/server/shared/ddl/neutral/typeguard/injected_expr.rs @@ -0,0 +1,110 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! DDL-time gate for a typeguard `DEFAULT` / `VALUE` expression. +//! +//! Both clauses store raw text and produce a field value on every write. The +//! Data Plane's `inject_defaults` reads that text with +//! `nodedb_query::expr_parse::parse_generated_expr` and evaluates it with +//! `SqlExpr::eval`, so a declaration the write path cannot evaluate is refused +//! here instead of at the first write. +//! +//! Two checks run, and neither evaluates the expression: +//! - The canonical declared-clause gate resolves every function name through +//! the `FunctionRegistry`. `nodedb_query::functions::eval_function` answers +//! `Null` for a name it does not know, so an unregistered name would store +//! `Null` on every write and report nothing. +//! - The write path's own parser refuses what it cannot evaluate, including +//! the non-deterministic functions a guard expression must not call. +//! +//! An unregistered function name raises SQLSTATE `42883`; every other +//! rejection raises SQLSTATE `42601`. `nextval` is refused outright, so +//! declaring a typeguard cannot advance a sequence. + +use super::super::super::result::DdlError; +use super::super::column_default::validate_clause_expr; +use super::parse::err; + +/// Refuse a typeguard `DEFAULT` or `VALUE` the engine cannot evaluate. +/// +/// `clause` names the keyword the author wrote, so the message points at it. +pub(super) fn validate_injected_expr( + field: &str, + clause: &str, + expr: &str, +) -> Result<(), DdlError> { + validate_clause_expr(clause, field, expr)?; + nodedb_query::expr_parse::parse_generated_expr(expr).map_err(|error| { + err( + "42601", + &format!("field '{field}': {clause} expression '{expr}' is invalid: {error}"), + ) + })?; + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + + /// Every clause form the write path evaluates stays accepted. + #[test] + fn evaluable_clause_expressions_are_accepted() { + for (clause, expr) in [ + ("DEFAULT", "'draft'"), + ("DEFAULT", "1"), + ("DEFAULT", "0"), + ("VALUE", "'server_computed'"), + ("VALUE", "name"), + ("VALUE", "LOWER(REPLACE(title, ' ', '-'))"), + ] { + validate_injected_expr("f", clause, expr) + .unwrap_or_else(|e| panic!("{clause} {expr} must be accepted: {}", e.message)); + } + } + + /// An unregistered function name raises `42883` rather than storing `Null` + /// on every write. + #[test] + fn an_unregistered_function_raises_undefined_function() { + let error = validate_injected_expr("status", "DEFAULT", "no_such_function()") + .expect_err("an unregistered function must refuse"); + assert_eq!(error.sqlstate, "42883"); + assert!( + error.message.contains("status"), + "the error must name the field: {}", + error.message + ); + } + + /// The same refusal covers a `VALUE` clause, and names that clause. + #[test] + fn an_unregistered_function_in_a_value_clause_names_the_clause() { + let error = validate_injected_expr("computed", "VALUE", "no_such_function()") + .expect_err("an unregistered function must refuse"); + assert_eq!(error.sqlstate, "42883"); + assert!( + error.message.contains("VALUE"), + "the error must name the clause: {}", + error.message + ); + } + + /// A function the write path refuses as non-deterministic is refused at + /// declaration with `42601`. + #[test] + fn a_non_deterministic_function_is_refused() { + for expr in ["now()", "uuid_v7()", "nextval('s')"] { + let error = validate_injected_expr("at", "DEFAULT", expr) + .expect_err("a non-deterministic function must refuse"); + assert_eq!(error.sqlstate, "42601", "on {expr}"); + } + } + + /// A malformed expression is a syntax error, not a stored clause. + #[test] + fn a_malformed_expression_is_refused() { + let error = validate_injected_expr("f", "DEFAULT", "'unterminated") + .expect_err("a malformed expression must refuse"); + assert_eq!(error.sqlstate, "42601"); + } +} diff --git a/nodedb/src/control/server/shared/ddl/neutral/typeguard/mod.rs b/nodedb/src/control/server/shared/ddl/neutral/typeguard/mod.rs index e1a509d6e..814d039f7 100644 --- a/nodedb/src/control/server/shared/ddl/neutral/typeguard/mod.rs +++ b/nodedb/src/control/server/shared/ddl/neutral/typeguard/mod.rs @@ -3,6 +3,7 @@ //! Protocol-neutral TYPEGUARD DDL family handlers. pub mod handlers; +mod injected_expr; pub mod parse; pub mod validate; diff --git a/nodedb/src/control/server/shared/ddl/neutral/typeguard/parse.rs b/nodedb/src/control/server/shared/ddl/neutral/typeguard/parse.rs index c7cdadfe0..f758ed234 100644 --- a/nodedb/src/control/server/shared/ddl/neutral/typeguard/parse.rs +++ b/nodedb/src/control/server/shared/ddl/neutral/typeguard/parse.rs @@ -9,9 +9,11 @@ use nodedb_sql::parser::preprocess::lex::{ find_ascii_case_insensitive, find_ascii_case_insensitive_from, }; +use nodedb_sql::parser::type_expr::parse_type_expr; use nodedb_types::TypeGuardFieldDef; use super::super::super::result::DdlError; +use super::injected_expr::validate_injected_expr; /// Extract collection name from `... TYPEGUARD [IF EXISTS] ON ...`. pub(super) fn extract_collection_name(sql: &str) -> Result { @@ -173,6 +175,11 @@ pub(super) fn parse_single_field(s: &str) -> Result )); } + // The declaration must name a type the engine resolves. `parse_type_expr` + // is the parser that enforces the guard on every write, so a spelling it + // rejects fails here instead of failing each later write. + parse_type_expr(&type_expr).map_err(|e| err("42601", &format!("field '{field}': {e}")))?; + // Extract DEFAULT expression if present. let default_expr = if let Some(def_pos) = find_word_boundary(rest, "DEFAULT") { let after_default = rest[def_pos + 7..].trim_start(); @@ -200,6 +207,15 @@ pub(super) fn parse_single_field(s: &str) -> Result )); } + // Both clauses produce a field value on every write, so a declaration the + // write path cannot evaluate is refused here. + if let Some(expr) = default_expr.as_deref() { + validate_injected_expr(&field, "DEFAULT", expr)?; + } + if let Some(expr) = value_expr.as_deref() { + validate_injected_expr(&field, "VALUE", expr)?; + } + Ok(TypeGuardFieldDef { field, type_expr, @@ -260,6 +276,49 @@ mod tests { assert_eq!(name, "metrics"); } + /// A declared type the engine cannot resolve stops the declaration. + /// + /// The guard would otherwise be stored and fail every later write. + /// `TIMESTAMP GARBAGE` covers the trailing-word case, which resolving the + /// leading token alone would read as `TIMESTAMP`. + #[test] + fn unresolvable_field_type_is_refused_at_declaration() { + for declaration in ["gadget WIDGET", "at TIMESTAMP GARBAGE"] { + let error = parse_single_field(declaration) + .expect_err("an unresolvable declared type must be refused"); + assert_eq!(error.sqlstate, "42601", "on {declaration}"); + } + } + + /// Every spelling the engine resolves stays accepted at declaration. + #[test] + fn every_resolvable_field_type_is_accepted_at_declaration() { + for declaration in [ + "a INT2", + "b INT4", + "c INT8", + "d SMALLINT", + "e FLOAT4", + "f FLOAT8", + "g FLOAT32", + "h DOUBLE PRECISION", + "i JSONB", + "j SYSTEM_TIMESTAMP", + "k VARCHAR(255)", + "l DECIMAL(10, 2)", + "m TIMESTAMP WITH TIME ZONE", + "n TIMESTAMP WITHOUT TIME ZONE", + "o OBJECT", + "p ARRAY", + "q STRING|NULL", + "r VECTOR(384)", + "s STRING REQUIRED CHECK (s <> '')", + ] { + parse_single_field(declaration) + .unwrap_or_else(|e| panic!("{declaration} must parse: {}", e.message)); + } + } + #[test] fn field_keywords_after_unicode_default_preserve_original_offsets() { let field = parse_single_field("label STRING DEFAULT 'ffff' CHECK (label <> '') REQUIRED") diff --git a/nodedb/src/control/server/shared/ddl/schema_validation.rs b/nodedb/src/control/server/shared/ddl/schema_validation.rs index bebbe40c8..0bf034974 100644 --- a/nodedb/src/control/server/shared/ddl/schema_validation.rs +++ b/nodedb/src/control/server/shared/ddl/schema_validation.rs @@ -2,18 +2,50 @@ //! Schema parsing and type validation helpers for collection DDL. +/// Expand a `SERIAL` / `BIGSERIAL` type token into its backing type and DEFAULT. +/// +/// `SERIAL` yields `INT DEFAULT nextval('{collection}_{field}_seq')` and +/// `BIGSERIAL` yields `BIGINT DEFAULT nextval(...)`. The sequence name matches +/// the one `create_serial_sequences` materializes, so the column reads the +/// sequence its own declaration created. Returns `None` for every other type. +/// +/// Trailing modifiers such as `PRIMARY KEY` follow the DEFAULT clause, where +/// `parse_column_type_str_full` expects them. +fn expand_serial_type(collection: &str, field: &str, type_str: &str) -> Option { + let trimmed = type_str.trim(); + let (head, tail) = match trimmed.find(char::is_whitespace) { + Some(index) => (&trimmed[..index], trimmed[index..].trim()), + None => (trimmed, ""), + }; + let base = if head.eq_ignore_ascii_case("SERIAL") { + "INT" + } else if head.eq_ignore_ascii_case("BIGSERIAL") { + "BIGINT" + } else { + return None; + }; + let expanded = format!("{base} DEFAULT nextval('{collection}_{field}_seq')"); + Some(if tail.is_empty() { + expanded + } else { + format!("{expanded} {tail}") + }) +} + /// Parse FIELDS clause from CREATE COLLECTION parts. /// /// Syntax: `CREATE COLLECTION name FIELDS (field1 type1, field2 type2, ...)` /// Returns empty vec if no FIELDS clause. /// -/// SERIAL and BIGSERIAL are expanded: -/// `id SERIAL` → `id INT` (caller creates implicit sequence) -/// `id BIGSERIAL` → `id BIGINT` (caller creates implicit sequence) +/// SERIAL and BIGSERIAL expand to their backing type plus a `nextval` DEFAULT +/// naming the sequence the caller creates for that column. /// /// The second return value lists field names that had SERIAL/BIGSERIAL types, /// so the caller can create the implicit sequences. -pub(crate) fn parse_fields_clause(parts: &[&str]) -> (Vec<(String, String)>, Vec) { +pub(crate) fn parse_fields_clause( + collection: &str, + parts: &[&str], +) -> (Vec<(String, String)>, Vec) { let fields_idx = parts.iter().position(|p| p.eq_ignore_ascii_case("FIELDS")); let fields_idx = match fields_idx { Some(i) => i, @@ -40,17 +72,13 @@ pub(crate) fn parse_fields_clause(parts: &[&str]) -> (Vec<(String, String)>, Vec let name = name.to_string(); let type_name = tokens.next().unwrap_or("text").to_uppercase(); - // Expand SERIAL/BIGSERIAL shorthand. - let actual_type = match type_name.as_str() { - "SERIAL" => { + // Expand SERIAL/BIGSERIAL shorthand into type + nextval DEFAULT. + let actual_type = match expand_serial_type(collection, &name, &type_name) { + Some(expanded) => { serial_fields.push(name.clone()); - "INT".to_string() + expanded } - "BIGSERIAL" => { - serial_fields.push(name.clone()); - "BIGINT".to_string() - } - other => other.to_string(), + None => type_name, }; fields.push((name, actual_type)); @@ -68,20 +96,19 @@ pub(crate) fn parse_fields_clause(parts: &[&str]) -> (Vec<(String, String)>, Vec /// of columns whose type was expanded from SERIAL / BIGSERIAL so the caller /// can create implicit sequences. pub(crate) fn parse_fields_clause_from_pairs( + collection: &str, columns: &[(String, String)], ) -> (Vec<(String, String)>, Vec) { let mut fields: Vec<(String, String)> = Vec::new(); let mut serial_fields: Vec = Vec::new(); for (name, type_str) in columns { - let actual_type = if type_str.eq_ignore_ascii_case("SERIAL") { - serial_fields.push(name.clone()); - "INT".to_string() - } else if type_str.eq_ignore_ascii_case("BIGSERIAL") { - serial_fields.push(name.clone()); - "BIGINT".to_string() - } else { - type_str.clone() + let actual_type = match expand_serial_type(collection, name, type_str) { + Some(expanded) => { + serial_fields.push(name.clone()); + expanded + } + None => type_str.clone(), }; fields.push((name.clone(), actual_type)); } diff --git a/nodedb/src/control/server/shared/plan_admission.rs b/nodedb/src/control/server/shared/plan_admission.rs index 3402f4f15..98f3a2c22 100644 --- a/nodedb/src/control/server/shared/plan_admission.rs +++ b/nodedb/src/control/server/shared/plan_admission.rs @@ -106,7 +106,7 @@ async fn plan_authorize_and_admit_once( permission_cache: Some(&*permission_cache), }; let (tasks, output_schema, versions, _cache_eligibility) = query_ctx - .plan_sql_with_rls_and_versions(sql, tenant_id, database_id, &security, false) + .plan_sql_with_rls_and_versions(sql, tenant_id, database_id, &security, None) .await?; (tasks, output_schema, versions) }; diff --git a/nodedb/src/control/server/shared/planning_overrides.rs b/nodedb/src/control/server/shared/planning_overrides.rs index 1ccef2e6c..e88d2e282 100644 --- a/nodedb/src/control/server/shared/planning_overrides.rs +++ b/nodedb/src/control/server/shared/planning_overrides.rs @@ -81,6 +81,10 @@ pub fn apply_planning_session_overrides( query_ctx.set_max_vector_dim(tenants.quota(tenant_id).max_vector_dim); } + // Sequence accessors resolve at plan time, so the planner needs this + // connection's `currval` map before it plans. + query_ctx.set_session_sequences(sessions.sequence_values(session_id)); + // Distributed shuffle-join override (`SET nodedb.force_shuffle_join = on` // and, optionally, `SET nodedb.shuffle_num_parts = N`). let force_shuffle_join = sessions diff --git a/nodedb/src/control/server/shared/session/mod.rs b/nodedb/src/control/server/shared/session/mod.rs index 17d3b93c2..42899b0bd 100644 --- a/nodedb/src/control/server/shared/session/mod.rs +++ b/nodedb/src/control/server/shared/session/mod.rs @@ -30,6 +30,7 @@ pub mod read_set; pub mod record_reads; mod reservation_release; pub mod savepoint_ops; +mod sequence_values; pub mod set_validation; pub mod staging_gate; mod state; diff --git a/nodedb/src/control/server/shared/session/sequence_values.rs b/nodedb/src/control/server/shared/session/sequence_values.rs new file mode 100644 index 000000000..f398ab991 --- /dev/null +++ b/nodedb/src/control/server/shared/session/sequence_values.rs @@ -0,0 +1,18 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! Session access to the per-connection `currval` map. + +use std::sync::Arc; + +use crate::control::sequence::SessionSequenceValues; + +use super::connection::SessionId; +use super::store::SessionStore; + +impl SessionStore { + /// The `currval` map of one connection. `None` when no session is + /// registered under `id` — a caller with no session gets no `currval`. + pub fn sequence_values(&self, id: impl Into) -> Option> { + self.read_session(id, |session| Arc::clone(&session.sequence_values)) + } +} diff --git a/nodedb/src/control/server/shared/session/state.rs b/nodedb/src/control/server/shared/session/state.rs index bdcd29837..2fe1834b8 100644 --- a/nodedb/src/control/server/shared/session/state.rs +++ b/nodedb/src/control/server/shared/session/state.rs @@ -215,6 +215,10 @@ pub struct ConnSession { /// GAP_FREE sequence reservations pending commit/rollback. /// On COMMIT: each reservation is finalized. On ROLLBACK: counter decremented. pub pending_sequence_reservations: Vec, + /// Last `nextval` value this session obtained per sequence — what SQL + /// `currval` returns. Shared with the plan-time catalog adapter, which + /// writes it on `nextval` and reads it on `currval`. + pub sequence_values: Arc, /// Millis-since-epoch of the last statement COMPLETION on this connection /// (also set to "now" at connection start). Read by the pgwire listener /// watchdog to decide idle eligibility: a connection is idle only when it @@ -321,6 +325,7 @@ impl ConnSession { temp_tables: super::temp_tables::TempTableRegistry::new(), plan_cache: crate::control::server::shared::session::plan_cache::PlanCache::new(128), pending_sequence_reservations: Vec::new(), + sequence_values: Arc::new(crate::control::sequence::SessionSequenceValues::new()), last_activity_ms: AtomicU64::new(now_unix_ms()), in_flight: AtomicU32::new(0), own_write_versions: HashMap::new(), diff --git a/nodedb/src/data/executor/core_loop/mod.rs b/nodedb/src/data/executor/core_loop/mod.rs index 0398e78e3..55e9e6211 100644 --- a/nodedb/src/data/executor/core_loop/mod.rs +++ b/nodedb/src/data/executor/core_loop/mod.rs @@ -30,6 +30,7 @@ pub use doc_config_seed::DocConfigSeedEntry; pub(in crate::data::executor) use segment_keks::SegmentKeks; pub use state::CoreLoop; pub use test_governor::test_governor; +pub(in crate::data::executor) use ts_declared_schema::TsGroupKeyKind; /// Shared test fixtures (`make_core_with_dir`, `make_default_task`), kept /// alongside the write-version-index tests that exercise the same `CoreLoop` /// apply chokepoints. Re-exported here so external test modules keep using diff --git a/nodedb/src/data/executor/core_loop/ts_declared_schema.rs b/nodedb/src/data/executor/core_loop/ts_declared_schema.rs index 1f6279ee0..acad6049d 100644 --- a/nodedb/src/data/executor/core_loop/ts_declared_schema.rs +++ b/nodedb/src/data/executor/core_loop/ts_declared_schema.rs @@ -116,6 +116,130 @@ impl CoreLoop { timestamp_idx, }) } + + /// How each named GROUP BY column of a timeseries collection renders. + /// + /// A grouped column must carry the type it carries ungrouped, so this + /// resolves the same declared shape row emission reads. A declared + /// collection answers from its DDL; a measurement ingested over the raw + /// ILP protocol has no DDL, so its resident memtable schema answers. + /// A column present in neither renders as text. + pub(in crate::data::executor) fn ts_group_key_kinds( + &self, + database_id: DatabaseId, + tid: TenantId, + collection: &str, + group_by: &[String], + ) -> Vec { + if let Some(declared) = self.declared_timeseries(database_id, tid, collection) { + let time_key_index = declared.time_key_index(); + return group_by + .iter() + .map(|name| { + let Some(index) = declared.columns.iter().position(|(c, _)| c == name) else { + return TsGroupKeyKind::Text; + }; + let declared_type = declared.columns[index].1.as_str(); + if declared_type_is_instant(declared_type) { + return TsGroupKeyKind::Instant; + } + kind_of_storage(memtable_column_type( + declared_type, + Some(index) == time_key_index, + )) + }) + .collect(); + } + + let key = (database_id, tid, collection.to_string()); + let Some(memtable) = self.columnar_memtables.get(&key) else { + return vec![TsGroupKeyKind::Text; group_by.len()]; + }; + let schema = memtable.schema(); + group_by + .iter() + .map(|name| { + schema + .columns + .iter() + .find(|(c, _)| c == name) + .map(|(_, ty)| kind_of_storage(*ty)) + .unwrap_or(TsGroupKeyKind::Text) + }) + .collect() + } + + /// Declared columns of a timeseries collection that carry an instant. + /// + /// A column declared `TIMESTAMP` or `TIMESTAMPTZ` is one. The memtable + /// keeps every timestamp column in epoch milliseconds, while a client + /// reads a `TIMESTAMP` cell as epoch microseconds, so row emission scales + /// exactly these columns. + /// + /// A `BIGINT TIME_KEY` shares the same millisecond column and is absent + /// from this list: its declared type is an integer, so it hands back the + /// number that was inserted. + /// + /// An undeclared measurement (raw ILP protocol ingest) has no entry and + /// yields an empty list — the planner types its columns as text, so no + /// cell of it is read as an instant. + pub(in crate::data::executor) fn ts_instant_columns( + &self, + database_id: DatabaseId, + tid: TenantId, + collection: &str, + ) -> Vec { + let Some(declared) = self.declared_timeseries(database_id, tid, collection) else { + return Vec::new(); + }; + declared + .columns + .iter() + .filter(|(_, type_str)| declared_type_is_instant(type_str)) + .map(|(name, _)| name.clone()) + .collect() + } +} + +/// How one grouped timeseries column renders in an aggregate result row. +/// +/// The grouped scan reduces every key to a string, so emission has to put the +/// column's own type back. The variants name the four shapes a stored +/// timeseries column can take on the wire. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(in crate::data::executor) enum TsGroupKeyKind { + /// A declared `TIMESTAMP` / `TIMESTAMPTZ` column: epoch microseconds. + Instant, + /// An integer column, including a `BIGINT TIME_KEY`, in its stored unit. + Integer, + /// A floating-point column. + Float, + /// A dictionary symbol, or a column with no resolvable storage type. + Text, +} + +/// The wire shape a memtable storage type renders as. +/// +/// `Timestamp` maps to `Integer` here: the instant case is decided from the +/// declared DDL type before this runs, so what reaches it is a `BIGINT` +/// time key or a system-time column, both of which render as the number +/// storage holds. +fn kind_of_storage(storage: ColumnType) -> TsGroupKeyKind { + match storage { + ColumnType::Int64 | ColumnType::Timestamp => TsGroupKeyKind::Integer, + ColumnType::Float64 => TsGroupKeyKind::Float, + ColumnType::Symbol => TsGroupKeyKind::Text, + } +} + +/// Whether a declared DDL type makes a column an instant on the wire. +/// +/// Both planes answer this from `nodedb_types::columnar::ColumnType`: the +/// Control Plane decides how a cell is READ, this decides the unit it is +/// WRITTEN in, and one classifier keeps them from naming different sets. +fn declared_type_is_instant(declared_type: &str) -> bool { + nodedb_types::columnar::ColumnType::from_declared_type(declared_type) + .is_some_and(|declared| declared.is_instant()) } /// Map a declared SQL type onto the memtable's storage type. @@ -124,20 +248,21 @@ impl CoreLoop { /// regardless of how it was spelled — `TIMESTAMP`, `TIMESTAMPTZ`, and /// `BIGINT` time keys all store epoch milliseconds. fn memtable_column_type(declared_type: &str, is_time_key: bool) -> ColumnType { + use nodedb_types::columnar::ColumnType as DeclaredType; + if is_time_key { return ColumnType::Timestamp; } - let bare = declared_type.split_whitespace().next().unwrap_or(""); - match bare.parse::() { - Ok(nodedb_types::columnar::ColumnType::Timestamp) - | Ok(nodedb_types::columnar::ColumnType::Timestamptz) - | Ok(nodedb_types::columnar::ColumnType::SystemTimestamp) => ColumnType::Timestamp, - Ok(nodedb_types::columnar::ColumnType::Int64) => ColumnType::Int64, - // The memtable has no boolean column; ILP ingest has always widened - // booleans to f64, so a declared BOOLEAN lands in the same place. - Ok(nodedb_types::columnar::ColumnType::Float64) - | Ok(nodedb_types::columnar::ColumnType::Bool) - | Ok(nodedb_types::columnar::ColumnType::Decimal { .. }) => ColumnType::Float64, + match DeclaredType::from_declared_type(declared_type) { + Some( + DeclaredType::Timestamp | DeclaredType::Timestamptz | DeclaredType::SystemTimestamp, + ) => ColumnType::Timestamp, + Some(DeclaredType::Int64) => ColumnType::Int64, + // The memtable has no boolean column; ILP ingest widens booleans to + // f64, so a declared BOOLEAN lands in the same place. + Some(DeclaredType::Float64 | DeclaredType::Bool | DeclaredType::Decimal { .. }) => { + ColumnType::Float64 + } // Everything else — TEXT, UUID, JSON, and any type the memtable // cannot represent natively — is stored as a dictionary symbol. _ => ColumnType::Symbol, @@ -189,6 +314,21 @@ mod tests { ); } + /// The declared types the planner reads back as instants are exactly the + /// ones emission scales. A `BIGINT` time key is stored in the same + /// millisecond column and must NOT be scaled. + #[test] + fn only_declared_timestamp_types_are_instants() { + assert!(declared_type_is_instant("TIMESTAMP TIME_KEY")); + assert!(declared_type_is_instant("TIMESTAMPTZ")); + assert!(declared_type_is_instant("timestamp")); + assert!(!declared_type_is_instant("BIGINT TIME_KEY")); + assert!(!declared_type_is_instant("INT")); + assert!(!declared_type_is_instant("TEXT")); + assert!(!declared_type_is_instant("SYSTEM_TIMESTAMP")); + assert!(!declared_type_is_instant("")); + } + #[test] fn unknown_types_fall_back_to_symbol() { assert_eq!( diff --git a/nodedb/src/data/executor/dispatch/query.rs b/nodedb/src/data/executor/dispatch/query.rs index 180a112d0..cbe383a73 100644 --- a/nodedb/src/data/executor/dispatch/query.rs +++ b/nodedb/src/data/executor/dispatch/query.rs @@ -8,6 +8,7 @@ use nodedb_physical::physical_plan::{GroupKeySpec, QueryOp}; use crate::data::executor::core_loop::CoreLoop; use crate::data::executor::handlers::join::{ HashJoinParams, JoinParams, NestedLoopJoinParams, ShuffleJoinInputs, SortMergeJoinParams, + instant_scale::JoinInstantSide, lateral::{LateralLoopParams, LateralTopKParams}, }; use crate::data::executor::task::ExecutionTask; @@ -110,7 +111,34 @@ impl CoreLoop { left_scan_filters, right_scan_filters, .. - } => self.execute_hash_join(HashJoinParams { + } => { + // Only a side this join scans itself carries stored + // milliseconds. A side supplied by a sub-plan was already + // rescaled by the handler that read it, so listing it here + // would scale one instant twice. + let mut local_sides = Vec::with_capacity(2); + if left_input.is_none() { + local_sides.push(JoinInstantSide { + collection: left_collection.as_str(), + qualifier: left_alias + .as_deref() + .unwrap_or_else(|| left_collection.as_str()), + }); + } + if right_input.is_none() { + local_sides.push(JoinInstantSide { + collection: right_collection.as_str(), + qualifier: right_alias + .as_deref() + .unwrap_or_else(|| right_collection.as_str()), + }); + } + let instant_columns = self.join_instant_columns( + task.request.database_id, + crate::types::TenantId::new(tid), + &local_sides, + ); + self.execute_hash_join(HashJoinParams { join: JoinParams { task, on, @@ -120,6 +148,7 @@ impl CoreLoop { computed_projection_bytes: computed_projection, join_filter_bytes: join_filters, post_filter_bytes: post_filters, + instant_columns: &instant_columns, }, tid, left_collection: left_collection.as_str(), @@ -134,7 +163,8 @@ impl CoreLoop { right_rls_filters, left_scan_filters, right_scan_filters, - }), + }) + } QueryOp::ShuffleJoinConsume { build_path, @@ -161,6 +191,10 @@ impl CoreLoop { computed_projection_bytes: &[], join_filter_bytes: &[], post_filter_bytes: &[], + // A shuffle consumer's rows are gathered by the + // coordinator, not sent to a client, and its staged + // frames were produced by the sides' own scans. + instant_columns: &[], }; let inputs = ShuffleJoinInputs { build_path: std::path::PathBuf::from(build_path), @@ -182,17 +216,34 @@ impl CoreLoop { limit, left_rls_filters, right_rls_filters, - } => self.execute_nested_loop_join(NestedLoopJoinParams { - task, - tid, - left_collection: left_collection.as_str(), - right_collection: right_collection.as_str(), - condition, - join_type, - limit: *limit, - left_rls_filters, - right_rls_filters, - }), + } => { + let instant_columns = self.join_instant_columns( + task.request.database_id, + crate::types::TenantId::new(tid), + &[ + JoinInstantSide { + collection: left_collection.as_str(), + qualifier: left_collection.as_str(), + }, + JoinInstantSide { + collection: right_collection.as_str(), + qualifier: right_collection.as_str(), + }, + ], + ); + self.execute_nested_loop_join(NestedLoopJoinParams { + task, + tid, + left_collection: left_collection.as_str(), + right_collection: right_collection.as_str(), + condition, + join_type, + limit: *limit, + left_rls_filters, + right_rls_filters, + instant_columns: &instant_columns, + }) + } QueryOp::SortMergeJoin { left_collection, @@ -203,18 +254,35 @@ impl CoreLoop { pre_sorted, left_rls_filters, right_rls_filters, - } => self.execute_sort_merge_join(SortMergeJoinParams { - task, - tid, - left_collection: left_collection.as_str(), - right_collection: right_collection.as_str(), - on, - join_type, - limit: *limit, - pre_sorted: *pre_sorted, - left_rls_filters, - right_rls_filters, - }), + } => { + let instant_columns = self.join_instant_columns( + task.request.database_id, + crate::types::TenantId::new(tid), + &[ + JoinInstantSide { + collection: left_collection.as_str(), + qualifier: left_collection.as_str(), + }, + JoinInstantSide { + collection: right_collection.as_str(), + qualifier: right_collection.as_str(), + }, + ], + ); + self.execute_sort_merge_join(SortMergeJoinParams { + task, + tid, + left_collection: left_collection.as_str(), + right_collection: right_collection.as_str(), + on, + join_type, + limit: *limit, + pre_sorted: *pre_sorted, + left_rls_filters, + right_rls_filters, + instant_columns: &instant_columns, + }) + } QueryOp::RecursiveScan { collection, diff --git a/nodedb/src/data/executor/handlers/join/instant_scale.rs b/nodedb/src/data/executor/handlers/join/instant_scale.rs new file mode 100644 index 000000000..4f2e41620 --- /dev/null +++ b/nodedb/src/data/executor/handlers/join/instant_scale.rs @@ -0,0 +1,86 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! Rescale the declared-instant cells a join materialized itself. +//! +//! A timeseries collection stores every timestamp column in epoch +//! milliseconds, and a client reads a `TIMESTAMP` cell as epoch microseconds. +//! A join scans its local sides through `scan_collection`, which hands back +//! the stored millisecond value, so the join rescales those cells as it emits +//! them — after every predicate has run, so the join itself still compares +//! milliseconds against milliseconds. +//! +//! Exactly once: a handler rescales ONLY the cells it read from a local +//! collection scan. Rows that arrive from a sub-plan response were already +//! rescaled by the handler that read them, and are never touched again. +//! +//! The rescale runs on the merged row, before any projection. A cell is named +//! by the key the join itself wrote, so a rename cannot hide it: the rename +//! happens strictly downstream, and a projected cell copies bytes the rescale +//! has already corrected. + +use crate::data::executor::core_loop::CoreLoop; +use crate::data::executor::handlers::timeseries::raw_scan::scale_instant_cells; +use crate::types::{DatabaseId, TenantId}; + +/// One locally-scanned join side: the collection its rows come from, and the +/// qualifier the merged row prefixes that side's keys with. +pub(in crate::data::executor) struct JoinInstantSide<'a> { + pub(in crate::data::executor) collection: &'a str, + pub(in crate::data::executor) qualifier: &'a str, +} + +impl CoreLoop { + /// The merged-row keys of the declared-instant cells `sides` contribute. + /// + /// A merged join row keys every cell `.`, so the + /// returned names are what the row carries before any projection runs. A + /// side that is not a timeseries collection contributes none, so a join + /// over ordinary collections returns an empty list and pays nothing. + pub(in crate::data::executor) fn join_instant_columns( + &self, + database_id: DatabaseId, + tid: TenantId, + sides: &[JoinInstantSide<'_>], + ) -> Vec { + let mut merged_keys = Vec::new(); + for side in sides { + for column in self.ts_instant_columns(database_id, tid, side.collection) { + merged_keys.push(format!("{}.{column}", side.qualifier)); + } + } + merged_keys + } +} + +/// Rescale the named cells of merged join rows, in place. +/// +/// Each row is a msgpack map. An empty `instant_columns` is a no-op, so a +/// join that touches no timeseries collection never decodes a row. +pub(in crate::data::executor) fn scale_join_instant_rows( + rows: &mut [Vec], + instant_columns: &[String], +) -> crate::Result<()> { + if instant_columns.is_empty() { + return Ok(()); + } + for row in rows.iter_mut() { + let mut decoded = [ + crate::util::bounded_msgpack::read_value(row.as_slice()).map_err(|e| { + crate::Error::Serialization { + format: "msgpack".into(), + detail: format!("decode join row for instant rescale: {e}"), + } + })?, + ]; + scale_instant_cells(&mut decoded, instant_columns)?; + let mut buf = Vec::with_capacity(row.len()); + rmpv::encode::write_value(&mut buf, &decoded[0]).map_err(|e| { + crate::Error::Serialization { + format: "msgpack".into(), + detail: format!("encode join row after instant rescale: {e}"), + } + })?; + *row = buf; + } + Ok(()) +} diff --git a/nodedb/src/data/executor/handlers/join/mod.rs b/nodedb/src/data/executor/handlers/join/mod.rs index f9d4f8ad7..a6dc9bff6 100644 --- a/nodedb/src/data/executor/handlers/join/mod.rs +++ b/nodedb/src/data/executor/handlers/join/mod.rs @@ -10,6 +10,7 @@ mod grace_repartition; mod grace_spill; pub mod hash; mod hash_handlers; +pub(in crate::data::executor) mod instant_scale; pub mod lateral; pub mod nested_loop; pub mod params; diff --git a/nodedb/src/data/executor/handlers/join/nested_loop.rs b/nodedb/src/data/executor/handlers/join/nested_loop.rs index e0e6db89a..d961bbc6a 100644 --- a/nodedb/src/data/executor/handlers/join/nested_loop.rs +++ b/nodedb/src/data/executor/handlers/join/nested_loop.rs @@ -30,6 +30,7 @@ impl CoreLoop { limit, left_rls_filters, right_rls_filters, + instant_columns, } = p; debug!( core = self.core_id, @@ -224,6 +225,18 @@ impl CoreLoop { return self.response_error(task, ErrorCode::ResourcesExhausted); } + // Last step before emission: the scans above compared the + // milliseconds storage holds, and the client reads microseconds. + if let Err(e) = super::instant_scale::scale_join_instant_rows(&mut results, instant_columns) + { + return self.response_error( + task, + ErrorCode::Internal { + detail: e.to_string(), + }, + ); + } + let payload = super::super::super::response_codec::encode_binary_rows(&results); self.response_with_payload(task, payload) } diff --git a/nodedb/src/data/executor/handlers/join/params.rs b/nodedb/src/data/executor/handlers/join/params.rs index c5620115e..8b14df27f 100644 --- a/nodedb/src/data/executor/handlers/join/params.rs +++ b/nodedb/src/data/executor/handlers/join/params.rs @@ -17,6 +17,11 @@ pub(crate) struct JoinParams<'a> { pub computed_projection_bytes: &'a [u8], pub join_filter_bytes: &'a [u8], pub post_filter_bytes: &'a [u8], + /// Merged-row keys of the declared-instant cells this join reads from its + /// own local scans. Empty when no timeseries collection is scanned + /// locally — including for every side supplied by a sub-plan, whose rows + /// were already rescaled by the handler that read them. + pub instant_columns: &'a [String], } /// Hash join: scans both sides from storage or executes resolved child sub-plans. @@ -76,6 +81,9 @@ pub(crate) struct NestedLoopJoinParams<'a> { pub left_rls_filters: &'a [u8], /// Row-level-security filters for the locally-scanned right side. pub right_rls_filters: &'a [u8], + /// Merged-row keys of the declared-instant cells the two local scans + /// contribute. Empty when neither side is a timeseries collection. + pub instant_columns: &'a [String], } /// Sort-merge join: O((N+M)·log N) equi-join with optional pre-sorted inputs. @@ -95,6 +103,9 @@ pub(crate) struct SortMergeJoinParams<'a> { pub left_rls_filters: &'a [u8], /// Row-level-security filters for the locally-scanned right side. pub right_rls_filters: &'a [u8], + /// Merged-row keys of the declared-instant cells the two local scans + /// contribute. Empty when neither side is a timeseries collection. + pub instant_columns: &'a [String], } // ── Test helpers ───────────────────────────────────────────────────────────── @@ -191,6 +202,11 @@ impl JoinParams<'_> { } } + // Every predicate above compared the milliseconds storage holds, and + // the client reads microseconds. Scale here, on the keys the join + // wrote, so a projection that renames a cell copies a corrected value. + super::instant_scale::scale_join_instant_rows(results, self.instant_columns)?; + if !self.computed_projection_bytes.is_empty() { let computed: Vec = zerompk::from_msgpack(self.computed_projection_bytes).map_err(|e| { @@ -258,6 +274,7 @@ mod tests { computed_projection_bytes: &[], join_filter_bytes: &[], post_filter_bytes: &[], + instant_columns: &[], }; let mut results = vec![vec![1u8, 2, 3], vec![4u8, 5, 6]]; assert!(params.filter_and_project(&mut results).is_ok()); @@ -280,6 +297,7 @@ mod tests { computed_projection_bytes: &[], join_filter_bytes: &[], post_filter_bytes: corrupt, + instant_columns: &[], }; let mut results = vec![vec![0u8; 8]]; // would be "leaked" under the old code let err = params.filter_and_project(&mut results); @@ -295,6 +313,50 @@ mod tests { ); } + /// A renaming projection cannot hide an instant cell from the rescale. + /// + /// The rescale names the merged key the join wrote, and the projection + /// renames the cell afterwards, so the emitted cell carries microseconds + /// under its output name. + #[test] + fn projection_rename_still_rescales_the_instant_cell() { + let task = make_dummy_task(); + let projection = vec![JoinProjection { + source: "e.captured_at".into(), + output: "ts".into(), + }]; + let params = JoinParams { + task: &task, + on: &[], + join_type: "inner", + limit: usize::MAX, + projection: &projection, + computed_projection_bytes: &[], + join_filter_bytes: &[], + post_filter_bytes: &[], + instant_columns: &["e.captured_at".to_string()], + }; + let mut results = vec![ + nodedb_types::json_to_msgpack( + &serde_json::json!({"e.captured_at": 1_583_402_400_000i64}), + ) + .expect("encode test row"), + ]; + params + .filter_and_project(&mut results) + .expect("filter_and_project"); + + let decoded = nodedb_types::value_from_msgpack(&results[0]).expect("decode emitted row"); + let nodedb_types::Value::Object(fields) = decoded else { + panic!("emitted row must be a map, got {decoded:?}"); + }; + assert_eq!( + fields.get("ts"), + Some(&nodedb_types::Value::Integer(1_583_402_400_000_000)), + "the renamed cell must carry epoch microseconds: {fields:?}" + ); + } + /// Valid encoded filters → matching rows retained, non-matching rows dropped /// (happy path unchanged). #[test] @@ -310,6 +372,7 @@ mod tests { computed_projection_bytes: &[], join_filter_bytes: &[], post_filter_bytes: &filter_bytes, + instant_columns: &[], }; let mut results = vec![ row_with_score(99), // should be kept diff --git a/nodedb/src/data/executor/handlers/join/shuffle_join.rs b/nodedb/src/data/executor/handlers/join/shuffle_join.rs index d1fceefad..2cbe7abfd 100644 --- a/nodedb/src/data/executor/handlers/join/shuffle_join.rs +++ b/nodedb/src/data/executor/handlers/join/shuffle_join.rs @@ -439,6 +439,7 @@ mod tests { computed_projection_bytes: &[], join_filter_bytes: &[], post_filter_bytes: &[], + instant_columns: &[], }; let inputs = ShuffleJoinInputs { build_path, diff --git a/nodedb/src/data/executor/handlers/join/sort_merge.rs b/nodedb/src/data/executor/handlers/join/sort_merge.rs index 286de26f6..c4803a8a5 100644 --- a/nodedb/src/data/executor/handlers/join/sort_merge.rs +++ b/nodedb/src/data/executor/handlers/join/sort_merge.rs @@ -51,6 +51,7 @@ impl CoreLoop { pre_sorted, left_rls_filters, right_rls_filters, + instant_columns, } = p; debug!( core = self.core_id, @@ -299,6 +300,18 @@ impl CoreLoop { return self.response_error(task, ErrorCode::ResourcesExhausted); } + // Last step before emission: the scans above compared the + // milliseconds storage holds, and the client reads microseconds. + if let Err(e) = super::instant_scale::scale_join_instant_rows(&mut results, instant_columns) + { + return self.response_error( + task, + ErrorCode::Internal { + detail: e.to_string(), + }, + ); + } + let payload = super::super::super::response_codec::encode_binary_rows(&results); self.response_with_payload(task, payload) } diff --git a/nodedb/src/data/executor/handlers/timeseries/aggregate.rs b/nodedb/src/data/executor/handlers/timeseries/aggregate.rs index 72b1c0453..7ee65f367 100644 --- a/nodedb/src/data/executor/handlers/timeseries/aggregate.rs +++ b/nodedb/src/data/executor/handlers/timeseries/aggregate.rs @@ -184,6 +184,10 @@ impl CoreLoop { }; // Phase 4: Encode response (MessagePack, no serde_json intermediate). + // A grouped column carries the type it carries ungrouped, so the + // encoder is told each key column's declared shape. + let group_key_kinds = + self.ts_group_key_kinds(task.request.database_id, tid, collection, group_by); let payload = match super::encode::encode_grouped_results( &merged, group_by, @@ -191,6 +195,7 @@ impl CoreLoop { limit, bucket_interval_ms, sort_keys, + &group_key_kinds, ) { Ok(p) => p, Err(e) => return self.response_error(task, e), diff --git a/nodedb/src/data/executor/handlers/timeseries/encode.rs b/nodedb/src/data/executor/handlers/timeseries/encode.rs index e28342bdb..922b4dfdc 100644 --- a/nodedb/src/data/executor/handlers/timeseries/encode.rs +++ b/nodedb/src/data/executor/handlers/timeseries/encode.rs @@ -4,6 +4,47 @@ use nodedb_query::agg_key::canonical_agg_key; +use crate::data::executor::core_loop::TsGroupKeyKind; + +/// Render one GROUP BY key part with the type its column carries ungrouped. +/// +/// The grouped scan reduces every key to a string, so the column's own type +/// is put back here. An empty part is SQL NULL. A declared instant is stored +/// in milliseconds and read in microseconds, exactly as row emission reads +/// it, so the two routes to one stored instant render it identically. +/// +/// A part that does not parse as its column's type falls back to the text it +/// holds: the key is data the scan produced, and dropping the group would +/// lose a row. +fn group_key_value(part: Option<&&str>, kind: TsGroupKeyKind) -> crate::Result { + let Some(text) = part.filter(|s| !s.is_empty()) else { + return Ok(rmpv::Value::Nil); + }; + let value = match kind { + TsGroupKeyKind::Instant => match text.parse::() { + Ok(millis) => { + let micros = nodedb_types::NdbDateTime::from_millis(millis) + .map_err(|e| crate::Error::Internal { + detail: format!("grouped timeseries key at {millis} ms: {e}"), + })? + .micros; + rmpv::Value::Integer(micros.into()) + } + Err(_) => rmpv::Value::String((*text).into()), + }, + TsGroupKeyKind::Integer => match text.parse::() { + Ok(n) => rmpv::Value::Integer(n.into()), + Err(_) => rmpv::Value::String((*text).into()), + }, + TsGroupKeyKind::Float => match text.parse::() { + Ok(f) => rmpv::Value::F64(f), + Err(_) => rmpv::Value::String((*text).into()), + }, + TsGroupKeyKind::Text => rmpv::Value::String((*text).into()), + }; + Ok(value) +} + /// Serialize GroupedAggResult directly to MessagePack bytes. /// /// Avoids building `Vec` (2M allocations for 2M groups). @@ -20,6 +61,7 @@ pub(in crate::data::executor) fn encode_grouped_results( limit: usize, bucket_interval_ms: i64, sort_keys: &[nodedb_physical::physical_plan::SortKeySpec], + group_key_kinds: &[TsGroupKeyKind], ) -> crate::Result> { let has_bucket = bucket_interval_ms > 0; // An ordered query has to see every group before cutting to `limit`: @@ -62,20 +104,20 @@ pub(in crate::data::executor) fn encode_grouped_results( )); for (i, field) in group_by.iter().enumerate() { - let val = parts - .get(i + 1) - .filter(|s| !s.is_empty()) - .map(|s| rmpv::Value::String((*s).into())) - .unwrap_or(rmpv::Value::Nil); + let kind = group_key_kinds + .get(i) + .copied() + .unwrap_or(TsGroupKeyKind::Text); + let val = group_key_value(parts.get(i + 1), kind)?; fields.push((rmpv::Value::String(field.as_str().into()), val)); } } else { for (i, field) in group_by.iter().enumerate() { - let val = parts + let kind = group_key_kinds .get(i) - .filter(|s| !s.is_empty()) - .map(|s| rmpv::Value::String((*s).into())) - .unwrap_or(rmpv::Value::Nil); + .copied() + .unwrap_or(TsGroupKeyKind::Text); + let val = group_key_value(parts.get(i), kind)?; fields.push((rmpv::Value::String(field.as_str().into()), val)); } } diff --git a/nodedb/src/data/executor/handlers/timeseries/ingest.rs b/nodedb/src/data/executor/handlers/timeseries/ingest.rs index 977c2c761..1135c32a6 100644 --- a/nodedb/src/data/executor/handlers/timeseries/ingest.rs +++ b/nodedb/src/data/executor/handlers/timeseries/ingest.rs @@ -327,7 +327,7 @@ impl CoreLoop { // missing float field is stored as NaN and both paths render it as SQL // NULL, which a hand-written projection over the ingest values would // have printed as "NaN". - let returned_rows: Vec = match returning { + let mut returned_rows: Vec = match returning { Some(_) => match self.columnar_memtables.get(&key) { Some(mt) => { super::raw_scan::emit_memtable_rows_at(mt, &outcome.accepted_row_indices) @@ -336,6 +336,13 @@ impl CoreLoop { }, None => Vec::new(), }; + // Same scan-unit rule `SELECT` applies: a declared `TIMESTAMP` cell + // leaves the engine as epoch microseconds, not the milliseconds + // storage holds. + let instant_columns = self.ts_instant_columns(task.request.database_id, tid, collection); + if let Err(e) = super::raw_scan::scale_instant_cells(&mut returned_rows, &instant_columns) { + return self.response_error(task, e); + } if accepted > 0 && let Some(lsn) = wal_lsn diff --git a/nodedb/src/data/executor/handlers/timeseries/raw_scan/mod.rs b/nodedb/src/data/executor/handlers/timeseries/raw_scan/mod.rs index 081da03b5..9b346ff12 100644 --- a/nodedb/src/data/executor/handlers/timeseries/raw_scan/mod.rs +++ b/nodedb/src/data/executor/handlers/timeseries/raw_scan/mod.rs @@ -8,5 +8,5 @@ pub mod partition_scan; pub mod row_emit; pub mod scan; -pub(in crate::data::executor) use row_emit::emit_memtable_rows_at; +pub(in crate::data::executor) use row_emit::{emit_memtable_rows_at, scale_instant_cells}; pub(in crate::data::executor) use scan::RawScanParams; diff --git a/nodedb/src/data/executor/handlers/timeseries/raw_scan/row_emit.rs b/nodedb/src/data/executor/handlers/timeseries/raw_scan/row_emit.rs index 1e50d4018..63180c16b 100644 --- a/nodedb/src/data/executor/handlers/timeseries/raw_scan/row_emit.rs +++ b/nodedb/src/data/executor/handlers/timeseries/raw_scan/row_emit.rs @@ -210,3 +210,128 @@ pub(super) fn nodedb_value_to_rmpv(v: &nodedb_types::Value) -> rmpv::Value { _ => rmpv::Value::Nil, } } + +/// Rescale every declared-instant cell of `rows` from the milliseconds the +/// memtable and the partitions store to the epoch microseconds a `TIMESTAMP` +/// cell carries on the wire. +/// +/// The engine's own unit stays milliseconds: partition ranges, retention, +/// `time_bucket` and every scan predicate read it. The scale therefore runs +/// once, as rows leave the scan — after filtering, sorting and computed +/// columns — so nothing inside the engine sees the wire unit. +/// +/// `instant_columns` comes from `CoreLoop::ts_instant_columns`, which lists +/// the columns declared `TIMESTAMP` or `TIMESTAMPTZ`. A `BIGINT TIME_KEY` +/// lives in the same millisecond column and is not in that list, so it keeps +/// the integer the client inserted. +/// +/// SQL NULL cells pass through untouched. A stored value that cannot be +/// expressed in microseconds fails the read rather than wrapping. +pub(in crate::data::executor) fn scale_instant_cells( + rows: &mut [rmpv::Value], + instant_columns: &[String], +) -> crate::Result<()> { + if instant_columns.is_empty() { + return Ok(()); + } + for row in rows.iter_mut() { + let rmpv::Value::Map(fields) = row else { + continue; + }; + for (key, value) in fields.iter_mut() { + let Some(name) = key.as_str() else { continue }; + if !instant_columns.iter().any(|c| c == name) { + continue; + } + let rmpv::Value::Integer(stored) = value else { + continue; + }; + let millis = stored.as_i64().ok_or_else(|| crate::Error::Internal { + detail: format!( + "timeseries column {name} holds {stored}, which is not a millisecond \ + count an instant can be read from" + ), + })?; + let micros = nodedb_types::NdbDateTime::from_millis(millis) + .map_err(|e| crate::Error::Internal { + detail: format!("timeseries column {name} at {millis} ms: {e}"), + })? + .micros; + *value = rmpv::Value::Integer(micros.into()); + } + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::scale_instant_cells; + + fn row(cells: &[(&str, i64)]) -> rmpv::Value { + rmpv::Value::Map( + cells + .iter() + .map(|(k, v)| { + ( + rmpv::Value::String((*k).into()), + rmpv::Value::Integer((*v).into()), + ) + }) + .collect(), + ) + } + + fn cell(row: &rmpv::Value, name: &str) -> Option { + let rmpv::Value::Map(fields) = row else { + return None; + }; + fields + .iter() + .find(|(k, _)| k.as_str() == Some(name)) + .and_then(|(_, v)| v.as_i64()) + } + + /// A declared `TIMESTAMP` column is read as epoch microseconds, so the + /// millisecond value storage holds is scaled on the way out. 2020-03-05 + /// stays 2020-03-05 instead of landing 50 years earlier. + #[test] + fn a_declared_instant_column_leaves_the_scan_in_microseconds() { + let mut rows = vec![row(&[("captured_at", 1_583_402_400_000)])]; + scale_instant_cells(&mut rows, &["captured_at".to_string()]).expect("scale"); + assert_eq!(cell(&rows[0], "captured_at"), Some(1_583_402_400_000_000)); + } + + /// A `BIGINT TIME_KEY` shares the millisecond storage column but is not a + /// declared instant, so its value is handed back exactly as inserted. + #[test] + fn a_column_that_is_not_a_declared_instant_keeps_its_value() { + let mut rows = vec![row(&[("ts", 1000), ("n", 7)])]; + scale_instant_cells(&mut rows, &["other".to_string()]).expect("scale"); + assert_eq!(cell(&rows[0], "ts"), Some(1000)); + assert_eq!(cell(&rows[0], "n"), Some(7)); + } + + /// A NULL instant cell stays NULL — there is no instant to scale. + #[test] + fn a_null_instant_cell_passes_through() { + let mut rows = vec![rmpv::Value::Map(vec![( + rmpv::Value::String("captured_at".into()), + rmpv::Value::Nil, + )])]; + scale_instant_cells(&mut rows, &["captured_at".to_string()]).expect("scale"); + assert_eq!(cell(&rows[0], "captured_at"), None); + } + + /// A stored millisecond count past the microsecond range fails the read. + /// Wrapping it would hand back an instant that is not the stored one. + #[test] + fn a_millisecond_value_beyond_the_microsecond_range_fails_the_read() { + let mut rows = vec![row(&[("captured_at", i64::MAX)])]; + let err = scale_instant_cells(&mut rows, &["captured_at".to_string()]) + .expect_err("i64::MAX ms cannot be expressed in microseconds"); + assert!( + err.to_string().contains("captured_at"), + "the error must name the column: {err}" + ); + } +} diff --git a/nodedb/src/data/executor/handlers/timeseries/raw_scan/scan.rs b/nodedb/src/data/executor/handlers/timeseries/raw_scan/scan.rs index 335d4f1a5..a51828b8f 100644 --- a/nodedb/src/data/executor/handlers/timeseries/raw_scan/scan.rs +++ b/nodedb/src/data/executor/handlers/timeseries/raw_scan/scan.rs @@ -271,6 +271,15 @@ impl CoreLoop { } results.truncate(limit); + // The engine stores its timestamp columns in milliseconds; a client + // reads a `TIMESTAMP` cell as epoch microseconds. Scale here, once + // every predicate, sort and computed column has run against the + // engine's own unit. + let instant_columns = self.ts_instant_columns(task.request.database_id, tid, collection); + if let Err(e) = super::row_emit::scale_instant_cells(&mut results, &instant_columns) { + return self.response_error(task, e); + } + let array = rmpv::Value::Array(results); let mut buf = Vec::new(); rmpv::encode::write_value(&mut buf, &array).unwrap_or(()); diff --git a/nodedb/src/error/types.rs b/nodedb/src/error/types.rs index ae23847b2..a4ed8d330 100644 --- a/nodedb/src/error/types.rs +++ b/nodedb/src/error/types.rs @@ -276,12 +276,33 @@ pub enum Error { #[error("query plan error: {detail}")] PlanError { detail: String }, + /// A statement asked for a SQL feature this server does not implement. + /// Propagated from a planner refusal such as + /// `SqlError::SequencePerRowUnsupported`; the pgwire layer renders this + /// as SQLSTATE `0A000` (feature_not_supported), so a client stops rather + /// than retries. + #[error("{detail}")] + FeatureNotSupported { detail: String }, + /// A function call in a query names no registered scalar, aggregate, or /// window function. Propagated from `SqlError::UndefinedFunction`; the /// pgwire layer renders this as SQLSTATE `42883` (undefined_function). #[error("function {name}(...) does not exist")] UndefinedFunction { name: String }, + /// A statement named a database object that does not exist — a sequence, + /// most commonly. Propagated from `SqlError::UndefinedObject`; the pgwire + /// layer renders this as SQLSTATE `42704` (undefined_object). + #[error("{kind} \"{name}\" does not exist")] + UndefinedObject { kind: &'static str, name: String }, + + /// An object exists but a prerequisite step has not run, such as `currval` + /// before this session called `nextval`. Propagated from + /// `SqlError::ObjectNotInPrerequisiteState`; the pgwire layer renders this + /// as SQLSTATE `55000` (object_not_in_prerequisite_state). + #[error("{detail}")] + ObjectNotInPrerequisiteState { object: String, detail: String }, + /// A column reference resolved against no relation, output alias, or /// synthetic column in scope. Propagated from `SqlError::UnknownColumn`; /// the pgwire layer renders it as SQLSTATE `42703` (undefined_column). diff --git a/nodedb/src/error_classify.rs b/nodedb/src/error_classify.rs index 3a853e9a1..aaf58663c 100644 --- a/nodedb/src/error_classify.rs +++ b/nodedb/src/error_classify.rs @@ -149,7 +149,16 @@ pub(crate) fn classify(e: &Error) -> NodeDbError { NodeDbError::quota_overcommit(field.clone(), detail) } Error::PlanError { detail } => NodeDbError::plan_error(detail), + // The native surface carries this as a plan error; the pgwire + // surface renders SQLSTATE `0A000`. + Error::FeatureNotSupported { detail } => NodeDbError::plan_error(detail), Error::UndefinedFunction { name } => NodeDbError::undefined_function(name.clone()), + Error::UndefinedObject { kind, name } => { + NodeDbError::undefined_object(format!("{kind} \"{name}\"")) + } + Error::ObjectNotInPrerequisiteState { object, detail } => { + NodeDbError::object_not_ready(object.clone(), detail.clone()) + } Error::UndefinedColumn { column } => NodeDbError::undefined_column(column.clone()), Error::AmbiguousColumn { column } => NodeDbError::ambiguous_column(column.clone()), Error::UnknownStrictField { column, .. } => NodeDbError::undefined_column(column.clone()), diff --git a/nodedb/tests/inproc/cases/executor_tests/test_group_by_alias.rs b/nodedb/tests/inproc/cases/executor_tests/test_group_by_alias.rs index 3b1cab4c5..d96111c4a 100644 --- a/nodedb/tests/inproc/cases/executor_tests/test_group_by_alias.rs +++ b/nodedb/tests/inproc/cases/executor_tests/test_group_by_alias.rs @@ -86,6 +86,7 @@ fn sql_to_physical(sql: &str) -> PhysicalPlan { shuffle_agg_num_parts: 0, broadcast_threshold_bytes: 8 * 1024 * 1024, shuffle_agg_threshold: 10_000, + sql_catalog: None, }; let tenant_id = nodedb::types::TenantId::new(1); let tasks = convert(&plans, tenant_id, &ctx).unwrap(); diff --git a/nodedb/tests/inproc/cases/system_task_call_sites.rs b/nodedb/tests/inproc/cases/system_task_call_sites.rs index 584f882d7..62a4779da 100644 --- a/nodedb/tests/inproc/cases/system_task_call_sites.rs +++ b/nodedb/tests/inproc/cases/system_task_call_sites.rs @@ -41,7 +41,7 @@ const ALLOWED: &[&str] = &[ "control/cluster/snapshot_applier.rs", // Committed DDL applied to engine state, and catalog maintenance. "control/server/shared/ddl/engine_apply.rs", - "control/server/shared/ddl/neutral/convert.rs", + "control/server/shared/ddl/neutral/convert/driver.rs", "control/server/shared/ddl/neutral/continuous_agg/create.rs", "control/server/shared/ddl/neutral/continuous_agg/drop.rs", "control/server/shared/ddl/neutral/continuous_agg/register.rs", diff --git a/nodedb/tests/wire/cases/mod.rs b/nodedb/tests/wire/cases/mod.rs index 92c02d3bd..49316db90 100644 --- a/nodedb/tests/wire/cases/mod.rs +++ b/nodedb/tests/wire/cases/mod.rs @@ -158,10 +158,14 @@ mod sql_bitemporal_document_visibility; mod sql_check_constraints; mod sql_collection_drop_index_cleanup; mod sql_conflict_policy; +mod sql_convert_column_defs; mod sql_copy_from; mod sql_copy_to; mod sql_cursors; +mod sql_declared_column_types; mod sql_default_expressions; +mod sql_default_vector_primary; +mod sql_default_volatility; mod sql_division_by_zero; mod sql_division_by_zero_composite; mod sql_dml_affected_counts; @@ -243,6 +247,7 @@ mod sql_transactions_unique_violation; mod sql_transactions_upsert_overlay; mod sql_transactions_vector_overlay; mod sql_trigger_fuel; +mod sql_typeguard_default_gate; mod sql_typeguard_defaults; mod sql_undefined_column; mod sql_undefined_column_dml; @@ -261,6 +266,7 @@ mod strict_bitemporal_audit_query; mod strict_bitemporal_select_star; mod strict_schema_restart; mod timeseries_declared_time_key; +mod timeseries_join_time_rendering; mod timeseries_write_row_level_security; mod transactional_ddl_atomicity; mod transactional_ddl_compensation; diff --git a/nodedb/tests/wire/cases/pgwire_returning_dml.rs b/nodedb/tests/wire/cases/pgwire_returning_dml.rs index dedfb9bb2..a8ba13163 100644 --- a/nodedb/tests/wire/cases/pgwire_returning_dml.rs +++ b/nodedb/tests/wire/cases/pgwire_returning_dml.rs @@ -482,10 +482,20 @@ async fn extended_query_returning_star_matches_the_announced_row_description() { // RowDescription announced for it. for (i, column) in row.columns().iter().enumerate() { let ty = column.type_(); + // One arm per numeric OID the RowDescription can carry. `score` is + // declared `INT`, so it is announced as int4 and must be read as + // `i32`: reading it as `String` is a client-side type error, not a + // fallback. The narrower widths are listed for the same reason. let value = if *ty == Type::INT8 { row.get::<_, i64>(i).to_string() + } else if *ty == Type::INT4 { + row.get::<_, i32>(i).to_string() + } else if *ty == Type::INT2 { + row.get::<_, i16>(i).to_string() } else if *ty == Type::FLOAT8 { row.get::<_, f64>(i).to_string() + } else if *ty == Type::FLOAT4 { + row.get::<_, f32>(i).to_string() } else { row.get::<_, String>(i) }; diff --git a/nodedb/tests/wire/cases/sql_convert_column_defs.rs b/nodedb/tests/wire/cases/sql_convert_column_defs.rs new file mode 100644 index 000000000..9844b0c86 --- /dev/null +++ b/nodedb/tests/wire/cases/sql_convert_column_defs.rs @@ -0,0 +1,88 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! Integration tests for the `CONVERT COLLECTION` column list. +//! +//! Covers the two properties a column definition must hold: +//! - A parameter list carrying a comma stays one column definition. +//! - A `DEFAULT` clause reaches the created column and fills a later insert. +//! +//! A `DEFAULT` the server cannot evaluate is refused at CONVERT with SQLSTATE +//! `42883`, the SQLSTATE `CREATE COLLECTION` raises for the same clause. + +use crate::harness::TestServer; + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn convert_accepts_a_parameter_list_with_a_space_after_the_comma() { + let server = TestServer::start().await; + + server.exec("CREATE COLLECTION conv_decimal").await.unwrap(); + + server + .exec( + "CONVERT COLLECTION conv_decimal TO document_strict \ + (id TEXT PRIMARY KEY, amount DECIMAL(10, 2))", + ) + .await + .unwrap(); + + server + .exec("INSERT INTO conv_decimal (id, amount) VALUES ('d1', 12.50)") + .await + .unwrap(); + + let rows = server + .query_text_joined("SELECT amount FROM conv_decimal") + .await + .unwrap(); + assert!( + rows.iter().any(|row| row.contains("12.5")), + "the parameterized column must be declared and hold its value: {rows:?}" + ); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn convert_carries_a_column_default_into_a_later_insert() { + let server = TestServer::start().await; + + server.exec("CREATE COLLECTION conv_default").await.unwrap(); + + server + .exec( + "CONVERT COLLECTION conv_default TO document_strict \ + (id TEXT PRIMARY KEY, status TEXT DEFAULT 'pending')", + ) + .await + .unwrap(); + + server + .exec("INSERT INTO conv_default (id) VALUES ('c1')") + .await + .unwrap(); + + let rows = server + .query_text_joined("SELECT status FROM conv_default") + .await + .unwrap(); + assert!( + rows.iter().any(|row| row.contains("pending")), + "the converted column must take its DEFAULT: {rows:?}" + ); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn convert_refuses_a_default_naming_an_unknown_function() { + let server = TestServer::start().await; + + server + .exec("CREATE COLLECTION conv_bad_default") + .await + .unwrap(); + + server + .expect_error( + "CONVERT COLLECTION conv_bad_default TO document_strict \ + (id TEXT PRIMARY KEY, status TEXT DEFAULT no_such_function())", + "42883", + ) + .await; +} diff --git a/nodedb/tests/wire/cases/sql_declared_column_types.rs b/nodedb/tests/wire/cases/sql_declared_column_types.rs new file mode 100644 index 000000000..d35ea21ac --- /dev/null +++ b/nodedb/tests/wire/cases/sql_declared_column_types.rs @@ -0,0 +1,240 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! A declared column type resolves from the type keyword alone. +//! +//! `CREATE COLLECTION` records each column as the raw DDL text after its +//! name, so a column written `n INT DEFAULT 5` is stored as the type string +//! `INT DEFAULT 5`. A resolver that matches that whole string recognizes no +//! keyword and falls through to text, which costs the column its numeric +//! `RowDescription` OID, its numeric comparison semantics, and its exact +//! integer round trip. +//! +//! Every test here pairs a column carrying a `DEFAULT` clause with a control +//! column of the same declared type carrying none, so the `DEFAULT` clause is +//! the only variable. +//! +//! Companion coverage: `ddl_int_width_aliases_strict_kv.rs` for declared +//! integer widths, `sql_default_expressions.rs` for DEFAULT evaluation itself. + +use crate::harness::TestServer; + +/// PostgreSQL type OID for `int4`. +const OID_INT4: u32 = 23; +/// PostgreSQL type OID for `int8`. +const OID_INT8: u32 = 20; + +/// A value above 2^53, which no `f64` and no decimal-rendering text path +/// carries back unchanged. +const BEYOND_F64_MANTISSA: i64 = 9_007_199_254_740_993; + +/// Assert the exact `RowDescription` OID of each named column. +/// +/// A missing column fails loudly rather than being skipped: a silent skip +/// turns this into a test that passes when the columns vanish. +fn assert_column_oids(row: &tokio_postgres::Row, expected: &[(&str, u32)]) { + for (col_name, expected_oid) in expected { + let col = row + .columns() + .iter() + .find(|c| c.name() == *col_name) + .unwrap_or_else(|| { + panic!( + "column '{col_name}' must appear in RowDescription; got {:?}", + row.columns().iter().map(|c| c.name()).collect::>() + ) + }); + assert_eq!( + col.type_().oid(), + *expected_oid, + "column '{col_name}' must advertise OID {expected_oid}, got {}", + col.type_().oid() + ); + } +} + +/// Asserts a rendered row carries a real value in place of an absent or NULL column. +fn assert_not_null(row: &str, label: &str) { + let trimmed = row.trim(); + assert!( + !trimmed.is_empty() && !trimmed.eq_ignore_ascii_case("null"), + "{label}: expected a value, got `{row}`" + ); +} + +/// Create one collection per engine carrying a defaulted and a control column +/// of each numeric width, then insert the single probe row. +/// +/// `n`/`big` declare a `DEFAULT`; `plain`/`plain_big` declare the same type +/// with none. `ts` covers a temporal column with a volatile `DEFAULT`. +async fn create_and_seed(server: &TestServer, name: &str, engine: &str) { + server + .exec(&format!( + "CREATE COLLECTION {name} (\ + id TEXT PRIMARY KEY, \ + n INT DEFAULT 5, \ + plain INT, \ + big BIGINT DEFAULT {BEYOND_F64_MANTISSA}, \ + plain_big BIGINT, \ + ts TIMESTAMP DEFAULT NOW()) WITH (engine='{engine}')" + )) + .await + .unwrap_or_else(|e| panic!("create {name} on {engine}: {e}")); + + server + .exec(&format!( + "INSERT INTO {name} (id, plain, plain_big) \ + VALUES ('k1', 5, {BEYOND_F64_MANTISSA})" + )) + .await + .unwrap_or_else(|e| panic!("insert into {name}: {e}")); +} + +/// Assert every declared-type behavior on a seeded collection. +/// +/// The defaulted column and its control column must agree on all three: +/// advertised OID, numeric comparison, and exact integer round trip. +async fn assert_declared_types_hold(server: &TestServer, name: &str) { + let stmt = server + .client + .prepare_typed( + &format!("SELECT n, plain, big, plain_big FROM {name} WHERE id = $1"), + &[tokio_postgres::types::Type::TEXT], + ) + .await + .unwrap_or_else(|e| panic!("prepare select on {name}: {e}")); + let rows = server + .client + .query(&stmt, &[&"k1"]) + .await + .unwrap_or_else(|e| panic!("execute select on {name}: {e}")); + assert_eq!(rows.len(), 1, "one row expected from {name}"); + + assert_column_oids( + &rows[0], + &[ + ("n", OID_INT4), + ("plain", OID_INT4), + ("big", OID_INT8), + ("plain_big", OID_INT8), + ], + ); + + // Typed getters matching the advertised widths: a wrong OID or a + // wrong-width binary payload panics inside `get` before the comparison. + assert_eq!(rows[0].get::<_, i32>("n"), 5); + assert_eq!(rows[0].get::<_, i32>("plain"), 5); + assert_eq!(rows[0].get::<_, i64>("big"), BEYOND_F64_MANTISSA); + assert_eq!(rows[0].get::<_, i64>("plain_big"), BEYOND_F64_MANTISSA); + + for column in ["n", "plain"] { + let matched = server + .query_text(&format!("SELECT id FROM {name} WHERE {column} > 4")) + .await + .unwrap_or_else(|e| panic!("{name}.{column} > 4: {e}")); + assert_eq!( + matched, + vec!["k1".to_string()], + "{name}.{column} > 4 must match the row, got {matched:?}" + ); + + let unmatched = server + .query_text(&format!("SELECT id FROM {name} WHERE {column} > 6")) + .await + .unwrap_or_else(|e| panic!("{name}.{column} > 6: {e}")); + assert!( + unmatched.is_empty(), + "{name}.{column} > 6 must match nothing, got {unmatched:?}" + ); + } + + for column in ["big", "plain_big"] { + let stored = server + .query_text(&format!("SELECT {column} FROM {name} WHERE id = 'k1'")) + .await + .unwrap_or_else(|e| panic!("{name}.{column} read: {e}")); + assert_eq!(stored.len(), 1, "one row expected for {name}.{column}"); + assert_eq!( + stored[0].trim(), + BEYOND_F64_MANTISSA.to_string(), + "{name}.{column} must round-trip exactly, got `{}`", + stored[0] + ); + } + + let stamps = server + .query_text(&format!("SELECT ts FROM {name} WHERE id = 'k1'")) + .await + .unwrap_or_else(|e| panic!("{name}.ts read: {e}")); + assert_eq!(stamps.len(), 1, "one row expected for {name}.ts"); + assert_not_null(&stamps[0], "ts"); +} + +/// A `document_strict` column keeps its declared type when a `DEFAULT` +/// clause follows it, matching a control column that declares none. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn strict_columns_keep_their_declared_type_across_a_default_clause() { + let server = TestServer::start().await; + create_and_seed(&server, "typed_defaults_strict", "document_strict").await; + assert_declared_types_hold(&server, "typed_defaults_strict").await; +} + +/// A `document_schemaless` column keeps its declared type when a `DEFAULT` +/// clause follows it, matching a control column that declares none. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn schemaless_columns_keep_their_declared_type_across_a_default_clause() { + let server = TestServer::start().await; + create_and_seed(&server, "typed_defaults_schemaless", "document_schemaless").await; + assert_declared_types_hold(&server, "typed_defaults_schemaless").await; +} + +/// A `columnar` column keeps its declared type when a `DEFAULT` clause +/// follows it, matching a control column that declares none. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn columnar_columns_keep_their_declared_type_across_a_default_clause() { + let server = TestServer::start().await; + create_and_seed(&server, "typed_defaults_columnar", "columnar").await; + assert_declared_types_hold(&server, "typed_defaults_columnar").await; +} + +/// A `NOT NULL` modifier leaves the declared integer type intact, the same +/// way a `DEFAULT` clause must. +/// +/// `NOT NULL` and `DEFAULT` are both trailing modifiers on the same stored +/// type string, so they share one resolution path. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn a_not_null_modifier_leaves_the_declared_integer_type_intact() { + let server = TestServer::start().await; + + server + .exec( + "CREATE COLLECTION typed_not_null (\ + id TEXT PRIMARY KEY, \ + n INT NOT NULL, \ + plain INT) WITH (engine='document_schemaless')", + ) + .await + .unwrap(); + server + .exec("INSERT INTO typed_not_null (id, n, plain) VALUES ('k1', 5, 5)") + .await + .unwrap(); + + let stmt = server + .client + .prepare_typed( + "SELECT n, plain FROM typed_not_null WHERE id = $1", + &[tokio_postgres::types::Type::TEXT], + ) + .await + .expect("prepare typed_not_null select"); + let rows = server + .client + .query(&stmt, &[&"k1"]) + .await + .expect("execute typed_not_null select"); + assert_eq!(rows.len(), 1, "one row expected from typed_not_null"); + + assert_column_oids(&rows[0], &[("n", OID_INT4), ("plain", OID_INT4)]); + assert_eq!(rows[0].get::<_, i32>("n"), 5); + assert_eq!(rows[0].get::<_, i32>("plain"), 5); +} diff --git a/nodedb/tests/wire/cases/sql_default_expressions.rs b/nodedb/tests/wire/cases/sql_default_expressions.rs index 7fbb86ed5..cee4bbd03 100644 --- a/nodedb/tests/wire/cases/sql_default_expressions.rs +++ b/nodedb/tests/wire/cases/sql_default_expressions.rs @@ -2,10 +2,9 @@ //! Integration coverage for DEFAULT expression evaluation in INSERT. //! -//! The planner's `evaluate_default_expr` recognizes only a fixed keyword list -//! (UUID_V7, NOW(), NANOID, literals). Any other expression returns None, -//! causing the column to be silently omitted. These tests verify that -//! expression-based defaults are evaluated, not dropped. +//! A declared DEFAULT expression evaluates on every engine, not only +//! `document_strict`. A DEFAULT the server cannot evaluate must be refused at +//! DDL time, never accepted and silently dropped at insert time. use crate::harness::TestServer; @@ -168,3 +167,277 @@ async fn default_recognized_expressions_still_work() { rows[0] ); } + +/// `DEFAULT nextval('seq')` fills a strict-engine primary key across two +/// inserts that omit the column. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn default_nextval_fills_a_strict_primary_key() { + let server = TestServer::start().await; + + server.exec("CREATE SEQUENCE seq_def_strict").await.unwrap(); + server + .exec( + "CREATE COLLECTION def_seq_strict (\ + id BIGINT DEFAULT nextval('seq_def_strict') PRIMARY KEY, \ + v TEXT) WITH (engine='document_strict')", + ) + .await + .unwrap(); + + server + .exec("INSERT INTO def_seq_strict (v) VALUES ('a')") + .await + .unwrap(); + server + .exec("INSERT INTO def_seq_strict (v) VALUES ('b')") + .await + .unwrap(); + + let rows = server + .query_text("SELECT id FROM def_seq_strict ORDER BY id") + .await + .unwrap(); + assert_eq!(rows.len(), 2, "two rows expected: {rows:?}"); + assert_not_null(&rows[0], "first id"); + assert_not_null(&rows[1], "second id"); + assert_eq!(rows, vec!["1".to_string(), "2".to_string()]); +} + +/// `DEFAULT nextval('seq')` fills a schemaless-engine primary key, the same +/// way it fills a strict one. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn default_nextval_fills_a_schemaless_primary_key() { + let server = TestServer::start().await; + + server + .exec("CREATE SEQUENCE seq_def_schemaless") + .await + .unwrap(); + server + .exec( + "CREATE COLLECTION def_seq_schemaless (\ + id BIGINT DEFAULT nextval('seq_def_schemaless') PRIMARY KEY, \ + v TEXT)", + ) + .await + .unwrap(); + + server + .exec("INSERT INTO def_seq_schemaless (v) VALUES ('a')") + .await + .unwrap(); + server + .exec("INSERT INTO def_seq_schemaless (v) VALUES ('b')") + .await + .unwrap(); + + let rows = server + .query_text("SELECT id FROM def_seq_schemaless ORDER BY id") + .await + .unwrap(); + assert_eq!(rows.len(), 2, "two rows expected: {rows:?}"); + assert_not_null(&rows[0], "first id"); + assert_not_null(&rows[1], "second id"); + assert_eq!(rows, vec!["1".to_string(), "2".to_string()]); +} + +/// `DEFAULT nextval('seq')` fills a KV-engine key column. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn default_nextval_fills_a_kv_key() { + let server = TestServer::start().await; + + server.exec("CREATE SEQUENCE seq_def_kv").await.unwrap(); + server + .exec( + "CREATE COLLECTION def_seq_kv (\ + id BIGINT DEFAULT nextval('seq_def_kv') PRIMARY KEY, \ + v TEXT) WITH (engine='kv')", + ) + .await + .unwrap(); + + server + .exec("INSERT INTO def_seq_kv (v) VALUES ('a')") + .await + .unwrap(); + server + .exec("INSERT INTO def_seq_kv (v) VALUES ('b')") + .await + .unwrap(); + + let rows = server + .query_text("SELECT id FROM def_seq_kv ORDER BY id") + .await + .unwrap(); + assert_eq!(rows.len(), 2, "two rows expected: {rows:?}"); + assert_not_null(&rows[0], "first id"); + assert_not_null(&rows[1], "second id"); + assert_eq!(rows, vec!["1".to_string(), "2".to_string()]); +} + +/// `DEFAULT nextval('seq')` fills a columnar-engine column. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn default_nextval_fills_a_columnar_column() { + let server = TestServer::start().await; + + server + .exec("CREATE SEQUENCE seq_def_columnar") + .await + .unwrap(); + server + .exec( + "CREATE COLLECTION def_seq_columnar (\ + id BIGINT DEFAULT nextval('seq_def_columnar') PRIMARY KEY, \ + v TEXT) WITH (engine='columnar')", + ) + .await + .unwrap(); + + server + .exec("INSERT INTO def_seq_columnar (v) VALUES ('a')") + .await + .unwrap(); + server + .exec("INSERT INTO def_seq_columnar (v) VALUES ('b')") + .await + .unwrap(); + + let rows = server + .query_text("SELECT id FROM def_seq_columnar ORDER BY id") + .await + .unwrap(); + assert_eq!(rows.len(), 2, "two rows expected: {rows:?}"); + assert_not_null(&rows[0], "first id"); + assert_not_null(&rows[1], "second id"); + assert_eq!(rows, vec!["1".to_string(), "2".to_string()]); +} + +/// `DEFAULT upper('x')` evaluates on a schemaless collection, isolating the +/// schemaless DEFAULT drop from the sequence-accessor problem. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn default_scalar_function_on_a_schemaless_collection() { + let server = TestServer::start().await; + + server + .exec( + "CREATE COLLECTION def_fn_schemaless (id TEXT PRIMARY KEY, a TEXT DEFAULT upper('x'))", + ) + .await + .unwrap(); + + server + .exec("INSERT INTO def_fn_schemaless (id) VALUES ('k1')") + .await + .unwrap(); + + let rows = server + .query_text("SELECT a FROM def_fn_schemaless WHERE id = 'k1'") + .await + .unwrap(); + assert_eq!(rows.len(), 1, "row should exist"); + assert_not_null(&rows[0], "a"); + assert!( + rows[0].contains('X'), + "DEFAULT upper('x') should produce 'X', got {:?}", + rows[0] + ); +} + +/// `DEFAULT UUID_V7()` evaluates on a schemaless collection and produces a +/// 36-character value. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn default_uuid_on_a_schemaless_collection() { + let server = TestServer::start().await; + + server + .exec( + "CREATE COLLECTION def_uuid_schemaless (id TEXT PRIMARY KEY, b TEXT DEFAULT UUID_V7())", + ) + .await + .unwrap(); + + server + .exec("INSERT INTO def_uuid_schemaless (id) VALUES ('k1')") + .await + .unwrap(); + + let rows = server + .query_text("SELECT b FROM def_uuid_schemaless WHERE id = 'k1'") + .await + .unwrap(); + assert_eq!(rows.len(), 1, "row should exist"); + assert_not_null(&rows[0], "b"); + assert_eq!( + rows[0].trim().len(), + 36, + "UUID_V7() must render as 36 characters, got `{}`", + rows[0] + ); +} + +/// A DEFAULT expression the server cannot evaluate is refused at DDL time +/// with `42883`, never accepted and silently dropped at insert time. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn a_default_that_cannot_be_evaluated_is_refused_at_ddl() { + let server = TestServer::start().await; + + server + .expect_error( + "CREATE COLLECTION def_unevaluable (\ + id TEXT PRIMARY KEY, \ + a TEXT DEFAULT no_such_function_here('x'))", + "42883", + ) + .await; +} + +/// `DEFAULT currval('seq')` fills a strict-engine column with the session's +/// last `nextval` result. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn default_currval_fills_a_column() { + let server = TestServer::start().await; + + server + .exec("CREATE SEQUENCE seq_def_currval") + .await + .unwrap(); + server + .query_text("SELECT nextval('seq_def_currval')") + .await + .unwrap(); + server + .exec( + "CREATE COLLECTION def_currval_strict (\ + id TEXT PRIMARY KEY, \ + n BIGINT DEFAULT currval('seq_def_currval')) WITH (engine='document_strict')", + ) + .await + .unwrap(); + + server + .exec("INSERT INTO def_currval_strict (id) VALUES ('k1')") + .await + .unwrap(); + + let rows = server + .query_text("SELECT n FROM def_currval_strict WHERE id = 'k1'") + .await + .unwrap(); + assert_eq!(rows.len(), 1, "row should exist"); + assert_not_null(&rows[0], "n"); + assert_eq!( + rows[0].trim(), + "1", + "currval-backed default must be 1, got `{}`", + rows[0] + ); +} + +/// Asserts a rendered row carries a real value in place of an absent or NULL column. +fn assert_not_null(row: &str, label: &str) { + let trimmed = row.trim(); + assert!( + !trimmed.is_empty() && !trimmed.eq_ignore_ascii_case("null"), + "{label}: expected a value, got `{row}`" + ); +} diff --git a/nodedb/tests/wire/cases/sql_default_vector_primary.rs b/nodedb/tests/wire/cases/sql_default_vector_primary.rs new file mode 100644 index 000000000..4a1be23fb --- /dev/null +++ b/nodedb/tests/wire/cases/sql_default_vector_primary.rs @@ -0,0 +1,165 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! Column DEFAULT evaluation on a `primary='vector'` collection. +//! +//! A vector-primary INSERT bypasses document encoding: the planner splits each +//! row into the vector field and a payload map before the shared DEFAULT pass +//! runs. A column the statement omits therefore reaches storage only if this +//! path materializes its declared DEFAULT itself. +//! +//! Companion coverage: `sql_default_expressions.rs` for DEFAULT evaluation on +//! the document, key-value, and columnar engines. + +use crate::harness::TestServer; + +/// Create the vector-primary collection every test in this file writes to. +async fn create_vector_primary(server: &TestServer, name: &str, key_type: &str) { + server + .exec(&format!( + "CREATE COLLECTION {name} (\ + id {key_type} PRIMARY KEY, \ + vec VECTOR(3), \ + owner STRING) \ + WITH (engine='vector', primary='vector', vector_field='vec', dim=3, \ + payload_indexes=['owner'])" + )) + .await + .unwrap_or_else(|e| panic!("create {name}: {e}")); +} + +/// Read the `id` column of every row, failing loudly on an absent or NULL value. +async fn sorted_ids(server: &TestServer, name: &str) -> Vec { + let rows = server + .query_named_rows(&format!("SELECT * FROM {name}")) + .await + .unwrap_or_else(|e| panic!("SELECT * FROM {name}: {e}")); + let mut ids: Vec = rows + .iter() + .map(|row| { + let id = row + .get("id") + .unwrap_or_else(|| panic!("row must carry an `id` column, got {row:?}")); + let trimmed = id.trim(); + assert!( + !trimmed.is_empty() && !trimmed.eq_ignore_ascii_case("null"), + "id: expected a value, got `{id}`" + ); + trimmed.to_string() + }) + .collect(); + ids.sort(); + ids +} + +/// A `DEFAULT nextval('seq')` key column allocates 1 then 2 across two +/// vector-primary inserts that omit it. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn nextval_default_fills_an_omitted_vector_primary_key() { + let server = TestServer::start().await; + + server + .exec("CREATE SEQUENCE seq_vector_default;") + .await + .unwrap(); + create_vector_primary( + &server, + "def_vec_seq", + "BIGINT DEFAULT nextval('seq_vector_default')", + ) + .await; + + server + .exec( + "INSERT INTO def_vec_seq (vec, owner) \ + VALUES (ARRAY[1.0, 0.0, 0.0], 'alice')", + ) + .await + .expect("vector-primary insert omitting a defaulted key must succeed"); + server + .exec( + "INSERT INTO def_vec_seq (vec, owner) \ + VALUES (ARRAY[0.0, 1.0, 0.0], 'bob')", + ) + .await + .expect("second vector-primary insert must succeed"); + + let ids = sorted_ids(&server, "def_vec_seq").await; + assert_eq!( + ids, + vec!["1".to_string(), "2".to_string()], + "nextval must allocate 1 then 2, got {ids:?}" + ); +} + +/// A `DEFAULT UUID_V7()` key column fills a distinct value per vector-primary +/// insert that omits it. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn uuid_default_fills_an_omitted_vector_primary_key() { + let server = TestServer::start().await; + + create_vector_primary(&server, "def_vec_uuid", "STRING DEFAULT UUID_V7()").await; + + server + .exec( + "INSERT INTO def_vec_uuid (vec, owner) \ + VALUES (ARRAY[1.0, 0.0, 0.0], 'alice')", + ) + .await + .expect("vector-primary insert omitting a defaulted key must succeed"); + server + .exec( + "INSERT INTO def_vec_uuid (vec, owner) \ + VALUES (ARRAY[0.0, 1.0, 0.0], 'bob')", + ) + .await + .expect("second vector-primary insert must succeed"); + + let ids = sorted_ids(&server, "def_vec_uuid").await; + assert_eq!(ids.len(), 2, "two rows expected: {ids:?}"); + let distinct: std::collections::HashSet<&str> = ids.iter().map(|s| s.as_str()).collect(); + assert_eq!( + distinct.len(), + 2, + "each row must carry a distinct id, got {ids:?}" + ); +} + +/// A non-key column DEFAULT fills on a vector-primary insert that omits it. +/// +/// The payload map carries every column but the vector field, so a defaulted +/// payload column shares the key column's materialization path. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn a_default_fills_an_omitted_vector_primary_payload_column() { + let server = TestServer::start().await; + + server + .exec( + "CREATE COLLECTION def_vec_payload (\ + id STRING PRIMARY KEY, \ + vec VECTOR(3), \ + owner STRING DEFAULT 'unassigned') \ + WITH (engine='vector', primary='vector', vector_field='vec', dim=3, \ + payload_indexes=['owner'])", + ) + .await + .unwrap(); + + server + .exec("INSERT INTO def_vec_payload (id, vec) VALUES ('r1', ARRAY[1.0, 0.0, 0.0])") + .await + .expect("vector-primary insert omitting a defaulted payload column must succeed"); + + let rows = server + .query_named_rows("SELECT * FROM def_vec_payload") + .await + .expect("SELECT * FROM def_vec_payload"); + assert_eq!(rows.len(), 1, "one row expected: {rows:?}"); + let owner = rows[0] + .get("owner") + .unwrap_or_else(|| panic!("row must carry an `owner` column, got {:?}", rows[0])); + assert_eq!( + owner.trim(), + "unassigned", + "the declared DEFAULT must fill the omitted column, got `{owner}`" + ); +} diff --git a/nodedb/tests/wire/cases/sql_default_volatility.rs b/nodedb/tests/wire/cases/sql_default_volatility.rs new file mode 100644 index 000000000..58861c822 --- /dev/null +++ b/nodedb/tests/wire/cases/sql_default_volatility.rs @@ -0,0 +1,154 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! Integration coverage for volatile DEFAULT expression re-evaluation. +//! A volatile DEFAULT (`UUID_V7()`, `NOW()`, `nextval(...)`) evaluates once +//! per execution, never once per cached plan. + +use crate::harness::TestServer; + +/// `DEFAULT UUID_V7()` produces a distinct value for each of three inserts. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn uuid_default_produces_a_distinct_value_per_row() { + let server = TestServer::start().await; + + server + .exec( + "CREATE COLLECTION vol_uuid_rows (\ + id TEXT PRIMARY KEY, \ + u TEXT DEFAULT UUID_V7()) WITH (engine='document_strict')", + ) + .await + .unwrap(); + + server + .exec("INSERT INTO vol_uuid_rows (id) VALUES ('k1')") + .await + .unwrap(); + server + .exec("INSERT INTO vol_uuid_rows (id) VALUES ('k2')") + .await + .unwrap(); + server + .exec("INSERT INTO vol_uuid_rows (id) VALUES ('k3')") + .await + .unwrap(); + + let rows = server + .query_text("SELECT u FROM vol_uuid_rows ORDER BY id") + .await + .unwrap(); + assert_eq!(rows.len(), 3, "three rows expected: {rows:?}"); + for row in &rows { + assert_not_null(row, "u"); + } + let distinct: std::collections::HashSet<&str> = rows.iter().map(|r| r.as_str()).collect(); + assert_eq!( + distinct.len(), + 3, + "each row must carry a distinct UUID, got {rows:?}" + ); +} + +/// Every volatile DEFAULT re-evaluates when the identical INSERT text runs +/// three times. A cached plan must never replay a frozen value. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn volatile_defaults_re_evaluate_for_repeated_identical_statements() { + let server = TestServer::start().await; + + server + .exec( + "CREATE COLLECTION vol_uuid_cached (\ + id TEXT DEFAULT UUID_V7() PRIMARY KEY, \ + t TIMESTAMP DEFAULT NOW(), \ + v TEXT) WITH (engine='document_strict')", + ) + .await + .unwrap(); + + // The three statements are byte-identical, so the plan cache keys them the same. + // NOW() renders at second granularity, so the gap must exceed one second. + for _ in 0..3 { + server + .exec("INSERT INTO vol_uuid_cached (v) VALUES ('same')") + .await + .unwrap(); + tokio::time::sleep(std::time::Duration::from_millis(1100)).await; + } + + let rows = server + .query_text("SELECT id FROM vol_uuid_cached") + .await + .unwrap(); + assert_eq!( + rows.len(), + 3, + "a repeated default collapses rows on a primary key, got {rows:?}" + ); + let distinct: std::collections::HashSet<&str> = rows.iter().map(|r| r.as_str()).collect(); + assert_eq!( + distinct.len(), + 3, + "each execution must produce a distinct id, got {rows:?}" + ); + + let stamps = server + .query_text("SELECT t FROM vol_uuid_cached") + .await + .unwrap(); + assert_eq!(stamps.len(), 3, "three rows expected: {stamps:?}"); + for stamp in &stamps { + assert_not_null(stamp, "t"); + } + let distinct_stamps: std::collections::HashSet<&str> = + stamps.iter().map(|r| r.as_str()).collect(); + assert!( + distinct_stamps.len() > 1, + "NOW() must advance across executions, got {stamps:?}" + ); +} + +/// `DEFAULT nextval('seq')` advances when the identical INSERT text runs +/// three times, filling a strict-engine primary key each time. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn nextval_default_advances_across_repeated_identical_statements() { + let server = TestServer::start().await; + + server + .exec("CREATE SEQUENCE seq_vol_cached;") + .await + .unwrap(); + server + .exec( + "CREATE COLLECTION vol_seq_cached (\ + id BIGINT DEFAULT nextval('seq_vol_cached') PRIMARY KEY, \ + v TEXT) WITH (engine='document_strict')", + ) + .await + .unwrap(); + + for _ in 0..3 { + server + .exec("INSERT INTO vol_seq_cached (v) VALUES ('same')") + .await + .unwrap(); + } + + let rows = server + .query_text("SELECT id FROM vol_seq_cached ORDER BY id") + .await + .unwrap(); + assert_eq!( + rows, + vec!["1".to_string(), "2".to_string(), "3".to_string()], + "nextval must advance across repeated executions, got {rows:?}" + ); +} + +/// Asserts a rendered row carries a real value in place of an absent or NULL column. +fn assert_not_null(row: &str, label: &str) { + let trimmed = row.trim(); + assert!( + !trimmed.is_empty() && !trimmed.eq_ignore_ascii_case("null"), + "{label}: expected a value, got `{row}`" + ); +} diff --git a/nodedb/tests/wire/cases/sql_sequences.rs b/nodedb/tests/wire/cases/sql_sequences.rs index 274ce0c1c..a14daefa6 100644 --- a/nodedb/tests/wire/cases/sql_sequences.rs +++ b/nodedb/tests/wire/cases/sql_sequences.rs @@ -74,3 +74,355 @@ async fn drop_sequence_if_exists() { .await .unwrap(); } + +/// `nextval('seq')` returns 1 on the first call and 2 on the second. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn nextval_returns_successive_values() { + let server = TestServer::start().await; + + server + .exec("CREATE SEQUENCE seq_nextval_succ") + .await + .unwrap(); + + let first = server + .query_text("SELECT nextval('seq_nextval_succ')") + .await + .unwrap(); + assert_eq!(first, vec!["1".to_string()], "first nextval must be 1"); + + let second = server + .query_text("SELECT nextval('seq_nextval_succ')") + .await + .unwrap(); + assert_eq!(second, vec!["2".to_string()], "second nextval must be 2"); +} + +/// `currval('seq')` returns the session's last `nextval` result, not a fresh +/// allocation. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn currval_returns_the_last_value_of_the_session() { + let server = TestServer::start().await; + + server + .exec("CREATE SEQUENCE seq_currval_session") + .await + .unwrap(); + + server + .query_text("SELECT nextval('seq_currval_session')") + .await + .unwrap(); + let first = server + .query_text("SELECT currval('seq_currval_session')") + .await + .unwrap(); + assert_eq!( + first, + vec!["1".to_string()], + "currval must echo the last nextval" + ); + + server + .query_text("SELECT nextval('seq_currval_session')") + .await + .unwrap(); + let second = server + .query_text("SELECT currval('seq_currval_session')") + .await + .unwrap(); + assert_eq!( + second, + vec!["2".to_string()], + "currval must track the second nextval" + ); +} + +/// `setval('seq', 10)` positions the sequence so the next `nextval` returns 11. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn setval_positions_the_next_allocation() { + let server = TestServer::start().await; + + server.exec("CREATE SEQUENCE seq_setval_pos").await.unwrap(); + + server + .query_text("SELECT setval('seq_setval_pos', 10)") + .await + .unwrap(); + let next = server + .query_text("SELECT nextval('seq_setval_pos')") + .await + .unwrap(); + assert_eq!( + next, + vec!["11".to_string()], + "nextval after setval(10) must be 11" + ); +} + +/// `nextval` on a sequence that was never created must fail with `42704` +/// (undefined_object), not `42883` (undefined_function) — the function +/// exists, the object does not. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn nextval_on_an_unknown_sequence_errors() { + let server = TestServer::start().await; + + server + .expect_error("SELECT nextval('seq_that_was_never_created')", "42704") + .await; +} + +/// A `SERIAL` column allocates 1 then 2 across two inserts that omit it. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn serial_column_allocates_successive_keys() { + let server = TestServer::start().await; + + server + .exec("CREATE COLLECTION seq_serial_alloc FIELDS (n SERIAL, v TEXT)") + .await + .unwrap(); + + server + .exec("INSERT INTO seq_serial_alloc (v) VALUES ('a')") + .await + .unwrap(); + server + .exec("INSERT INTO seq_serial_alloc (v) VALUES ('b')") + .await + .unwrap(); + + let rows = server + .query_text("SELECT n FROM seq_serial_alloc ORDER BY n") + .await + .unwrap(); + assert_eq!(rows.len(), 2, "two rows expected: {rows:?}"); + for row in &rows { + let trimmed = row.trim(); + assert!( + !trimmed.is_empty() && !trimmed.eq_ignore_ascii_case("null"), + "SERIAL column must not be empty or NULL, got `{row}`" + ); + } + assert_eq!(rows, vec!["1".to_string(), "2".to_string()]); +} + +/// Preparing an INSERT must not consume a `nextval` allocation. +/// Only executing the statement advances the sequence. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn preparing_an_insert_does_not_advance_a_sequence_default() { + let server = TestServer::start().await; + + server + .exec("CREATE SEQUENCE seq_plan_side_effect;") + .await + .unwrap(); + server + .exec( + "CREATE COLLECTION seq_plan_probe (\ + id BIGINT DEFAULT nextval('seq_plan_side_effect') PRIMARY KEY, \ + v TEXT) WITH (engine='document_strict')", + ) + .await + .unwrap(); + + const INSERT: &str = "INSERT INTO seq_plan_probe (v) VALUES ('a')"; + + // Parse/Describe only. Each round trip plans the statement without + // executing it, so none of them can allocate a sequence value. + for _ in 0..3 { + server + .client + .prepare(INSERT) + .await + .expect("prepare INSERT with a nextval default must succeed"); + } + + server.exec(INSERT).await.unwrap(); + + let rows = server + .query_text("SELECT id FROM seq_plan_probe") + .await + .unwrap(); + assert_eq!(rows.len(), 1, "one row expected: {rows:?}"); + assert_eq!( + rows[0].trim(), + "1", + "planning must leave the sequence at its start, got id `{}`", + rows[0] + ); + + // The next allocation is 2 when exactly one value was consumed. + let next = server + .query_text("SELECT nextval('seq_plan_side_effect')") + .await + .unwrap(); + assert_eq!( + next, + vec!["2".to_string()], + "one execution must consume exactly one value, got {next:?}" + ); +} + +/// Describing an INSERT that omits a `nextval` DEFAULT reports its columns +/// without consuming a sequence value. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn describing_an_insert_leaves_the_sequence_untouched() { + let server = TestServer::start().await; + + server + .exec("CREATE SEQUENCE seq_describe_probe;") + .await + .unwrap(); + server + .exec( + "CREATE COLLECTION seq_describe_target (\ + id BIGINT DEFAULT nextval('seq_describe_probe') PRIMARY KEY, \ + v TEXT) WITH (engine='document_strict')", + ) + .await + .unwrap(); + + server + .client + .prepare("INSERT INTO seq_describe_target (v) VALUES ($1) RETURNING id") + .await + .expect("prepare INSERT ... RETURNING with a nextval default must succeed"); + + // The sequence was never executed against, so the first allocation is 1. + let first = server + .query_text("SELECT nextval('seq_describe_probe')") + .await + .unwrap(); + assert_eq!( + first, + vec!["1".to_string()], + "describe must not allocate, got {first:?}" + ); +} + +/// Create a two-row KV collection and a sequence, both named after `label`. +/// +/// Returns `(sequence_name, collection_name)`. +async fn two_row_collection_with_sequence(server: &TestServer, label: &str) -> (String, String) { + let sequence = format!("seq_row_ctx_{label}"); + let collection = format!("row_ctx_{label}"); + + server + .exec(&format!("CREATE SEQUENCE {sequence}")) + .await + .unwrap(); + server + .exec(&format!( + "CREATE COLLECTION {collection} (id BIGINT PRIMARY KEY, v TEXT) \ + WITH (engine = 'kv')" + )) + .await + .unwrap(); + server + .exec(&format!("INSERT INTO {collection} (id, v) VALUES (1, 'a')")) + .await + .unwrap(); + server + .exec(&format!("INSERT INTO {collection} (id, v) VALUES (2, 'b')")) + .await + .unwrap(); + + (sequence, collection) +} + +/// `nextval` in a SELECT list over a FROM relation raises `0A000` +/// (feature_not_supported). Per-row allocation is not implemented, and a +/// silent NULL for every row is worse than the refusal. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn nextval_in_a_select_list_over_a_scan_is_refused() { + let server = TestServer::start().await; + let (sequence, collection) = two_row_collection_with_sequence(&server, "nextval").await; + + server + .expect_error( + &format!("SELECT nextval('{sequence}') FROM {collection}"), + "0A000", + ) + .await; +} + +/// `currval` in a SELECT list over a FROM relation raises `0A000`, the same +/// as `nextval`: both read session sequence state the row evaluator lacks. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn currval_in_a_select_list_over_a_scan_is_refused() { + let server = TestServer::start().await; + let (sequence, collection) = two_row_collection_with_sequence(&server, "currval").await; + + server + .query_text(&format!("SELECT nextval('{sequence}')")) + .await + .unwrap(); + server + .expect_error( + &format!("SELECT currval('{sequence}') FROM {collection}"), + "0A000", + ) + .await; +} + +/// A projection mixing a plain column with a sequence accessor is refused as +/// a whole. Returning the column and a NULL beside it would ship the silent +/// cell this refusal exists to prevent. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn a_projection_mixing_a_column_and_an_accessor_is_refused() { + let server = TestServer::start().await; + let (sequence, collection) = two_row_collection_with_sequence(&server, "mixed").await; + + server + .expect_error( + &format!("SELECT id, nextval('{sequence}') FROM {collection}"), + "0A000", + ) + .await; +} + +/// A sequence accessor in a WHERE clause over a FROM relation is refused for +/// the same reason as one in the SELECT list. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn a_sequence_accessor_in_a_where_clause_is_refused() { + let server = TestServer::start().await; + let (sequence, collection) = two_row_collection_with_sequence(&server, "where").await; + + server + .expect_error( + &format!("SELECT id FROM {collection} WHERE id = nextval('{sequence}')"), + "0A000", + ) + .await; +} + +/// The refusal yields no result set at all, so no row can carry an empty or +/// NULL cell, and the statement allocates nothing from the sequence. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn a_refused_accessor_produces_no_row_and_no_allocation() { + let server = TestServer::start().await; + let (sequence, collection) = two_row_collection_with_sequence(&server, "guard").await; + + let result = server + .query_text(&format!("SELECT nextval('{sequence}') FROM {collection}")) + .await; + let message = match result { + Ok(rows) => panic!("per-row nextval must not return rows, got {rows:?}"), + Err(message) => message, + }; + assert!( + message.contains("0A000"), + "refusal must carry SQLSTATE 0A000, got: {message}" + ); + + // A refused statement consumes nothing, so the first real allocation is 1. + let first = server + .query_text(&format!("SELECT nextval('{sequence}')")) + .await + .unwrap(); + assert_eq!( + first, + vec!["1".to_string()], + "the refused statement must not have allocated, got {first:?}" + ); +} diff --git a/nodedb/tests/wire/cases/sql_typeguard_default_gate.rs b/nodedb/tests/wire/cases/sql_typeguard_default_gate.rs new file mode 100644 index 000000000..7f621680c --- /dev/null +++ b/nodedb/tests/wire/cases/sql_typeguard_default_gate.rs @@ -0,0 +1,100 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! A typeguard `DEFAULT` / `VALUE` the engine cannot evaluate is refused at +//! declaration. +//! +//! Both clauses produce a field value on every write. An unregistered function +//! name would otherwise store `NULL` on every write and report nothing, and a +//! non-deterministic call would fail every write instead of the declaration. + +use crate::harness::TestServer; + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn typeguard_unevaluable_default_is_refused_at_declaration() { + let server = TestServer::start().await; + + server.exec("CREATE COLLECTION tg_gate").await.unwrap(); + + // A function name no registry knows. + server + .expect_error( + "CREATE TYPEGUARD ON tg_gate (status STRING DEFAULT no_such_function())", + "42883", + ) + .await; + + // The same refusal covers the VALUE clause. + server + .expect_error( + "CREATE TYPEGUARD ON tg_gate (computed STRING VALUE no_such_function())", + "42883", + ) + .await; + + // A call the write path refuses as non-deterministic. + server + .expect_error( + "CREATE TYPEGUARD ON tg_gate (at STRING DEFAULT now())", + "42601", + ) + .await; + + // ALTER carries the same refusal. + server + .expect_error( + "ALTER TYPEGUARD ON tg_gate ADD status STRING DEFAULT no_such_function()", + "42883", + ) + .await; + + // A refused declaration reaches no storage. + let rows = server + .query_text("SHOW TYPEGUARD ON tg_gate") + .await + .unwrap(); + assert_eq!(rows.len(), 0, "refused guard must not be stored: {rows:?}"); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn typeguard_evaluable_defaults_stay_accepted() { + let server = TestServer::start().await; + + server.exec("CREATE COLLECTION tg_gate_ok").await.unwrap(); + + server + .exec( + "CREATE TYPEGUARD ON tg_gate_ok (\ + status STRING DEFAULT 'draft',\ + version INT REQUIRED DEFAULT 1\ + )", + ) + .await + .unwrap(); + + server + .exec("ALTER TYPEGUARD ON tg_gate_ok ADD slug STRING VALUE LOWER(status)") + .await + .unwrap(); + + let rows = server + .query_text("SHOW TYPEGUARD ON tg_gate_ok") + .await + .unwrap(); + assert_eq!(rows.len(), 3, "every accepted guard is stored: {rows:?}"); + + // The accepted DEFAULT still injects at write time. + server + .exec("INSERT INTO tg_gate_ok { id: 'g1', name: 'Alice' }") + .await + .unwrap(); + + let stored = server + .query_text_joined("SELECT * FROM tg_gate_ok WHERE id = 'g1'") + .await + .unwrap(); + assert_eq!(stored.len(), 1); + assert!( + stored[0].contains("draft"), + "DEFAULT must still inject: {stored:?}" + ); +} diff --git a/nodedb/tests/wire/cases/sql_typeguard_defaults.rs b/nodedb/tests/wire/cases/sql_typeguard_defaults.rs index 9c966f4fa..726716db5 100644 --- a/nodedb/tests/wire/cases/sql_typeguard_defaults.rs +++ b/nodedb/tests/wire/cases/sql_typeguard_defaults.rs @@ -422,3 +422,37 @@ async fn strict_default_now() { .unwrap(); assert_eq!(rows.len(), 1, "should have 1 row: {rows:?}"); } + +// ── Unresolvable declared type ── + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn typeguard_unresolvable_type_is_refused_at_declaration() { + let server = TestServer::start().await; + + server.exec("CREATE COLLECTION tg_bad_type").await.unwrap(); + + // A type name the engine resolves to nothing. + server + .expect_error("CREATE TYPEGUARD ON tg_bad_type (gadget WIDGET)", "42601") + .await; + + // A trailing word that reading the leading token alone would ignore. + server + .expect_error( + "CREATE TYPEGUARD ON tg_bad_type (at TIMESTAMP GARBAGE)", + "42601", + ) + .await; + + // ALTER carries the same refusal. + server + .expect_error("ALTER TYPEGUARD ON tg_bad_type ADD gadget WIDGET", "42601") + .await; + + // A refused declaration reaches no storage. + let rows = server + .query_text("SHOW TYPEGUARD ON tg_bad_type") + .await + .unwrap(); + assert_eq!(rows.len(), 0, "refused guard must not be stored: {rows:?}"); +} diff --git a/nodedb/tests/wire/cases/timeseries_declared_time_key.rs b/nodedb/tests/wire/cases/timeseries_declared_time_key.rs index 34a79507f..464ecd0f6 100644 --- a/nodedb/tests/wire/cases/timeseries_declared_time_key.rs +++ b/nodedb/tests/wire/cases/timeseries_declared_time_key.rs @@ -17,7 +17,11 @@ //! `BIGINT` time keys. A name the engine happens to recognise internally //! (`ts`, `timestamp`, `time`) is not special: it is the user's column. //! -//! Time-key values read back as the epoch milliseconds the engine stores. +//! A declared `TIMESTAMP` time key reads back as the instant that was +//! inserted: the engine stores epoch milliseconds and a `TIMESTAMP` cell is +//! read as epoch microseconds, so the two units meet as the row leaves the +//! scan. A `BIGINT` time key shares that storage column and is not a +//! timestamp, so it reads back as the number that was inserted. use crate::harness::TestServer; @@ -29,9 +33,14 @@ const LATE: &str = "2020-03-05 13:00:00"; /// Upper bound that both event times fall below but no ingest-assigned /// wall-clock time ever will. const AFTER_BOTH: &str = "2021-01-01 00:00:00"; -/// `EARLY` as the epoch milliseconds the timeseries engine stores and reads -/// back for a time-key column. -const EARLY_MS: &str = "1583402400000"; +/// `EARLY` as a declared `TIMESTAMP` column renders it. The engine stores +/// 1583402400000 epoch milliseconds; a `TIMESTAMP` cell carries epoch +/// microseconds, which the pgwire encoder writes as ISO-8601 UTC. +const EARLY_ISO: &str = "2020-03-05T10:00:00.000000Z"; +/// `EARLY` as `SELECT *` renders it: a star projection announces no catalog +/// type, so its cells stay the raw stored number — here the same instant in +/// epoch microseconds. +const EARLY_MICROS: &str = "1583402400000000"; #[tokio::test] async fn time_key_named_ts_round_trips() { @@ -58,7 +67,10 @@ async fn time_key_named_ts_round_trips() { !ts.is_empty(), "declared TIME_KEY column `ts` must not read back NULL: {rows:?}" ); - assert_eq!(ts, EARLY_MS, "`ts` must round-trip the inserted event time"); + assert_eq!( + ts, EARLY_ISO, + "`ts` must round-trip the inserted event time" + ); } #[tokio::test] @@ -114,7 +126,7 @@ async fn select_star_projects_declared_columns_only() { ); assert_eq!( rows[0].get("ts").map(String::as_str), - Some(EARLY_MS), + Some(EARLY_MICROS), "`SELECT *` must carry the inserted event time under `ts`: {rows:?}" ); } @@ -141,7 +153,7 @@ async fn custom_named_time_key_round_trips() { assert_eq!(rows.len(), 1, "one inserted row must read back: {rows:?}"); assert_eq!( rows[0].get("captured_at").map(String::as_str), - Some(EARLY_MS), + Some(EARLY_ISO), "`captured_at` must round-trip the inserted event time: {rows:?}" ); } @@ -289,7 +301,7 @@ async fn non_time_key_column_named_timestamp_keeps_its_value() { ); assert_eq!( rows[0].get("captured_at").map(String::as_str), - Some(EARLY_MS), + Some(EARLY_ISO), "the declared time key must still round-trip: {rows:?}" ); } diff --git a/nodedb/tests/wire/cases/timeseries_join_time_rendering.rs b/nodedb/tests/wire/cases/timeseries_join_time_rendering.rs new file mode 100644 index 000000000..ecc4d4e71 --- /dev/null +++ b/nodedb/tests/wire/cases/timeseries_join_time_rendering.rs @@ -0,0 +1,216 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! A stored instant renders the same however a query reaches it. +//! +//! A timeseries `TIMESTAMP` time key can be read three ways: a direct +//! `SELECT` of the column, the same column projected through a JOIN, and the +//! same column used as a `GROUP BY` key. Each route runs through its own +//! scan and its own encoder. All three name one stored instant, so all three +//! must render it identically. +//! +//! Every assertion here compares two reads of the SAME row against each +//! other. A hardcoded rendering would pin whichever route was written down +//! first and call the other one wrong, so the comparison states the +//! invariant instead. One absolute check anchors the pair, so the two routes +//! cannot agree on a value that denotes the wrong instant. + +use crate::harness::TestServer; + +/// One event time, years in the past, so a value the engine substitutes at +/// ingest (wall-clock "now") is separable from the value the INSERT supplied. +const EARLY: &str = "2020-03-05 10:00:00"; +/// `EARLY` as a declared `TIMESTAMP` column renders it. The engine stores +/// 1583402400000 epoch milliseconds; a `TIMESTAMP` cell carries epoch +/// microseconds, which the pgwire encoder writes as ISO-8601 UTC. +const EARLY_ISO: &str = "2020-03-05T10:00:00.000000Z"; +/// `EARLY` as epoch microseconds — 1583402400000 milliseconds times 1000. +/// A projection that announces no catalog type leaves its cells this number. +const EARLY_MICROS: &str = "1583402400000000"; + +/// Create a timeseries collection and a document collection that join on the +/// event's `host`, then insert exactly one row into each so the join yields +/// one row. +async fn setup(server: &TestServer, events: &str, hosts: &str) { + server + .exec(&format!( + "CREATE COLLECTION {events} \ + (captured_at TIMESTAMP TIME_KEY, host TEXT, v FLOAT) \ + WITH (engine='timeseries')" + )) + .await + .unwrap_or_else(|e| panic!("create {events}: {e}")); + server + .exec(&format!( + "CREATE COLLECTION {hosts} (id TEXT PRIMARY KEY, region TEXT) \ + WITH (engine='document_strict')" + )) + .await + .unwrap_or_else(|e| panic!("create {hosts}: {e}")); + + server + .exec(&format!( + "INSERT INTO {events} (captured_at, host, v) VALUES ('{EARLY}', 'h1', 1.5)" + )) + .await + .unwrap_or_else(|e| panic!("insert into {events}: {e}")); + server + .exec(&format!( + "INSERT INTO {hosts} (id, region) VALUES ('h1', 'eu')" + )) + .await + .unwrap_or_else(|e| panic!("insert into {hosts}: {e}")); +} + +/// A time key projected through a JOIN renders as the direct `SELECT` renders +/// it. Both reads name one stored row, so a divergence is a rendering defect. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn a_time_key_renders_the_same_through_a_join_as_through_a_select() { + let server = TestServer::start().await; + setup(&server, "tsj_join_events", "tsj_join_hosts").await; + + let direct = server + .query_text("SELECT captured_at FROM tsj_join_events") + .await + .expect("direct SELECT of the declared time key must succeed"); + assert_eq!(direct.len(), 1, "one stored point: {direct:?}"); + + let joined = server + .query_text( + "SELECT tsj_join_events.captured_at FROM tsj_join_events \ + INNER JOIN tsj_join_hosts ON tsj_join_events.host = tsj_join_hosts.id", + ) + .await + .expect("the same column projected through a JOIN must succeed"); + assert_eq!( + joined.len(), + 1, + "one event matches one host, so the join yields one row: {joined:?}" + ); + + assert_eq!( + joined[0], direct[0], + "the time key must render the same through a JOIN as through a SELECT: \ + joined={joined:?} direct={direct:?}" + ); +} + +/// A time key used as a `GROUP BY` key renders as the direct `SELECT` renders +/// it. The aggregate encoder writes the group key itself, so it is a third +/// route to the same stored instant. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn a_time_key_renders_the_same_through_an_aggregate_as_through_a_select() { + let server = TestServer::start().await; + setup(&server, "tsj_agg_events", "tsj_agg_hosts").await; + + let direct = server + .query_text("SELECT captured_at FROM tsj_agg_events") + .await + .expect("direct SELECT of the declared time key must succeed"); + assert_eq!(direct.len(), 1, "one stored point: {direct:?}"); + + let grouped = server + .query_text("SELECT captured_at, COUNT(*) FROM tsj_agg_events GROUP BY captured_at") + .await + .expect("GROUP BY on the declared time key must succeed"); + assert_eq!( + grouped.len(), + 1, + "one stored point falls in one group: {grouped:?}" + ); + + assert_eq!( + grouped[0], direct[0], + "the time key must render the same as a GROUP BY key as through a SELECT: \ + grouped={grouped:?} direct={direct:?}" + ); +} + +/// A time key read through a JOIN denotes the instant the INSERT supplied. +/// This anchors the comparisons above, which two wrong routes can otherwise +/// pass by agreeing with each other. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn a_joined_time_key_denotes_the_stored_instant() { + let server = TestServer::start().await; + setup(&server, "tsj_abs_events", "tsj_abs_hosts").await; + + let joined = server + .query_text( + "SELECT tsj_abs_events.captured_at FROM tsj_abs_events \ + INNER JOIN tsj_abs_hosts ON tsj_abs_events.host = tsj_abs_hosts.id", + ) + .await + .expect("the time key projected through a JOIN must succeed"); + assert_eq!(joined.len(), 1, "the join yields one row: {joined:?}"); + + // Two renderings denote 2020-03-05T10:00:00Z, and the expected value is + // whichever one the join announces a type for. EARLY_ISO is that instant + // written as ISO-8601 UTC, which a cell typed TIMESTAMP produces. + // EARLY_MICROS is the same instant in epoch microseconds — the unit a + // TIMESTAMP cell carries — which an untyped cell leaves as a number. + // Epoch MILLISECONDS denote 1970-01-19 read either way, so a millisecond + // value fails both arms. + assert!( + joined[0] == EARLY_ISO || joined[0] == EARLY_MICROS, + "a joined time key must denote {EARLY}: expected {EARLY_ISO} \ + or {EARLY_MICROS}, got {joined:?}" + ); +} + +/// A computed projection of a time key renders it as a direct read does. +/// `COALESCE` returns the stored cell untouched, so both reads name one instant. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn a_computed_projection_of_a_time_key_matches_a_direct_read() { + let server = TestServer::start().await; + setup(&server, "tsj_comp_events", "tsj_comp_hosts").await; + + let direct = server + .query_text("SELECT COALESCE(captured_at, captured_at) AS captured_at FROM tsj_comp_events") + .await + .expect("a computed projection of the time key must succeed"); + assert_eq!(direct.len(), 1, "one stored point: {direct:?}"); + + let joined = server + .query_text( + "SELECT COALESCE(tsj_comp_events.captured_at, tsj_comp_events.captured_at) \ + AS captured_at FROM tsj_comp_events \ + INNER JOIN tsj_comp_hosts ON tsj_comp_events.host = tsj_comp_hosts.id", + ) + .await + .expect("the same computed projection over a JOIN must succeed"); + assert_eq!(joined.len(), 1, "the join yields one row: {joined:?}"); + + assert_eq!( + joined[0], direct[0], + "a computed projection of the time key must render the same through a JOIN \ + as through a SELECT: joined={joined:?} direct={direct:?}" + ); +} + +/// An aliased joined time key denotes the instant the INSERT supplied. +/// The alias renames the cell, so the emitted name must still carry the instant. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn an_aliased_joined_time_key_denotes_the_stored_instant() { + let server = TestServer::start().await; + setup(&server, "tsj_alias_events", "tsj_alias_hosts").await; + + let joined = server + .query_text( + "SELECT tsj_alias_events.captured_at AS ts FROM tsj_alias_events \ + INNER JOIN tsj_alias_hosts ON tsj_alias_events.host = tsj_alias_hosts.id", + ) + .await + .expect("an aliased time key projected through a JOIN must succeed"); + assert_eq!(joined.len(), 1, "the join yields one row: {joined:?}"); + + // The expected value is EARLY, the inserted instant, in whichever unit the + // alias announces a type for. EARLY_ISO is 2020-03-05T10:00:00Z as a cell + // typed TIMESTAMP renders it; EARLY_MICROS is the same instant in the epoch + // microseconds a TIMESTAMP cell carries, which an untyped cell leaves as a + // number. The stored 1583402400000 milliseconds denote 1970-01-19 read + // either way, so a millisecond value fails both arms. + assert!( + joined[0] == EARLY_ISO || joined[0] == EARLY_MICROS, + "an aliased joined time key must denote {EARLY}: expected {EARLY_ISO} \ + or {EARLY_MICROS}, got {joined:?}" + ); +}