From 4b00406d33c71210c1050f7650d1456ba0f5929e Mon Sep 17 00:00:00 2001 From: Farhan Syah Date: Wed, 9 Sep 2026 20:23:49 +0800 Subject: [PATCH 01/23] test(sql): cover sequence functions and DEFAULT expressions across engines Add coverage for nextval/currval/setval, SERIAL column allocation, and DEFAULT expressions (nextval, currval, scalar functions, UUID_V7) across strict, schemaless, KV, and columnar engines. Also cover the unknown-sequence and unevaluable-DEFAULT error paths. --- .../wire/cases/sql_default_expressions.rs | 281 +++++++++++++++++- nodedb/tests/wire/cases/sql_sequences.rs | 131 ++++++++ 2 files changed, 408 insertions(+), 4 deletions(-) diff --git a/nodedb/tests/wire/cases/sql_default_expressions.rs b/nodedb/tests/wire/cases/sql_default_expressions.rs index 7fbb86ed5..cee4bbd03 100644 --- a/nodedb/tests/wire/cases/sql_default_expressions.rs +++ b/nodedb/tests/wire/cases/sql_default_expressions.rs @@ -2,10 +2,9 @@ //! Integration coverage for DEFAULT expression evaluation in INSERT. //! -//! The planner's `evaluate_default_expr` recognizes only a fixed keyword list -//! (UUID_V7, NOW(), NANOID, literals). Any other expression returns None, -//! causing the column to be silently omitted. These tests verify that -//! expression-based defaults are evaluated, not dropped. +//! A declared DEFAULT expression evaluates on every engine, not only +//! `document_strict`. A DEFAULT the server cannot evaluate must be refused at +//! DDL time, never accepted and silently dropped at insert time. use crate::harness::TestServer; @@ -168,3 +167,277 @@ async fn default_recognized_expressions_still_work() { rows[0] ); } + +/// `DEFAULT nextval('seq')` fills a strict-engine primary key across two +/// inserts that omit the column. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn default_nextval_fills_a_strict_primary_key() { + let server = TestServer::start().await; + + server.exec("CREATE SEQUENCE seq_def_strict").await.unwrap(); + server + .exec( + "CREATE COLLECTION def_seq_strict (\ + id BIGINT DEFAULT nextval('seq_def_strict') PRIMARY KEY, \ + v TEXT) WITH (engine='document_strict')", + ) + .await + .unwrap(); + + server + .exec("INSERT INTO def_seq_strict (v) VALUES ('a')") + .await + .unwrap(); + server + .exec("INSERT INTO def_seq_strict (v) VALUES ('b')") + .await + .unwrap(); + + let rows = server + .query_text("SELECT id FROM def_seq_strict ORDER BY id") + .await + .unwrap(); + assert_eq!(rows.len(), 2, "two rows expected: {rows:?}"); + assert_not_null(&rows[0], "first id"); + assert_not_null(&rows[1], "second id"); + assert_eq!(rows, vec!["1".to_string(), "2".to_string()]); +} + +/// `DEFAULT nextval('seq')` fills a schemaless-engine primary key, the same +/// way it fills a strict one. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn default_nextval_fills_a_schemaless_primary_key() { + let server = TestServer::start().await; + + server + .exec("CREATE SEQUENCE seq_def_schemaless") + .await + .unwrap(); + server + .exec( + "CREATE COLLECTION def_seq_schemaless (\ + id BIGINT DEFAULT nextval('seq_def_schemaless') PRIMARY KEY, \ + v TEXT)", + ) + .await + .unwrap(); + + server + .exec("INSERT INTO def_seq_schemaless (v) VALUES ('a')") + .await + .unwrap(); + server + .exec("INSERT INTO def_seq_schemaless (v) VALUES ('b')") + .await + .unwrap(); + + let rows = server + .query_text("SELECT id FROM def_seq_schemaless ORDER BY id") + .await + .unwrap(); + assert_eq!(rows.len(), 2, "two rows expected: {rows:?}"); + assert_not_null(&rows[0], "first id"); + assert_not_null(&rows[1], "second id"); + assert_eq!(rows, vec!["1".to_string(), "2".to_string()]); +} + +/// `DEFAULT nextval('seq')` fills a KV-engine key column. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn default_nextval_fills_a_kv_key() { + let server = TestServer::start().await; + + server.exec("CREATE SEQUENCE seq_def_kv").await.unwrap(); + server + .exec( + "CREATE COLLECTION def_seq_kv (\ + id BIGINT DEFAULT nextval('seq_def_kv') PRIMARY KEY, \ + v TEXT) WITH (engine='kv')", + ) + .await + .unwrap(); + + server + .exec("INSERT INTO def_seq_kv (v) VALUES ('a')") + .await + .unwrap(); + server + .exec("INSERT INTO def_seq_kv (v) VALUES ('b')") + .await + .unwrap(); + + let rows = server + .query_text("SELECT id FROM def_seq_kv ORDER BY id") + .await + .unwrap(); + assert_eq!(rows.len(), 2, "two rows expected: {rows:?}"); + assert_not_null(&rows[0], "first id"); + assert_not_null(&rows[1], "second id"); + assert_eq!(rows, vec!["1".to_string(), "2".to_string()]); +} + +/// `DEFAULT nextval('seq')` fills a columnar-engine column. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn default_nextval_fills_a_columnar_column() { + let server = TestServer::start().await; + + server + .exec("CREATE SEQUENCE seq_def_columnar") + .await + .unwrap(); + server + .exec( + "CREATE COLLECTION def_seq_columnar (\ + id BIGINT DEFAULT nextval('seq_def_columnar') PRIMARY KEY, \ + v TEXT) WITH (engine='columnar')", + ) + .await + .unwrap(); + + server + .exec("INSERT INTO def_seq_columnar (v) VALUES ('a')") + .await + .unwrap(); + server + .exec("INSERT INTO def_seq_columnar (v) VALUES ('b')") + .await + .unwrap(); + + let rows = server + .query_text("SELECT id FROM def_seq_columnar ORDER BY id") + .await + .unwrap(); + assert_eq!(rows.len(), 2, "two rows expected: {rows:?}"); + assert_not_null(&rows[0], "first id"); + assert_not_null(&rows[1], "second id"); + assert_eq!(rows, vec!["1".to_string(), "2".to_string()]); +} + +/// `DEFAULT upper('x')` evaluates on a schemaless collection, isolating the +/// schemaless DEFAULT drop from the sequence-accessor problem. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn default_scalar_function_on_a_schemaless_collection() { + let server = TestServer::start().await; + + server + .exec( + "CREATE COLLECTION def_fn_schemaless (id TEXT PRIMARY KEY, a TEXT DEFAULT upper('x'))", + ) + .await + .unwrap(); + + server + .exec("INSERT INTO def_fn_schemaless (id) VALUES ('k1')") + .await + .unwrap(); + + let rows = server + .query_text("SELECT a FROM def_fn_schemaless WHERE id = 'k1'") + .await + .unwrap(); + assert_eq!(rows.len(), 1, "row should exist"); + assert_not_null(&rows[0], "a"); + assert!( + rows[0].contains('X'), + "DEFAULT upper('x') should produce 'X', got {:?}", + rows[0] + ); +} + +/// `DEFAULT UUID_V7()` evaluates on a schemaless collection and produces a +/// 36-character value. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn default_uuid_on_a_schemaless_collection() { + let server = TestServer::start().await; + + server + .exec( + "CREATE COLLECTION def_uuid_schemaless (id TEXT PRIMARY KEY, b TEXT DEFAULT UUID_V7())", + ) + .await + .unwrap(); + + server + .exec("INSERT INTO def_uuid_schemaless (id) VALUES ('k1')") + .await + .unwrap(); + + let rows = server + .query_text("SELECT b FROM def_uuid_schemaless WHERE id = 'k1'") + .await + .unwrap(); + assert_eq!(rows.len(), 1, "row should exist"); + assert_not_null(&rows[0], "b"); + assert_eq!( + rows[0].trim().len(), + 36, + "UUID_V7() must render as 36 characters, got `{}`", + rows[0] + ); +} + +/// A DEFAULT expression the server cannot evaluate is refused at DDL time +/// with `42883`, never accepted and silently dropped at insert time. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn a_default_that_cannot_be_evaluated_is_refused_at_ddl() { + let server = TestServer::start().await; + + server + .expect_error( + "CREATE COLLECTION def_unevaluable (\ + id TEXT PRIMARY KEY, \ + a TEXT DEFAULT no_such_function_here('x'))", + "42883", + ) + .await; +} + +/// `DEFAULT currval('seq')` fills a strict-engine column with the session's +/// last `nextval` result. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn default_currval_fills_a_column() { + let server = TestServer::start().await; + + server + .exec("CREATE SEQUENCE seq_def_currval") + .await + .unwrap(); + server + .query_text("SELECT nextval('seq_def_currval')") + .await + .unwrap(); + server + .exec( + "CREATE COLLECTION def_currval_strict (\ + id TEXT PRIMARY KEY, \ + n BIGINT DEFAULT currval('seq_def_currval')) WITH (engine='document_strict')", + ) + .await + .unwrap(); + + server + .exec("INSERT INTO def_currval_strict (id) VALUES ('k1')") + .await + .unwrap(); + + let rows = server + .query_text("SELECT n FROM def_currval_strict WHERE id = 'k1'") + .await + .unwrap(); + assert_eq!(rows.len(), 1, "row should exist"); + assert_not_null(&rows[0], "n"); + assert_eq!( + rows[0].trim(), + "1", + "currval-backed default must be 1, got `{}`", + rows[0] + ); +} + +/// Asserts a rendered row carries a real value in place of an absent or NULL column. +fn assert_not_null(row: &str, label: &str) { + let trimmed = row.trim(); + assert!( + !trimmed.is_empty() && !trimmed.eq_ignore_ascii_case("null"), + "{label}: expected a value, got `{row}`" + ); +} diff --git a/nodedb/tests/wire/cases/sql_sequences.rs b/nodedb/tests/wire/cases/sql_sequences.rs index 274ce0c1c..5e45704c0 100644 --- a/nodedb/tests/wire/cases/sql_sequences.rs +++ b/nodedb/tests/wire/cases/sql_sequences.rs @@ -74,3 +74,134 @@ async fn drop_sequence_if_exists() { .await .unwrap(); } + +/// `nextval('seq')` returns 1 on the first call and 2 on the second. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn nextval_returns_successive_values() { + let server = TestServer::start().await; + + server + .exec("CREATE SEQUENCE seq_nextval_succ") + .await + .unwrap(); + + let first = server + .query_text("SELECT nextval('seq_nextval_succ')") + .await + .unwrap(); + assert_eq!(first, vec!["1".to_string()], "first nextval must be 1"); + + let second = server + .query_text("SELECT nextval('seq_nextval_succ')") + .await + .unwrap(); + assert_eq!(second, vec!["2".to_string()], "second nextval must be 2"); +} + +/// `currval('seq')` returns the session's last `nextval` result, not a fresh +/// allocation. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn currval_returns_the_last_value_of_the_session() { + let server = TestServer::start().await; + + server + .exec("CREATE SEQUENCE seq_currval_session") + .await + .unwrap(); + + server + .query_text("SELECT nextval('seq_currval_session')") + .await + .unwrap(); + let first = server + .query_text("SELECT currval('seq_currval_session')") + .await + .unwrap(); + assert_eq!( + first, + vec!["1".to_string()], + "currval must echo the last nextval" + ); + + server + .query_text("SELECT nextval('seq_currval_session')") + .await + .unwrap(); + let second = server + .query_text("SELECT currval('seq_currval_session')") + .await + .unwrap(); + assert_eq!( + second, + vec!["2".to_string()], + "currval must track the second nextval" + ); +} + +/// `setval('seq', 10)` positions the sequence so the next `nextval` returns 11. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn setval_positions_the_next_allocation() { + let server = TestServer::start().await; + + server.exec("CREATE SEQUENCE seq_setval_pos").await.unwrap(); + + server + .query_text("SELECT setval('seq_setval_pos', 10)") + .await + .unwrap(); + let next = server + .query_text("SELECT nextval('seq_setval_pos')") + .await + .unwrap(); + assert_eq!( + next, + vec!["11".to_string()], + "nextval after setval(10) must be 11" + ); +} + +/// `nextval` on a sequence that was never created must fail with `42704` +/// (undefined_object), not `42883` (undefined_function) — the function +/// exists, the object does not. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn nextval_on_an_unknown_sequence_errors() { + let server = TestServer::start().await; + + server + .expect_error("SELECT nextval('seq_that_was_never_created')", "42704") + .await; +} + +/// A `SERIAL` column allocates 1 then 2 across two inserts that omit it. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn serial_column_allocates_successive_keys() { + let server = TestServer::start().await; + + server + .exec("CREATE COLLECTION seq_serial_alloc FIELDS (n SERIAL, v TEXT)") + .await + .unwrap(); + + server + .exec("INSERT INTO seq_serial_alloc (v) VALUES ('a')") + .await + .unwrap(); + server + .exec("INSERT INTO seq_serial_alloc (v) VALUES ('b')") + .await + .unwrap(); + + let rows = server + .query_text("SELECT n FROM seq_serial_alloc ORDER BY n") + .await + .unwrap(); + assert_eq!(rows.len(), 2, "two rows expected: {rows:?}"); + for row in &rows { + let trimmed = row.trim(); + assert!( + !trimmed.is_empty() && !trimmed.eq_ignore_ascii_case("null"), + "SERIAL column must not be empty or NULL, got `{row}`" + ); + } + assert_eq!(rows, vec!["1".to_string(), "2".to_string()]); +} From 589ce2ebe46ec825560ad90b7597e0cdd3b83426 Mon Sep 17 00:00:00 2001 From: Farhan Syah Date: Wed, 9 Sep 2026 21:07:54 +0800 Subject: [PATCH 02/23] test(sql): cover volatile DEFAULT expression re-evaluation Add wire-level integration tests asserting UUID_V7(), NOW(), and nextval() DEFAULT expressions produce fresh values per execution rather than replaying a cached plan's frozen result, including across repeated byte-identical INSERT statements. --- nodedb/tests/wire/cases/mod.rs | 1 + .../wire/cases/sql_default_volatility.rs | 154 ++++++++++++++++++ 2 files changed, 155 insertions(+) create mode 100644 nodedb/tests/wire/cases/sql_default_volatility.rs diff --git a/nodedb/tests/wire/cases/mod.rs b/nodedb/tests/wire/cases/mod.rs index 92c02d3bd..73958e2fa 100644 --- a/nodedb/tests/wire/cases/mod.rs +++ b/nodedb/tests/wire/cases/mod.rs @@ -162,6 +162,7 @@ mod sql_copy_from; mod sql_copy_to; mod sql_cursors; mod sql_default_expressions; +mod sql_default_volatility; mod sql_division_by_zero; mod sql_division_by_zero_composite; mod sql_dml_affected_counts; diff --git a/nodedb/tests/wire/cases/sql_default_volatility.rs b/nodedb/tests/wire/cases/sql_default_volatility.rs new file mode 100644 index 000000000..58861c822 --- /dev/null +++ b/nodedb/tests/wire/cases/sql_default_volatility.rs @@ -0,0 +1,154 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! Integration coverage for volatile DEFAULT expression re-evaluation. +//! A volatile DEFAULT (`UUID_V7()`, `NOW()`, `nextval(...)`) evaluates once +//! per execution, never once per cached plan. + +use crate::harness::TestServer; + +/// `DEFAULT UUID_V7()` produces a distinct value for each of three inserts. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn uuid_default_produces_a_distinct_value_per_row() { + let server = TestServer::start().await; + + server + .exec( + "CREATE COLLECTION vol_uuid_rows (\ + id TEXT PRIMARY KEY, \ + u TEXT DEFAULT UUID_V7()) WITH (engine='document_strict')", + ) + .await + .unwrap(); + + server + .exec("INSERT INTO vol_uuid_rows (id) VALUES ('k1')") + .await + .unwrap(); + server + .exec("INSERT INTO vol_uuid_rows (id) VALUES ('k2')") + .await + .unwrap(); + server + .exec("INSERT INTO vol_uuid_rows (id) VALUES ('k3')") + .await + .unwrap(); + + let rows = server + .query_text("SELECT u FROM vol_uuid_rows ORDER BY id") + .await + .unwrap(); + assert_eq!(rows.len(), 3, "three rows expected: {rows:?}"); + for row in &rows { + assert_not_null(row, "u"); + } + let distinct: std::collections::HashSet<&str> = rows.iter().map(|r| r.as_str()).collect(); + assert_eq!( + distinct.len(), + 3, + "each row must carry a distinct UUID, got {rows:?}" + ); +} + +/// Every volatile DEFAULT re-evaluates when the identical INSERT text runs +/// three times. A cached plan must never replay a frozen value. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn volatile_defaults_re_evaluate_for_repeated_identical_statements() { + let server = TestServer::start().await; + + server + .exec( + "CREATE COLLECTION vol_uuid_cached (\ + id TEXT DEFAULT UUID_V7() PRIMARY KEY, \ + t TIMESTAMP DEFAULT NOW(), \ + v TEXT) WITH (engine='document_strict')", + ) + .await + .unwrap(); + + // The three statements are byte-identical, so the plan cache keys them the same. + // NOW() renders at second granularity, so the gap must exceed one second. + for _ in 0..3 { + server + .exec("INSERT INTO vol_uuid_cached (v) VALUES ('same')") + .await + .unwrap(); + tokio::time::sleep(std::time::Duration::from_millis(1100)).await; + } + + let rows = server + .query_text("SELECT id FROM vol_uuid_cached") + .await + .unwrap(); + assert_eq!( + rows.len(), + 3, + "a repeated default collapses rows on a primary key, got {rows:?}" + ); + let distinct: std::collections::HashSet<&str> = rows.iter().map(|r| r.as_str()).collect(); + assert_eq!( + distinct.len(), + 3, + "each execution must produce a distinct id, got {rows:?}" + ); + + let stamps = server + .query_text("SELECT t FROM vol_uuid_cached") + .await + .unwrap(); + assert_eq!(stamps.len(), 3, "three rows expected: {stamps:?}"); + for stamp in &stamps { + assert_not_null(stamp, "t"); + } + let distinct_stamps: std::collections::HashSet<&str> = + stamps.iter().map(|r| r.as_str()).collect(); + assert!( + distinct_stamps.len() > 1, + "NOW() must advance across executions, got {stamps:?}" + ); +} + +/// `DEFAULT nextval('seq')` advances when the identical INSERT text runs +/// three times, filling a strict-engine primary key each time. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn nextval_default_advances_across_repeated_identical_statements() { + let server = TestServer::start().await; + + server + .exec("CREATE SEQUENCE seq_vol_cached;") + .await + .unwrap(); + server + .exec( + "CREATE COLLECTION vol_seq_cached (\ + id BIGINT DEFAULT nextval('seq_vol_cached') PRIMARY KEY, \ + v TEXT) WITH (engine='document_strict')", + ) + .await + .unwrap(); + + for _ in 0..3 { + server + .exec("INSERT INTO vol_seq_cached (v) VALUES ('same')") + .await + .unwrap(); + } + + let rows = server + .query_text("SELECT id FROM vol_seq_cached ORDER BY id") + .await + .unwrap(); + assert_eq!( + rows, + vec!["1".to_string(), "2".to_string(), "3".to_string()], + "nextval must advance across repeated executions, got {rows:?}" + ); +} + +/// Asserts a rendered row carries a real value in place of an absent or NULL column. +fn assert_not_null(row: &str, label: &str) { + let trimmed = row.trim(); + assert!( + !trimmed.is_empty() && !trimmed.eq_ignore_ascii_case("null"), + "{label}: expected a value, got `{row}`" + ); +} From aaf4a879ef179262b4f0a1b9d992ee00922b4f81 Mon Sep 17 00:00:00 2001 From: Farhan Syah Date: Wed, 9 Sep 2026 21:18:02 +0800 Subject: [PATCH 03/23] fix(sql): recover the DEFAULT clause on schemaless collections The schemaless-document arm dropped every column's DEFAULT clause, the primary key's included, when building catalog ColumnInfo. A declared default never reached evaluation, so the column was written absent and read back as NULL. Recover each field's default from its declared type in stored.fields, the way the columnar arm already does. --- .../control/planner/catalog_adapter/type_convert.rs | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/nodedb/src/control/planner/catalog_adapter/type_convert.rs b/nodedb/src/control/planner/catalog_adapter/type_convert.rs index 02c8f8af6..1c74810ad 100644 --- a/nodedb/src/control/planner/catalog_adapter/type_convert.rs +++ b/nodedb/src/control/planner/catalog_adapter/type_convert.rs @@ -61,12 +61,20 @@ pub(super) fn convert_collection_type( .declared_primary_key .clone() .unwrap_or_else(|| "id".to_string()); + // `stored.fields` carries every declared column by name, the pk + // included, so its DEFAULT clause is recovered the same way the + // columnar arm recovers one for its synthetic-pk field. + let pk_default = stored + .fields + .iter() + .find(|(name, _)| name.eq_ignore_ascii_case(&pk_name)) + .and_then(|(_, type_str)| declared_default(type_str)); let mut columns = vec![ColumnInfo { name: pk_name.clone(), data_type: SqlDataType::String, nullable: false, is_primary_key: true, - default: None, + default: pk_default, raw_type: None, int_width: None, float_width: None, @@ -81,7 +89,7 @@ pub(super) fn convert_collection_type( data_type: parse_type_str(type_str), nullable: true, is_primary_key: false, - default: None, + default: declared_default(type_str), raw_type: None, int_width: IntWidth::from_declared_type(type_str), float_width: FloatWidth::from_declared_type(type_str), From f795153510392256f4d008add8510889a4830fb3 Mon Sep 17 00:00:00 2001 From: Farhan Syah Date: Wed, 9 Sep 2026 22:46:43 +0800 Subject: [PATCH 04/23] feat(sql): add nextval/currval/setval sequence functions Introduce nextval, currval, and setval as volatile scalar functions routed at plan time to SqlCatalog sequence methods, with per-session last-value tracking for currval. Add a plan-level volatility scan and a Volatile variant so a plan containing a volatile call (sequence functions, UUID/ULID/nanoid generators, DEFAULT expressions) is excluded from the plan cache and re-evaluated on every execution instead of freezing a value. Wrap lowered physical plans in a LoweredPlan that carries this cache verdict through the gateway, and thread the new SQLSTATEs and error constructors needed to reject invalid sequence access. --- .../common_suite/cases/gateway_execute.rs | 6 +- .../cases/http_gateway_migration.rs | 6 +- .../cases/pgwire_gateway_migration.rs | 6 +- nodedb-sql/src/catalog.rs | 48 ++++++++++ nodedb-sql/src/error.rs | 13 +++ nodedb-sql/src/functions/arg_types.rs | 8 ++ nodedb-sql/src/functions/builtins/helpers.rs | 1 + nodedb-sql/src/functions/builtins/scalars.rs | 2 + .../functions/builtins/scalars/datetime.rs | 6 +- .../src/functions/builtins/scalars/id_fn.rs | 21 +++-- .../functions/builtins/scalars/sequence_fn.rs | 51 +++++++++++ .../builtins/scalars/spatial/registrations.rs | 1 + nodedb-sql/src/functions/registry.rs | 20 +++++ nodedb-sql/src/planner/catalog_expr_fold.rs | 51 ++++++++++- nodedb-sql/src/planner/const_fold.rs | 90 +++++++++++++++---- nodedb-sql/src/planner/defaults.rs | 9 +- .../src/planner/dml_helpers/kv_insert.rs | 13 ++- nodedb-sql/src/planner/select/helpers.rs | 17 ---- nodedb-sql/src/planner/select/select_stmt.rs | 9 +- nodedb-sql/src/types/plan/cacheability.rs | 18 +++- nodedb-sql/src/types/plan/mod.rs | 2 + nodedb-sql/src/types/plan/variants.rs | 9 ++ nodedb-sql/src/types/plan/volatility_scan.rs | 80 +++++++++++++++++ .../src/visitor/plan_visitor/dispatch.rs | 5 +- .../src/error/ctors/read_query_auth.rs | 26 ++++++ nodedb-types/src/error/sqlstate.rs | 5 ++ nodedb-types/src/lib.rs | 2 + nodedb-types/src/volatility.rs | 30 +++++++ nodedb/src/control/gateway/lowered_plan.rs | 48 ++++++++++ nodedb/src/control/gateway/mod.rs | 2 + nodedb/src/control/gateway/sql_execute.rs | 36 +++++--- .../planner/catalog_adapter/adapter.rs | 21 +++++ .../control/planner/catalog_adapter/mod.rs | 1 + .../catalog_adapter/sequence_access.rs | 84 +++++++++++++++++ .../catalog_adapter/sql_catalog_impl.rs | 31 +++++++ .../control/planner/context/query/context.rs | 34 +++++++ .../control/planner/context/query/planning.rs | 19 ++-- .../planner/sql_plan_convert/cache_verdict.rs | 21 +++++ .../control/planner/sql_plan_convert/mod.rs | 2 + .../planner/sql_plan_convert/output_schema.rs | 1 + nodedb/src/control/sequence/mod.rs | 2 + nodedb/src/control/sequence/range_alloc.rs | 2 +- nodedb/src/control/sequence/registry.rs | 12 +-- nodedb/src/control/sequence/session_values.rs | 52 +++++++++++ .../control/server/pgwire/types/error_map.rs | 8 ++ .../server/shared/planning_overrides.rs | 4 + .../src/control/server/shared/session/mod.rs | 1 + .../server/shared/session/sequence_values.rs | 18 ++++ .../control/server/shared/session/state.rs | 5 ++ nodedb/src/error/types.rs | 13 +++ nodedb/src/error_classify.rs | 6 ++ 51 files changed, 894 insertions(+), 84 deletions(-) create mode 100644 nodedb-sql/src/functions/builtins/scalars/sequence_fn.rs create mode 100644 nodedb-sql/src/types/plan/volatility_scan.rs create mode 100644 nodedb-types/src/volatility.rs create mode 100644 nodedb/src/control/gateway/lowered_plan.rs create mode 100644 nodedb/src/control/planner/catalog_adapter/sequence_access.rs create mode 100644 nodedb/src/control/planner/sql_plan_convert/cache_verdict.rs create mode 100644 nodedb/src/control/sequence/session_values.rs create mode 100644 nodedb/src/control/server/shared/session/sequence_values.rs diff --git a/nodedb-cluster-tests/tests/common_suite/cases/gateway_execute.rs b/nodedb-cluster-tests/tests/common_suite/cases/gateway_execute.rs index a2b02d8e6..c9ee4a8ab 100644 --- a/nodedb-cluster-tests/tests/common_suite/cases/gateway_execute.rs +++ b/nodedb-cluster-tests/tests/common_suite/cases/gateway_execute.rs @@ -20,7 +20,7 @@ use nodedb::control::gateway::core::QueryContext; use nodedb::control::gateway::plan_cache::PlanCacheKey; use nodedb::control::gateway::plan_cache::{hash_placeholder_types, hash_sql}; use nodedb::control::gateway::version_set::GatewayVersionSet; -use nodedb::control::gateway::{Gateway, PlanCache}; +use nodedb::control::gateway::{Gateway, LoweredPlan, PlanCache}; use nodedb::types::TenantId; use nodedb_physical::physical_plan::{KvOp, PhysicalPlan}; use nodedb_types::QualifiedCollection; @@ -135,7 +135,7 @@ async fn gateway_execute_sql_plan_cache_populated() { let sql = "GET gw_cache_smoke smoke-key"; let make_plan = || { - Ok(PhysicalPlan::Kv(KvOp::Get { + Ok(LoweredPlan::cacheable(PhysicalPlan::Kv(KvOp::Get { collection: QualifiedCollection::new( nodedb_types::id::DatabaseId::DEFAULT, "gw_cache_smoke", @@ -143,7 +143,7 @@ async fn gateway_execute_sql_plan_cache_populated() { key: b"smoke-key".to_vec(), rls_filters: vec![], surrogate_ceiling: None, - })) + }))) }; // Cache starts empty. diff --git a/nodedb-cluster-tests/tests/common_suite/cases/http_gateway_migration.rs b/nodedb-cluster-tests/tests/common_suite/cases/http_gateway_migration.rs index 59007d2ca..ef889768d 100644 --- a/nodedb-cluster-tests/tests/common_suite/cases/http_gateway_migration.rs +++ b/nodedb-cluster-tests/tests/common_suite/cases/http_gateway_migration.rs @@ -17,9 +17,9 @@ use std::sync::Arc; use std::time::Duration; use nodedb::Error; -use nodedb::control::gateway::Gateway; use nodedb::control::gateway::GatewayErrorMap; use nodedb::control::gateway::core::QueryContext; +use nodedb::control::gateway::{Gateway, LoweredPlan}; use nodedb::types::TenantId; use nodedb_physical::physical_plan::{KvOp, PhysicalPlan}; use nodedb_types::QualifiedCollection; @@ -184,7 +184,7 @@ async fn http_gateway_migration_cross_node_query() { get_sql, &[], || { - Ok(PhysicalPlan::Kv(KvOp::Get { + Ok(LoweredPlan::cacheable(PhysicalPlan::Kv(KvOp::Get { collection: QualifiedCollection::new( nodedb_types::id::DatabaseId::DEFAULT, "http_gw_cross_node", @@ -192,7 +192,7 @@ async fn http_gateway_migration_cross_node_query() { key: b"cross-key".to_vec(), rls_filters: vec![], surrogate_ceiling: None, - })) + }))) }, |plan| async { Ok(common::authorize_gateway_plan(&follower.shared, &ctx, plan).await) diff --git a/nodedb-cluster-tests/tests/common_suite/cases/pgwire_gateway_migration.rs b/nodedb-cluster-tests/tests/common_suite/cases/pgwire_gateway_migration.rs index c9867ae3f..d02c841b5 100644 --- a/nodedb-cluster-tests/tests/common_suite/cases/pgwire_gateway_migration.rs +++ b/nodedb-cluster-tests/tests/common_suite/cases/pgwire_gateway_migration.rs @@ -20,9 +20,9 @@ use crate::common; use std::sync::Arc; use std::time::Duration; -use nodedb::control::gateway::Gateway; use nodedb::control::gateway::core::QueryContext; use nodedb::control::gateway::version_set::GatewayVersionSet; +use nodedb::control::gateway::{Gateway, LoweredPlan}; use nodedb::types::TenantId; use nodedb_physical::physical_plan::{KvOp, PhysicalPlan}; use nodedb_types::QualifiedCollection; @@ -200,7 +200,7 @@ async fn pgwire_gateway_migration_plan_cache_hits() { let sql = "GET pgwire_gw_cache cache-key"; let make_plan = || { - Ok(PhysicalPlan::Kv(KvOp::Get { + Ok(LoweredPlan::cacheable(PhysicalPlan::Kv(KvOp::Get { collection: QualifiedCollection::new( nodedb_types::id::DatabaseId::DEFAULT, "pgwire_gw_cache", @@ -208,7 +208,7 @@ async fn pgwire_gateway_migration_plan_cache_hits() { key: b"cache-key".to_vec(), rls_filters: vec![], surrogate_ceiling: None, - })) + }))) }; let authorize_plan = |plan: PhysicalPlan| async { Ok(common::authorize_gateway_plan(&node.shared, &ctx, plan).await) diff --git a/nodedb-sql/src/catalog.rs b/nodedb-sql/src/catalog.rs index 41aa91772..4638193ea 100644 --- a/nodedb-sql/src/catalog.rs +++ b/nodedb-sql/src/catalog.rs @@ -192,6 +192,54 @@ pub trait SqlCatalog { fn resolve_regtype(&self, _name: &str) -> Option { None } + + /// Advance a sequence and return the allocated value, recording it as this + /// session's `currval`. Scoping mirrors `resolve_regclass`. + /// + /// An unknown name returns [`SqlError::UndefinedObject`] (SQLSTATE + /// `42704`). The default returns "sequence access unavailable" so + /// implementors with no sequence state compile without change. + fn sequence_nextval( + &self, + _database_id: nodedb_types::DatabaseId, + _tenant_id: u64, + _name: &str, + ) -> Result { + Err(sequence_access_unavailable()) + } + + /// Return the last value this session obtained from `sequence_nextval`. + /// + /// A sequence this session never advanced returns + /// [`SqlError::ObjectNotInPrerequisiteState`] (SQLSTATE `55000`). + fn sequence_currval( + &self, + _database_id: nodedb_types::DatabaseId, + _tenant_id: u64, + _name: &str, + ) -> Result { + Err(sequence_access_unavailable()) + } + + /// Position a sequence so the next `sequence_nextval` returns + /// `value + increment`. Returns `value`. + fn sequence_setval( + &self, + _database_id: nodedb_types::DatabaseId, + _tenant_id: u64, + _name: &str, + _value: i64, + ) -> Result { + Err(sequence_access_unavailable()) + } +} + +/// The error a catalog with no sequence state returns for every accessor. +fn sequence_access_unavailable() -> crate::SqlError { + crate::SqlError::ObjectNotInPrerequisiteState { + object: "sequence".into(), + detail: "sequence access unavailable: this catalog carries no sequence registry".into(), + } } /// View of a registered array, surfaced to the SQL planner. Decoded by diff --git a/nodedb-sql/src/error.rs b/nodedb-sql/src/error.rs index b31a34c5b..d38424c0e 100644 --- a/nodedb-sql/src/error.rs +++ b/nodedb-sql/src/error.rs @@ -19,6 +19,19 @@ pub enum SqlError { #[error("function {name}(...) does not exist")] UndefinedFunction { name: String }, + /// A statement names a database object that does not exist — a sequence, + /// most commonly. Distinct from [`SqlError::UndefinedFunction`]: the + /// function exists, the object it names does not. PostgreSQL rejects the + /// same input with SQLSTATE `42704` (`undefined_object`). + #[error("{kind} \"{name}\" does not exist")] + UndefinedObject { kind: &'static str, name: String }, + + /// An object exists but a prerequisite step has not run, such as `currval` + /// before this session called `nextval`. PostgreSQL rejects the same input + /// with SQLSTATE `55000` (`object_not_in_prerequisite_state`). + #[error("{detail}")] + ObjectNotInPrerequisiteState { object: String, detail: String }, + #[error("unknown column '{column}' in table '{table}'")] UnknownColumn { table: String, column: String }, diff --git a/nodedb-sql/src/functions/arg_types.rs b/nodedb-sql/src/functions/arg_types.rs index eac91261d..652aa8f17 100644 --- a/nodedb-sql/src/functions/arg_types.rs +++ b/nodedb-sql/src/functions/arg_types.rs @@ -391,3 +391,11 @@ pub static JSON_CONTAINS_ARGS: &[ArgTypeSpec] = &[any("container"), any("needle" /// `json_merge(base, overlay)` / `json_patch(base, overlay)`. pub static JSON_MERGE_ARGS: &[ArgTypeSpec] = &[any("base"), any("overlay")]; + +// ── Sequence accessors ─────────────────────────────────────────────────────── + +/// `nextval(name)` / `currval(name)` — one sequence name. +pub static SEQUENCE_NAME_ARGS: &[ArgTypeSpec] = &[typed("sequence", TEXT)]; + +/// `setval(name, value)` — a sequence name and the value it must take. +pub static SETVAL_ARGS: &[ArgTypeSpec] = &[typed("sequence", TEXT), typed("value", INT64_ONLY)]; diff --git a/nodedb-sql/src/functions/builtins/helpers.rs b/nodedb-sql/src/functions/builtins/helpers.rs index 6757194da..1eba3822a 100644 --- a/nodedb-sql/src/functions/builtins/helpers.rs +++ b/nodedb-sql/src/functions/builtins/helpers.rs @@ -30,5 +30,6 @@ pub(super) fn m( return_type, arg_types, since: V0_1_0, + volatility: nodedb_types::Volatility::Immutable, } } diff --git a/nodedb-sql/src/functions/builtins/scalars.rs b/nodedb-sql/src/functions/builtins/scalars.rs index f73556d45..bd23528e6 100644 --- a/nodedb-sql/src/functions/builtins/scalars.rs +++ b/nodedb-sql/src/functions/builtins/scalars.rs @@ -12,6 +12,7 @@ mod math; mod misc; mod pg_fts; mod pg_json; +mod sequence_fn; mod spatial; mod string; mod vector; @@ -32,6 +33,7 @@ pub(super) fn scalar_functions() -> Vec { fns.extend(array_fn::array_fn_functions()); fns.extend(array_elem::array_elem_functions()); fns.extend(id_fn::id_fn_functions()); + fns.extend(sequence_fn::sequence_fn_functions()); fns.extend(misc::misc_functions()); fns } diff --git a/nodedb-sql/src/functions/builtins/scalars/datetime.rs b/nodedb-sql/src/functions/builtins/scalars/datetime.rs index ec0dfcd08..18e8da509 100644 --- a/nodedb-sql/src/functions/builtins/scalars/datetime.rs +++ b/nodedb-sql/src/functions/builtins/scalars/datetime.rs @@ -44,7 +44,8 @@ pub(super) fn datetime_functions() -> Vec { no_trigger(), Some(ColumnType::Timestamptz), arg_types::NO_ARGS, - ), + ) + .volatile(), m( "current_timestamp", Scalar, @@ -53,7 +54,8 @@ pub(super) fn datetime_functions() -> Vec { no_trigger(), Some(ColumnType::Timestamptz), arg_types::NO_ARGS, - ), + ) + .volatile(), m( "datetime", Scalar, diff --git a/nodedb-sql/src/functions/builtins/scalars/id_fn.rs b/nodedb-sql/src/functions/builtins/scalars/id_fn.rs index 6b22aa059..bd5b2b737 100644 --- a/nodedb-sql/src/functions/builtins/scalars/id_fn.rs +++ b/nodedb-sql/src/functions/builtins/scalars/id_fn.rs @@ -31,7 +31,8 @@ pub(super) fn id_fn_functions() -> Vec { no_trigger(), Some(ColumnType::Uuid), arg_types::NO_ARGS, - ), + ) + .volatile(), // `uuid_v4` and `gen_random_uuid` are aliases for `uuid` — same // `nodedb_query::functions::id::try_eval` match arm // (`"uuid" | "uuid_v4" | "gen_random_uuid"`). @@ -43,7 +44,8 @@ pub(super) fn id_fn_functions() -> Vec { no_trigger(), Some(ColumnType::Uuid), arg_types::NO_ARGS, - ), + ) + .volatile(), m( "gen_random_uuid", Scalar, @@ -52,7 +54,8 @@ pub(super) fn id_fn_functions() -> Vec { no_trigger(), Some(ColumnType::Uuid), arg_types::NO_ARGS, - ), + ) + .volatile(), m( "uuid_v7", Scalar, @@ -61,7 +64,8 @@ pub(super) fn id_fn_functions() -> Vec { no_trigger(), Some(ColumnType::Uuid), arg_types::NO_ARGS, - ), + ) + .volatile(), m( "ulid", Scalar, @@ -70,7 +74,8 @@ pub(super) fn id_fn_functions() -> Vec { no_trigger(), Some(ColumnType::Ulid), arg_types::NO_ARGS, - ), + ) + .volatile(), m( "cuid2", Scalar, @@ -79,7 +84,8 @@ pub(super) fn id_fn_functions() -> Vec { no_trigger(), Some(ColumnType::String), arg_types::NO_ARGS, - ), + ) + .volatile(), // `nanoid(length?)` — the optional length argument means max_args is // 1 even though the common call form `nanoid()` takes none. m( @@ -90,7 +96,8 @@ pub(super) fn id_fn_functions() -> Vec { no_trigger(), Some(ColumnType::String), arg_types::NANOID_ARGS, - ), + ) + .volatile(), m( "is_uuid", Scalar, diff --git a/nodedb-sql/src/functions/builtins/scalars/sequence_fn.rs b/nodedb-sql/src/functions/builtins/scalars/sequence_fn.rs new file mode 100644 index 000000000..2a72468d9 --- /dev/null +++ b/nodedb-sql/src/functions/builtins/scalars/sequence_fn.rs @@ -0,0 +1,51 @@ +// SPDX-License-Identifier: Apache-2.0 + +//! Sequence accessor registrations: `nextval`, `currval`, `setval`. +//! +//! These are `Scalar` and `Volatile` at once. `Volatile` keeps the constant +//! folder from freezing a call into a literal and keeps a plan holding one +//! out of the plan cache, so each execution allocates a fresh value. +//! `nodedb_sql::planner::catalog_expr_fold` routes each call to the +//! `SqlCatalog` sequence methods at plan time. + +use nodedb_types::columnar::ColumnType; + +use crate::functions::arg_types; +use crate::functions::registry::{FunctionCategory::Scalar, FunctionMeta}; + +use super::super::helpers::{m, no_trigger}; + +pub(super) fn sequence_fn_functions() -> Vec { + vec![ + m( + "nextval", + Scalar, + 1, + 1, + no_trigger(), + Some(ColumnType::Int64), + arg_types::SEQUENCE_NAME_ARGS, + ) + .volatile(), + m( + "currval", + Scalar, + 1, + 1, + no_trigger(), + Some(ColumnType::Int64), + arg_types::SEQUENCE_NAME_ARGS, + ) + .volatile(), + m( + "setval", + Scalar, + 2, + 2, + no_trigger(), + Some(ColumnType::Int64), + arg_types::SETVAL_ARGS, + ) + .volatile(), + ] +} diff --git a/nodedb-sql/src/functions/builtins/scalars/spatial/registrations.rs b/nodedb-sql/src/functions/builtins/scalars/spatial/registrations.rs index c8b69027f..7ba32e813 100644 --- a/nodedb-sql/src/functions/builtins/scalars/spatial/registrations.rs +++ b/nodedb-sql/src/functions/builtins/scalars/spatial/registrations.rs @@ -31,6 +31,7 @@ pub(in crate::functions::builtins::scalars) fn spatial_functions() -> Vec Self { + self.volatility = Volatility::Volatile; + self + } } /// The function registry. @@ -135,6 +148,13 @@ impl FunctionRegistry { .is_some_and(|f| f.category == FunctionCategory::Aggregate) } + /// Whether a function must re-evaluate on every execution. + /// An unknown name is not volatile — the existence gate rejects it first. + pub fn is_volatile(&self, name: &str) -> bool { + self.lookup(name) + .is_some_and(|f| f.volatility.is_volatile()) + } + /// Check if a function is a window function. pub fn is_window(&self, name: &str) -> bool { self.lookup(name) diff --git a/nodedb-sql/src/planner/catalog_expr_fold.rs b/nodedb-sql/src/planner/catalog_expr_fold.rs index 4a7339cbe..97e92ddb5 100644 --- a/nodedb-sql/src/planner/catalog_expr_fold.rs +++ b/nodedb-sql/src/planner/catalog_expr_fold.rs @@ -6,6 +6,7 @@ use nodedb_types::DatabaseId; use crate::catalog::SqlCatalog; use crate::functions::registry::FunctionRegistry; +use crate::planner::const_fold::FoldScope; use crate::types::{SqlExpr, SqlValue}; pub(super) fn eval_catalog_constant( @@ -29,7 +30,55 @@ pub(super) fn eval_catalog_constant( })?; return Ok(SqlValue::String(normalized)); } - super::select::helpers::eval_constant_expr(expr, functions) + if let Some(value) = eval_sequence_accessor(expr, catalog)? { + return Ok(value); + } + // A from-less SELECT's plan is marked volatile when it holds a volatile + // call, so it is never cached. Evaluating the call here therefore serves + // this execution only, and the next execution re-plans and re-evaluates. + Ok( + crate::planner::const_fold::fold_constant_scoped(expr, functions, FoldScope::Once)? + .unwrap_or(SqlValue::Null), + ) +} + +/// Route `nextval` / `currval` / `setval` to the catalog's sequence state. +/// +/// Returns `Ok(None)` for every other expression. These calls are `Volatile`, +/// so the constant folder never reaches them and the plan holding the result +/// is never cached — each execution re-plans and allocates again. +fn eval_sequence_accessor( + expr: &SqlExpr, + catalog: &dyn SqlCatalog, +) -> crate::Result> { + let SqlExpr::Function { name, args, .. } = expr else { + return Ok(None); + }; + let lowered = name.to_ascii_lowercase(); + if !matches!(lowered.as_str(), "nextval" | "currval" | "setval") { + return Ok(None); + } + let sequence = match args.first() { + Some(SqlExpr::Literal(SqlValue::String(name))) => name.as_str(), + _ => { + return Err(crate::SqlError::Arity { + detail: format!("{lowered} requires a literal sequence name"), + }); + } + }; + let value = match lowered.as_str() { + "nextval" => catalog.sequence_nextval(DatabaseId::DEFAULT, 0, sequence)?, + "currval" => catalog.sequence_currval(DatabaseId::DEFAULT, 0, sequence)?, + _ => { + let Some(SqlExpr::Literal(SqlValue::Int(target))) = args.get(1) else { + return Err(crate::SqlError::Arity { + detail: "setval requires a literal bigint second argument".into(), + }); + }; + catalog.sequence_setval(DatabaseId::DEFAULT, 0, sequence, *target)? + } + }; + Ok(Some(SqlValue::Int(value))) } pub(super) fn validate_expr( diff --git a/nodedb-sql/src/planner/const_fold.rs b/nodedb-sql/src/planner/const_fold.rs index 8f94a81ea..c06a9fbbb 100644 --- a/nodedb-sql/src/planner/const_fold.rs +++ b/nodedb-sql/src/planner/const_fold.rs @@ -51,15 +51,38 @@ pub fn fold_constant_default(expr: &SqlExpr) -> FoldResult { fold_constant(expr, default_registry()) } +/// Whether a fold is allowed to evaluate `Volatile` functions. +/// +/// `Reuse` is plan-time folding whose result outlives this execution, so a +/// volatile call is never evaluated. `Once` is folding for a plan that is +/// itself marked volatile and never cached, so the call is evaluated here and +/// re-evaluated on the next execution's re-plan. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum FoldScope { + /// The folded value can be reused by a later execution. + Reuse, + /// The folded value serves this execution only. + Once, +} + /// Fold a `SqlExpr` to a literal `SqlValue` at plan time. See [`FoldResult`] -/// for what each outcome means. +/// for what each outcome means. Volatile calls are never folded. pub fn fold_constant(expr: &SqlExpr, registry: &FunctionRegistry) -> FoldResult { + fold_constant_scoped(expr, registry, FoldScope::Reuse) +} + +/// Fold a `SqlExpr` under an explicit [`FoldScope`]. +pub fn fold_constant_scoped( + expr: &SqlExpr, + registry: &FunctionRegistry, + scope: FoldScope, +) -> FoldResult { match expr { SqlExpr::Literal(v) => Ok(Some(v.clone())), SqlExpr::ArrayLiteral(items) => { let mut folded = Vec::with_capacity(items.len()); for item in items { - match fold_constant(item, registry)? { + match fold_constant_scoped(item, registry, scope)? { Some(v) => folded.push(v), None => return Ok(None), } @@ -69,7 +92,7 @@ pub fn fold_constant(expr: &SqlExpr, registry: &FunctionRegistry) -> FoldResult SqlExpr::UnaryOp { op: UnaryOp::Neg, expr, - } => Ok(match fold_constant(expr, registry)? { + } => Ok(match fold_constant_scoped(expr, registry, scope)? { // `checked_neg` so negating `i64::MIN` declines to fold rather // than wrapping (release) or panicking (debug). Some(SqlValue::Int(i)) => i.checked_neg().map(SqlValue::Int), @@ -79,17 +102,18 @@ pub fn fold_constant(expr: &SqlExpr, registry: &FunctionRegistry) -> FoldResult }), SqlExpr::BinaryOp { left, op, right } => { let (Some(l), Some(r)) = ( - fold_constant(left, registry)?, - fold_constant(right, registry)?, + fold_constant_scoped(left, registry, scope)?, + fold_constant_scoped(right, registry, scope)?, ) else { return Ok(None); }; fold_binary(l, *op, r) } - SqlExpr::Function { name, args, .. } => fold_function_call(name, args, registry), - SqlExpr::Cast { expr, to_type } => { - Ok(fold_constant(expr, registry)?.and_then(|inner| fold_cast(inner, to_type))) + SqlExpr::Function { name, args, .. } => { + fold_function_call_scoped(name, args, registry, scope) } + SqlExpr::Cast { expr, to_type } => Ok(fold_constant_scoped(expr, registry, scope)? + .and_then(|inner| fold_cast(inner, to_type))), _ => Ok(None), } } @@ -310,12 +334,23 @@ fn fold_binary(l: SqlValue, op: BinaryOp, r: SqlValue) -> FoldResult { /// through the shared scalar evaluator, and converting the result back to /// `SqlValue`. Only folds functions that are present in `registry`, so /// callers can distinguish "unknown function" from "known function, all -/// args folded". +/// args folded". A `Volatile` function is never folded. pub fn fold_function_call(name: &str, args: &[SqlExpr], registry: &FunctionRegistry) -> FoldResult { + fold_function_call_scoped(name, args, registry, FoldScope::Reuse) +} + +/// Fold a function call under an explicit [`FoldScope`]. +pub fn fold_function_call_scoped( + name: &str, + args: &[SqlExpr], + registry: &FunctionRegistry, + scope: FoldScope, +) -> FoldResult { // Gate on registry so unknown-function paths keep their existing // fallbacks instead of collapsing to SqlValue::Null. Aggregates and // window functions aren't foldable — they need a row stream. - let Some(meta) = registry.lookup(name) else { + let name_lower = name.to_lowercase(); + let Some(meta) = registry.lookup(&name_lower) else { return Ok(None); }; if matches!( @@ -324,10 +359,22 @@ pub fn fold_function_call(name: &str, args: &[SqlExpr], registry: &FunctionRegis ) { return Ok(None); } + // A volatile call must run per execution, so folding it into a reusable + // literal would freeze the first result into every later execution of the + // same plan. Same "not folded" outcome the category guard above produces. + if meta.volatility.is_volatile() && scope == FoldScope::Reuse { + return Ok(None); + } + // Sequence accessors read catalog state this folder has no handle on. + // `planner::catalog_expr_fold::eval_catalog_constant` resolves them + // through `SqlCatalog` before any fold runs. + if matches!(name_lower.as_str(), "nextval" | "currval" | "setval") { + return Ok(None); + } let mut folded_args = Vec::with_capacity(args.len()); for arg in args { - match fold_constant(arg, registry)? { + match fold_constant_scoped(arg, registry, scope)? { Some(v) => folded_args.push(sql_to_ndb_value(v)), None => return Ok(None), } @@ -339,7 +386,7 @@ pub fn fold_function_call(name: &str, args: &[SqlExpr], registry: &FunctionRegis // clause; `SELECT mod(5, 0)` has no row scope and became NULL. // Exhaustive on purpose: a new `EvalError` variant must be classified // here rather than defaulting to "defer to a runtime that may not exist". - match nodedb_query::functions::eval_function(&name.to_lowercase(), &folded_args) { + match nodedb_query::functions::eval_function(&name_lower, &folded_args) { Ok(result) => Ok(Some(ndb_to_sql_value(result))), Err(nodedb_query::EvalError::DivisionByZero) => Err(SqlError::DivisionByZero), } @@ -402,16 +449,21 @@ mod tests { use super::*; #[test] - fn fold_now_produces_timestamptz() { + fn fold_now_produces_timestamptz_once_only() { let registry = FunctionRegistry::new(); let expr = SqlExpr::Function { name: "now".into(), args: vec![], distinct: false, }; - let val = fold_constant(&expr, ®istry) + // `now` is volatile, so a reusable plan must never carry its value. + assert!( + matches!(fold_constant(&expr, ®istry), Ok(None)), + "now() must not fold into a reusable plan" + ); + let val = fold_constant_scoped(&expr, ®istry, FoldScope::Once) .expect("fold must not error") - .expect("now() should fold"); + .expect("now() must fold for a single execution"); match val { SqlValue::Timestamptz(dt) => { // Sanity: must not be epoch (year 1970). @@ -422,15 +474,19 @@ mod tests { } #[test] - fn fold_current_timestamp_produces_timestamptz() { + fn fold_current_timestamp_produces_timestamptz_once_only() { let registry = FunctionRegistry::new(); let expr = SqlExpr::Function { name: "current_timestamp".into(), args: vec![], distinct: false, }; + assert!( + matches!(fold_constant(&expr, ®istry), Ok(None)), + "current_timestamp must not fold into a reusable plan" + ); assert!(matches!( - fold_constant(&expr, ®istry), + fold_constant_scoped(&expr, ®istry, FoldScope::Once), Ok(Some(SqlValue::Timestamptz(_))) )); } diff --git a/nodedb-sql/src/planner/defaults.rs b/nodedb-sql/src/planner/defaults.rs index 5fb914978..d17a66095 100644 --- a/nodedb-sql/src/planner/defaults.rs +++ b/nodedb-sql/src/planner/defaults.rs @@ -99,7 +99,14 @@ fn parse_parametric_or_literal( /// Attempt to parse the DEFAULT expression as SQL, then const-fold it. fn try_const_fold_default(expr: &str) -> Option { let sql_expr = crate::parse_expr_string(expr).ok()?; - let folded = crate::planner::const_fold::fold_constant_default(&sql_expr).ok()??; + // `Once`: a materialized DEFAULT serves this insert only, and an INSERT + // plan carrying a volatile DEFAULT is never admitted to the plan cache. + let folded = crate::planner::const_fold::fold_constant_scoped( + &sql_expr, + crate::planner::const_fold::default_registry(), + crate::planner::const_fold::FoldScope::Once, + ) + .ok()??; Some(sql_value_to_ndb(folded)) } diff --git a/nodedb-sql/src/planner/dml_helpers/kv_insert.rs b/nodedb-sql/src/planner/dml_helpers/kv_insert.rs index e3a7d0a3a..1d970312c 100644 --- a/nodedb-sql/src/planner/dml_helpers/kv_insert.rs +++ b/nodedb-sql/src/planner/dml_helpers/kv_insert.rs @@ -71,13 +71,14 @@ pub(crate) fn build_kv_insert_plan( // the read path can only encode SQL NULL for it. See // `declared_type_coerce` for the full rationale. let mut coerced_rows: Vec> = Vec::with_capacity(rows_ast.len()); + let mut volatile_defaults = false; for row_exprs in rows_ast { let mut row: Vec<(String, SqlValue)> = Vec::with_capacity(columns.len()); for (i, col) in columns.iter().enumerate() { let Some(expr) = row_exprs.get(i) else { break }; row.push((col.clone(), expr_to_sql_value(expr)?)); } - materialize_declared_defaults(declared_columns, &mut row)?; + volatile_defaults |= materialize_declared_defaults(declared_columns, &mut row)?; // The key column is exempt — see `coerce_rows_to_declared_types`. coerce_row_to_declared_types(declared_columns, &mut row, Some(key_col_name))?; coerced_rows.push(row); @@ -130,6 +131,7 @@ pub(crate) fn build_kv_insert_plan( ttl_secs, intent, on_conflict_updates, + volatile_defaults, }]) } @@ -151,10 +153,14 @@ pub(crate) fn build_kv_insert_plan( /// supplied literal. Filling them in afterwards would make `DEFAULT 999999` /// on a `SMALLINT` column a way to store a value the same literal is /// rejected for. +/// +/// Returns whether any materialized default came from a `Volatile` +/// expression, so the caller can keep the plan out of the plan cache. fn materialize_declared_defaults( declared_columns: &[ColumnInfo], row: &mut Vec<(String, SqlValue)>, -) -> Result<()> { +) -> Result { + let mut volatile = false; for column in declared_columns { let Some(default_expr) = column.default.as_deref() else { continue; @@ -170,9 +176,10 @@ fn materialize_declared_defaults( })?; let Some(evaluated) = evaluated else { continue }; let value = nodedb_value_to_sql_value(&column.name, evaluated)?; + volatile |= crate::types::plan::default_expr_is_volatile(default_expr); row.push((column.name.clone(), value)); } - Ok(()) + Ok(volatile) } /// Convert an evaluated default back into the planner's literal type. diff --git a/nodedb-sql/src/planner/select/helpers.rs b/nodedb-sql/src/planner/select/helpers.rs index 08631f839..d8bcc2f2c 100644 --- a/nodedb-sql/src/planner/select/helpers.rs +++ b/nodedb-sql/src/planner/select/helpers.rs @@ -6,7 +6,6 @@ use sqlparser::ast; use crate::error::{Result, SqlError}; -use crate::functions::registry::FunctionRegistry; use crate::parser::normalize::{SCHEMA_QUALIFIED_MSG, normalize_ident}; use crate::resolver::ColumnScope; use crate::resolver::columns::TableScope; @@ -184,22 +183,6 @@ pub fn extract_func_args(func: &ast::Function) -> Result> { } } -/// Evaluate a constant SqlExpr to a SqlValue. Delegates to the shared -/// `const_fold::fold_constant` helper so that zero-arg scalar functions -/// like `now()` and `current_timestamp` go through the same evaluator -/// as the runtime expression path. -/// -/// An expression that is not constant yields NULL — this is the from-less -/// SELECT path, which has no row scope to defer to. An expression that *is* -/// constant and failed to evaluate raises instead: it can never succeed, so -/// NULL would be a wrong answer rather than an unknown one. -pub(crate) fn eval_constant_expr( - expr: &SqlExpr, - functions: &FunctionRegistry, -) -> crate::Result { - Ok(crate::planner::const_fold::fold_constant(expr, functions)?.unwrap_or(SqlValue::Null)) -} - pub(super) fn extract_column_name(expr: &ast::Expr) -> Result { match expr { ast::Expr::Identifier(ident) => Ok(normalize_ident(ident)), diff --git a/nodedb-sql/src/planner/select/select_stmt.rs b/nodedb-sql/src/planner/select/select_stmt.rs index e6b6fa18e..3ab107f9e 100644 --- a/nodedb-sql/src/planner/select/select_stmt.rs +++ b/nodedb-sql/src/planner/select/select_stmt.rs @@ -75,10 +75,12 @@ pub(super) fn plan_select( let projection = convert_projection(&select.projection, &scope)?; let mut columns = Vec::new(); let mut values = Vec::new(); + let mut volatile = false; for (i, proj) in projection.iter().enumerate() { match proj { Projection::Computed { expr, alias } => { columns.push(alias.clone()); + volatile |= crate::types::plan::expr_is_volatile(expr); values.push(crate::planner::catalog_expr_fold::eval_catalog_constant( expr, catalog, functions, )?); @@ -94,7 +96,11 @@ pub(super) fn plan_select( } } return Ok(PlannedSelect { - plan: SqlPlan::ConstantResult { columns, values }, + plan: SqlPlan::ConstantResult { + columns, + values, + volatile, + }, scope, }); } @@ -188,6 +194,7 @@ pub(super) fn plan_select( Box::new(SqlPlan::ConstantResult { columns: Vec::new(), values: Vec::new(), + volatile: false, }), ); for sq in subquery_joins diff --git a/nodedb-sql/src/types/plan/cacheability.rs b/nodedb-sql/src/types/plan/cacheability.rs index 5f5e1f24c..6ef8d40c1 100644 --- a/nodedb-sql/src/types/plan/cacheability.rs +++ b/nodedb-sql/src/types/plan/cacheability.rs @@ -9,7 +9,8 @@ use super::SqlPlan; pub enum PlanCacheEligibility { /// Lowering depends only on schema/catalog descriptors tracked by the cache. Cacheable, - /// Lowering consults mutable row identity and must run for every execution. + /// Lowering consults mutable row identity, or the plan holds a volatile + /// expression, and must run for every execution. DataDependent, } @@ -34,10 +35,25 @@ impl SqlPlan { /// Document point operations resolve primary-key bytes to a surrogate while /// lowering. That binding can appear after an earlier miss without any /// schema-version change, so those physical tasks cannot be cached. + /// + /// A plan holding a volatile call is `DataDependent` for the same reason: + /// the call was evaluated while the plan was built, so caching it would + /// replay one execution's value into every later one. pub fn cache_eligibility(&self) -> PlanCacheEligibility { use PlanCacheEligibility::{Cacheable, DataDependent}; match self { + Self::ConstantResult { volatile: true, .. } => DataDependent, + Self::Insert { + column_defaults, .. + } + | Self::Upsert { + column_defaults, .. + } if super::volatility_scan::defaults_are_volatile(column_defaults) => DataDependent, + Self::KvInsert { + volatile_defaults: true, + .. + } => DataDependent, Self::PointGet { engine: EngineType::DocumentSchemaless | EngineType::DocumentStrict, .. diff --git a/nodedb-sql/src/types/plan/mod.rs b/nodedb-sql/src/types/plan/mod.rs index 2f3388f7f..b1d21ba1c 100644 --- a/nodedb-sql/src/types/plan/mod.rs +++ b/nodedb-sql/src/types/plan/mod.rs @@ -8,9 +8,11 @@ mod row_types; mod variant_name; mod variants; mod vector_opts; +mod volatility_scan; pub use cacheability::PlanCacheEligibility; pub use merge_types::{MergeClauseKind, MergePlanAction, MergePlanClause}; pub use row_types::{KvInsertIntent, VectorPrimaryRow}; pub use variants::{DistanceMetric, SqlPlan}; pub use vector_opts::{ArrayPrefilter, VectorAnnOptions, VectorQuantization}; +pub use volatility_scan::{default_expr_is_volatile, defaults_are_volatile, expr_is_volatile}; diff --git a/nodedb-sql/src/types/plan/variants.rs b/nodedb-sql/src/types/plan/variants.rs index 5e539e7b8..305b9f019 100644 --- a/nodedb-sql/src/types/plan/variants.rs +++ b/nodedb-sql/src/types/plan/variants.rs @@ -27,6 +27,10 @@ pub enum SqlPlan { ConstantResult { columns: Vec, values: Vec, + /// Whether any projected expression called a `Volatile` function. + /// The values were evaluated while this plan was built, so a cached + /// plan would replay them; a volatile plan is never cached. + volatile: bool, }, // ── Reads ── @@ -138,6 +142,11 @@ pub enum SqlPlan { /// Empty for plain UPSERT (whole-value overwrite) and for INSERT /// variants. on_conflict_updates: Vec<(String, SqlExpr)>, + /// Whether any DEFAULT materialized into `entries` came from a + /// `Volatile` expression. The key-value planner evaluates declared + /// defaults while building this plan, so a cached plan would replay + /// one execution's value; a volatile plan is never cached. + volatile_defaults: bool, }, /// UPSERT: insert or merge if document exists. Upsert { diff --git a/nodedb-sql/src/types/plan/volatility_scan.rs b/nodedb-sql/src/types/plan/volatility_scan.rs new file mode 100644 index 000000000..338f7a81f --- /dev/null +++ b/nodedb-sql/src/types/plan/volatility_scan.rs @@ -0,0 +1,80 @@ +// SPDX-License-Identifier: Apache-2.0 + +//! Volatility scanning over plan expressions and column DEFAULT strings. +//! +//! A plan holding a volatile call must re-plan per execution, so the plan +//! cache never replays a frozen value. + +use crate::planner::const_fold::default_registry; +use crate::types_expr::SqlExpr; + +/// Whether an expression tree contains a call to a `Volatile` function. +pub fn expr_is_volatile(expr: &SqlExpr) -> bool { + match expr { + SqlExpr::Function { name, args, .. } => { + default_registry().is_volatile(name) || args.iter().any(expr_is_volatile) + } + SqlExpr::BinaryOp { left, right, .. } => expr_is_volatile(left) || expr_is_volatile(right), + SqlExpr::UnaryOp { expr, .. } + | SqlExpr::Cast { expr, .. } + | SqlExpr::IsNull { expr, .. } => expr_is_volatile(expr), + SqlExpr::Case { + operand, + when_then, + else_expr, + } => { + operand.as_deref().is_some_and(expr_is_volatile) + || when_then + .iter() + .any(|(when, then)| expr_is_volatile(when) || expr_is_volatile(then)) + || else_expr.as_deref().is_some_and(expr_is_volatile) + } + SqlExpr::InList { expr, list, .. } => { + expr_is_volatile(expr) || list.iter().any(expr_is_volatile) + } + SqlExpr::Between { + expr, low, high, .. + } => expr_is_volatile(expr) || expr_is_volatile(low) || expr_is_volatile(high), + SqlExpr::Like { expr, pattern, .. } => expr_is_volatile(expr) || expr_is_volatile(pattern), + SqlExpr::ArrayLiteral(items) => items.iter().any(expr_is_volatile), + SqlExpr::Column { .. } | SqlExpr::Literal(_) | SqlExpr::Subquery(_) | SqlExpr::Wildcard => { + false + } + } +} + +/// Column DEFAULT spellings that generate a fresh value per row but name no +/// registered function. Kept in step with the generator arms of +/// `crate::planner::defaults::evaluate_default_expr`. +const VOLATILE_DEFAULT_ALIASES: &[&str] = + &["uuidv7", "uuidv4", "gen_uuid_v7", "gen_uuid_v4", "gen_ulid"]; + +/// Whether a stored column DEFAULT expression re-evaluates per execution. +/// +/// The catalog stores a DEFAULT as text, in either a bare form (`UUID_V7`) or +/// a call form (`uuid_v7()`, `nextval('s')`), so the leading identifier is +/// checked before the string is parsed as an expression. +pub fn default_expr_is_volatile(expr: &str) -> bool { + let trimmed = expr.trim(); + let head: String = trimmed + .chars() + .take_while(|c| c.is_alphanumeric() || *c == '_') + .collect::() + .to_ascii_lowercase(); + if !head.is_empty() + && (default_registry().is_volatile(&head) + || VOLATILE_DEFAULT_ALIASES.contains(&head.as_str())) + { + return true; + } + crate::parse_expr_string(trimmed) + .ok() + .is_some_and(|parsed| expr_is_volatile(&parsed)) +} + +/// Whether any `(column, default_expr)` pair re-evaluates per execution. +pub fn defaults_are_volatile(defaults: &[(String, String)]) -> bool { + defaults + .iter() + .any(|(_, expr)| default_expr_is_volatile(expr)) +} diff --git a/nodedb-sql/src/visitor/plan_visitor/dispatch.rs b/nodedb-sql/src/visitor/plan_visitor/dispatch.rs index 1357c1b1d..fe4bff77c 100644 --- a/nodedb-sql/src/visitor/plan_visitor/dispatch.rs +++ b/nodedb-sql/src/visitor/plan_visitor/dispatch.rs @@ -17,7 +17,9 @@ use crate::types::SqlPlan; pub fn dispatch(visitor: &mut V, plan: &SqlPlan) -> Result { match plan { - SqlPlan::ConstantResult { columns, values } => visitor.constant_result(columns, values), + SqlPlan::ConstantResult { + columns, values, .. + } => visitor.constant_result(columns, values), SqlPlan::Scan { collection, alias, @@ -113,6 +115,7 @@ pub fn dispatch(visitor: &mut V, plan: &SqlPlan) -> Result visitor.kv_insert(collection, entries, *ttl_secs, *intent, on_conflict_updates), SqlPlan::Upsert { collection, diff --git a/nodedb-types/src/error/ctors/read_query_auth.rs b/nodedb-types/src/error/ctors/read_query_auth.rs index 446d5fd97..9ac2d70a5 100644 --- a/nodedb-types/src/error/ctors/read_query_auth.rs +++ b/nodedb-types/src/error/ctors/read_query_auth.rs @@ -134,6 +134,32 @@ impl NodeDbError { } } + /// A statement names a database object that does not exist. Distinct from + /// `undefined_function`: the function exists, the object it names does + /// not. Renders as SQLSTATE `42704` (`undefined_object`). + pub fn undefined_object(object: impl Into) -> Self { + let object = object.into(); + Self { + code: ErrorCode::UNDEFINED_OBJECT, + message: format!("{object} does not exist"), + details: ErrorDetails::UndefinedObject { object }, + cause: None, + } + } + + /// An object exists but a prerequisite step has not run, such as `currval` + /// before this session called `nextval`. Renders as SQLSTATE `55000` + /// (`object_not_in_prerequisite_state`). + pub fn object_not_ready(object: impl Into, detail: impl Into) -> Self { + let object = object.into(); + Self { + code: ErrorCode::OBJECT_NOT_READY, + message: detail.into(), + details: ErrorDetails::ObjectNotReady { object }, + cause: None, + } + } + /// A column reference names no column of any relation in scope. Distinct /// from `plan_error` so clients match on the code rather than parsing the /// message. diff --git a/nodedb-types/src/error/sqlstate.rs b/nodedb-types/src/error/sqlstate.rs index ffea17029..021e979ac 100644 --- a/nodedb-types/src/error/sqlstate.rs +++ b/nodedb-types/src/error/sqlstate.rs @@ -175,6 +175,10 @@ pub const STATEMENT_TOO_COMPLEX: &str = "54001"; // ── Class 55 — Object Not In Prerequisite State ────────────────────────────── +/// `55000` — `object_not_in_prerequisite_state` (the object exists but the +/// step this statement needs has not run, e.g. `currval` before `nextval`) +pub const OBJECT_NOT_IN_PREREQUISITE_STATE: &str = "55000"; + /// `55P03` — `lock_not_available` (no cluster leader) pub const LOCK_NOT_AVAILABLE: &str = "55P03"; @@ -362,6 +366,7 @@ mod tests { CONFIGURATION_LIMIT_EXCEEDED, PROGRAM_LIMIT_EXCEEDED, STATEMENT_TOO_COMPLEX, + OBJECT_NOT_IN_PREREQUISITE_STATE, LOCK_NOT_AVAILABLE, QUERY_CANCELED, CANNOT_CONNECT_NOW, diff --git a/nodedb-types/src/lib.rs b/nodedb-types/src/lib.rs index 7d34bd951..afcad8014 100644 --- a/nodedb-types/src/lib.rs +++ b/nodedb-types/src/lib.rs @@ -69,6 +69,7 @@ pub mod vector_dtype; pub mod vector_index_params; pub mod vector_index_stats; pub mod vector_model; +pub mod volatility; pub mod wire_version; pub use approx::{CountMinSketch, HyperLogLog, SpaceSaving, TDigest}; @@ -139,3 +140,4 @@ pub use vector_dtype::VectorStorageDtype; pub use vector_index_params::StoredVectorIndexParams; pub use vector_index_stats::{VectorIndexQuantization, VectorIndexStats, VectorIndexType}; pub use vector_model::{VectorModelEntry, VectorModelMetadata}; +pub use volatility::Volatility; diff --git a/nodedb-types/src/volatility.rs b/nodedb-types/src/volatility.rs new file mode 100644 index 000000000..8dfad6180 --- /dev/null +++ b/nodedb-types/src/volatility.rs @@ -0,0 +1,30 @@ +// SPDX-License-Identifier: Apache-2.0 + +//! Function volatility: how far a call's result can be reused. +//! +//! Volatility is orthogonal to a function's category. A function is +//! `Scalar` and `Volatile` at the same time. + +/// How far a function call's result can be reused. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] +pub enum Volatility { + /// Same arguments always give the same result. Foldable at plan time. + #[default] + Immutable, + /// Constant within one statement, not across statements. + Stable, + /// Can change per call, or has side effects. + Volatile, +} + +impl Volatility { + /// Whether a call can be folded to a literal at plan time. + pub fn is_foldable(self) -> bool { + matches!(self, Self::Immutable) + } + + /// Whether a call must re-evaluate on every execution. + pub fn is_volatile(self) -> bool { + matches!(self, Self::Volatile) + } +} diff --git a/nodedb/src/control/gateway/lowered_plan.rs b/nodedb/src/control/gateway/lowered_plan.rs new file mode 100644 index 000000000..e0a5d3b1c --- /dev/null +++ b/nodedb/src/control/gateway/lowered_plan.rs @@ -0,0 +1,48 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! The lowered plan the gateway executes, paired with its plan-cache verdict. + +use nodedb_physical::physical_plan::PhysicalPlan; +use nodedb_sql::types::PlanCacheEligibility; + +/// A `SqlPlan` lowered to a `PhysicalPlan`, carrying the verdict that decides +/// admission to the gateway plan cache. +/// +/// The verdict rides beside the plan instead of inside it. `PhysicalPlan` is +/// the Data Plane wire enum; cacheability is a Control Plane planning property +/// the Data Plane never reads. A field on the enum would widen the wire shape +/// and every match on it for no execution benefit. +#[derive(Debug, Clone)] +pub struct LoweredPlan { + /// The physical operation to execute. + pub plan: PhysicalPlan, + /// Whether the gateway can admit this plan to its cache. + pub cache_eligibility: PlanCacheEligibility, +} + +impl LoweredPlan { + /// Pair a lowered plan with the verdict its `SqlPlan` batch produced. + /// + /// Pass the value returned by + /// `crate::control::planner::sql_plan_convert::batch_cache_eligibility`. + /// Never derive a second volatility verdict here. + pub fn new(plan: PhysicalPlan, cache_eligibility: PlanCacheEligibility) -> Self { + Self { + plan, + cache_eligibility, + } + } + + /// Pair a plan built directly, with no `SqlPlan` behind it. + /// + /// A directly constructed operation carries no expression tree, so it + /// carries no volatile call. + pub fn cacheable(plan: PhysicalPlan) -> Self { + Self::new(plan, PlanCacheEligibility::Cacheable) + } + + /// Whether the gateway can admit this plan to its cache. + pub fn is_cacheable(&self) -> bool { + self.cache_eligibility.is_cacheable() + } +} diff --git a/nodedb/src/control/gateway/mod.rs b/nodedb/src/control/gateway/mod.rs index fc73a4c54..50058b703 100644 --- a/nodedb/src/control/gateway/mod.rs +++ b/nodedb/src/control/gateway/mod.rs @@ -9,6 +9,7 @@ pub mod error_map; pub mod fuser; pub mod invalidation; pub mod key_extractor; +pub mod lowered_plan; pub mod plan_cache; pub mod retry; pub mod route; @@ -22,6 +23,7 @@ pub use core::Gateway; pub use error_map::GatewayErrorMap; pub use invalidation::PlanCacheInvalidator; pub use key_extractor::{KeyExtractor, UnwiredKeyExtractor}; +pub use lowered_plan::LoweredPlan; pub use plan_cache::PlanCache; pub use route::{RouteDecision, TaskRoute}; pub use version_check::{DescriptorCheckError, check_descriptor_versions}; diff --git a/nodedb/src/control/gateway/sql_execute.rs b/nodedb/src/control/gateway/sql_execute.rs index 7fbe6f4f3..fc063cf2b 100644 --- a/nodedb/src/control/gateway/sql_execute.rs +++ b/nodedb/src/control/gateway/sql_execute.rs @@ -11,6 +11,7 @@ use crate::control::server::shared::clone_write::CloneCheckedTask; use nodedb_physical::physical_plan::PhysicalPlan; use super::core::{Gateway, QueryContext, authorized_plan_for_context}; +use super::lowered_plan::LoweredPlan; use super::plan_cache::{PlanCacheKey, SqlKey, hash_placeholder_types, hash_sql}; use super::version_set::{permission_tree_version_key, rls_version_key}; @@ -20,12 +21,17 @@ impl Gateway { /// `plan_fn` runs at most once on a cache miss. `authorize_fn` clone-checks /// and authorizes the exact plan — the cached plan or the newly planned /// value — the same way every other dispatch entry point does. + /// + /// `plan_fn` returns the plan together with its + /// [`nodedb_sql::types::PlanCacheEligibility`] verdict. A `DataDependent` + /// plan enters neither the plan cache nor the version-set side cache, so + /// the next execution re-plans and re-evaluates its volatile calls. pub async fn execute_sql( &self, ctx: &QueryContext, sql: &str, placeholder_types: &[&str], - plan_fn: impl FnOnce() -> Result, + plan_fn: impl FnOnce() -> Result, authorize_fn: Authorize, ) -> Result>, Error> where @@ -88,19 +94,29 @@ impl Gateway { } } - let plan = plan_fn()?; + let lowered = plan_fn()?; + let cacheable = lowered.is_cacheable(); + let plan = lowered.plan; let actual_vs = self .collect_version_set(&plan, ctx.tenant_id.as_u64(), ctx.database_id) .await?; - let actual_key = PlanCacheKey { - sql_text_hash: sql_hash, - placeholder_types_hash: ph_hash, - version_set: actual_vs.clone(), - }; - self.plan_cache - .insert_version_set(sql_key, actual_vs.clone()); - self.plan_cache.insert(actual_key, Arc::new(plan.clone())); + // A volatile plan froze its `nextval` / `now()` / UUID values while it + // was built. Admitting it would replay this execution's values into + // every later one, so it skips both caches and re-plans next time. + // The side cache is skipped too: its entry is pruned only alongside the + // plan entry it maps to, so storing one without a plan leaves an orphan + // that survives every DDL invalidation and buys no hit. + if cacheable { + let actual_key = PlanCacheKey { + sql_text_hash: sql_hash, + placeholder_types_hash: ph_hash, + version_set: actual_vs.clone(), + }; + self.plan_cache + .insert_version_set(sql_key, actual_vs.clone()); + self.plan_cache.insert(actual_key, Arc::new(plan.clone())); + } let checked = authorize_fn(plan.clone()).await?; let plan = authorized_plan_for_context(ctx, checked)?; diff --git a/nodedb/src/control/planner/catalog_adapter/adapter.rs b/nodedb/src/control/planner/catalog_adapter/adapter.rs index 8b6842f5b..73f120e4b 100644 --- a/nodedb/src/control/planner/catalog_adapter/adapter.rs +++ b/nodedb/src/control/planner/catalog_adapter/adapter.rs @@ -54,6 +54,14 @@ pub struct OriginCatalog { /// negligible — `get_collection` is called only a handful /// of times per plan. pub(super) recorded_versions: Mutex, + /// Node-wide sequence counters. `None` for adapters built without a + /// `SharedState`; every sequence accessor then reports that sequence + /// access is unavailable. + pub(super) sequence_registry: Option>, + /// The calling session's `currval` map. `nextval` records into it and + /// `currval` reads only from it, so one session never sees another + /// session's allocation. `None` for planning with no session behind it. + pub(super) session_sequences: Option>, } impl OriginCatalog { @@ -83,6 +91,8 @@ impl OriginCatalog { drain_tracker: None, recorded_versions: Mutex::new(DescriptorVersionSet::new()), array_catalog: None, + sequence_registry: None, + session_sequences: None, } } @@ -107,9 +117,20 @@ impl OriginCatalog { drain_tracker: Some(Arc::clone(&shared.lease_drain)), recorded_versions: Mutex::new(DescriptorVersionSet::new()), array_catalog: Some(shared.array_catalog.clone()), + sequence_registry: Some(Arc::clone(&shared.sequence_registry)), + session_sequences: None, } } + /// Bind the calling session's `currval` map to this adapter. + pub fn with_session_sequences( + mut self, + values: Option>, + ) -> Self { + self.session_sequences = values; + self + } + /// Drain the recorded descriptor-version set and return it. /// Callers capture this after planning finishes and use it /// as the plan cache key + freshness witness. diff --git a/nodedb/src/control/planner/catalog_adapter/mod.rs b/nodedb/src/control/planner/catalog_adapter/mod.rs index 84514f2db..fe5c8beca 100644 --- a/nodedb/src/control/planner/catalog_adapter/mod.rs +++ b/nodedb/src/control/planner/catalog_adapter/mod.rs @@ -31,6 +31,7 @@ //! queries. mod adapter; +mod sequence_access; mod sql_catalog_impl; mod type_convert; diff --git a/nodedb/src/control/planner/catalog_adapter/sequence_access.rs b/nodedb/src/control/planner/catalog_adapter/sequence_access.rs new file mode 100644 index 000000000..020fdd543 --- /dev/null +++ b/nodedb/src/control/planner/catalog_adapter/sequence_access.rs @@ -0,0 +1,84 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! Sequence accessor bodies behind `OriginCatalog`'s `SqlCatalog` methods. +//! +//! `nextval` advances the node counter AND records the value in the calling +//! session's map. `currval` reads only that session map, so one session never +//! reports another session's allocation. + +use nodedb_sql::SqlError; + +use crate::control::sequence::{SequenceError, SequenceRegistry}; + +use super::adapter::OriginCatalog; + +impl OriginCatalog { + /// Advance `name` and record the value as this session's `currval`. + pub(super) fn advance_sequence(&self, name: &str) -> Result { + let registry = self.require_sequence_registry()?; + let database_id = self.database_id.as_u64(); + let value = registry + .nextval(database_id, self.tenant_id, name) + .map_err(|e| map_sequence_error(name, e))?; + if let Some(session) = &self.session_sequences { + session.record(database_id, self.tenant_id, name, value); + } + Ok(value) + } + + /// The last value THIS SESSION obtained from `nextval` on `name`. + pub(super) fn session_sequence_value(&self, name: &str) -> Result { + // The sequence must exist before its absence from the session map can + // mean "not yet called in this session" rather than "no such object". + let registry = self.require_sequence_registry()?; + if !registry.exists(self.database_id.as_u64(), self.tenant_id, name) { + return Err(undefined_sequence(name)); + } + self.session_sequences + .as_ref() + .and_then(|session| session.last(self.database_id.as_u64(), self.tenant_id, name)) + .ok_or_else(|| SqlError::ObjectNotInPrerequisiteState { + object: name.to_string(), + detail: format!( + "currval of sequence \"{name}\" is not yet defined in this session" + ), + }) + } + + /// Position `name` so the next `nextval` returns `value + increment`. + pub(super) fn position_sequence(&self, name: &str, value: i64) -> Result { + let registry = self.require_sequence_registry()?; + registry + .setval(self.database_id.as_u64(), self.tenant_id, name, value) + .map_err(|e| map_sequence_error(name, e)) + } + + fn require_sequence_registry(&self) -> Result<&SequenceRegistry, SqlError> { + self.sequence_registry + .as_deref() + .ok_or_else(|| SqlError::ObjectNotInPrerequisiteState { + object: "sequence".into(), + detail: "sequence access unavailable: this planner holds no sequence registry" + .into(), + }) + } +} + +/// The function exists, the object does not — SQLSTATE `42704`, never `42883`. +fn undefined_sequence(name: &str) -> SqlError { + SqlError::UndefinedObject { + kind: "sequence", + name: name.to_string(), + } +} + +/// Map a registry error onto its planner equivalent. +fn map_sequence_error(name: &str, error: SequenceError) -> SqlError { + match error { + SequenceError::NotFound { .. } => undefined_sequence(name), + other => SqlError::ObjectNotInPrerequisiteState { + object: name.to_string(), + detail: other.to_string(), + }, + } +} diff --git a/nodedb/src/control/planner/catalog_adapter/sql_catalog_impl.rs b/nodedb/src/control/planner/catalog_adapter/sql_catalog_impl.rs index e9c7db299..4bac4b2e6 100644 --- a/nodedb/src/control/planner/catalog_adapter/sql_catalog_impl.rs +++ b/nodedb/src/control/planner/catalog_adapter/sql_catalog_impl.rs @@ -220,6 +220,37 @@ impl SqlCatalog for OriginCatalog { ) } + /// Database and tenant come from the adapter's own scope, mirroring + /// `resolve_regclass`; the trait's arguments are the caller's plan-time + /// defaults and carry no session scope. + fn sequence_nextval( + &self, + _database_id: nodedb_types::DatabaseId, + _tenant_id: u64, + name: &str, + ) -> Result { + self.advance_sequence(name) + } + + fn sequence_currval( + &self, + _database_id: nodedb_types::DatabaseId, + _tenant_id: u64, + name: &str, + ) -> Result { + self.session_sequence_value(name) + } + + fn sequence_setval( + &self, + _database_id: nodedb_types::DatabaseId, + _tenant_id: u64, + name: &str, + value: i64, + ) -> Result { + self.position_sequence(name, value) + } + fn resolve_regtype(&self, name: &str) -> Option { crate::control::server::pgwire::catalog::tables::pg_type::type_oid_map() .get(name) diff --git a/nodedb/src/control/planner/context/query/context.rs b/nodedb/src/control/planner/context/query/context.rs index a7cee59a2..c9f96e67f 100644 --- a/nodedb/src/control/planner/context/query/context.rs +++ b/nodedb/src/control/planner/context/query/context.rs @@ -98,6 +98,13 @@ pub struct QueryContext { /// mirrors `broadcast_threshold_bytes` so `&self` plan calls read it without /// an exclusive borrow. pub(super) shuffle_agg_threshold: std::sync::atomic::AtomicUsize, + /// The calling connection's `currval` map, forwarded into the catalog + /// adapter each plan call. Written per request by + /// `apply_planning_session_overrides`, exactly like the knobs above. + /// `None` for planning with no session behind it, which then reports + /// that sequence access is unavailable rather than crossing sessions. + pub(super) session_sequences: + std::sync::Mutex>>, } impl QueryContext { @@ -122,6 +129,7 @@ impl QueryContext { shuffle_agg_threshold: std::sync::atomic::AtomicUsize::new( super::tuning::DEFAULT_SHUFFLE_AGG_THRESHOLD, ), + session_sequences: std::sync::Mutex::new(None), } } @@ -191,6 +199,7 @@ impl QueryContext { shuffle_agg_threshold: std::sync::atomic::AtomicUsize::new( super::tuning::DEFAULT_SHUFFLE_AGG_THRESHOLD, ), + session_sequences: std::sync::Mutex::new(None), } } @@ -228,10 +237,35 @@ impl QueryContext { shuffle_agg_threshold: std::sync::atomic::AtomicUsize::new( super::tuning::DEFAULT_SHUFFLE_AGG_THRESHOLD, ), + session_sequences: std::sync::Mutex::new(None), } } } +impl QueryContext { + /// Bind the calling connection's `currval` map for the next plan call. + pub fn set_session_sequences( + &self, + values: Option>, + ) { + let mut slot = self + .session_sequences + .lock() + .unwrap_or_else(|p| p.into_inner()); + *slot = values; + } + + /// The `currval` map bound for the current plan call. + pub(super) fn session_sequences( + &self, + ) -> Option> { + self.session_sequences + .lock() + .unwrap_or_else(|p| p.into_inner()) + .clone() + } +} + /// The node-default broadcast threshold (bytes) for fixtures that have no /// `SharedState` tuning to read. Sourced from `ClusterTransportTuning::default()` /// so the planner default and the config default never drift. diff --git a/nodedb/src/control/planner/context/query/planning.rs b/nodedb/src/control/planner/context/query/planning.rs index 8a845dcd7..76e2630ad 100644 --- a/nodedb/src/control/planner/context/query/planning.rs +++ b/nodedb/src/control/planner/context/query/planning.rs @@ -42,6 +42,12 @@ fn map_plan_error(error: nodedb_sql::SqlError, tenant_id: crate::types::TenantId nodedb_sql::SqlError::UndefinedFunction { name } => { crate::Error::UndefinedFunction { name } } + nodedb_sql::SqlError::UndefinedObject { kind, name } => { + crate::Error::UndefinedObject { kind, name } + } + nodedb_sql::SqlError::ObjectNotInPrerequisiteState { object, detail } => { + crate::Error::ObjectNotInPrerequisiteState { object, detail } + } // A constant expression that divides by zero is the same condition the // row-scope evaluator raises, so it carries the same code. nodedb_sql::SqlError::DivisionByZero => crate::Error::DivisionByZero, @@ -128,6 +134,9 @@ impl QueryContext { } else { inputs.build_adapter(tenant_id.as_u64(), database_id) }; + // `nextval` records into the calling session's map and `currval` reads + // only from it, so the adapter must know which session is planning. + let catalog = catalog.with_session_sequences(self.session_sequences()); let plans = nodedb_sql::plan_sql(sql, &catalog).map_err(|e| map_plan_error(e, tenant_id))?; // Fold catalog-dependent cast expressions (::regclass, ::regtype) to @@ -189,14 +198,8 @@ impl QueryContext { &catalog, database_id, ); - let cache_eligibility = if plans - .iter() - .all(|plan| plan.cache_eligibility().is_cacheable()) - { - nodedb_sql::types::PlanCacheEligibility::Cacheable - } else { - nodedb_sql::types::PlanCacheEligibility::DataDependent - }; + let cache_eligibility = + crate::control::planner::sql_plan_convert::batch_cache_eligibility(&plans); let tasks = crate::control::planner::sql_plan_convert::convert(&plans, tenant_id, &ctx)?; Ok((tasks, output_schema, version_set, cache_eligibility)) } diff --git a/nodedb/src/control/planner/sql_plan_convert/cache_verdict.rs b/nodedb/src/control/planner/sql_plan_convert/cache_verdict.rs new file mode 100644 index 000000000..92e94a8c9 --- /dev/null +++ b/nodedb/src/control/planner/sql_plan_convert/cache_verdict.rs @@ -0,0 +1,21 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! Plan-cache verdict for a batch of logical plans. + +use nodedb_sql::types::{PlanCacheEligibility, SqlPlan}; + +/// Fold one statement batch into a single plan-cache verdict. +/// +/// The batch is cacheable only when every plan in it is cacheable. +/// `SqlPlan::cache_eligibility` is the one source of truth for volatile calls +/// and for row identity bound while lowering. Never derive a second verdict. +pub fn batch_cache_eligibility(plans: &[SqlPlan]) -> PlanCacheEligibility { + if plans + .iter() + .all(|plan| plan.cache_eligibility().is_cacheable()) + { + PlanCacheEligibility::Cacheable + } else { + PlanCacheEligibility::DataDependent + } +} diff --git a/nodedb/src/control/planner/sql_plan_convert/mod.rs b/nodedb/src/control/planner/sql_plan_convert/mod.rs index bdfb309b6..ba341a353 100644 --- a/nodedb/src/control/planner/sql_plan_convert/mod.rs +++ b/nodedb/src/control/planner/sql_plan_convert/mod.rs @@ -4,6 +4,7 @@ pub mod aggregate; pub mod array_alter_convert; pub mod array_convert; pub mod array_fn_convert; +pub mod cache_verdict; pub mod convert; pub mod dml; pub mod expr; @@ -19,4 +20,5 @@ pub mod set_ops; pub mod value; pub mod visitor; +pub use cache_verdict::batch_cache_eligibility; pub use convert::{ConvertContext, PlanningPurpose, convert}; diff --git a/nodedb/src/control/planner/sql_plan_convert/output_schema.rs b/nodedb/src/control/planner/sql_plan_convert/output_schema.rs index 4e5df4633..d42d9e6f6 100644 --- a/nodedb/src/control/planner/sql_plan_convert/output_schema.rs +++ b/nodedb/src/control/planner/sql_plan_convert/output_schema.rs @@ -568,6 +568,7 @@ mod tests { let plans = vec![SqlPlan::ConstantResult { columns: vec!["a".to_string(), "b".to_string()], values: vec![], + volatile: false, }]; let schema = build_output_schema(&plans, &NoCatalog, nodedb_types::DatabaseId::DEFAULT); assert_eq!(schema.columns.len(), 2); diff --git a/nodedb/src/control/sequence/mod.rs b/nodedb/src/control/sequence/mod.rs index a4f7525a2..16ba6f427 100644 --- a/nodedb/src/control/sequence/mod.rs +++ b/nodedb/src/control/sequence/mod.rs @@ -6,10 +6,12 @@ pub mod gap_free; pub mod log; pub mod range_alloc; pub mod registry; +pub mod session_values; pub mod types; pub use self::format::{FormatToken, ResetScope}; pub use self::gap_free::GapFreeManager; pub use self::range_alloc::RangeAllocator; pub use self::registry::SequenceRegistry; +pub use self::session_values::SessionSequenceValues; pub use self::types::SequenceError; diff --git a/nodedb/src/control/sequence/range_alloc.rs b/nodedb/src/control/sequence/range_alloc.rs index abff0d486..f469eacc9 100644 --- a/nodedb/src/control/sequence/range_alloc.rs +++ b/nodedb/src/control/sequence/range_alloc.rs @@ -141,7 +141,7 @@ impl RangeAllocator { let current_val = state .sequence_registry - .currval(database_id, tenant_id, sequence_name) + .node_current_value(database_id, tenant_id, sequence_name) .unwrap_or(0); let range_start = current_val + increment; diff --git a/nodedb/src/control/sequence/registry.rs b/nodedb/src/control/sequence/registry.rs index 8edb090b3..3b4b7d4a2 100644 --- a/nodedb/src/control/sequence/registry.rs +++ b/nodedb/src/control/sequence/registry.rs @@ -239,8 +239,10 @@ impl SequenceRegistry { handle.nextval_batch(n) } - /// Get the current value (last nextval result on this node). - pub fn currval( + /// Read this NODE's counter — the last value any session on this node + /// allocated. SQL `currval` is session-scoped and does not read this; + /// range allocation and diagnostics do. + pub fn node_current_value( &self, database_id: u64, tenant_id: u64, @@ -419,7 +421,7 @@ pub enum SequenceValue { Formatted(String), } -fn registry_key(database_id: u64, tenant_id: u64, name: &str) -> String { +pub(crate) fn registry_key(database_id: u64, tenant_id: u64, name: &str) -> String { format!("{database_id}:{tenant_id}:{name}") } @@ -452,9 +454,9 @@ mod tests { ); assert_eq!(registry.nextval(4, 1, "orders_seq").unwrap(), 1); assert_eq!(registry.nextval(4, 1, "orders_seq").unwrap(), 2); - assert_eq!(registry.currval(4, 1, "orders_seq").unwrap(), 2); + assert_eq!(registry.node_current_value(4, 1, "orders_seq").unwrap(), 2); assert_eq!(registry.setval(4, 1, "orders_seq", 10).unwrap(), 10); - assert_eq!(registry.currval(4, 1, "orders_seq").unwrap(), 10); + assert_eq!(registry.node_current_value(4, 1, "orders_seq").unwrap(), 10); }) .await; } diff --git a/nodedb/src/control/sequence/session_values.rs b/nodedb/src/control/sequence/session_values.rs new file mode 100644 index 000000000..09c738a4b --- /dev/null +++ b/nodedb/src/control/sequence/session_values.rs @@ -0,0 +1,52 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! Per-session record of the last `nextval` each sequence handed this session. +//! +//! SQL `currval` returns the last value THIS SESSION obtained from `nextval`, +//! never the node-wide counter that `SequenceRegistry::node_current_value` +//! reads. Keys match `SequenceRegistry`'s: `"{database_id}:{tenant_id}:{name}"`. + +use std::collections::HashMap; +use std::sync::Mutex; + +use super::registry::registry_key; + +/// One connection's `currval` state, shared with the plan-time catalog adapter. +#[derive(Default)] +pub struct SessionSequenceValues { + values: Mutex>, +} + +impl SessionSequenceValues { + pub fn new() -> Self { + Self::default() + } + + /// Record the value `nextval` just handed this session. + pub fn record(&self, database_id: u64, tenant_id: u64, name: &str, value: i64) { + let key = registry_key(database_id, tenant_id, name); + self.values + .lock() + .unwrap_or_else(|p| p.into_inner()) + .insert(key, value); + } + + /// The last value this session obtained for `name`, or `None` when this + /// session never called `nextval` on it. + pub fn last(&self, database_id: u64, tenant_id: u64, name: &str) -> Option { + let key = registry_key(database_id, tenant_id, name); + self.values + .lock() + .unwrap_or_else(|p| p.into_inner()) + .get(&key) + .copied() + } + + /// Drop every recorded value. Called on a full session reset. + pub fn clear(&self) { + self.values + .lock() + .unwrap_or_else(|p| p.into_inner()) + .clear(); + } +} diff --git a/nodedb/src/control/server/pgwire/types/error_map.rs b/nodedb/src/control/server/pgwire/types/error_map.rs index d4299d39c..3cd3fb262 100644 --- a/nodedb/src/control/server/pgwire/types/error_map.rs +++ b/nodedb/src/control/server/pgwire/types/error_map.rs @@ -50,6 +50,14 @@ pub fn error_to_sqlstate(err: &crate::Error) -> (&'static str, &'static str, Str sqlstate::UNDEFINED_FUNCTION, format!("function {name}(...) does not exist"), ), + crate::Error::UndefinedObject { .. } => { + ("ERROR", sqlstate::UNDEFINED_OBJECT, err.to_string()) + } + crate::Error::ObjectNotInPrerequisiteState { detail, .. } => ( + "ERROR", + sqlstate::OBJECT_NOT_IN_PREREQUISITE_STATE, + detail.clone(), + ), crate::Error::UndefinedColumn { column } => ( "ERROR", sqlstate::UNDEFINED_COLUMN, diff --git a/nodedb/src/control/server/shared/planning_overrides.rs b/nodedb/src/control/server/shared/planning_overrides.rs index 1ccef2e6c..e88d2e282 100644 --- a/nodedb/src/control/server/shared/planning_overrides.rs +++ b/nodedb/src/control/server/shared/planning_overrides.rs @@ -81,6 +81,10 @@ pub fn apply_planning_session_overrides( query_ctx.set_max_vector_dim(tenants.quota(tenant_id).max_vector_dim); } + // Sequence accessors resolve at plan time, so the planner needs this + // connection's `currval` map before it plans. + query_ctx.set_session_sequences(sessions.sequence_values(session_id)); + // Distributed shuffle-join override (`SET nodedb.force_shuffle_join = on` // and, optionally, `SET nodedb.shuffle_num_parts = N`). let force_shuffle_join = sessions diff --git a/nodedb/src/control/server/shared/session/mod.rs b/nodedb/src/control/server/shared/session/mod.rs index 17d3b93c2..42899b0bd 100644 --- a/nodedb/src/control/server/shared/session/mod.rs +++ b/nodedb/src/control/server/shared/session/mod.rs @@ -30,6 +30,7 @@ pub mod read_set; pub mod record_reads; mod reservation_release; pub mod savepoint_ops; +mod sequence_values; pub mod set_validation; pub mod staging_gate; mod state; diff --git a/nodedb/src/control/server/shared/session/sequence_values.rs b/nodedb/src/control/server/shared/session/sequence_values.rs new file mode 100644 index 000000000..f398ab991 --- /dev/null +++ b/nodedb/src/control/server/shared/session/sequence_values.rs @@ -0,0 +1,18 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! Session access to the per-connection `currval` map. + +use std::sync::Arc; + +use crate::control::sequence::SessionSequenceValues; + +use super::connection::SessionId; +use super::store::SessionStore; + +impl SessionStore { + /// The `currval` map of one connection. `None` when no session is + /// registered under `id` — a caller with no session gets no `currval`. + pub fn sequence_values(&self, id: impl Into) -> Option> { + self.read_session(id, |session| Arc::clone(&session.sequence_values)) + } +} diff --git a/nodedb/src/control/server/shared/session/state.rs b/nodedb/src/control/server/shared/session/state.rs index bdcd29837..2fe1834b8 100644 --- a/nodedb/src/control/server/shared/session/state.rs +++ b/nodedb/src/control/server/shared/session/state.rs @@ -215,6 +215,10 @@ pub struct ConnSession { /// GAP_FREE sequence reservations pending commit/rollback. /// On COMMIT: each reservation is finalized. On ROLLBACK: counter decremented. pub pending_sequence_reservations: Vec, + /// Last `nextval` value this session obtained per sequence — what SQL + /// `currval` returns. Shared with the plan-time catalog adapter, which + /// writes it on `nextval` and reads it on `currval`. + pub sequence_values: Arc, /// Millis-since-epoch of the last statement COMPLETION on this connection /// (also set to "now" at connection start). Read by the pgwire listener /// watchdog to decide idle eligibility: a connection is idle only when it @@ -321,6 +325,7 @@ impl ConnSession { temp_tables: super::temp_tables::TempTableRegistry::new(), plan_cache: crate::control::server::shared::session::plan_cache::PlanCache::new(128), pending_sequence_reservations: Vec::new(), + sequence_values: Arc::new(crate::control::sequence::SessionSequenceValues::new()), last_activity_ms: AtomicU64::new(now_unix_ms()), in_flight: AtomicU32::new(0), own_write_versions: HashMap::new(), diff --git a/nodedb/src/error/types.rs b/nodedb/src/error/types.rs index ae23847b2..e15239bb8 100644 --- a/nodedb/src/error/types.rs +++ b/nodedb/src/error/types.rs @@ -282,6 +282,19 @@ pub enum Error { #[error("function {name}(...) does not exist")] UndefinedFunction { name: String }, + /// A statement named a database object that does not exist — a sequence, + /// most commonly. Propagated from `SqlError::UndefinedObject`; the pgwire + /// layer renders this as SQLSTATE `42704` (undefined_object). + #[error("{kind} \"{name}\" does not exist")] + UndefinedObject { kind: &'static str, name: String }, + + /// An object exists but a prerequisite step has not run, such as `currval` + /// before this session called `nextval`. Propagated from + /// `SqlError::ObjectNotInPrerequisiteState`; the pgwire layer renders this + /// as SQLSTATE `55000` (object_not_in_prerequisite_state). + #[error("{detail}")] + ObjectNotInPrerequisiteState { object: String, detail: String }, + /// A column reference resolved against no relation, output alias, or /// synthetic column in scope. Propagated from `SqlError::UnknownColumn`; /// the pgwire layer renders it as SQLSTATE `42703` (undefined_column). diff --git a/nodedb/src/error_classify.rs b/nodedb/src/error_classify.rs index 3a853e9a1..9607ee789 100644 --- a/nodedb/src/error_classify.rs +++ b/nodedb/src/error_classify.rs @@ -150,6 +150,12 @@ pub(crate) fn classify(e: &Error) -> NodeDbError { } Error::PlanError { detail } => NodeDbError::plan_error(detail), Error::UndefinedFunction { name } => NodeDbError::undefined_function(name.clone()), + Error::UndefinedObject { kind, name } => { + NodeDbError::undefined_object(format!("{kind} \"{name}\"")) + } + Error::ObjectNotInPrerequisiteState { object, detail } => { + NodeDbError::object_not_ready(object.clone(), detail.clone()) + } Error::UndefinedColumn { column } => NodeDbError::undefined_column(column.clone()), Error::AmbiguousColumn { column } => NodeDbError::ambiguous_column(column.clone()), Error::UnknownStrictField { column, .. } => NodeDbError::undefined_column(column.clone()), From 33987ebf6b491b2aa5de3c3152554a9ea30ebdb5 Mon Sep 17 00:00:00 2001 From: Farhan Syah Date: Thu, 10 Sep 2026 00:11:38 +0800 Subject: [PATCH 05/23] fix(sql): evaluate sequence-backed column DEFAULTs at plan time nextval/currval DEFAULTs now resolve through the catalog instead of being silently dropped. evaluate_default_expr and materialize_row_defaults require a SqlCatalog handle, and ConvertContext carries an optional Arc so INSERT and UPSERT conversion can reach it after planning returns. Consolidate the four drifted copies of the SqlError -> Control-Plane Error mapping (only one mapped RetryableSchemaChanged) into a single plan_error_map module shared by query planning and DEFAULT materialization. Extract engine routing for row-shaped INSERT/UPSERT converters into a dedicated route module, and DML helper parameters into a params module. --- nodedb-sql/src/error.rs | 19 ++ nodedb-sql/src/planner/catalog_expr_fold.rs | 2 +- nodedb-sql/src/planner/defaults.rs | 197 ++++++++++++++---- nodedb-sql/src/planner/dml.rs | 54 ++--- .../src/planner/dml_helpers/kv_insert.rs | 111 ++++------ nodedb-sql/src/planner/dml_helpers/mod.rs | 3 + nodedb-sql/src/planner/dml_helpers/params.rs | 27 +++ .../control/planner/context/query/planning.rs | 86 ++------ nodedb/src/control/planner/mod.rs | 1 + nodedb/src/control/planner/plan_error_map.rs | 63 ++++++ .../array_fn_convert/aggregate.rs | 1 + .../array_fn_convert/slice.rs | 1 + .../planner/sql_plan_convert/convert.rs | 23 ++ .../planner/sql_plan_convert/dml/insert.rs | 74 ++----- .../sql_plan_convert/dml/kv_and_vector.rs | 1 + .../planner/sql_plan_convert/dml/mod.rs | 1 + .../planner/sql_plan_convert/dml/route.rs | 55 +++++ .../dml/update_delete/delete.rs | 1 + .../dml/update_delete/update.rs | 1 + .../planner/sql_plan_convert/dml/upsert.rs | 30 +-- .../planner/sql_plan_convert/set_ops.rs | 2 + .../sql_plan_convert/value/defaults.rs | 41 ++++ .../planner/sql_plan_convert/value/mod.rs | 10 +- .../planner/sql_plan_convert/value/rows.rs | 20 +- .../executor_tests/test_group_by_alias.rs | 1 + 25 files changed, 535 insertions(+), 290 deletions(-) create mode 100644 nodedb-sql/src/planner/dml_helpers/params.rs create mode 100644 nodedb/src/control/planner/plan_error_map.rs create mode 100644 nodedb/src/control/planner/sql_plan_convert/dml/route.rs create mode 100644 nodedb/src/control/planner/sql_plan_convert/value/defaults.rs diff --git a/nodedb-sql/src/error.rs b/nodedb-sql/src/error.rs index d38424c0e..7cad3031a 100644 --- a/nodedb-sql/src/error.rs +++ b/nodedb-sql/src/error.rs @@ -98,6 +98,25 @@ pub enum SqlError { #[error("unsupported: {detail}")] Unsupported { detail: String }, + /// A declared column DEFAULT the server cannot evaluate to a value. + /// + /// The column is never omitted instead. A DEFAULT that disappears stores + /// NULL where the declaration promised a value, and nothing reports it. + #[error("DEFAULT for column '{column}' cannot be evaluated: {expr}")] + UnevaluableDefault { column: String, expr: String }, + + /// `setval` appeared inside a column DEFAULT. + /// + /// A DEFAULT runs once per row, so evaluating `setval` there will move the + /// sequence's position on every inserted row. PostgreSQL reports a + /// function used in a context that forbids it as SQLSTATE `42601` + /// (`syntax_error`), and this refusal carries the same code. + #[error( + "setval() is not allowed in the DEFAULT for column '{column}'; \ + a DEFAULT must not move a sequence's position" + )] + SetvalInColumnDefault { column: String }, + #[error("invalid function call: {detail}")] InvalidFunction { detail: String }, diff --git a/nodedb-sql/src/planner/catalog_expr_fold.rs b/nodedb-sql/src/planner/catalog_expr_fold.rs index 97e92ddb5..4c5562c52 100644 --- a/nodedb-sql/src/planner/catalog_expr_fold.rs +++ b/nodedb-sql/src/planner/catalog_expr_fold.rs @@ -47,7 +47,7 @@ pub(super) fn eval_catalog_constant( /// Returns `Ok(None)` for every other expression. These calls are `Volatile`, /// so the constant folder never reaches them and the plan holding the result /// is never cached — each execution re-plans and allocates again. -fn eval_sequence_accessor( +pub(super) fn eval_sequence_accessor( expr: &SqlExpr, catalog: &dyn SqlCatalog, ) -> crate::Result> { diff --git a/nodedb-sql/src/planner/defaults.rs b/nodedb-sql/src/planner/defaults.rs index d17a66095..389e34a9d 100644 --- a/nodedb-sql/src/planner/defaults.rs +++ b/nodedb-sql/src/planner/defaults.rs @@ -3,8 +3,12 @@ //! Column DEFAULT expression evaluation at insert time. //! //! Supports ID generation functions (UUIDv4/v7, ULID, CUID2, NANOID), `NOW()`, -//! and literal values. More complex defaults (arbitrary expressions) fall -//! through to the plan-time const-folder. +//! sequence accessors (`nextval`, `currval`), and literal values. Anything +//! else routes through the plan-time const-folder. +//! +//! A DEFAULT that cannot be evaluated raises [`SqlError::UnevaluableDefault`]. +//! The column is never omitted: an omitted column stores NULL where the +//! declaration promised a value, and nothing reports it. //! //! Lives in the SQL crate rather than beside one engine's converter because //! every engine that materializes a DEFAULT has to produce the SAME value for @@ -14,91 +18,106 @@ //! coercion and range checks run, so a materialized default is validated //! exactly like a supplied one. -use nodedb_types::NodeDbError; +use crate::catalog::SqlCatalog; +use crate::error::SqlError; +use crate::types::{SqlExpr, SqlValue}; -pub fn evaluate_default_expr(expr: &str) -> Result, NodeDbError> { +/// Evaluate `expr`, the DEFAULT declared on `column`, to one value. +/// +/// `catalog` resolves the sequence accessors `nextval` and `currval`. It is +/// required rather than optional: a catalog-free evaluator silently dropped +/// every sequence-backed DEFAULT. +/// +/// The absence of a DEFAULT is the caller's `ColumnInfo::default` being `None` +/// and never reaches here. Every call therefore either yields a value or +/// raises. +pub fn evaluate_default_expr( + expr: &str, + column: &str, + catalog: &dyn SqlCatalog, +) -> crate::Result { let upper = expr.trim().to_uppercase(); match upper.as_str() { - "UUID_V7" | "UUIDV7" | "GEN_UUID_V7()" | "UUID_V7()" => Ok(Some( - nodedb_types::Value::String(nodedb_types::id_gen::uuid_v7()), - )), - "UUID_V4" | "UUIDV4" | "UUID" | "GEN_UUID_V4()" | "UUID_V4()" => Ok(Some( - nodedb_types::Value::String(nodedb_types::id_gen::uuid_v4()), - )), - "ULID" | "GEN_ULID()" | "ULID()" => Ok(Some(nodedb_types::Value::String( - nodedb_types::id_gen::ulid(), - ))), - "CUID2" | "CUID2()" => Ok(Some(nodedb_types::Value::String( - nodedb_types::id_gen::cuid2(), - ))), - "NANOID" | "NANOID()" => Ok(Some(nodedb_types::Value::String( - nodedb_types::id_gen::nanoid(), - ))), + "UUID_V7" | "UUIDV7" | "GEN_UUID_V7()" | "UUID_V7()" => { + Ok(nodedb_types::Value::String(nodedb_types::id_gen::uuid_v7())) + } + "UUID_V4" | "UUIDV4" | "UUID" | "GEN_UUID_V4()" | "UUID_V4()" => { + Ok(nodedb_types::Value::String(nodedb_types::id_gen::uuid_v4())) + } + "ULID" | "GEN_ULID()" | "ULID()" => { + Ok(nodedb_types::Value::String(nodedb_types::id_gen::ulid())) + } + "CUID2" | "CUID2()" => Ok(nodedb_types::Value::String(nodedb_types::id_gen::cuid2())), + "NANOID" | "NANOID()" => Ok(nodedb_types::Value::String(nodedb_types::id_gen::nanoid())), "NOW()" => { let now = std::time::SystemTime::now() .duration_since(std::time::UNIX_EPOCH) .unwrap_or_default(); - Ok(Some(nodedb_types::Value::String( + Ok(nodedb_types::Value::String( chrono::DateTime::from_timestamp_millis(now.as_millis() as i64) .map(|dt| dt.to_rfc3339()) .unwrap_or_else(|| now.as_millis().to_string()), - ))) + )) } - _ => parse_parametric_or_literal(expr, &upper), + _ => parse_parametric_or_literal(expr, &upper, column, catalog), } } fn parse_parametric_or_literal( expr: &str, upper: &str, -) -> Result, NodeDbError> { + column: &str, + catalog: &dyn SqlCatalog, +) -> crate::Result { // NANOID(N) — custom length. if upper.starts_with("NANOID(") && upper.ends_with(')') { let len_str = &upper[7..upper.len() - 1]; if let Ok(len) = len_str.parse::() { - return Ok(Some(nodedb_types::Value::String( + return Ok(nodedb_types::Value::String( nodedb_types::id_gen::nanoid_with_length(len), - ))); + )); } } // CUID2(N) — custom length; validates length range and surfaces planning errors. if upper.starts_with("CUID2(") && upper.ends_with(')') { let len_str = &upper[6..upper.len() - 1]; if let Ok(len) = len_str.parse::() { - let id = nodedb_types::id_gen::cuid2_with_length(len).map_err(|e| { - NodeDbError::plan_error_at( - "defaults", - format!("CUID2({len}) default expression is invalid: {e}"), - ) + let id = nodedb_types::id_gen::cuid2_with_length(len).map_err(|e| SqlError::Parse { + detail: format!("CUID2({len}) default expression is invalid: {e}"), })?; - return Ok(Some(nodedb_types::Value::String(id))); + return Ok(nodedb_types::Value::String(id)); } } // Numeric literal. if let Ok(i) = expr.trim().parse::() { - return Ok(Some(nodedb_types::Value::Integer(i))); + return Ok(nodedb_types::Value::Integer(i)); } if let Ok(f) = expr.trim().parse::() { - return Ok(Some(nodedb_types::Value::Float(f))); + return Ok(nodedb_types::Value::Float(f)); } // Quoted string literal. let trimmed = expr.trim(); if (trimmed.starts_with('\'') && trimmed.ends_with('\'')) || (trimmed.starts_with('"') && trimmed.ends_with('"')) { - return Ok(Some(nodedb_types::Value::String( + return Ok(nodedb_types::Value::String( trimmed[1..trimmed.len() - 1].to_string(), - ))); + )); } - // Fallback: try the plan-time const-folder for arbitrary expressions - // (e.g. `upper('x')`, `1 + 2`, `concat('a', 'b')`). - Ok(try_const_fold_default(expr)) + evaluate_parsed_default(expr, column, catalog) } -/// Attempt to parse the DEFAULT expression as SQL, then const-fold it. -fn try_const_fold_default(expr: &str) -> Option { - let sql_expr = crate::parse_expr_string(expr).ok()?; +/// Parse the DEFAULT as SQL, then resolve it against the catalog or the folder. +fn evaluate_parsed_default( + expr: &str, + column: &str, + catalog: &dyn SqlCatalog, +) -> crate::Result { + let sql_expr = crate::parse_expr_string(expr).map_err(|_| unevaluable(column, expr))?; + if let Some(value) = evaluate_sequence_default(&sql_expr, column, catalog)? { + return Ok(sql_value_to_ndb(value)); + } // `Once`: a materialized DEFAULT serves this insert only, and an INSERT // plan carrying a volatile DEFAULT is never admitted to the plan cache. let folded = crate::planner::const_fold::fold_constant_scoped( @@ -106,12 +125,37 @@ fn try_const_fold_default(expr: &str) -> Option { crate::planner::const_fold::default_registry(), crate::planner::const_fold::FoldScope::Once, ) - .ok()??; - Some(sql_value_to_ndb(folded)) + .map_err(|_| unevaluable(column, expr))? + .ok_or_else(|| unevaluable(column, expr))?; + Ok(sql_value_to_ndb(folded)) +} + +/// Resolve `nextval` / `currval` through the catalog; refuse `setval`. +/// +/// Returns `Ok(None)` for every other expression, leaving it to the folder. +fn evaluate_sequence_default( + expr: &SqlExpr, + column: &str, + catalog: &dyn SqlCatalog, +) -> crate::Result> { + if let SqlExpr::Function { name, .. } = expr + && name.eq_ignore_ascii_case("setval") + { + return Err(SqlError::SetvalInColumnDefault { + column: column.to_string(), + }); + } + super::catalog_expr_fold::eval_sequence_accessor(expr, catalog) } -fn sql_value_to_ndb(v: crate::types::SqlValue) -> nodedb_types::Value { - use crate::types::SqlValue; +fn unevaluable(column: &str, expr: &str) -> SqlError { + SqlError::UnevaluableDefault { + column: column.to_string(), + expr: expr.to_string(), + } +} + +fn sql_value_to_ndb(v: SqlValue) -> nodedb_types::Value { match v { SqlValue::Null => nodedb_types::Value::Null, SqlValue::Bool(b) => nodedb_types::Value::Bool(b), @@ -127,3 +171,66 @@ fn sql_value_to_ndb(v: crate::types::SqlValue) -> nodedb_types::Value { SqlValue::Timestamptz(dt) => nodedb_types::Value::DateTime(dt), } } + +/// Convert an evaluated default back into the planner's literal type. +/// +/// The inverse of `sql_value_to_ndb` above, which is the only producer of +/// these values — so every shape the evaluator can emit has an exact +/// counterpart here. Anything else raises rather than rendering through +/// `Debug`: a `DEFAULT` that stored `Uuid("…")` as its own debug text is the +/// same class of defect as dropping it, and harder to notice because the +/// column looks populated. +pub fn default_value_to_sql(column: &str, value: nodedb_types::Value) -> crate::Result { + Ok(match value { + nodedb_types::Value::Null => SqlValue::Null, + nodedb_types::Value::Bool(b) => SqlValue::Bool(b), + nodedb_types::Value::Integer(i) => SqlValue::Int(i), + nodedb_types::Value::Float(f) => SqlValue::Float(f), + nodedb_types::Value::Decimal(d) => SqlValue::Decimal(d), + nodedb_types::Value::String(s) => SqlValue::String(s), + nodedb_types::Value::Bytes(b) => SqlValue::Bytes(b), + nodedb_types::Value::NaiveDateTime(dt) => SqlValue::Timestamp(dt), + nodedb_types::Value::DateTime(dt) => SqlValue::Timestamptz(dt), + nodedb_types::Value::Array(items) => SqlValue::Array( + items + .into_iter() + .map(|item| default_value_to_sql(column, item)) + .collect::>>()?, + ), + other => { + return Err(SqlError::Unsupported { + detail: format!( + "default for column '{column}' evaluates to a value with no SQL literal \ + form: {other:?}" + ), + }); + } + }) +} + +/// Fill in every column of `row` that declares a DEFAULT and the statement omitted. +/// +/// `column_defaults` is the catalog's `(column_name, default_expr)` list. +/// Each entry evaluates at most once per row, so a `nextval` DEFAULT allocates +/// exactly one value per row. +/// +/// A column the statement supplied stays untouched, an explicit `NULL` +/// included: `NULL` is a value the author chose, and overwriting it with the +/// default makes storing one impossible. +/// +/// A DEFAULT the evaluator cannot resolve raises +/// [`SqlError::UnevaluableDefault`] rather than leaving the column out. +pub fn materialize_row_defaults( + row: &mut Vec<(String, SqlValue)>, + column_defaults: &[(String, String)], + catalog: &dyn SqlCatalog, +) -> crate::Result<()> { + for (column, default_expr) in column_defaults { + if row.iter().any(|(name, _)| name == column) { + continue; + } + let evaluated = evaluate_default_expr(default_expr, column, catalog)?; + row.push((column.clone(), default_value_to_sql(column, evaluated)?)); + } + Ok(()) +} diff --git a/nodedb-sql/src/planner/dml.rs b/nodedb-sql/src/planner/dml.rs index 5ee92a914..d5b5c769e 100644 --- a/nodedb-sql/src/planner/dml.rs +++ b/nodedb-sql/src/planner/dml.rs @@ -6,9 +6,10 @@ use nodedb_types::DatabaseId; use sqlparser::ast::{self}; use super::dml_helpers::{ - bind_insert_select_columns, build_kv_insert_plan, build_vector_primary_insert_plan, - check_declared_float_ranges_in_assignments, check_declared_int_ranges_in_assignments, - coerce_and_check_rows, convert_value_rows, resolve_insert_columns, + KvInsertParams, bind_insert_select_columns, build_kv_insert_plan, + build_vector_primary_insert_plan, check_declared_float_ranges_in_assignments, + check_declared_int_ranges_in_assignments, coerce_and_check_rows, convert_value_rows, + resolve_insert_columns, }; use crate::engine_rules::{self, InsertParams}; use crate::error::{Result, SqlError}; @@ -177,15 +178,16 @@ pub fn plan_insert(ins: &ast::Insert, catalog: &dyn SqlCatalog) -> Result Result>], - intent: KvInsertIntent, - mut on_conflict_updates: Vec<(String, SqlExpr)>, - pk_col: Option<&str>, - declared_columns: &[ColumnInfo], -) -> Result> { +pub(crate) fn build_kv_insert_plan(params: KvInsertParams<'_>) -> Result> { + let KvInsertParams { + collection: table_name, + columns, + rows_ast, + intent, + mut on_conflict_updates, + pk_col, + declared_columns, + catalog, + } = params; // Positional KV insert (no column list): the key/value split below is // driven entirely by matching column *names* against `key_col_name`/ // `"ttl"`. With an empty `columns` list there is no key to bind to, so @@ -78,7 +80,7 @@ pub(crate) fn build_kv_insert_plan( let Some(expr) = row_exprs.get(i) else { break }; row.push((col.clone(), expr_to_sql_value(expr)?)); } - volatile_defaults |= materialize_declared_defaults(declared_columns, &mut row)?; + volatile_defaults |= materialize_declared_defaults(declared_columns, &mut row, catalog)?; // The key column is exempt — see `coerce_rows_to_declared_types`. coerce_row_to_declared_types(declared_columns, &mut row, Some(key_col_name))?; coerced_rows.push(row); @@ -154,11 +156,15 @@ pub(crate) fn build_kv_insert_plan( /// on a `SMALLINT` column a way to store a value the same literal is /// rejected for. /// +/// A DEFAULT the evaluator cannot resolve raises `SqlError::UnevaluableDefault` +/// rather than leaving the column out. `catalog` resolves `nextval` / `currval`. +/// /// Returns whether any materialized default came from a `Volatile` /// expression, so the caller can keep the plan out of the plan cache. fn materialize_declared_defaults( declared_columns: &[ColumnInfo], row: &mut Vec<(String, SqlValue)>, + catalog: &dyn SqlCatalog, ) -> Result { let mut volatile = false; for column in declared_columns { @@ -169,63 +175,37 @@ fn materialize_declared_defaults( continue; } let evaluated = - crate::planner::defaults::evaluate_default_expr(default_expr).map_err(|e| { - SqlError::Parse { - detail: format!("default for column '{}' is invalid: {e}", column.name), - } - })?; - let Some(evaluated) = evaluated else { continue }; - let value = nodedb_value_to_sql_value(&column.name, evaluated)?; + crate::planner::defaults::evaluate_default_expr(default_expr, &column.name, catalog)?; + let value = crate::planner::defaults::default_value_to_sql(&column.name, evaluated)?; volatile |= crate::types::plan::default_expr_is_volatile(default_expr); row.push((column.name.clone(), value)); } Ok(volatile) } -/// Convert an evaluated default back into the planner's literal type. -/// -/// The inverse of `sql_value_to_ndb` in `planner::defaults`, which is the only -/// producer of these values — so every shape the evaluator can emit has an -/// exact counterpart here. Anything else is rejected rather than rendered -/// through `Debug`: a `DEFAULT` that silently stored `Uuid("…")` as its own -/// debug text would be the same class of defect as dropping it entirely, but -/// harder to notice because the column would look populated. -fn nodedb_value_to_sql_value(column: &str, value: nodedb_types::Value) -> Result { - Ok(match value { - nodedb_types::Value::Null => SqlValue::Null, - nodedb_types::Value::Bool(b) => SqlValue::Bool(b), - nodedb_types::Value::Integer(i) => SqlValue::Int(i), - nodedb_types::Value::Float(f) => SqlValue::Float(f), - nodedb_types::Value::Decimal(d) => SqlValue::Decimal(d), - nodedb_types::Value::String(s) => SqlValue::String(s), - nodedb_types::Value::Bytes(b) => SqlValue::Bytes(b), - nodedb_types::Value::NaiveDateTime(dt) => SqlValue::Timestamp(dt), - nodedb_types::Value::DateTime(dt) => SqlValue::Timestamptz(dt), - nodedb_types::Value::Array(items) => SqlValue::Array( - items - .into_iter() - .map(|item| nodedb_value_to_sql_value(column, item)) - .collect::>>()?, - ), - other => { - return Err(SqlError::Unsupported { - detail: format!( - "default for column '{column}' evaluates to a value with no SQL literal \ - form: {other:?}" - ), - }); - } - }) -} - #[cfg(test)] mod kv_on_conflict_range_tests { + use sqlparser::ast; use sqlparser::ast::{Expr, Value, ValueWithSpan}; use sqlparser::tokenizer::Span; use super::*; use nodedb_types::columnar::{FloatWidth, IntWidth}; + /// A catalog with no collections and no sequence state. These cases + /// declare no DEFAULT, so no accessor is ever reached. + struct NoCatalog; + + impl SqlCatalog for NoCatalog { + fn get_collection( + &self, + _database_id: nodedb_types::DatabaseId, + _name: &str, + ) -> std::result::Result, crate::catalog::SqlCatalogError> { + Ok(None) + } + } + fn string_column(name: &str) -> ColumnInfo { ColumnInfo { name: name.to_string(), @@ -281,15 +261,16 @@ mod kv_on_conflict_range_tests { int_column("n", Some(IntWidth::I32)), float_column("r", Some(FloatWidth::F32)), ]; - build_kv_insert_plan( - "t".to_string(), - &["key".to_string()], - &[ast::Parens::with_empty_span(vec![key_value_expr("a")])], - KvInsertIntent::Put, - updates, - Some("key"), - &declared, - ) + build_kv_insert_plan(KvInsertParams { + collection: "t".to_string(), + columns: &["key".to_string()], + rows_ast: &[ast::Parens::with_empty_span(vec![key_value_expr("a")])], + intent: KvInsertIntent::Put, + on_conflict_updates: updates, + pk_col: Some("key"), + declared_columns: &declared, + catalog: &NoCatalog, + }) } #[test] diff --git a/nodedb-sql/src/planner/dml_helpers/mod.rs b/nodedb-sql/src/planner/dml_helpers/mod.rs index 40fdeb125..978eec9a5 100644 --- a/nodedb-sql/src/planner/dml_helpers/mod.rs +++ b/nodedb-sql/src/planner/dml_helpers/mod.rs @@ -8,11 +8,13 @@ //! - [`vector_primary_insert`] — vector-primary collection insert plans //! - [`kv_insert`] — KV engine insert plans //! - [`insert_select_bind`] — `INSERT ... SELECT` target-column binding +//! - [`params`] — parameter structs for the helpers above mod ast_extract; mod insert_columns; mod insert_select_bind; mod kv_insert; +mod params; mod range_check; mod value_convert; mod vector_primary_insert; @@ -22,6 +24,7 @@ pub(super) use ast_extract::extract_table_name_from_table_with_joins; pub(super) use insert_columns::resolve_insert_columns; pub(super) use insert_select_bind::bind_insert_select_columns; pub(super) use kv_insert::build_kv_insert_plan; +pub(super) use params::KvInsertParams; pub(super) use range_check::{ check_declared_float_ranges_in_assignments, check_declared_int_ranges_in_assignments, coerce_and_check_rows, diff --git a/nodedb-sql/src/planner/dml_helpers/params.rs b/nodedb-sql/src/planner/dml_helpers/params.rs new file mode 100644 index 000000000..db7a678e9 --- /dev/null +++ b/nodedb-sql/src/planner/dml_helpers/params.rs @@ -0,0 +1,27 @@ +// SPDX-License-Identifier: Apache-2.0 + +//! Parameter structs passed into DML planning helpers. + +use sqlparser::ast; + +use crate::catalog::SqlCatalog; +use crate::types::*; + +/// Parameters for [`super::build_kv_insert_plan`], shared by plain `INSERT`, +/// `UPSERT`, and `INSERT ... ON CONFLICT (key) DO UPDATE` against the KV +/// engine — the three paths differ only in `intent` and +/// `on_conflict_updates`. +pub(crate) struct KvInsertParams<'a> { + pub collection: String, + pub columns: &'a [String], + pub rows_ast: &'a [ast::Parens>], + pub intent: KvInsertIntent, + pub on_conflict_updates: Vec<(String, SqlExpr)>, + /// Schema-defined primary-key column name from `CollectionInfo::primary_key`. + /// When supplied, that column is used as the KV key regardless of + /// whether it is named `"key"`. Falls back to the literal name `"key"` + /// when `None` (legacy / generic KV collections). + pub pk_col: Option<&'a str>, + pub declared_columns: &'a [ColumnInfo], + pub catalog: &'a dyn SqlCatalog, +} diff --git a/nodedb/src/control/planner/context/query/planning.rs b/nodedb/src/control/planner/context/query/planning.rs index 76e2630ad..69a483b17 100644 --- a/nodedb/src/control/planner/context/query/planning.rs +++ b/nodedb/src/control/planner/context/query/planning.rs @@ -11,64 +11,10 @@ use std::sync::Arc; use super::QueryContext; use crate::control::planner::context::security::PlanSecurityContext; +use crate::control::planner::plan_error_map::map_plan_error; use crate::control::planner::sql_plan_convert::PlanningPurpose; use crate::control::server::response_shape::schema::OutputSchema; -/// Map a planner error onto its Control-Plane equivalent. -/// -/// One mapping for every `plan_sql*` call site. Four copies of this match -/// existed and had already drifted — only one of them mapped -/// `RetryableSchemaChanged`, so the same condition was retryable on one path -/// and a flat plan error on the others. A new variant added to `SqlError` -/// reaches every site through here or none. -fn map_plan_error(error: nodedb_sql::SqlError, tenant_id: crate::types::TenantId) -> crate::Error { - match error { - nodedb_sql::SqlError::RetryableSchemaChanged { descriptor } => { - crate::Error::RetryableSchemaChanged { descriptor } - } - nodedb_sql::SqlError::CollectionDeactivated { - name, - retention_expires_at_ns, - .. - } => crate::Error::CollectionDeactivated { - tenant_id, - collection: name, - retention_expires_at_ns, - }, - nodedb_sql::SqlError::UnknownTable { name } => crate::Error::CollectionNotFound { - tenant_id, - collection: name, - }, - nodedb_sql::SqlError::UndefinedFunction { name } => { - crate::Error::UndefinedFunction { name } - } - nodedb_sql::SqlError::UndefinedObject { kind, name } => { - crate::Error::UndefinedObject { kind, name } - } - nodedb_sql::SqlError::ObjectNotInPrerequisiteState { object, detail } => { - crate::Error::ObjectNotInPrerequisiteState { object, detail } - } - // A constant expression that divides by zero is the same condition the - // row-scope evaluator raises, so it carries the same code. - nodedb_sql::SqlError::DivisionByZero => crate::Error::DivisionByZero, - nodedb_sql::SqlError::InvalidLimitValue { clause, value } => { - crate::Error::InvalidLimitValue { clause, value } - } - nodedb_sql::SqlError::UnknownColumn { column, .. } => { - crate::Error::UndefinedColumn { column } - } - nodedb_sql::SqlError::AmbiguousColumn { column } => { - crate::Error::AmbiguousColumn { column } - } - // A target/expression count mismatch is a syntax error in PostgreSQL, - // so it renders 42601 through `BadRequest`. - nodedb_sql::SqlError::Arity { detail } => crate::Error::BadRequest { detail }, - other => crate::Error::PlanError { - detail: other.to_string(), - }, - } -} - /// Bundled arguments for [`QueryContext::plan_sql_with_rls`]. pub struct PlanSqlWithRlsParams<'a> { pub sql: &'a str, @@ -136,9 +82,11 @@ impl QueryContext { }; // `nextval` records into the calling session's map and `currval` reads // only from it, so the adapter must know which session is planning. - let catalog = catalog.with_session_sequences(self.session_sequences()); - let plans = - nodedb_sql::plan_sql(sql, &catalog).map_err(|e| map_plan_error(e, tenant_id))?; + // `Arc` because the converters evaluate column DEFAULTs that read the + // catalog — `nextval` and `currval` — after planning returns. + let catalog = Arc::new(catalog.with_session_sequences(self.session_sequences())); + let plans = nodedb_sql::plan_sql(sql, catalog.as_ref()) + .map_err(|e| map_plan_error(e, tenant_id))?; // Fold catalog-dependent cast expressions (::regclass, ::regtype) to // constant OID literals at plan time, before crossing the bridge. // The data-plane evaluator is pure and has no catalog access. @@ -147,7 +95,7 @@ impl QueryContext { .map(|p| { nodedb_sql::planner::catalog_fold::fold_catalog_exprs_in_plan( p, - &catalog, + catalog.as_ref(), database_id, tenant_id.as_u64(), ) @@ -191,11 +139,12 @@ impl QueryContext { .load(std::sync::atomic::Ordering::Relaxed), database_id, tenant_id, + sql_catalog: Some(Arc::clone(&catalog) as _), }; let output_schema = crate::control::planner::sql_plan_convert::output_schema::build_output_schema( &plans, - &catalog, + catalog.as_ref(), database_id, ); let cache_eligibility = @@ -399,15 +348,23 @@ impl QueryContext { // Fresh adapter per plan call: same rationale as // `plan_with_nodedb_sql_for_purpose`. Its recorded version set is returned to the // caller so parameterized plans participate in descriptor admission. - let catalog = inputs.build_adapter(tenant_id.as_u64(), database_id); - let raw_plans = nodedb_sql::plan_sql_with_params(sql, params, &catalog) + // `nextval` records into the calling session's map and `currval` reads + // only from it, so the adapter must know which session is planning. + // `Arc` because the converters evaluate column DEFAULTs that read the + // catalog after planning returns. + let catalog = Arc::new( + inputs + .build_adapter(tenant_id.as_u64(), database_id) + .with_session_sequences(self.session_sequences()), + ); + let raw_plans = nodedb_sql::plan_sql_with_params(sql, params, catalog.as_ref()) .map_err(|error| map_plan_error(error, tenant_id))?; let plans: Vec<_> = raw_plans .into_iter() .map(|p| { nodedb_sql::planner::catalog_fold::fold_catalog_exprs_in_plan( p, - &catalog, + catalog.as_ref(), database_id, tenant_id.as_u64(), ) @@ -450,11 +407,12 @@ impl QueryContext { .load(std::sync::atomic::Ordering::Relaxed), database_id, tenant_id, + sql_catalog: Some(Arc::clone(&catalog) as _), }; let output_schema = crate::control::planner::sql_plan_convert::output_schema::build_output_schema( &plans, - &catalog, + catalog.as_ref(), database_id, ); let mut tasks = diff --git a/nodedb/src/control/planner/mod.rs b/nodedb/src/control/planner/mod.rs index bb5951700..ac07b12b7 100644 --- a/nodedb/src/control/planner/mod.rs +++ b/nodedb/src/control/planner/mod.rs @@ -7,6 +7,7 @@ pub mod context; pub mod descriptor_set; pub mod implicit_edges; pub mod materialized_sum; +pub(crate) mod plan_error_map; pub mod procedural; pub mod redaction_refusal; pub mod rls_injection; diff --git a/nodedb/src/control/planner/plan_error_map.rs b/nodedb/src/control/planner/plan_error_map.rs new file mode 100644 index 000000000..dfd3f957d --- /dev/null +++ b/nodedb/src/control/planner/plan_error_map.rs @@ -0,0 +1,63 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! `nodedb_sql::SqlError` to Control-Plane `crate::Error` mapping. + +/// Map a planner error onto its Control-Plane equivalent. +/// +/// One mapping for every site that surfaces a `nodedb_sql` error: the +/// `plan_sql*` calls and the `SqlPlan` -> `PhysicalPlan` converters that +/// evaluate column DEFAULTs. Four copies of this match existed and had +/// already drifted — only one of them mapped `RetryableSchemaChanged`, so the +/// same condition was retryable on one path and a flat plan error on the +/// others. A new variant added to `SqlError` reaches every site through here +/// or none. +pub(crate) fn map_plan_error( + error: nodedb_sql::SqlError, + tenant_id: crate::types::TenantId, +) -> crate::Error { + match error { + nodedb_sql::SqlError::RetryableSchemaChanged { descriptor } => { + crate::Error::RetryableSchemaChanged { descriptor } + } + nodedb_sql::SqlError::CollectionDeactivated { + name, + retention_expires_at_ns, + .. + } => crate::Error::CollectionDeactivated { + tenant_id, + collection: name, + retention_expires_at_ns, + }, + nodedb_sql::SqlError::UnknownTable { name } => crate::Error::CollectionNotFound { + tenant_id, + collection: name, + }, + nodedb_sql::SqlError::UndefinedFunction { name } => { + crate::Error::UndefinedFunction { name } + } + nodedb_sql::SqlError::UndefinedObject { kind, name } => { + crate::Error::UndefinedObject { kind, name } + } + nodedb_sql::SqlError::ObjectNotInPrerequisiteState { object, detail } => { + crate::Error::ObjectNotInPrerequisiteState { object, detail } + } + // A constant expression that divides by zero is the same condition the + // row-scope evaluator raises, so it carries the same code. + nodedb_sql::SqlError::DivisionByZero => crate::Error::DivisionByZero, + nodedb_sql::SqlError::InvalidLimitValue { clause, value } => { + crate::Error::InvalidLimitValue { clause, value } + } + nodedb_sql::SqlError::UnknownColumn { column, .. } => { + crate::Error::UndefinedColumn { column } + } + nodedb_sql::SqlError::AmbiguousColumn { column } => { + crate::Error::AmbiguousColumn { column } + } + // A target/expression count mismatch is a syntax error in PostgreSQL, + // so it renders 42601 through `BadRequest`. + nodedb_sql::SqlError::Arity { detail } => crate::Error::BadRequest { detail }, + other => crate::Error::PlanError { + detail: other.to_string(), + }, + } +} diff --git a/nodedb/src/control/planner/sql_plan_convert/array_fn_convert/aggregate.rs b/nodedb/src/control/planner/sql_plan_convert/array_fn_convert/aggregate.rs index d52fc535b..c61cf62bc 100644 --- a/nodedb/src/control/planner/sql_plan_convert/array_fn_convert/aggregate.rs +++ b/nodedb/src/control/planner/sql_plan_convert/array_fn_convert/aggregate.rs @@ -153,6 +153,7 @@ mod tests { shuffle_agg_num_parts: 0, broadcast_threshold_bytes: 8 * 1024 * 1024, shuffle_agg_threshold: 10_000, + sql_catalog: None, database_id: crate::types::DatabaseId::DEFAULT, tenant_id: crate::types::TenantId::new(0), } diff --git a/nodedb/src/control/planner/sql_plan_convert/array_fn_convert/slice.rs b/nodedb/src/control/planner/sql_plan_convert/array_fn_convert/slice.rs index ed997b9f4..845605a93 100644 --- a/nodedb/src/control/planner/sql_plan_convert/array_fn_convert/slice.rs +++ b/nodedb/src/control/planner/sql_plan_convert/array_fn_convert/slice.rs @@ -255,6 +255,7 @@ mod tests { shuffle_agg_num_parts: 0, broadcast_threshold_bytes: 8 * 1024 * 1024, shuffle_agg_threshold: 10_000, + sql_catalog: None, database_id: crate::types::DatabaseId::DEFAULT, tenant_id: crate::types::TenantId::new(0), }; diff --git a/nodedb/src/control/planner/sql_plan_convert/convert.rs b/nodedb/src/control/planner/sql_plan_convert/convert.rs index 7f2a67670..7887e2056 100644 --- a/nodedb/src/control/planner/sql_plan_convert/convert.rs +++ b/nodedb/src/control/planner/sql_plan_convert/convert.rs @@ -129,6 +129,12 @@ pub struct ConvertContext { /// `DEFAULT_SHUFFLE_AGG_THRESHOLD`; overridable per-session via /// `nodedb.shuffle_agg_threshold` for operator control and test determinism. pub shuffle_agg_threshold: usize, + /// The catalog the plan was built against. INSERT/UPSERT conversion + /// evaluates each declared column DEFAULT here, and a sequence-backed + /// DEFAULT (`nextval`, `currval`) reads its value through this handle. + /// `None` for converters built by sub-planners that hold no catalog; a + /// catalog-reading DEFAULT then raises instead of dropping the column. + pub sql_catalog: Option>, } impl ConvertContext { @@ -136,6 +142,22 @@ impl ConvertContext { self.purpose == PlanningPurpose::Metadata } + /// The catalog a column DEFAULT is evaluated against. + /// + /// Raises when the converter holds none. A DEFAULT that reads the catalog + /// must fail loudly: omitting the column stores NULL where the + /// declaration promised a value. + pub fn sql_catalog( + &self, + ) -> crate::Result<&(dyn nodedb_sql::catalog::SqlCatalog + Send + Sync)> { + self.sql_catalog + .as_deref() + .ok_or_else(|| crate::Error::PlanError { + detail: "plan conversion holds no catalog, so a column DEFAULT that reads one cannot be evaluated" + .into(), + }) + } + /// Resolve an existing surrogate without creating a mapping while planning /// metadata. Execute planning retains the allocating assignment behavior. pub fn surrogate_for_pk( @@ -292,6 +314,7 @@ mod tests { shuffle_agg_num_parts: 0, broadcast_threshold_bytes: 0, shuffle_agg_threshold: 0, + sql_catalog: None, } } diff --git a/nodedb/src/control/planner/sql_plan_convert/dml/insert.rs b/nodedb/src/control/planner/sql_plan_convert/dml/insert.rs index 57bcb697e..693b323cb 100644 --- a/nodedb/src/control/planner/sql_plan_convert/dml/insert.rs +++ b/nodedb/src/control/planner/sql_plan_convert/dml/insert.rs @@ -10,7 +10,10 @@ use nodedb_physical::physical_plan::ColumnarInsertIntent; use nodedb_physical::physical_plan::*; use super::super::convert::ConvertContext; -use super::super::value::{row_to_msgpack, rows_to_msgpack_array, sql_value_to_string}; +use super::super::value::{ + expand_row_defaults, row_to_msgpack, rows_to_msgpack_array, sql_value_to_string, +}; +use super::route::{WriteRoute, insert_route}; use nodedb_physical::physical_task::{PhysicalTask, PostSetOp}; /// Build a `ColumnarSchema` from raw catalog column-type strings. @@ -213,17 +216,6 @@ pub(super) fn columnar_row_surrogates( Ok(out) } -pub(in super::super) fn nodedb_value_to_sql(val: nodedb_types::Value) -> SqlValue { - match val { - nodedb_types::Value::Integer(n) => SqlValue::Int(n), - nodedb_types::Value::Float(f) => SqlValue::Float(f), - nodedb_types::Value::String(s) => SqlValue::String(s), - nodedb_types::Value::Bool(b) => SqlValue::Bool(b), - nodedb_types::Value::Null => SqlValue::Null, - _ => SqlValue::String(format!("{val:?}")), - } -} - /// Bundled arguments for [`convert_insert`]. pub(in super::super) struct ConvertInsertArgs<'a> { pub collection: &'a str, @@ -257,6 +249,9 @@ pub(in super::super) fn convert_insert( let vshard = VShardId::from_collection_in_database(ctx.database_id, collection); let mut tasks = Vec::new(); let mut columnar_rows: Vec<&Vec<(String, SqlValue)>> = Vec::new(); + // Resolved once per statement, before any DEFAULT is materialized: an + // engine with no INSERT lowering here must not burn a sequence value. + let route = insert_route(engine, collection)?; // Both INSERT routing gates, read from the catalog once for the whole // statement (never re-hit per row). @@ -295,44 +290,17 @@ pub(in super::super) fn convert_insert( let mut balanced_documents: Vec<(String, Vec)> = Vec::new(); let mut balanced_surrogates: Vec = Vec::new(); - let mut expanded_rows: Vec> = Vec::with_capacity(rows.len()); - for row in rows { - if column_defaults.is_empty() { - expanded_rows.push(row.clone()); - continue; - } - let mut expanded = row.clone(); - for (col_name, default_expr) in column_defaults { - if !expanded.iter().any(|(k, _)| k == col_name) - && let Some(val) = super::super::value::evaluate_default_expr(default_expr) - .map_err(|e| crate::Error::PlanError { - detail: format!("default for column '{col_name}': {e}"), - })? - { - expanded.push((col_name.clone(), nodedb_value_to_sql(val))); - } - } - expanded_rows.push(expanded); - } + // Every engine's rows expand their DEFAULTs here, ahead of identity + // derivation. A DEFAULT materialized after the primary-key NOT NULL gate + // refuses a key the declaration supplies. + let expanded_rows = expand_row_defaults(rows, column_defaults, tenant_id, ctx)?; - for (i, row) in expanded_rows.iter().enumerate() { - match engine { - EngineType::KeyValue => { - return Err(crate::Error::PlanError { - detail: "KV INSERT must use SqlPlan::KvInsert path".into(), - }); - } - EngineType::Timeseries => { - return Err(crate::Error::PlanError { - detail: format!( - "INSERT into '{collection}': timeseries collections use TimeseriesIngest, not Insert" - ), - }); + for row in &expanded_rows { + match route { + WriteRoute::ColumnarFamily => { + columnar_rows.push(row); } - EngineType::Columnar | EngineType::Spatial => { - columnar_rows.push(&rows[i]); - } - EngineType::DocumentSchemaless | EngineType::DocumentStrict => { + WriteRoute::Document => { let value_bytes = row_to_msgpack(row)?; let (doc_id, surrogate) = resolve_doc_identity(ctx, collection, primary_key, row)?; // One page for the whole statement: the rows of a balanced @@ -381,13 +349,6 @@ pub(in super::super) fn convert_insert( txn_id: None, }); } - EngineType::Array => { - return Err(crate::Error::PlanError { - detail: format!( - "INSERT into '{collection}': array engine uses INSERT INTO ARRAY syntax" - ), - }); - } } } @@ -405,7 +366,7 @@ pub(in super::super) fn convert_insert( } if !columnar_rows.is_empty() { - let payload = rows_to_msgpack_array(&columnar_rows, column_defaults)?; + let payload = rows_to_msgpack_array(&columnar_rows)?; let intent = if if_absent { ColumnarInsertIntent::InsertIfAbsent } else { @@ -495,6 +456,7 @@ mod tests { shuffle_agg_num_parts: 0, broadcast_threshold_bytes: 8 * 1024 * 1024, shuffle_agg_threshold: 10_000, + sql_catalog: None, database_id: crate::types::DatabaseId::DEFAULT, tenant_id: crate::types::TenantId::new(0), }; diff --git a/nodedb/src/control/planner/sql_plan_convert/dml/kv_and_vector.rs b/nodedb/src/control/planner/sql_plan_convert/dml/kv_and_vector.rs index 43c2b03f5..c5c03f023 100644 --- a/nodedb/src/control/planner/sql_plan_convert/dml/kv_and_vector.rs +++ b/nodedb/src/control/planner/sql_plan_convert/dml/kv_and_vector.rs @@ -223,6 +223,7 @@ mod tests { shuffle_agg_num_parts: 0, broadcast_threshold_bytes: 8 * 1024 * 1024, shuffle_agg_threshold: 10_000, + sql_catalog: None, database_id: crate::types::DatabaseId::DEFAULT, tenant_id: crate::types::TenantId::new(0), } diff --git a/nodedb/src/control/planner/sql_plan_convert/dml/mod.rs b/nodedb/src/control/planner/sql_plan_convert/dml/mod.rs index d87cf86b8..f2271a9c9 100644 --- a/nodedb/src/control/planner/sql_plan_convert/dml/mod.rs +++ b/nodedb/src/control/planner/sql_plan_convert/dml/mod.rs @@ -5,6 +5,7 @@ mod crdt_gate; mod insert; mod kv_and_vector; mod merge; +mod route; mod update_delete; mod upsert; diff --git a/nodedb/src/control/planner/sql_plan_convert/dml/route.rs b/nodedb/src/control/planner/sql_plan_convert/dml/route.rs new file mode 100644 index 000000000..af87a7006 --- /dev/null +++ b/nodedb/src/control/planner/sql_plan_convert/dml/route.rs @@ -0,0 +1,55 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! Engine routing for the row-shaped write converters. + +use nodedb_sql::types::EngineType; + +/// The lowering a statement's rows take. +pub(super) enum WriteRoute { + /// One task per row, carrying a `DocumentOp` or a `CrdtOp`. + Document, + /// One batched `ColumnarOp` task for the whole statement. + ColumnarFamily, +} + +/// Resolve an INSERT's route, refusing engines that lower elsewhere. +/// +/// Routing runs once per statement, ahead of DEFAULT materialization, so a +/// refused engine never allocates a sequence value it discards. +pub(super) fn insert_route(engine: &EngineType, collection: &str) -> crate::Result { + match engine { + EngineType::DocumentSchemaless | EngineType::DocumentStrict => Ok(WriteRoute::Document), + EngineType::Columnar | EngineType::Spatial => Ok(WriteRoute::ColumnarFamily), + EngineType::KeyValue => Err(crate::Error::PlanError { + detail: "KV INSERT must use SqlPlan::KvInsert path".into(), + }), + EngineType::Timeseries => Err(crate::Error::PlanError { + detail: format!( + "INSERT into '{collection}': timeseries collections use TimeseriesIngest, not Insert" + ), + }), + EngineType::Array => Err(crate::Error::PlanError { + detail: format!( + "INSERT into '{collection}': array engine uses INSERT INTO ARRAY syntax" + ), + }), + } +} + +/// Resolve an UPSERT's route, refusing engines with no upsert lowering. +/// +/// Runs ahead of DEFAULT materialization for the same reason as +/// [`insert_route`]. +pub(super) fn upsert_route(engine: &EngineType, collection: &str) -> crate::Result { + match engine { + EngineType::DocumentSchemaless | EngineType::DocumentStrict => Ok(WriteRoute::Document), + EngineType::Columnar | EngineType::Spatial => Ok(WriteRoute::ColumnarFamily), + EngineType::Timeseries | EngineType::KeyValue | EngineType::Array => { + Err(crate::Error::PlanError { + detail: format!( + "UPSERT into '{collection}': engine type {engine:?} does not support upsert" + ), + }) + } + } +} diff --git a/nodedb/src/control/planner/sql_plan_convert/dml/update_delete/delete.rs b/nodedb/src/control/planner/sql_plan_convert/dml/update_delete/delete.rs index 29246ede5..073e58865 100644 --- a/nodedb/src/control/planner/sql_plan_convert/dml/update_delete/delete.rs +++ b/nodedb/src/control/planner/sql_plan_convert/dml/update_delete/delete.rs @@ -259,6 +259,7 @@ mod tests { shuffle_agg_num_parts: 0, broadcast_threshold_bytes: 8 * 1024 * 1024, shuffle_agg_threshold: 10_000, + sql_catalog: None, database_id: crate::types::DatabaseId::DEFAULT, tenant_id: crate::types::TenantId::new(0), }; diff --git a/nodedb/src/control/planner/sql_plan_convert/dml/update_delete/update.rs b/nodedb/src/control/planner/sql_plan_convert/dml/update_delete/update.rs index f9c75d93f..bfd2894fb 100644 --- a/nodedb/src/control/planner/sql_plan_convert/dml/update_delete/update.rs +++ b/nodedb/src/control/planner/sql_plan_convert/dml/update_delete/update.rs @@ -386,6 +386,7 @@ mod tests { shuffle_agg_num_parts: 0, broadcast_threshold_bytes: 8 * 1024 * 1024, shuffle_agg_threshold: 10_000, + sql_catalog: None, database_id: crate::types::DatabaseId::DEFAULT, tenant_id: crate::types::TenantId::new(0), }; diff --git a/nodedb/src/control/planner/sql_plan_convert/dml/upsert.rs b/nodedb/src/control/planner/sql_plan_convert/dml/upsert.rs index a9ec7435b..7586b436e 100644 --- a/nodedb/src/control/planner/sql_plan_convert/dml/upsert.rs +++ b/nodedb/src/control/planner/sql_plan_convert/dml/upsert.rs @@ -14,8 +14,11 @@ use nodedb_physical::physical_plan::ColumnarInsertIntent; use nodedb_physical::physical_plan::*; use super::super::convert::ConvertContext; -use super::super::value::{assignments_to_update_values, row_to_msgpack, rows_to_msgpack_array}; +use super::super::value::{ + assignments_to_update_values, expand_row_defaults, row_to_msgpack, rows_to_msgpack_array, +}; use super::insert::{build_schema_bytes, columnar_row_surrogates, resolve_doc_identity}; +use super::route::{WriteRoute, upsert_route}; use nodedb_physical::physical_task::{PhysicalTask, PostSetOp}; /// Bundled arguments for [`convert_upsert`]. @@ -50,6 +53,9 @@ pub(in super::super) fn convert_upsert( let collection = coll_qualified.as_str(); let vshard = VShardId::from_collection_in_database(ctx.database_id, collection); let mut tasks = Vec::new(); + // Resolved once per statement, before any DEFAULT is materialized: an + // engine with no UPSERT lowering must not burn a sequence value. + let route = upsert_route(engine, collection)?; // Detect CRDT document collections once. An explicit `ON CONFLICT DO UPDATE // SET ...` cannot be honored: CRDT conflict resolution IS the LWW @@ -73,9 +79,14 @@ pub(in super::super) fn convert_upsert( let mut columnar_rows: Vec<&Vec<(String, SqlValue)>> = Vec::new(); - for row in rows { - match engine { - EngineType::DocumentSchemaless | EngineType::DocumentStrict => { + // Every engine's rows expand their DEFAULTs here, ahead of identity + // derivation, so the primary-key NOT NULL gate reads the row the + // declaration promises — see `expand_row_defaults`. + let expanded_rows = expand_row_defaults(rows, column_defaults, tenant_id, ctx)?; + + for row in &expanded_rows { + match route { + WriteRoute::Document => { let value_bytes = row_to_msgpack(row)?; let (doc_id, surrogate) = resolve_doc_identity(ctx, collection, primary_key, row)?; let plan = if is_crdt { @@ -114,21 +125,14 @@ pub(in super::super) fn convert_upsert( txn_id: None, }); } - EngineType::Columnar | EngineType::Spatial => { + WriteRoute::ColumnarFamily => { columnar_rows.push(row); } - EngineType::Timeseries | EngineType::KeyValue | EngineType::Array => { - return Err(crate::Error::PlanError { - detail: format!( - "UPSERT into '{collection}': engine type {engine:?} does not support upsert" - ), - }); - } } } if !columnar_rows.is_empty() { - let payload = rows_to_msgpack_array(&columnar_rows, column_defaults)?; + let payload = rows_to_msgpack_array(&columnar_rows)?; let surrogates = columnar_row_surrogates(ctx, collection, &columnar_rows, primary_key)?; let schema_bytes = build_schema_bytes(column_schema); tasks.push(PhysicalTask { diff --git a/nodedb/src/control/planner/sql_plan_convert/set_ops.rs b/nodedb/src/control/planner/sql_plan_convert/set_ops.rs index 8b4d6dacb..4493c7f02 100644 --- a/nodedb/src/control/planner/sql_plan_convert/set_ops.rs +++ b/nodedb/src/control/planner/sql_plan_convert/set_ops.rs @@ -410,6 +410,7 @@ mod tests { shuffle_agg_num_parts: 0, broadcast_threshold_bytes: 8 * 1024 * 1024, shuffle_agg_threshold: 10_000, + sql_catalog: None, database_id: crate::types::DatabaseId::DEFAULT, tenant_id: crate::types::TenantId::new(0), }, @@ -469,6 +470,7 @@ mod tests { shuffle_agg_num_parts: 0, broadcast_threshold_bytes: 8 * 1024 * 1024, shuffle_agg_threshold: 10_000, + sql_catalog: None, database_id: crate::types::DatabaseId::DEFAULT, tenant_id: crate::types::TenantId::new(0), }, diff --git a/nodedb/src/control/planner/sql_plan_convert/value/defaults.rs b/nodedb/src/control/planner/sql_plan_convert/value/defaults.rs new file mode 100644 index 000000000..43dfaf105 --- /dev/null +++ b/nodedb/src/control/planner/sql_plan_convert/value/defaults.rs @@ -0,0 +1,41 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! The one DEFAULT materialization point for the row-shaped DML converters. + +use nodedb_sql::types::SqlValue; + +use super::super::convert::ConvertContext; +use crate::control::planner::plan_error_map::map_plan_error; +use crate::types::TenantId; + +/// Expand each row's omitted DEFAULT columns, before engine dispatch. +/// +/// INSERT and UPSERT call this once per statement, so identity derivation, +/// the primary-key NOT NULL gate, and the stored payload all read the same +/// row. A DEFAULT materialized per engine after that gate refuses a key the +/// declaration supplies. +/// +/// Each DEFAULT evaluates once per row, so a `nextval` DEFAULT allocates +/// exactly one value per row of a multi-row VALUES clause. +/// +/// `column_defaults` empty means nothing to expand: the rows pass through and +/// the catalog is never read. +pub(in super::super) fn expand_row_defaults( + rows: &[Vec<(String, SqlValue)>], + column_defaults: &[(String, String)], + tenant_id: TenantId, + ctx: &ConvertContext, +) -> crate::Result>> { + if column_defaults.is_empty() { + return Ok(rows.to_vec()); + } + let catalog = ctx.sql_catalog()?; + let mut expanded = Vec::with_capacity(rows.len()); + for row in rows { + let mut row = row.clone(); + nodedb_sql::planner::defaults::materialize_row_defaults(&mut row, column_defaults, catalog) + .map_err(|e| map_plan_error(e, tenant_id))?; + expanded.push(row); + } + Ok(expanded) +} diff --git a/nodedb/src/control/planner/sql_plan_convert/value/mod.rs b/nodedb/src/control/planner/sql_plan_convert/value/mod.rs index 778074ead..851ae2e54 100644 --- a/nodedb/src/control/planner/sql_plan_convert/value/mod.rs +++ b/nodedb/src/control/planner/sql_plan_convert/value/mod.rs @@ -1,10 +1,11 @@ // SPDX-License-Identifier: BUSL-1.1 -//! Value conversion utilities: SqlValue ↔ nodedb_types::Value, msgpack encoding, -//! and the re-export of the shared column-default evaluator. +//! Value conversion utilities: SqlValue ↔ nodedb_types::Value, msgpack +//! encoding, and the one DEFAULT materialization point. pub(super) mod assignments; pub(super) mod convert; +pub(super) mod defaults; pub(super) mod msgpack_write; pub(super) mod rows; @@ -14,12 +15,9 @@ pub(super) use assignments::{ pub(super) use convert::{ sql_value_to_bytes, sql_value_to_msgpack, sql_value_to_nodedb_value, sql_value_to_string, }; -// The evaluator lives in the SQL crate so every engine that materializes a -// DEFAULT produces the same value for the same expression; re-exported here so -// the document/columnar converters keep their existing import path. +pub(super) use defaults::expand_row_defaults; pub(super) use msgpack_write::{ row_to_msgpack, write_msgpack_array_header, write_msgpack_map_header, write_msgpack_str, write_msgpack_value, }; -pub(super) use nodedb_sql::planner::defaults::evaluate_default_expr; pub(super) use rows::rows_to_msgpack_array; diff --git a/nodedb/src/control/planner/sql_plan_convert/value/rows.rs b/nodedb/src/control/planner/sql_plan_convert/value/rows.rs index 2fcf789af..ab35416aa 100644 --- a/nodedb/src/control/planner/sql_plan_convert/value/rows.rs +++ b/nodedb/src/control/planner/sql_plan_convert/value/rows.rs @@ -5,28 +5,18 @@ use nodedb_sql::types::SqlValue; use super::convert::sql_value_to_nodedb_value; -use nodedb_sql::planner::defaults::evaluate_default_expr; -pub(crate) fn rows_to_msgpack_array( - rows: &[&Vec<(String, SqlValue)>], - column_defaults: &[(String, String)], -) -> crate::Result> { +/// Encode already-expanded rows as one msgpack array of maps. +/// +/// Callers materialize DEFAULTs through `expand_row_defaults` before routing, +/// so every column the declaration promises is already present in `rows`. +pub(crate) fn rows_to_msgpack_array(rows: &[&Vec<(String, SqlValue)>]) -> crate::Result> { let mut arr: Vec = Vec::with_capacity(rows.len()); for row in rows { let mut map = std::collections::HashMap::new(); for (key, val) in row.iter() { map.insert(key.clone(), sql_value_to_nodedb_value(val)); } - for (col_name, default_expr) in column_defaults { - if !map.contains_key(col_name) - && let Some(val) = - evaluate_default_expr(default_expr).map_err(|e| crate::Error::PlanError { - detail: format!("default for column '{col_name}': {e}"), - })? - { - map.insert(col_name.clone(), val); - } - } arr.push(nodedb_types::Value::Object(map)); } let val = nodedb_types::Value::Array(arr); diff --git a/nodedb/tests/inproc/cases/executor_tests/test_group_by_alias.rs b/nodedb/tests/inproc/cases/executor_tests/test_group_by_alias.rs index 3b1cab4c5..d96111c4a 100644 --- a/nodedb/tests/inproc/cases/executor_tests/test_group_by_alias.rs +++ b/nodedb/tests/inproc/cases/executor_tests/test_group_by_alias.rs @@ -86,6 +86,7 @@ fn sql_to_physical(sql: &str) -> PhysicalPlan { shuffle_agg_num_parts: 0, broadcast_threshold_bytes: 8 * 1024 * 1024, shuffle_agg_threshold: 10_000, + sql_catalog: None, }; let tenant_id = nodedb::types::TenantId::new(1); let tasks = convert(&plans, tenant_id, &ctx).unwrap(); From 49cb1390037c2477e0cad688a3eebb97e071f599 Mon Sep 17 00:00:00 2001 From: Farhan Syah Date: Thu, 10 Sep 2026 01:41:00 +0800 Subject: [PATCH 06/23] feat(sql): expand SERIAL into a nextval DEFAULT and gate declared DEFAULTs SERIAL and BIGSERIAL columns now expand to their backing type plus a DEFAULT nextval('{collection}_{field}_seq') naming the sequence the caller creates, instead of only the bare type. parse_fields_clause and parse_fields_clause_from_pairs take the collection name to build that sequence name. Add a DDL-time gate that classifies and parses every declared column DEFAULT through the same paths evaluate_default_expr uses, so a call to an unregistered function is refused at CREATE with SQLSTATE 42883 instead of surfacing at the first INSERT. The gate parses a setval or sequence-accessor DEFAULT without evaluating it, so declaring a column never advances a sequence. --- nodedb-sql/src/planner/defaults.rs | 94 ++++++++++++++----- .../ddl/neutral/collection/create/build.rs | 33 +++++-- .../create/build_column_defaults.rs | 49 ++++++++++ .../ddl/neutral/collection/create/mod.rs | 5 +- .../shared/ddl/neutral/collection/register.rs | 2 +- .../server/shared/ddl/schema_validation.rs | 69 +++++++++----- 6 files changed, 198 insertions(+), 54 deletions(-) create mode 100644 nodedb/src/control/server/shared/ddl/neutral/collection/create/build_column_defaults.rs diff --git a/nodedb-sql/src/planner/defaults.rs b/nodedb-sql/src/planner/defaults.rs index 389e34a9d..7799354f2 100644 --- a/nodedb-sql/src/planner/defaults.rs +++ b/nodedb-sql/src/planner/defaults.rs @@ -37,45 +37,84 @@ pub fn evaluate_default_expr( catalog: &dyn SqlCatalog, ) -> crate::Result { let upper = expr.trim().to_uppercase(); - match upper.as_str() { + if let Some(value) = eval_keyword_default(&upper) { + return Ok(value); + } + if let Some(value) = eval_parametric_or_literal(expr, &upper)? { + return Ok(value); + } + evaluate_parsed_default(expr, column, catalog) +} + +/// Check that `expr`, the DEFAULT declared on `column`, can be evaluated. +/// +/// DDL calls this to refuse an unevaluable DEFAULT at declaration time. It +/// classifies the expression through the same arms `evaluate_default_expr` +/// uses, then parses anything left over. Parsing runs the resolver's +/// `FunctionRegistry` gate, so an unregistered function name raises +/// [`SqlError::UndefinedFunction`]. +/// +/// A sequence accessor is parsed, never called, so declaring a column must +/// never advance a sequence. +pub fn validate_default_expr(expr: &str, column: &str) -> crate::Result<()> { + let upper = expr.trim().to_uppercase(); + if eval_keyword_default(&upper).is_some() { + return Ok(()); + } + if eval_parametric_or_literal(expr, &upper)?.is_some() { + return Ok(()); + } + let sql_expr = crate::parse_expr_string(expr)?; + reject_setval_default(&sql_expr, column) +} + +/// Evaluate the keyword-spelled defaults: the ID generators and `NOW()`. +/// +/// Returns `None` for every other expression. This is the one list of +/// keyword forms; the DDL gate classifies through it rather than repeating it. +fn eval_keyword_default(upper: &str) -> Option { + let value = match upper { "UUID_V7" | "UUIDV7" | "GEN_UUID_V7()" | "UUID_V7()" => { - Ok(nodedb_types::Value::String(nodedb_types::id_gen::uuid_v7())) + nodedb_types::Value::String(nodedb_types::id_gen::uuid_v7()) } "UUID_V4" | "UUIDV4" | "UUID" | "GEN_UUID_V4()" | "UUID_V4()" => { - Ok(nodedb_types::Value::String(nodedb_types::id_gen::uuid_v4())) + nodedb_types::Value::String(nodedb_types::id_gen::uuid_v4()) } "ULID" | "GEN_ULID()" | "ULID()" => { - Ok(nodedb_types::Value::String(nodedb_types::id_gen::ulid())) + nodedb_types::Value::String(nodedb_types::id_gen::ulid()) } - "CUID2" | "CUID2()" => Ok(nodedb_types::Value::String(nodedb_types::id_gen::cuid2())), - "NANOID" | "NANOID()" => Ok(nodedb_types::Value::String(nodedb_types::id_gen::nanoid())), + "CUID2" | "CUID2()" => nodedb_types::Value::String(nodedb_types::id_gen::cuid2()), + "NANOID" | "NANOID()" => nodedb_types::Value::String(nodedb_types::id_gen::nanoid()), "NOW()" => { let now = std::time::SystemTime::now() .duration_since(std::time::UNIX_EPOCH) .unwrap_or_default(); - Ok(nodedb_types::Value::String( + nodedb_types::Value::String( chrono::DateTime::from_timestamp_millis(now.as_millis() as i64) .map(|dt| dt.to_rfc3339()) .unwrap_or_else(|| now.as_millis().to_string()), - )) + ) } - _ => parse_parametric_or_literal(expr, &upper, column, catalog), - } + _ => return None, + }; + Some(value) } -fn parse_parametric_or_literal( +/// Evaluate the parametric ID generators and the bare literals. +/// +/// Returns `Ok(None)` when `expr` is none of them, leaving it to the parser. +/// This is the one list of literal forms; the DDL gate reuses it. +fn eval_parametric_or_literal( expr: &str, upper: &str, - column: &str, - catalog: &dyn SqlCatalog, -) -> crate::Result { +) -> crate::Result> { // NANOID(N) — custom length. if upper.starts_with("NANOID(") && upper.ends_with(')') { let len_str = &upper[7..upper.len() - 1]; if let Ok(len) = len_str.parse::() { - return Ok(nodedb_types::Value::String( + return Ok(Some(nodedb_types::Value::String( nodedb_types::id_gen::nanoid_with_length(len), - )); + ))); } } // CUID2(N) — custom length; validates length range and surfaces planning errors. @@ -85,27 +124,27 @@ fn parse_parametric_or_literal( let id = nodedb_types::id_gen::cuid2_with_length(len).map_err(|e| SqlError::Parse { detail: format!("CUID2({len}) default expression is invalid: {e}"), })?; - return Ok(nodedb_types::Value::String(id)); + return Ok(Some(nodedb_types::Value::String(id))); } } // Numeric literal. if let Ok(i) = expr.trim().parse::() { - return Ok(nodedb_types::Value::Integer(i)); + return Ok(Some(nodedb_types::Value::Integer(i))); } if let Ok(f) = expr.trim().parse::() { - return Ok(nodedb_types::Value::Float(f)); + return Ok(Some(nodedb_types::Value::Float(f))); } // Quoted string literal. let trimmed = expr.trim(); if (trimmed.starts_with('\'') && trimmed.ends_with('\'')) || (trimmed.starts_with('"') && trimmed.ends_with('"')) { - return Ok(nodedb_types::Value::String( + return Ok(Some(nodedb_types::Value::String( trimmed[1..trimmed.len() - 1].to_string(), - )); + ))); } - evaluate_parsed_default(expr, column, catalog) + Ok(None) } /// Parse the DEFAULT as SQL, then resolve it against the catalog or the folder. @@ -138,6 +177,15 @@ fn evaluate_sequence_default( column: &str, catalog: &dyn SqlCatalog, ) -> crate::Result> { + reject_setval_default(expr, column)?; + super::catalog_expr_fold::eval_sequence_accessor(expr, catalog) +} + +/// Refuse `setval` as a column DEFAULT. +/// +/// `setval` moves a sequence rather than reading one, so a column cannot take +/// its result as a value. Both the evaluator and the DDL gate call this. +fn reject_setval_default(expr: &SqlExpr, column: &str) -> crate::Result<()> { if let SqlExpr::Function { name, .. } = expr && name.eq_ignore_ascii_case("setval") { @@ -145,7 +193,7 @@ fn evaluate_sequence_default( column: column.to_string(), }); } - super::catalog_expr_fold::eval_sequence_accessor(expr, catalog) + Ok(()) } fn unevaluable(column: &str, expr: &str) -> SqlError { diff --git a/nodedb/src/control/server/shared/ddl/neutral/collection/create/build.rs b/nodedb/src/control/server/shared/ddl/neutral/collection/create/build.rs index 8d695562e..2c81900df 100644 --- a/nodedb/src/control/server/shared/ddl/neutral/collection/create/build.rs +++ b/nodedb/src/control/server/shared/ddl/neutral/collection/create/build.rs @@ -34,6 +34,7 @@ use super::super::enforcement::{ use super::engine_option::validate_engine_name; use super::request::CreateCollectionRequest; +use super::build_column_defaults::validate_column_defaults; use super::build_flags::{err, resolve_crdt_flag, validate_crdt_signing_storage, validate_name}; use super::build_post_create::{create_serial_sequences, log_vector_fields}; use super::build_primary_engine::resolve_primary_engine; @@ -86,6 +87,13 @@ pub async fn build_and_persist( )); } + // Refuse a DEFAULT the server cannot evaluate here, not at the first + // INSERT. It runs before any lifecycle guard or predecessor purge, so a + // rejected declaration leaves the existing state untouched. A SERIAL + // column carries no DEFAULT text yet; the `nextval` this build generates + // for it names a registered function and clears the same gate. + validate_column_defaults(columns)?; + let tenant_id = identity.tenant_id; // Metadata Raft serializes clustered DDL. Without it, hold an exclusive @@ -179,10 +187,22 @@ pub async fn build_and_persist( let canonical_engine = validate_engine_name(engine, options)?; let bitemporal_flag = flags.iter().any(|f| f == "BITEMPORAL"); + // Expand SERIAL / BIGSERIAL first, so every later reader of the column + // list sees the backing type plus the `nextval` DEFAULT naming the + // sequence `create_serial_sequences` materializes below. + let (expanded_columns, serial_fields) = + crate::control::server::shared::ddl::schema_validation::parse_fields_clause_from_pairs( + name, columns, + ); + // Resolve user-defined type names to TEXT for schema building. // Original names are preserved in `fields` for drop-protection. - let resolved_columns: Vec<(String, String)> = - resolve_custom_type_columns(columns, state, database_id.as_u64(), tenant_id.as_u64()); + let resolved_columns: Vec<(String, String)> = resolve_custom_type_columns( + &expanded_columns, + state, + database_id.as_u64(), + tenant_id.as_u64(), + ); let (collection_type, columnar_schema_columns) = nodedb_sql::ddl_ast::build_collection_type( canonical_engine, @@ -193,10 +213,7 @@ pub async fn build_and_persist( ) .map_err(|e| err("42601", e.to_string()))?; - let (mut fields, serial_fields) = - crate::control::server::shared::ddl::schema_validation::parse_fields_clause_from_pairs( - columns, - ); + let mut fields = expanded_columns.clone(); if fields.is_empty() && !columnar_schema_columns.is_empty() { fields = columnar_schema_columns; } @@ -210,7 +227,7 @@ pub async fn build_and_persist( }; let (primary, vector_primary) = - resolve_primary_engine(options, columns, &fields, &collection_type)?; + resolve_primary_engine(options, &expanded_columns, &fields, &collection_type)?; let append_only = flags.iter().any(|f| f == "APPEND_ONLY"); let hash_chain = flags.iter().any(|f| f == "HASH_CHAIN"); @@ -245,7 +262,7 @@ pub async fn build_and_persist( // column list. Recorded on every engine so schemaless collections can // key their document id off it instead of the hardcoded `id` field; // harmless for strict/KV, which already track the PK on their schema. - let declared_primary_key = columns.iter().find_map(|(col_name, type_str)| { + let declared_primary_key = expanded_columns.iter().find_map(|(col_name, type_str)| { let (_, is_pk, _, _) = nodedb_sql::ddl_ast::collection_type::parse_column_type_str_full(type_str); is_pk.then(|| col_name.clone()) diff --git a/nodedb/src/control/server/shared/ddl/neutral/collection/create/build_column_defaults.rs b/nodedb/src/control/server/shared/ddl/neutral/collection/create/build_column_defaults.rs new file mode 100644 index 000000000..ae201f2d1 --- /dev/null +++ b/nodedb/src/control/server/shared/ddl/neutral/collection/create/build_column_defaults.rs @@ -0,0 +1,49 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! DDL-time gate for column `DEFAULT` expressions. +//! +//! A `DEFAULT` clause is stored as raw text on the column type string and is +//! evaluated only when an INSERT omits the column. Without this gate a +//! `CREATE` accepts a call to a function that does not exist, and the author +//! learns about it at the first insert instead of at the declaration. +//! +//! The check runs `nodedb_sql`'s DEFAULT classifier, which consults the same +//! `FunctionRegistry` the resolver's undefined-function gate consults. No +//! second registry and no second name list exist here. + +use nodedb_sql::SqlError; +use nodedb_sql::ddl_ast::collection_type::parse_column_type_str_full; +use nodedb_types::error::sqlstate; + +use super::super::super::super::result::DdlError; + +/// Refuse a declared column `DEFAULT` the server cannot evaluate. +/// +/// The expression is classified and parsed, never evaluated, so a +/// `DEFAULT nextval('s')` column never advances its sequence at `CREATE`. +/// +/// An unregistered function name raises SQLSTATE `42883`; every other +/// rejection raises SQLSTATE `42601`. +pub(super) fn validate_column_defaults(columns: &[(String, String)]) -> Result<(), DdlError> { + for (column, type_str) in columns { + let (_, _, _, default_expr) = parse_column_type_str_full(type_str); + let Some(expr) = default_expr else { + continue; + }; + nodedb_sql::planner::defaults::validate_default_expr(&expr, column) + .map_err(|error| default_error(column, &error))?; + } + Ok(()) +} + +/// Map a DEFAULT validation error onto its SQLSTATE. +fn default_error(column: &str, error: &SqlError) -> DdlError { + let sqlstate = match error { + SqlError::UndefinedFunction { .. } => sqlstate::UNDEFINED_FUNCTION, + _ => sqlstate::SYNTAX_ERROR, + }; + DdlError::new( + sqlstate, + format!("DEFAULT for column '{column}' is invalid: {error}"), + ) +} diff --git a/nodedb/src/control/server/shared/ddl/neutral/collection/create/mod.rs b/nodedb/src/control/server/shared/ddl/neutral/collection/create/mod.rs index c59a645ad..ee032daf9 100644 --- a/nodedb/src/control/server/shared/ddl/neutral/collection/create/mod.rs +++ b/nodedb/src/control/server/shared/ddl/neutral/collection/create/mod.rs @@ -4,6 +4,7 @@ //! //! Relocated from `pgwire::ddl::collection::create` (now deleted): //! - [`build`] — the shared `build_and_persist` body + `Variant` +//! - [`build_column_defaults`] — DDL-time column `DEFAULT` gate for `build` //! - [`build_flags`] — name / flag validation for `build` //! - [`build_primary_engine`] — vector-primary resolution for `build` //! - [`build_post_create`] — post-create side effects for `build` @@ -12,12 +13,14 @@ //! - [`table`] — the `create_table` entry point //! - [`request`] — `CreateCollectionRequest` //! -//! `build_flags`, `build_primary_engine`, and `build_post_create` are +//! `build_column_defaults`, `build_flags`, `build_primary_engine`, and +//! `build_post_create` are //! internal to [`build`] — declared here (siblings must be declared by the //! parent module, not by `build` itself) but scoped no wider than `build` //! needs them. pub mod build; +pub(in crate::control::server::shared::ddl::neutral::collection::create) mod build_column_defaults; pub(in crate::control::server::shared::ddl::neutral::collection::create) mod build_flags; pub(in crate::control::server::shared::ddl::neutral::collection::create) mod build_post_create; pub(in crate::control::server::shared::ddl::neutral::collection::create) mod build_primary_engine; diff --git a/nodedb/src/control/server/shared/ddl/neutral/collection/register.rs b/nodedb/src/control/server/shared/ddl/neutral/collection/register.rs index 6d3f8e789..f0674723a 100644 --- a/nodedb/src/control/server/shared/ddl/neutral/collection/register.rs +++ b/nodedb/src/control/server/shared/ddl/neutral/collection/register.rs @@ -62,7 +62,7 @@ pub async fn dispatch_register_if_needed( return Ok(()); }; let (fields, _serial_fields) = - crate::control::server::shared::ddl::schema_validation::parse_fields_clause(parts); + crate::control::server::shared::ddl::schema_validation::parse_fields_clause(&name, parts); let mut indexes = derive_auto_indexes(fields.iter().map(|(n, _)| n.as_str())); extend_with_catalog_indexes(&mut indexes, &coll); // `sql` is unused on this leader-side path: index derivation reads diff --git a/nodedb/src/control/server/shared/ddl/schema_validation.rs b/nodedb/src/control/server/shared/ddl/schema_validation.rs index bebbe40c8..0bf034974 100644 --- a/nodedb/src/control/server/shared/ddl/schema_validation.rs +++ b/nodedb/src/control/server/shared/ddl/schema_validation.rs @@ -2,18 +2,50 @@ //! Schema parsing and type validation helpers for collection DDL. +/// Expand a `SERIAL` / `BIGSERIAL` type token into its backing type and DEFAULT. +/// +/// `SERIAL` yields `INT DEFAULT nextval('{collection}_{field}_seq')` and +/// `BIGSERIAL` yields `BIGINT DEFAULT nextval(...)`. The sequence name matches +/// the one `create_serial_sequences` materializes, so the column reads the +/// sequence its own declaration created. Returns `None` for every other type. +/// +/// Trailing modifiers such as `PRIMARY KEY` follow the DEFAULT clause, where +/// `parse_column_type_str_full` expects them. +fn expand_serial_type(collection: &str, field: &str, type_str: &str) -> Option { + let trimmed = type_str.trim(); + let (head, tail) = match trimmed.find(char::is_whitespace) { + Some(index) => (&trimmed[..index], trimmed[index..].trim()), + None => (trimmed, ""), + }; + let base = if head.eq_ignore_ascii_case("SERIAL") { + "INT" + } else if head.eq_ignore_ascii_case("BIGSERIAL") { + "BIGINT" + } else { + return None; + }; + let expanded = format!("{base} DEFAULT nextval('{collection}_{field}_seq')"); + Some(if tail.is_empty() { + expanded + } else { + format!("{expanded} {tail}") + }) +} + /// Parse FIELDS clause from CREATE COLLECTION parts. /// /// Syntax: `CREATE COLLECTION name FIELDS (field1 type1, field2 type2, ...)` /// Returns empty vec if no FIELDS clause. /// -/// SERIAL and BIGSERIAL are expanded: -/// `id SERIAL` → `id INT` (caller creates implicit sequence) -/// `id BIGSERIAL` → `id BIGINT` (caller creates implicit sequence) +/// SERIAL and BIGSERIAL expand to their backing type plus a `nextval` DEFAULT +/// naming the sequence the caller creates for that column. /// /// The second return value lists field names that had SERIAL/BIGSERIAL types, /// so the caller can create the implicit sequences. -pub(crate) fn parse_fields_clause(parts: &[&str]) -> (Vec<(String, String)>, Vec) { +pub(crate) fn parse_fields_clause( + collection: &str, + parts: &[&str], +) -> (Vec<(String, String)>, Vec) { let fields_idx = parts.iter().position(|p| p.eq_ignore_ascii_case("FIELDS")); let fields_idx = match fields_idx { Some(i) => i, @@ -40,17 +72,13 @@ pub(crate) fn parse_fields_clause(parts: &[&str]) -> (Vec<(String, String)>, Vec let name = name.to_string(); let type_name = tokens.next().unwrap_or("text").to_uppercase(); - // Expand SERIAL/BIGSERIAL shorthand. - let actual_type = match type_name.as_str() { - "SERIAL" => { + // Expand SERIAL/BIGSERIAL shorthand into type + nextval DEFAULT. + let actual_type = match expand_serial_type(collection, &name, &type_name) { + Some(expanded) => { serial_fields.push(name.clone()); - "INT".to_string() + expanded } - "BIGSERIAL" => { - serial_fields.push(name.clone()); - "BIGINT".to_string() - } - other => other.to_string(), + None => type_name, }; fields.push((name, actual_type)); @@ -68,20 +96,19 @@ pub(crate) fn parse_fields_clause(parts: &[&str]) -> (Vec<(String, String)>, Vec /// of columns whose type was expanded from SERIAL / BIGSERIAL so the caller /// can create implicit sequences. pub(crate) fn parse_fields_clause_from_pairs( + collection: &str, columns: &[(String, String)], ) -> (Vec<(String, String)>, Vec) { let mut fields: Vec<(String, String)> = Vec::new(); let mut serial_fields: Vec = Vec::new(); for (name, type_str) in columns { - let actual_type = if type_str.eq_ignore_ascii_case("SERIAL") { - serial_fields.push(name.clone()); - "INT".to_string() - } else if type_str.eq_ignore_ascii_case("BIGSERIAL") { - serial_fields.push(name.clone()); - "BIGINT".to_string() - } else { - type_str.clone() + let actual_type = match expand_serial_type(collection, name, type_str) { + Some(expanded) => { + serial_fields.push(name.clone()); + expanded + } + None => type_str.clone(), }; fields.push((name.clone(), actual_type)); } From 6e166efdcfe1660cdc8f09b40c39c73091e00a69 Mon Sep 17 00:00:00 2001 From: Farhan Syah Date: Thu, 10 Sep 2026 01:53:29 +0800 Subject: [PATCH 07/23] test(sql): cover plan-only prepare/describe leaving sequences untouched Prepare and describe round trips for an INSERT with a nextval DEFAULT must plan the statement without allocating from the sequence; only executing the statement advances it. --- nodedb/tests/wire/cases/sql_sequences.rs | 94 ++++++++++++++++++++++++ 1 file changed, 94 insertions(+) diff --git a/nodedb/tests/wire/cases/sql_sequences.rs b/nodedb/tests/wire/cases/sql_sequences.rs index 5e45704c0..a86949309 100644 --- a/nodedb/tests/wire/cases/sql_sequences.rs +++ b/nodedb/tests/wire/cases/sql_sequences.rs @@ -205,3 +205,97 @@ async fn serial_column_allocates_successive_keys() { } assert_eq!(rows, vec!["1".to_string(), "2".to_string()]); } + +/// Preparing an INSERT must not consume a `nextval` allocation. +/// Only executing the statement advances the sequence. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn preparing_an_insert_does_not_advance_a_sequence_default() { + let server = TestServer::start().await; + + server + .exec("CREATE SEQUENCE seq_plan_side_effect;") + .await + .unwrap(); + server + .exec( + "CREATE COLLECTION seq_plan_probe (\ + id BIGINT DEFAULT nextval('seq_plan_side_effect') PRIMARY KEY, \ + v TEXT) WITH (engine='document_strict')", + ) + .await + .unwrap(); + + const INSERT: &str = "INSERT INTO seq_plan_probe (v) VALUES ('a')"; + + // Parse/Describe only. Each round trip plans the statement without + // executing it, so none of them can allocate a sequence value. + for _ in 0..3 { + server + .client + .prepare(INSERT) + .await + .expect("prepare INSERT with a nextval default must succeed"); + } + + server.exec(INSERT).await.unwrap(); + + let rows = server + .query_text("SELECT id FROM seq_plan_probe") + .await + .unwrap(); + assert_eq!(rows.len(), 1, "one row expected: {rows:?}"); + assert_eq!( + rows[0].trim(), + "1", + "planning must leave the sequence at its start, got id `{}`", + rows[0] + ); + + // The next allocation is 2 when exactly one value was consumed. + let next = server + .query_text("SELECT nextval('seq_plan_side_effect')") + .await + .unwrap(); + assert_eq!( + next, + vec!["2".to_string()], + "one execution must consume exactly one value, got {next:?}" + ); +} + +/// Describing an INSERT that omits a `nextval` DEFAULT reports its columns +/// without consuming a sequence value. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn describing_an_insert_leaves_the_sequence_untouched() { + let server = TestServer::start().await; + + server + .exec("CREATE SEQUENCE seq_describe_probe;") + .await + .unwrap(); + server + .exec( + "CREATE COLLECTION seq_describe_target (\ + id BIGINT DEFAULT nextval('seq_describe_probe') PRIMARY KEY, \ + v TEXT) WITH (engine='document_strict')", + ) + .await + .unwrap(); + + server + .client + .prepare("INSERT INTO seq_describe_target (v) VALUES ($1) RETURNING id") + .await + .expect("prepare INSERT ... RETURNING with a nextval default must succeed"); + + // The sequence was never executed against, so the first allocation is 1. + let first = server + .query_text("SELECT nextval('seq_describe_probe')") + .await + .unwrap(); + assert_eq!( + first, + vec!["1".to_string()], + "describe must not allocate, got {first:?}" + ); +} From dd79709088052f351ba280731cbead855cbe388d Mon Sep 17 00:00:00 2001 From: Farhan Syah Date: Thu, 10 Sep 2026 01:53:57 +0800 Subject: [PATCH 08/23] test(sql): cover declared column types across DEFAULT and NOT NULL A column declared with a numeric type keeps its OID, comparison semantics, and exact integer round trip when followed by a DEFAULT clause or a NOT NULL modifier, across the document (strict and schemaless) and columnar engines. --- nodedb/tests/wire/cases/mod.rs | 1 + .../wire/cases/sql_declared_column_types.rs | 240 ++++++++++++++++++ 2 files changed, 241 insertions(+) create mode 100644 nodedb/tests/wire/cases/sql_declared_column_types.rs diff --git a/nodedb/tests/wire/cases/mod.rs b/nodedb/tests/wire/cases/mod.rs index 73958e2fa..d156abedc 100644 --- a/nodedb/tests/wire/cases/mod.rs +++ b/nodedb/tests/wire/cases/mod.rs @@ -161,6 +161,7 @@ mod sql_conflict_policy; mod sql_copy_from; mod sql_copy_to; mod sql_cursors; +mod sql_declared_column_types; mod sql_default_expressions; mod sql_default_volatility; mod sql_division_by_zero; diff --git a/nodedb/tests/wire/cases/sql_declared_column_types.rs b/nodedb/tests/wire/cases/sql_declared_column_types.rs new file mode 100644 index 000000000..d35ea21ac --- /dev/null +++ b/nodedb/tests/wire/cases/sql_declared_column_types.rs @@ -0,0 +1,240 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! A declared column type resolves from the type keyword alone. +//! +//! `CREATE COLLECTION` records each column as the raw DDL text after its +//! name, so a column written `n INT DEFAULT 5` is stored as the type string +//! `INT DEFAULT 5`. A resolver that matches that whole string recognizes no +//! keyword and falls through to text, which costs the column its numeric +//! `RowDescription` OID, its numeric comparison semantics, and its exact +//! integer round trip. +//! +//! Every test here pairs a column carrying a `DEFAULT` clause with a control +//! column of the same declared type carrying none, so the `DEFAULT` clause is +//! the only variable. +//! +//! Companion coverage: `ddl_int_width_aliases_strict_kv.rs` for declared +//! integer widths, `sql_default_expressions.rs` for DEFAULT evaluation itself. + +use crate::harness::TestServer; + +/// PostgreSQL type OID for `int4`. +const OID_INT4: u32 = 23; +/// PostgreSQL type OID for `int8`. +const OID_INT8: u32 = 20; + +/// A value above 2^53, which no `f64` and no decimal-rendering text path +/// carries back unchanged. +const BEYOND_F64_MANTISSA: i64 = 9_007_199_254_740_993; + +/// Assert the exact `RowDescription` OID of each named column. +/// +/// A missing column fails loudly rather than being skipped: a silent skip +/// turns this into a test that passes when the columns vanish. +fn assert_column_oids(row: &tokio_postgres::Row, expected: &[(&str, u32)]) { + for (col_name, expected_oid) in expected { + let col = row + .columns() + .iter() + .find(|c| c.name() == *col_name) + .unwrap_or_else(|| { + panic!( + "column '{col_name}' must appear in RowDescription; got {:?}", + row.columns().iter().map(|c| c.name()).collect::>() + ) + }); + assert_eq!( + col.type_().oid(), + *expected_oid, + "column '{col_name}' must advertise OID {expected_oid}, got {}", + col.type_().oid() + ); + } +} + +/// Asserts a rendered row carries a real value in place of an absent or NULL column. +fn assert_not_null(row: &str, label: &str) { + let trimmed = row.trim(); + assert!( + !trimmed.is_empty() && !trimmed.eq_ignore_ascii_case("null"), + "{label}: expected a value, got `{row}`" + ); +} + +/// Create one collection per engine carrying a defaulted and a control column +/// of each numeric width, then insert the single probe row. +/// +/// `n`/`big` declare a `DEFAULT`; `plain`/`plain_big` declare the same type +/// with none. `ts` covers a temporal column with a volatile `DEFAULT`. +async fn create_and_seed(server: &TestServer, name: &str, engine: &str) { + server + .exec(&format!( + "CREATE COLLECTION {name} (\ + id TEXT PRIMARY KEY, \ + n INT DEFAULT 5, \ + plain INT, \ + big BIGINT DEFAULT {BEYOND_F64_MANTISSA}, \ + plain_big BIGINT, \ + ts TIMESTAMP DEFAULT NOW()) WITH (engine='{engine}')" + )) + .await + .unwrap_or_else(|e| panic!("create {name} on {engine}: {e}")); + + server + .exec(&format!( + "INSERT INTO {name} (id, plain, plain_big) \ + VALUES ('k1', 5, {BEYOND_F64_MANTISSA})" + )) + .await + .unwrap_or_else(|e| panic!("insert into {name}: {e}")); +} + +/// Assert every declared-type behavior on a seeded collection. +/// +/// The defaulted column and its control column must agree on all three: +/// advertised OID, numeric comparison, and exact integer round trip. +async fn assert_declared_types_hold(server: &TestServer, name: &str) { + let stmt = server + .client + .prepare_typed( + &format!("SELECT n, plain, big, plain_big FROM {name} WHERE id = $1"), + &[tokio_postgres::types::Type::TEXT], + ) + .await + .unwrap_or_else(|e| panic!("prepare select on {name}: {e}")); + let rows = server + .client + .query(&stmt, &[&"k1"]) + .await + .unwrap_or_else(|e| panic!("execute select on {name}: {e}")); + assert_eq!(rows.len(), 1, "one row expected from {name}"); + + assert_column_oids( + &rows[0], + &[ + ("n", OID_INT4), + ("plain", OID_INT4), + ("big", OID_INT8), + ("plain_big", OID_INT8), + ], + ); + + // Typed getters matching the advertised widths: a wrong OID or a + // wrong-width binary payload panics inside `get` before the comparison. + assert_eq!(rows[0].get::<_, i32>("n"), 5); + assert_eq!(rows[0].get::<_, i32>("plain"), 5); + assert_eq!(rows[0].get::<_, i64>("big"), BEYOND_F64_MANTISSA); + assert_eq!(rows[0].get::<_, i64>("plain_big"), BEYOND_F64_MANTISSA); + + for column in ["n", "plain"] { + let matched = server + .query_text(&format!("SELECT id FROM {name} WHERE {column} > 4")) + .await + .unwrap_or_else(|e| panic!("{name}.{column} > 4: {e}")); + assert_eq!( + matched, + vec!["k1".to_string()], + "{name}.{column} > 4 must match the row, got {matched:?}" + ); + + let unmatched = server + .query_text(&format!("SELECT id FROM {name} WHERE {column} > 6")) + .await + .unwrap_or_else(|e| panic!("{name}.{column} > 6: {e}")); + assert!( + unmatched.is_empty(), + "{name}.{column} > 6 must match nothing, got {unmatched:?}" + ); + } + + for column in ["big", "plain_big"] { + let stored = server + .query_text(&format!("SELECT {column} FROM {name} WHERE id = 'k1'")) + .await + .unwrap_or_else(|e| panic!("{name}.{column} read: {e}")); + assert_eq!(stored.len(), 1, "one row expected for {name}.{column}"); + assert_eq!( + stored[0].trim(), + BEYOND_F64_MANTISSA.to_string(), + "{name}.{column} must round-trip exactly, got `{}`", + stored[0] + ); + } + + let stamps = server + .query_text(&format!("SELECT ts FROM {name} WHERE id = 'k1'")) + .await + .unwrap_or_else(|e| panic!("{name}.ts read: {e}")); + assert_eq!(stamps.len(), 1, "one row expected for {name}.ts"); + assert_not_null(&stamps[0], "ts"); +} + +/// A `document_strict` column keeps its declared type when a `DEFAULT` +/// clause follows it, matching a control column that declares none. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn strict_columns_keep_their_declared_type_across_a_default_clause() { + let server = TestServer::start().await; + create_and_seed(&server, "typed_defaults_strict", "document_strict").await; + assert_declared_types_hold(&server, "typed_defaults_strict").await; +} + +/// A `document_schemaless` column keeps its declared type when a `DEFAULT` +/// clause follows it, matching a control column that declares none. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn schemaless_columns_keep_their_declared_type_across_a_default_clause() { + let server = TestServer::start().await; + create_and_seed(&server, "typed_defaults_schemaless", "document_schemaless").await; + assert_declared_types_hold(&server, "typed_defaults_schemaless").await; +} + +/// A `columnar` column keeps its declared type when a `DEFAULT` clause +/// follows it, matching a control column that declares none. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn columnar_columns_keep_their_declared_type_across_a_default_clause() { + let server = TestServer::start().await; + create_and_seed(&server, "typed_defaults_columnar", "columnar").await; + assert_declared_types_hold(&server, "typed_defaults_columnar").await; +} + +/// A `NOT NULL` modifier leaves the declared integer type intact, the same +/// way a `DEFAULT` clause must. +/// +/// `NOT NULL` and `DEFAULT` are both trailing modifiers on the same stored +/// type string, so they share one resolution path. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn a_not_null_modifier_leaves_the_declared_integer_type_intact() { + let server = TestServer::start().await; + + server + .exec( + "CREATE COLLECTION typed_not_null (\ + id TEXT PRIMARY KEY, \ + n INT NOT NULL, \ + plain INT) WITH (engine='document_schemaless')", + ) + .await + .unwrap(); + server + .exec("INSERT INTO typed_not_null (id, n, plain) VALUES ('k1', 5, 5)") + .await + .unwrap(); + + let stmt = server + .client + .prepare_typed( + "SELECT n, plain FROM typed_not_null WHERE id = $1", + &[tokio_postgres::types::Type::TEXT], + ) + .await + .expect("prepare typed_not_null select"); + let rows = server + .client + .query(&stmt, &[&"k1"]) + .await + .expect("execute typed_not_null select"); + assert_eq!(rows.len(), 1, "one row expected from typed_not_null"); + + assert_column_oids(&rows[0], &[("n", OID_INT4), ("plain", OID_INT4)]); + assert_eq!(rows[0].get::<_, i32>("n"), 5); + assert_eq!(rows[0].get::<_, i32>("plain"), 5); +} From 95428a47f4ecdcf526b62e0a1bcc1f7c413975ff Mon Sep 17 00:00:00 2001 From: Farhan Syah Date: Thu, 10 Sep 2026 01:54:17 +0800 Subject: [PATCH 09/23] test(sql): cover DEFAULT evaluation on primary='vector' collections A vector-primary insert splits each row into the vector field and a payload map before the shared DEFAULT pass runs. Cover a nextval and a UUID_V7 key DEFAULT and a non-key payload DEFAULT, each on a column the insert omits. --- nodedb/tests/wire/cases/mod.rs | 1 + .../wire/cases/sql_default_vector_primary.rs | 165 ++++++++++++++++++ 2 files changed, 166 insertions(+) create mode 100644 nodedb/tests/wire/cases/sql_default_vector_primary.rs diff --git a/nodedb/tests/wire/cases/mod.rs b/nodedb/tests/wire/cases/mod.rs index d156abedc..e51303c21 100644 --- a/nodedb/tests/wire/cases/mod.rs +++ b/nodedb/tests/wire/cases/mod.rs @@ -163,6 +163,7 @@ mod sql_copy_to; mod sql_cursors; mod sql_declared_column_types; mod sql_default_expressions; +mod sql_default_vector_primary; mod sql_default_volatility; mod sql_division_by_zero; mod sql_division_by_zero_composite; diff --git a/nodedb/tests/wire/cases/sql_default_vector_primary.rs b/nodedb/tests/wire/cases/sql_default_vector_primary.rs new file mode 100644 index 000000000..4a1be23fb --- /dev/null +++ b/nodedb/tests/wire/cases/sql_default_vector_primary.rs @@ -0,0 +1,165 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! Column DEFAULT evaluation on a `primary='vector'` collection. +//! +//! A vector-primary INSERT bypasses document encoding: the planner splits each +//! row into the vector field and a payload map before the shared DEFAULT pass +//! runs. A column the statement omits therefore reaches storage only if this +//! path materializes its declared DEFAULT itself. +//! +//! Companion coverage: `sql_default_expressions.rs` for DEFAULT evaluation on +//! the document, key-value, and columnar engines. + +use crate::harness::TestServer; + +/// Create the vector-primary collection every test in this file writes to. +async fn create_vector_primary(server: &TestServer, name: &str, key_type: &str) { + server + .exec(&format!( + "CREATE COLLECTION {name} (\ + id {key_type} PRIMARY KEY, \ + vec VECTOR(3), \ + owner STRING) \ + WITH (engine='vector', primary='vector', vector_field='vec', dim=3, \ + payload_indexes=['owner'])" + )) + .await + .unwrap_or_else(|e| panic!("create {name}: {e}")); +} + +/// Read the `id` column of every row, failing loudly on an absent or NULL value. +async fn sorted_ids(server: &TestServer, name: &str) -> Vec { + let rows = server + .query_named_rows(&format!("SELECT * FROM {name}")) + .await + .unwrap_or_else(|e| panic!("SELECT * FROM {name}: {e}")); + let mut ids: Vec = rows + .iter() + .map(|row| { + let id = row + .get("id") + .unwrap_or_else(|| panic!("row must carry an `id` column, got {row:?}")); + let trimmed = id.trim(); + assert!( + !trimmed.is_empty() && !trimmed.eq_ignore_ascii_case("null"), + "id: expected a value, got `{id}`" + ); + trimmed.to_string() + }) + .collect(); + ids.sort(); + ids +} + +/// A `DEFAULT nextval('seq')` key column allocates 1 then 2 across two +/// vector-primary inserts that omit it. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn nextval_default_fills_an_omitted_vector_primary_key() { + let server = TestServer::start().await; + + server + .exec("CREATE SEQUENCE seq_vector_default;") + .await + .unwrap(); + create_vector_primary( + &server, + "def_vec_seq", + "BIGINT DEFAULT nextval('seq_vector_default')", + ) + .await; + + server + .exec( + "INSERT INTO def_vec_seq (vec, owner) \ + VALUES (ARRAY[1.0, 0.0, 0.0], 'alice')", + ) + .await + .expect("vector-primary insert omitting a defaulted key must succeed"); + server + .exec( + "INSERT INTO def_vec_seq (vec, owner) \ + VALUES (ARRAY[0.0, 1.0, 0.0], 'bob')", + ) + .await + .expect("second vector-primary insert must succeed"); + + let ids = sorted_ids(&server, "def_vec_seq").await; + assert_eq!( + ids, + vec!["1".to_string(), "2".to_string()], + "nextval must allocate 1 then 2, got {ids:?}" + ); +} + +/// A `DEFAULT UUID_V7()` key column fills a distinct value per vector-primary +/// insert that omits it. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn uuid_default_fills_an_omitted_vector_primary_key() { + let server = TestServer::start().await; + + create_vector_primary(&server, "def_vec_uuid", "STRING DEFAULT UUID_V7()").await; + + server + .exec( + "INSERT INTO def_vec_uuid (vec, owner) \ + VALUES (ARRAY[1.0, 0.0, 0.0], 'alice')", + ) + .await + .expect("vector-primary insert omitting a defaulted key must succeed"); + server + .exec( + "INSERT INTO def_vec_uuid (vec, owner) \ + VALUES (ARRAY[0.0, 1.0, 0.0], 'bob')", + ) + .await + .expect("second vector-primary insert must succeed"); + + let ids = sorted_ids(&server, "def_vec_uuid").await; + assert_eq!(ids.len(), 2, "two rows expected: {ids:?}"); + let distinct: std::collections::HashSet<&str> = ids.iter().map(|s| s.as_str()).collect(); + assert_eq!( + distinct.len(), + 2, + "each row must carry a distinct id, got {ids:?}" + ); +} + +/// A non-key column DEFAULT fills on a vector-primary insert that omits it. +/// +/// The payload map carries every column but the vector field, so a defaulted +/// payload column shares the key column's materialization path. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn a_default_fills_an_omitted_vector_primary_payload_column() { + let server = TestServer::start().await; + + server + .exec( + "CREATE COLLECTION def_vec_payload (\ + id STRING PRIMARY KEY, \ + vec VECTOR(3), \ + owner STRING DEFAULT 'unassigned') \ + WITH (engine='vector', primary='vector', vector_field='vec', dim=3, \ + payload_indexes=['owner'])", + ) + .await + .unwrap(); + + server + .exec("INSERT INTO def_vec_payload (id, vec) VALUES ('r1', ARRAY[1.0, 0.0, 0.0])") + .await + .expect("vector-primary insert omitting a defaulted payload column must succeed"); + + let rows = server + .query_named_rows("SELECT * FROM def_vec_payload") + .await + .expect("SELECT * FROM def_vec_payload"); + assert_eq!(rows.len(), 1, "one row expected: {rows:?}"); + let owner = rows[0] + .get("owner") + .unwrap_or_else(|| panic!("row must carry an `owner` column, got {:?}", rows[0])); + assert_eq!( + owner.trim(), + "unassigned", + "the declared DEFAULT must fill the omitted column, got `{owner}`" + ); +} From e643437d80df80a5d423ef6f94a9611cfb9f8936 Mon Sep 17 00:00:00 2001 From: Farhan Syah Date: Thu, 10 Sep 2026 05:03:57 +0800 Subject: [PATCH 10/23] feat(sql): materialize declared DEFAULTs for KV and vector-primary inserts Extract the KV engine's DEFAULT-materialization helper into a shared declared_defaults module and reuse it for vector-primary collection inserts, which previously bypassed EngineRules::plan_insert and never expanded a declared DEFAULT. Materialization runs before the row's declared-type coercion and range checks, so a default is validated exactly like a supplied literal. Track whether any materialized default came from a volatile expression (e.g. nextval) on VectorPrimaryInsert, matching the KV insert plan's existing gate, so a plan containing one is excluded from the plan cache instead of replaying a frozen value. --- nodedb-sql/src/planner/dml.rs | 18 ++++- .../planner/dml_helpers/declared_defaults.rs | 69 +++++++++++++++++++ .../src/planner/dml_helpers/kv_insert.rs | 49 +------------ nodedb-sql/src/planner/dml_helpers/mod.rs | 3 + .../dml_helpers/vector_primary_insert.rs | 7 ++ nodedb-sql/src/types/plan/cacheability.rs | 4 ++ nodedb-sql/src/types/plan/variants.rs | 5 ++ .../src/visitor/plan_visitor/dispatch_rest.rs | 2 + 8 files changed, 108 insertions(+), 49 deletions(-) create mode 100644 nodedb-sql/src/planner/dml_helpers/declared_defaults.rs diff --git a/nodedb-sql/src/planner/dml.rs b/nodedb-sql/src/planner/dml.rs index d5b5c769e..350e078fd 100644 --- a/nodedb-sql/src/planner/dml.rs +++ b/nodedb-sql/src/planner/dml.rs @@ -9,7 +9,7 @@ use super::dml_helpers::{ KvInsertParams, bind_insert_select_columns, build_kv_insert_plan, build_vector_primary_insert_plan, check_declared_float_ranges_in_assignments, check_declared_int_ranges_in_assignments, coerce_and_check_rows, convert_value_rows, - resolve_insert_columns, + materialize_defaults_in_rows, resolve_insert_columns, }; use crate::engine_rules::{self, InsertParams}; use crate::error::{Result, SqlError}; @@ -196,12 +196,26 @@ pub fn plan_insert(ins: &ast::Insert, catalog: &dyn SqlCatalog) -> Result, + catalog: &dyn SqlCatalog, +) -> Result { + let mut volatile = false; + for column in declared_columns { + let Some(default_expr) = column.default.as_deref() else { + continue; + }; + if row.iter().any(|(name, _)| name == &column.name) { + continue; + } + let evaluated = + crate::planner::defaults::evaluate_default_expr(default_expr, &column.name, catalog)?; + let value = crate::planner::defaults::default_value_to_sql(&column.name, evaluated)?; + volatile |= crate::types::plan::default_expr_is_volatile(default_expr); + row.push((column.name.clone(), value)); + } + Ok(volatile) +} + +/// Materialize declared DEFAULTs across a whole `VALUES` row set. +/// +/// Returns whether any materialized default was volatile. +pub(crate) fn materialize_defaults_in_rows( + declared_columns: &[ColumnInfo], + rows: &mut [Vec<(String, SqlValue)>], + catalog: &dyn SqlCatalog, +) -> Result { + let mut volatile = false; + for row in rows.iter_mut() { + volatile |= materialize_declared_defaults(declared_columns, row, catalog)?; + } + Ok(volatile) +} diff --git a/nodedb-sql/src/planner/dml_helpers/kv_insert.rs b/nodedb-sql/src/planner/dml_helpers/kv_insert.rs index d936a789f..0c486e5b1 100644 --- a/nodedb-sql/src/planner/dml_helpers/kv_insert.rs +++ b/nodedb-sql/src/planner/dml_helpers/kv_insert.rs @@ -3,13 +3,13 @@ //! Plan construction for the KV engine's `VALUES`-clause insert paths //! (plain `INSERT`, `UPSERT`, and `INSERT ... ON CONFLICT DO UPDATE`). +use super::declared_defaults::materialize_declared_defaults; use super::params::KvInsertParams; use super::range_check::{ check_declared_float_ranges, check_declared_float_ranges_in_assignments, check_declared_int_ranges, check_declared_int_ranges_in_assignments, }; use super::value_convert::expr_to_sql_value; -use crate::catalog::SqlCatalog; use crate::error::{Result, SqlError}; use crate::planner::declared_type_coerce::{ coerce_assignments_to_declared_types, coerce_row_to_declared_types, @@ -137,52 +137,6 @@ pub(crate) fn build_kv_insert_plan(params: KvInsertParams<'_>) -> Result, - catalog: &dyn SqlCatalog, -) -> Result { - let mut volatile = false; - for column in declared_columns { - let Some(default_expr) = column.default.as_deref() else { - continue; - }; - if row.iter().any(|(name, _)| name == &column.name) { - continue; - } - let evaluated = - crate::planner::defaults::evaluate_default_expr(default_expr, &column.name, catalog)?; - let value = crate::planner::defaults::default_value_to_sql(&column.name, evaluated)?; - volatile |= crate::types::plan::default_expr_is_volatile(default_expr); - row.push((column.name.clone(), value)); - } - Ok(volatile) -} - #[cfg(test)] mod kv_on_conflict_range_tests { use sqlparser::ast; @@ -190,6 +144,7 @@ mod kv_on_conflict_range_tests { use sqlparser::tokenizer::Span; use super::*; + use crate::catalog::SqlCatalog; use nodedb_types::columnar::{FloatWidth, IntWidth}; /// A catalog with no collections and no sequence state. These cases diff --git a/nodedb-sql/src/planner/dml_helpers/mod.rs b/nodedb-sql/src/planner/dml_helpers/mod.rs index 978eec9a5..242266de8 100644 --- a/nodedb-sql/src/planner/dml_helpers/mod.rs +++ b/nodedb-sql/src/planner/dml_helpers/mod.rs @@ -5,12 +5,14 @@ //! - [`range_check`] — declared-width coercion + range validation //! - [`insert_columns`] — positional-insert column resolution //! - [`ast_extract`] — table-name / primary-key point-lookup extraction +//! - [`declared_defaults`] — declared column DEFAULT materialization //! - [`vector_primary_insert`] — vector-primary collection insert plans //! - [`kv_insert`] — KV engine insert plans //! - [`insert_select_bind`] — `INSERT ... SELECT` target-column binding //! - [`params`] — parameter structs for the helpers above mod ast_extract; +mod declared_defaults; mod insert_columns; mod insert_select_bind; mod kv_insert; @@ -21,6 +23,7 @@ mod vector_primary_insert; pub use ast_extract::extract_point_keys; pub(super) use ast_extract::extract_table_name_from_table_with_joins; +pub(super) use declared_defaults::materialize_defaults_in_rows; pub(super) use insert_columns::resolve_insert_columns; pub(super) use insert_select_bind::bind_insert_select_columns; pub(super) use kv_insert::build_kv_insert_plan; diff --git a/nodedb-sql/src/planner/dml_helpers/vector_primary_insert.rs b/nodedb-sql/src/planner/dml_helpers/vector_primary_insert.rs index 934888e84..1b48ba1ad 100644 --- a/nodedb-sql/src/planner/dml_helpers/vector_primary_insert.rs +++ b/nodedb-sql/src/planner/dml_helpers/vector_primary_insert.rs @@ -10,11 +10,17 @@ use crate::types::*; /// Extracts the vector-field column into `vector: Vec` and collects /// all remaining columns into `payload_fields`. Rows missing the vector /// column are rejected. +/// +/// `rows` arrive with every declared DEFAULT already materialized, so a +/// defaulted key or payload column reaches `payload_fields` like a supplied +/// one. `volatile_defaults` reports whether any of those defaults was volatile, +/// which keeps the plan out of the physical-plan cache. pub(crate) fn build_vector_primary_insert_plan( collection: &str, vpc: &nodedb_types::VectorPrimaryConfig, _columns: &[String], rows: Vec>, + volatile_defaults: bool, ) -> Result> { let mut result_rows = Vec::with_capacity(rows.len()); for row in rows { @@ -82,5 +88,6 @@ pub(crate) fn build_vector_primary_insert_plan( storage_dtype: vpc.storage_dtype, payload_indexes: vpc.payload_indexes.clone(), rows: result_rows, + volatile_defaults, }]) } diff --git a/nodedb-sql/src/types/plan/cacheability.rs b/nodedb-sql/src/types/plan/cacheability.rs index 6ef8d40c1..c03d0b509 100644 --- a/nodedb-sql/src/types/plan/cacheability.rs +++ b/nodedb-sql/src/types/plan/cacheability.rs @@ -53,6 +53,10 @@ impl SqlPlan { Self::KvInsert { volatile_defaults: true, .. + } + | Self::VectorPrimaryInsert { + volatile_defaults: true, + .. } => DataDependent, Self::PointGet { engine: EngineType::DocumentSchemaless | EngineType::DocumentStrict, diff --git a/nodedb-sql/src/types/plan/variants.rs b/nodedb-sql/src/types/plan/variants.rs index 305b9f019..7e11a6d5b 100644 --- a/nodedb-sql/src/types/plan/variants.rs +++ b/nodedb-sql/src/types/plan/variants.rs @@ -726,6 +726,11 @@ pub enum SqlPlan { /// via `payload.add_index` on the first DirectUpsert. payload_indexes: Vec<(String, nodedb_types::PayloadIndexKind)>, rows: Vec, + /// Whether any row carries a value materialized from a volatile + /// DEFAULT such as `nextval`. The value was evaluated while the plan + /// was built, so caching the lowered tasks would replay one + /// execution's value into every later one. + volatile_defaults: bool, }, // ── Index DDL ─────────────────────────────────────────────────────── diff --git a/nodedb-sql/src/visitor/plan_visitor/dispatch_rest.rs b/nodedb-sql/src/visitor/plan_visitor/dispatch_rest.rs index 2572ae135..0f5f61452 100644 --- a/nodedb-sql/src/visitor/plan_visitor/dispatch_rest.rs +++ b/nodedb-sql/src/visitor/plan_visitor/dispatch_rest.rs @@ -162,6 +162,8 @@ pub(super) fn dispatch_rest( storage_dtype, payload_indexes, rows, + // Cache eligibility only; lowering does not read it. + volatile_defaults: _, } => visitor.vector_primary_insert( collection, field, From ce641407d60157801c0236856259e111528fd195 Mon Sep 17 00:00:00 2001 From: Farhan Syah Date: Thu, 10 Sep 2026 05:04:15 +0800 Subject: [PATCH 11/23] feat(sql): announce RETURNING output columns instead of an empty schema A write plan's OutputSchema previously carried no columns, so the simple-query protocol fell back to deriving a RowDescription from the row payload rather than the catalog. Thread the parsed RETURNING spec through every planning entry point down to build_output_schema, which now announces the target collection's declared columns for a named clause and defers to the row-derived shape only for RETURNING *. Propagate the same announced schema through the Calvin dispatch and response paths and the neutral DDL DML path, so a RETURNING write renders identically regardless of which route it took. Split output_schema.rs into a directory of build/columns/returning modules along the way, and fix parse_type_str to strip DEFAULT/NOT NULL modifiers off a catalog column's raw declared-type text before resolving its wire type. Propagate a vector-primary insert's msgpack serialization error instead of discarding it into an empty payload. --- nodedb/src/control/event_trigger.rs | 2 +- .../planner/catalog_adapter/type_convert.rs | 20 +- .../control/planner/context/query/planning.rs | 37 +- .../procedural/executor/core/dispatch.rs | 2 +- .../sql_plan_convert/dml/kv_and_vector.rs | 5 +- .../build.rs} | 400 +++++------------- .../sql_plan_convert/output_schema/columns.rs | 270 ++++++++++++ .../sql_plan_convert/output_schema/mod.rs | 8 + .../output_schema/returning.rs | 217 ++++++++++ nodedb/src/control/scatter_gather/hop.rs | 2 +- .../server/pgwire/handler/cursor_query.rs | 2 +- .../server/pgwire/handler/prepared/parser.rs | 5 +- .../pgwire/handler/routing/calvin_dispatch.rs | 11 +- .../pgwire/handler/routing/calvin_response.rs | 8 +- .../server/pgwire/handler/routing/execute.rs | 6 +- .../server/pgwire/handler/routing/planning.rs | 16 +- .../pgwire/handler/routing/pre_dispatch.rs | 9 +- .../server/response_shape/returning.rs | 14 +- .../neutral/collection/dml/parse/dispatch.rs | 18 +- .../server/shared/ddl/neutral/planning.rs | 2 +- .../control/server/shared/plan_admission.rs | 2 +- 21 files changed, 711 insertions(+), 345 deletions(-) rename nodedb/src/control/planner/sql_plan_convert/{output_schema.rs => output_schema/build.rs} (65%) create mode 100644 nodedb/src/control/planner/sql_plan_convert/output_schema/columns.rs create mode 100644 nodedb/src/control/planner/sql_plan_convert/output_schema/mod.rs create mode 100644 nodedb/src/control/planner/sql_plan_convert/output_schema/returning.rs diff --git a/nodedb/src/control/event_trigger.rs b/nodedb/src/control/event_trigger.rs index 35f387348..217e58a09 100644 --- a/nodedb/src/control/event_trigger.rs +++ b/nodedb/src/control/event_trigger.rs @@ -340,7 +340,7 @@ pub async fn run_event_action_sql( tenant_id, database_id, &security.context(&shared), - false, + None, ) .await .map_err(|source| TriggerActionError::Plan { source })?; diff --git a/nodedb/src/control/planner/catalog_adapter/type_convert.rs b/nodedb/src/control/planner/catalog_adapter/type_convert.rs index 1c74810ad..c0bfcdd90 100644 --- a/nodedb/src/control/planner/catalog_adapter/type_convert.rs +++ b/nodedb/src/control/planner/catalog_adapter/type_convert.rs @@ -269,8 +269,17 @@ fn convert_column_type(ct: &nodedb_types::columnar::ColumnType) -> SqlDataType { } } +/// Resolve the declared SQL type of a catalog `fields` entry. +/// +/// The catalog records the raw DDL text that followed the column name, so an +/// entry reads `INT DEFAULT 5` or `INT NOT NULL`, not `INT`. The bare type +/// token comes from `parse_column_type_str_full`, the same splitter +/// `declared_default` uses and the same boundary `IntWidth::from_declared_type` +/// and `FloatWidth::from_declared_type` respect. A trailing modifier therefore +/// never changes the resolved type. fn parse_type_str(s: &str) -> SqlDataType { - let upper = s.to_uppercase(); + let (bare, _, _, _) = nodedb_sql::ddl_ast::collection_type::parse_column_type_str_full(s); + let upper = bare.to_uppercase(); // Handle DECIMAL/NUMERIC with optional (p,s) params. if upper.starts_with("DECIMAL") || upper.starts_with("NUMERIC") { return SqlDataType::Decimal; @@ -287,9 +296,12 @@ fn parse_type_str(s: &str) -> SqlDataType { // Same contract as the integer arm above, for the float family: every // spelling `FloatWidth::from_declared_type` recognizes must appear // here, or the column falls through to `_ => String` and advertises - // OID 25 (text) no matter what width was declared. - "FLOAT" | "FLOAT4" | "FLOAT8" | "FLOAT32" | "FLOAT64" | "DOUBLE" | "DOUBLE PRECISION" - | "REAL" => SqlDataType::Float64, + // OID 25 (text) no matter what width was declared. `DOUBLE PRECISION` + // arrives as the bare token `DOUBLE`, matching how + // `FloatWidth::from_declared_type` recognizes it. + "FLOAT" | "FLOAT4" | "FLOAT8" | "FLOAT32" | "FLOAT64" | "DOUBLE" | "REAL" => { + SqlDataType::Float64 + } "BOOL" | "BOOLEAN" => SqlDataType::Bool, "BYTES" | "BYTEA" | "BLOB" => SqlDataType::Bytes, "TIMESTAMP" | "TIMESTAMPTZ" => SqlDataType::Timestamp, diff --git a/nodedb/src/control/planner/context/query/planning.rs b/nodedb/src/control/planner/context/query/planning.rs index 69a483b17..6d46558ef 100644 --- a/nodedb/src/control/planner/context/query/planning.rs +++ b/nodedb/src/control/planner/context/query/planning.rs @@ -14,6 +14,7 @@ use crate::control::planner::context::security::PlanSecurityContext; use crate::control::planner::plan_error_map::map_plan_error; use crate::control::planner::sql_plan_convert::PlanningPurpose; use crate::control::server::response_shape::schema::OutputSchema; +use nodedb_physical::physical_plan::ReturningSpec; /// Bundled arguments for [`QueryContext::plan_sql_with_rls`]. pub struct PlanSqlWithRlsParams<'a> { @@ -49,6 +50,7 @@ impl QueryContext { tenant_id: crate::types::TenantId, database_id: crate::types::DatabaseId, purpose: PlanningPurpose, + returning: Option<&ReturningSpec>, ) -> crate::Result<( Vec, OutputSchema, @@ -146,6 +148,7 @@ impl QueryContext { &plans, catalog.as_ref(), database_id, + returning, ); let cache_eligibility = crate::control::planner::sql_plan_convert::batch_cache_eligibility(&plans); @@ -169,18 +172,22 @@ impl QueryContext { database_id, sec, } = params; - self.plan_sql_with_rls_returning(sql, tenant_id, database_id, sec, false) + self.plan_sql_with_rls_returning(sql, tenant_id, database_id, sec, None) .await } - /// Plan SQL with RLS injection, optionally propagating a RETURNING flag. + /// Plan SQL with RLS injection, announcing a DML `RETURNING` clause. + /// + /// `returning` is the spec `strip_returning` parsed off the statement, so + /// the write announces the columns it projects. `None` for a statement + /// that carries no clause. pub async fn plan_sql_with_rls_returning( &self, sql: &str, tenant_id: crate::types::TenantId, database_id: crate::types::DatabaseId, sec: &PlanSecurityContext<'_>, - returning: bool, + returning: Option<&ReturningSpec>, ) -> crate::Result<( Vec, OutputSchema, @@ -202,7 +209,7 @@ impl QueryContext { tenant_id: crate::types::TenantId, database_id: crate::types::DatabaseId, sec: &PlanSecurityContext<'_>, - returning: bool, + returning: Option<&ReturningSpec>, ) -> crate::Result<( Vec, OutputSchema, @@ -241,7 +248,7 @@ impl QueryContext { tenant_id, database_id, sec, - false, + None, PlanningPurpose::Metadata, ) .await @@ -254,7 +261,7 @@ impl QueryContext { tenant_id: crate::types::TenantId, database_id: crate::types::DatabaseId, sec: &PlanSecurityContext<'_>, - _returning: bool, + returning: Option<&ReturningSpec>, purpose: PlanningPurpose, ) -> crate::Result<( Vec, @@ -263,7 +270,7 @@ impl QueryContext { nodedb_sql::types::PlanCacheEligibility, )> { let (mut tasks, output_schema, mut version_set, cache_eligibility) = - self.plan_with_nodedb_sql_for_purpose(sql, tenant_id, database_id, purpose)?; + self.plan_with_nodedb_sql_for_purpose(sql, tenant_id, database_id, purpose, returning)?; // Versions read BEFORE injection, never after: injection reads live // policy/grant state under its own lock, and a mutation racing in @@ -313,13 +320,21 @@ impl QueryContext { tenant_id: crate::types::TenantId, database_id: crate::types::DatabaseId, sec: &PlanSecurityContext<'_>, + returning: Option<&ReturningSpec>, ) -> crate::Result<( Vec, OutputSchema, )> { - self.plan_sql_with_params_and_rls_and_versions(sql, params, tenant_id, database_id, sec) - .await - .map(|(tasks, schema, _)| (tasks, schema)) + self.plan_sql_with_params_and_rls_and_versions( + sql, + params, + tenant_id, + database_id, + sec, + returning, + ) + .await + .map(|(tasks, schema, _)| (tasks, schema)) } /// Parameterized RLS planning plus the descriptor versions observed by its @@ -332,6 +347,7 @@ impl QueryContext { tenant_id: crate::types::TenantId, database_id: crate::types::DatabaseId, sec: &PlanSecurityContext<'_>, + returning: Option<&ReturningSpec>, ) -> crate::Result<( Vec, OutputSchema, @@ -414,6 +430,7 @@ impl QueryContext { &plans, catalog.as_ref(), database_id, + returning, ); let mut tasks = crate::control::planner::sql_plan_convert::convert(&plans, tenant_id, &ctx)?; diff --git a/nodedb/src/control/planner/procedural/executor/core/dispatch.rs b/nodedb/src/control/planner/procedural/executor/core/dispatch.rs index 44bcf3bed..0606e485e 100644 --- a/nodedb/src/control/planner/procedural/executor/core/dispatch.rs +++ b/nodedb/src/control/planner/procedural/executor/core/dispatch.rs @@ -94,7 +94,7 @@ impl<'a> StatementExecutor<'a> { self.tenant_id, self.database_id, &security.context(self.state), - false, + None, ) .await?; let lease_scope = self.state.acquire_plan_lease_scope(&versions)?; diff --git a/nodedb/src/control/planner/sql_plan_convert/dml/kv_and_vector.rs b/nodedb/src/control/planner/sql_plan_convert/dml/kv_and_vector.rs index c5c03f023..5b3130117 100644 --- a/nodedb/src/control/planner/sql_plan_convert/dml/kv_and_vector.rs +++ b/nodedb/src/control/planner/sql_plan_convert/dml/kv_and_vector.rs @@ -171,7 +171,10 @@ pub(in super::super) fn convert_vector_primary_insert( .iter() .map(|(k, v)| (k.clone(), sql_value_to_nodedb_value(v))) .collect(); - zerompk::to_msgpack_vec(&value_map).unwrap_or_default() + zerompk::to_msgpack_vec(&value_map).map_err(|e| crate::Error::Serialization { + format: "msgpack".into(), + detail: format!("vector-primary payload: {e}"), + })? }; tasks.push(PhysicalTask { diff --git a/nodedb/src/control/planner/sql_plan_convert/output_schema.rs b/nodedb/src/control/planner/sql_plan_convert/output_schema/build.rs similarity index 65% rename from nodedb/src/control/planner/sql_plan_convert/output_schema.rs rename to nodedb/src/control/planner/sql_plan_convert/output_schema/build.rs index d42d9e6f6..b0dbd54dd 100644 --- a/nodedb/src/control/planner/sql_plan_convert/output_schema.rs +++ b/nodedb/src/control/planner/sql_plan_convert/output_schema/build.rs @@ -4,234 +4,39 @@ //! `SqlPlan` list, threaded into response shaping so the pgwire encoder can //! advertise correct RowDescription type OIDs. //! -//! Bare columns carry their real catalog type; computed SELECT expressions, -//! GROUP BY keys, and aggregate results are typed conservatively via -//! [`output_schema_types`](super::output_schema_types). A wrong non-TEXT OID -//! makes clients fail to parse the text value, so every uncertain case falls -//! back to `DdlColType::Text`, the safe default. +//! A read plan announces its projection. A write plan announces the columns +//! its `RETURNING` clause projects, and nothing when it carries none — see +//! [`build_returning_schema`](super::returning::build_returning_schema). use std::collections::HashMap; +use nodedb_physical::physical_plan::ReturningSpec; use nodedb_sql::catalog::SqlCatalog; use nodedb_sql::types::SqlPlan; -use nodedb_sql::types::query::{AggOutputSlot, Projection}; -use nodedb_sql::types_expr::SqlExpr; +use nodedb_sql::types::query::AggOutputSlot; -use super::lateral::collection_name_from_plan; -use super::output_schema_types::{infer_aggregate_type, infer_computed_expr_type}; -use crate::control::server::response_shape::schema::{ - OutputColumn, OutputSchema, sql_data_type_to_ddl_col_type_with_width, -}; +use crate::control::planner::sql_plan_convert::lateral::collection_name_from_plan; +use crate::control::planner::sql_plan_convert::output_schema_types::infer_aggregate_type; +use crate::control::server::response_shape::schema::{OutputColumn, OutputSchema}; use crate::control::server::response_shape::types::DdlColType; -/// Maps one `Projection` entry to an `OutputColumn`, given a map of bare -/// column name -> resolved wire type for the collection in scope. -/// -/// This is the authoritative derivation rule (see also `schema_from_projection` -/// in this module): for a qualified `table.column` reference, `lookup_key` keeps the full -/// dot-joined form (the join executor prefixes every key with its source -/// collection name) while `display_name` is the last segment. For a bare -/// column both are identical. -/// -/// `Projection::Star` / `Projection::QualifiedStar` have no single concrete -/// column and return `None`; the caller sets `is_star` instead. -fn projection_to_column( - p: &Projection, - types: &HashMap, -) -> Option { - match p { - Projection::Column(qname) => { - let display_name = qname - .rsplit('.') - .next() - .map(str::to_string) - .unwrap_or_else(|| qname.clone()); - let ty = types - .get(&display_name) - .copied() - .unwrap_or(DdlColType::Text); - Some(OutputColumn { - display_name, - lookup_key: qname.clone(), - ty, - }) - } - Projection::Computed { expr, alias } => { - // For an aliased column reference (`o.id AS oid`) the Data Plane - // keys the value by the underlying column, not the alias — so the - // lookup_key must be the qualified expression (matching the join - // executor's prefixed keys) while the alias is only the display - // name. A genuine computed expression (`price * qty AS total`) is - // emitted by the executor under its alias, so that stays the key. - let lookup_key = match expr { - SqlExpr::Column { - table: Some(t), - name, - } => format!("{t}.{name}"), - SqlExpr::Column { table: None, name } => name.clone(), - _ => alias.clone(), - }; - Some(OutputColumn { - display_name: alias.clone(), - lookup_key, - ty: infer_computed_expr_type(expr, types), - }) - } - Projection::Star | Projection::QualifiedStar(_) => None, - } -} - -/// Builds a `HashMap` of bare column name -> resolved wire type for -/// `collection`, via a best-effort catalog lookup. Returns an empty map -/// (never an error) when the lookup fails or the collection is unknown — -/// callers fall back to `DdlColType::Text` for every column in that case. -fn column_types_for( - catalog: &C, - database_id: nodedb_types::DatabaseId, - collection: &str, -) -> HashMap { - match catalog.get_collection(database_id, collection) { - Ok(Some(info)) => info - .columns - .iter() - .map(|c| { - ( - c.name.clone(), - sql_data_type_to_ddl_col_type_with_width( - &c.data_type, - c.int_width, - c.float_width, - ), - ) - }) - .collect(), - _ => HashMap::new(), - } -} - -/// Derives an `OutputColumn` for one GROUP BY key expression. -/// -/// The `display_name` is the SELECT-list output name: the explicit alias when -/// the projection aliased the key (`SELECT k AS label ... GROUP BY k` yields -/// output column `label`, matching Postgres), otherwise the key's own column -/// name. The `lookup_key` always stays the raw grouped column name (the key -/// the aggregate executor emits the value under), so `project_row` still finds -/// the value. -/// -/// The output type is the grouped column's catalog type when the key is a bare -/// column (resolved from `types`, default `Text`); a computed-expression key is -/// typed conservatively via [`infer_computed_expr_type`], defaulting to `Text`. -/// -/// A non-`Column` GROUP BY key (a computed expression) derives its `lookup_key` -/// from the shared index-based `computed_group_key_name` rule — the exact name -/// the aggregate spec emits the evaluated value under, so the two can never -/// diverge. Its `display_name` is the SELECT-list alias when present -/// (`UPPER(label) AS u` shows column `u`), else the same placeholder. -fn group_by_key_column( - expr: &SqlExpr, - index: usize, - alias: Option<&str>, - types: &HashMap, -) -> OutputColumn { - match expr { - SqlExpr::Column { table, name } => { - let lookup_key = match table { - Some(t) => format!("{t}.{name}"), - None => name.clone(), - }; - let display_name = alias.map(str::to_string).unwrap_or_else(|| name.clone()); - let ty = types.get(name).copied().unwrap_or(DdlColType::Text); - OutputColumn { - display_name, - lookup_key, - ty, - } - } - _ => { - // The executor emits the evaluated value under the shared - // index-based name (see `group_by_to_specs`), so `lookup_key` MUST - // equal it. `display_name` is the SELECT-list alias when present - // (`UPPER(label) AS u` shows column `u`), else the same placeholder. - let lookup_key = super::group_key_name::computed_group_key_name(index); - let display_name = alias - .map(str::to_string) - .unwrap_or_else(|| lookup_key.clone()); - OutputColumn { - display_name, - lookup_key, - ty: infer_computed_expr_type(expr, types), - } - } - } -} - -/// Returns the collection's columns in declared catalog order, mapped to -/// `OutputColumn`s (`display_name` = `lookup_key` = column name). Returns an -/// empty `Vec` when the catalog/collection lookup fails or the collection has -/// no declared columns (e.g. a schemaless collection) — so a schemaless -/// `SELECT *` still yields empty columns, deriving its shape from the rows. -fn ordered_columns_for( - catalog: &C, - database_id: nodedb_types::DatabaseId, - collection: &str, -) -> Vec { - match catalog.get_collection(database_id, collection) { - Ok(Some(info)) => info - .columns - .iter() - .map(|c| OutputColumn { - display_name: c.name.clone(), - lookup_key: c.name.clone(), - ty: sql_data_type_to_ddl_col_type_with_width( - &c.data_type, - c.int_width, - c.float_width, - ), - }) - .collect(), - _ => Vec::new(), - } -} - -/// Maps a projection list to an `OutputSchema` fragment using `types`. -/// -/// A `Star` / `QualifiedStar` in the projection sets `is_star` and expands -/// into `ordered_cols` (the collection's catalog columns in declared order), -/// appending only entries not already produced by a named projection. When -/// the projection has no star, `ordered_cols` is ignored and behavior is the -/// named-columns-only, `is_star=false` case. -fn schema_from_projection( - projection: &[Projection], - types: &HashMap, - ordered_cols: &[OutputColumn], -) -> OutputSchema { - let mut columns = Vec::with_capacity(projection.len()); - let mut is_star = false; - for p in projection { - match projection_to_column(p, types) { - Some(col) => columns.push(col), - None => { - is_star = true; - for oc in ordered_cols { - if !columns.iter().any(|c| c.lookup_key == oc.lookup_key) { - columns.push(oc.clone()); - } - } - } - } - } - OutputSchema { columns, is_star } -} +use super::columns::{ + column_types_for, group_by_key_column, ordered_columns_for, schema_from_projection, +}; +use super::returning::build_returning_schema; /// Derives the planner-authoritative output schema of a compiled plan list. /// -/// Only the plan variants that carry a resolvable projection against a -/// single named collection are handled directly; other plan variants are -/// handled by later units in this effort and fall back to an empty schema. -pub fn build_output_schema( +/// A read plan announces the columns its projection names. A write plan +/// announces the columns `returning` projects: the clause is stripped from the +/// statement text before planning, so the plan itself carries no column list +/// and the caller supplies the parsed spec. `None` means the statement carries +/// no `RETURNING` clause, and a write then announces nothing. +pub fn build_output_schema( plans: &[SqlPlan], catalog: &C, database_id: nodedb_types::DatabaseId, + returning: Option<&ReturningSpec>, ) -> OutputSchema { let Some(plan) = plans.first() else { return OutputSchema { @@ -397,12 +202,17 @@ pub fn build_output_schema( // Set operations take their column names/types from the first // (left) branch, matching standard SQL set-op semantics. SqlPlan::Union { inputs, .. } => match inputs.first() { - Some(first) => build_output_schema(std::slice::from_ref(first), catalog, database_id), + Some(first) => { + build_output_schema(std::slice::from_ref(first), catalog, database_id, returning) + } None => OutputSchema::default(), }, - SqlPlan::Intersect { left, .. } | SqlPlan::Except { left, .. } => { - build_output_schema(std::slice::from_ref(left.as_ref()), catalog, database_id) - } + SqlPlan::Intersect { left, .. } | SqlPlan::Except { left, .. } => build_output_schema( + std::slice::from_ref(left.as_ref()), + catalog, + database_id, + returning, + ), SqlPlan::RecursiveValue { columns, .. } => OutputSchema { columns: columns .iter() @@ -416,9 +226,12 @@ pub fn build_output_schema( }, // The outer query determines the final projected shape; the CTE // definitions themselves are only inputs to it. - SqlPlan::Cte { outer, .. } => { - build_output_schema(std::slice::from_ref(outer.as_ref()), catalog, database_id) - } + SqlPlan::Cte { outer, .. } => build_output_schema( + std::slice::from_ref(outer.as_ref()), + catalog, + database_id, + returning, + ), // A post-processor's projected shape is its outer projection; an empty // projection (SELECT *) inherits the body's columns. (Synthesized during // conversion, so this is normally unreached — the schema is derived from @@ -428,7 +241,12 @@ pub fn build_output_schema( input, projection, .. } => { if projection.is_empty() { - build_output_schema(std::slice::from_ref(input.as_ref()), catalog, database_id) + build_output_schema( + std::slice::from_ref(input.as_ref()), + catalog, + database_id, + returning, + ) } else { let types = HashMap::new(); schema_from_projection(projection, &types, &[]) @@ -458,38 +276,40 @@ pub fn build_output_schema( .collect(), is_star: false, }, - // Writes / DDL: no output rows, nothing to shape. - SqlPlan::Insert { .. } - | SqlPlan::KvInsert { .. } - | SqlPlan::Upsert { .. } - | SqlPlan::Update { .. } - | SqlPlan::UpdateFrom { .. } - | SqlPlan::Delete { .. } - | SqlPlan::Truncate { .. } - | SqlPlan::TimeseriesIngest { .. } - | SqlPlan::InsertSelect { .. } + // A write announces exactly what its `RETURNING` clause projects, from + // the target collection's declared columns. `RETURNING` is a + // projection, so it is typed like one: a `SELECT ts, host, v` and an + // `INSERT ... RETURNING ts, host, v` announce the same three types and + // render the same stored row identically. + SqlPlan::Insert { collection, .. } + | SqlPlan::KvInsert { collection, .. } + | SqlPlan::Upsert { collection, .. } + | SqlPlan::Update { collection, .. } + | SqlPlan::UpdateFrom { collection, .. } + | SqlPlan::Delete { collection, .. } + | SqlPlan::TimeseriesIngest { collection, .. } + | SqlPlan::VectorPrimaryInsert { collection, .. } => { + build_returning_schema(returning, collection, catalog, database_id) + } + // Same rule, for the two writes that name their target `target`. + SqlPlan::Merge { target, .. } | SqlPlan::InsertSelect { target, .. } => { + build_returning_schema(returning, target, catalog, database_id) + } + // No rows to shape. `TRUNCATE`, index DDL, and the whole `CREATE ARRAY` + // family answer with a command tag; the array DML ops answer with an + // affected count, and `inject_returning_spec` attaches no spec to them, + // so announcing columns for one would hold a count payload to a row + // shape it does not have. + SqlPlan::Truncate { .. } | SqlPlan::CreateArray { .. } | SqlPlan::DropArray { .. } | SqlPlan::AlterArray { .. } | SqlPlan::InsertArray { .. } | SqlPlan::DeleteArray { .. } - | SqlPlan::VectorPrimaryInsert { .. } | SqlPlan::CreateIndex { .. } | SqlPlan::DropIndex { .. } | SqlPlan::ArrayFlush { .. } | SqlPlan::ArrayCompact { .. } => OutputSchema::default(), - // `Merge` (with or without RETURNING) and `Update`/`UpdateFrom` with - // `returning: true` are shaped downstream via - // `PlanKind::ReturningRows` -> `shape_returning_rows`, which reads - // column names/values directly out of the response payload - // (`RowsPayload` msgpack) when no columns were announced to the - // client. An empty schema here is therefore correct, not a - // placeholder: it says "nothing announced", which is exactly true of - // the simple-query path, where the RowDescription is built from those - // same payload-derived rows. The extended-query path announces its - // columns at Describe time and supplies them as the projection - // instead, and the shaper then holds the rows to them. - SqlPlan::Merge { .. } => OutputSchema::default(), // `ArrayAgg` / `ArrayElementwise` compile to `ArrayOp::Aggregate` / // `ArrayOp::Elementwise`, which `describe_plan` classifies as // `PlanKind::MultiRow`. `MultiRow` responses are shaped by @@ -503,49 +323,8 @@ pub fn build_output_schema( #[cfg(test)] mod tests { use super::*; - - #[test] - fn bare_column_uses_matching_type_from_map() { - let mut types = HashMap::new(); - types.insert("foo".to_string(), DdlColType::Int8); - let p = Projection::Column("foo".to_string()); - let col = projection_to_column(&p, &types).expect("Some for Column"); - assert_eq!(col.lookup_key, "foo"); - assert_eq!(col.display_name, "foo"); - assert_eq!(col.ty, DdlColType::Int8); - } - - #[test] - fn qualified_column_display_is_last_segment() { - let types = HashMap::new(); - let p = Projection::Column("t.bar".to_string()); - let col = projection_to_column(&p, &types).expect("Some for Column"); - assert_eq!(col.lookup_key, "t.bar"); - assert_eq!(col.display_name, "bar"); - assert_eq!(col.ty, DdlColType::Text); - } - - #[test] - fn computed_uses_alias_for_both_and_defaults_to_text() { - let types = HashMap::new(); - let p = Projection::Computed { - expr: nodedb_sql::types_expr::SqlExpr::Wildcard, - alias: "total".to_string(), - }; - let col = projection_to_column(&p, &types).expect("Some for Computed"); - assert_eq!(col.lookup_key, "total"); - assert_eq!(col.display_name, "total"); - assert_eq!(col.ty, DdlColType::Text); - } - - #[test] - fn star_returns_none() { - let types = HashMap::new(); - assert!(projection_to_column(&Projection::Star, &types).is_none()); - assert!( - projection_to_column(&Projection::QualifiedStar("t".to_string()), &types).is_none() - ); - } + use nodedb_sql::types::query::Projection; + use nodedb_sql::types_expr::SqlExpr; /// Catalog stub whose `get_collection` is never called by the /// `ConstantResult` branch under test; only required to satisfy the @@ -570,7 +349,8 @@ mod tests { values: vec![], volatile: false, }]; - let schema = build_output_schema(&plans, &NoCatalog, nodedb_types::DatabaseId::DEFAULT); + let schema = + build_output_schema(&plans, &NoCatalog, nodedb_types::DatabaseId::DEFAULT, None); assert_eq!(schema.columns.len(), 2); assert_eq!(schema.columns[0].display_name, "a"); assert_eq!(schema.columns[0].lookup_key, "a"); @@ -638,7 +418,8 @@ mod tests { sort_keys: Vec::new(), }]; - let schema = build_output_schema(&plans, &NoCatalog, nodedb_types::DatabaseId::DEFAULT); + let schema = + build_output_schema(&plans, &NoCatalog, nodedb_types::DatabaseId::DEFAULT, None); assert_eq!(schema.columns.len(), 3); // Group-key display_name is the SELECT-list alias; lookup_key stays // the raw grouped column name (the executor's emitted value key). @@ -665,7 +446,8 @@ mod tests { ], distinct: false, }]; - let schema = build_output_schema(&plans, &NoCatalog, nodedb_types::DatabaseId::DEFAULT); + let schema = + build_output_schema(&plans, &NoCatalog, nodedb_types::DatabaseId::DEFAULT, None); assert_eq!(schema.columns.len(), 1); assert_eq!(schema.columns[0].display_name, "id"); } @@ -681,7 +463,8 @@ mod tests { max_depth: 100, distinct: false, }]; - let schema = build_output_schema(&plans, &NoCatalog, nodedb_types::DatabaseId::DEFAULT); + let schema = + build_output_schema(&plans, &NoCatalog, nodedb_types::DatabaseId::DEFAULT, None); assert_eq!(schema.columns.len(), 1); assert_eq!(schema.columns[0].display_name, "n"); assert_eq!(schema.columns[0].lookup_key, "n"); @@ -722,7 +505,8 @@ mod tests { key_value: nodedb_sql::types_expr::SqlValue::Null, projection: id_and_dist_projection(), }]; - let schema = build_output_schema(&plans, &NoCatalog, nodedb_types::DatabaseId::DEFAULT); + let schema = + build_output_schema(&plans, &NoCatalog, nodedb_types::DatabaseId::DEFAULT, None); assert_id_and_dist_schema(&schema); } @@ -742,7 +526,8 @@ mod tests { payload_filters: Vec::new(), projection: id_and_dist_projection(), }]; - let schema = build_output_schema(&plans, &NoCatalog, nodedb_types::DatabaseId::DEFAULT); + let schema = + build_output_schema(&plans, &NoCatalog, nodedb_types::DatabaseId::DEFAULT, None); assert_id_and_dist_schema(&schema); } @@ -759,7 +544,8 @@ mod tests { score_alias: None, projection: id_and_dist_projection(), }]; - let schema = build_output_schema(&plans, &NoCatalog, nodedb_types::DatabaseId::DEFAULT); + let schema = + build_output_schema(&plans, &NoCatalog, nodedb_types::DatabaseId::DEFAULT, None); assert_id_and_dist_schema(&schema); } @@ -776,7 +562,8 @@ mod tests { score_alias: None, projection: id_and_dist_projection(), }]; - let schema = build_output_schema(&plans, &NoCatalog, nodedb_types::DatabaseId::DEFAULT); + let schema = + build_output_schema(&plans, &NoCatalog, nodedb_types::DatabaseId::DEFAULT, None); assert_id_and_dist_schema(&schema); } @@ -874,7 +661,12 @@ mod tests { grouping_sets: None, sort_keys: Vec::new(), }]; - let schema = build_output_schema(&plans, &TypedCatalog, nodedb_types::DatabaseId::DEFAULT); + let schema = build_output_schema( + &plans, + &TypedCatalog, + nodedb_types::DatabaseId::DEFAULT, + None, + ); // group-keys-first fallback (empty output_order): region, then aggs. assert_eq!(schema.columns.len(), 5); // GROUP BY text column -> the text column's catalog type. @@ -914,7 +706,12 @@ mod tests { grouping_sets: None, sort_keys: Vec::new(), }]; - let schema = build_output_schema(&plans, &TypedCatalog, nodedb_types::DatabaseId::DEFAULT); + let schema = build_output_schema( + &plans, + &TypedCatalog, + nodedb_types::DatabaseId::DEFAULT, + None, + ); assert_eq!(schema.columns.len(), 1); assert_eq!(schema.columns[0].display_name, "u"); assert_eq!(schema.columns[0].ty, DdlColType::Text); @@ -939,7 +736,12 @@ mod tests { }, ]; let plans = vec![scan_plan("metrics", projection)]; - let schema = build_output_schema(&plans, &TypedCatalog, nodedb_types::DatabaseId::DEFAULT); + let schema = build_output_schema( + &plans, + &TypedCatalog, + nodedb_types::DatabaseId::DEFAULT, + None, + ); assert_eq!(schema.columns.len(), 2); // Bare-column-passthrough computed expr -> the column's catalog type. assert_eq!(schema.columns[0].display_name, "aliased_n"); diff --git a/nodedb/src/control/planner/sql_plan_convert/output_schema/columns.rs b/nodedb/src/control/planner/sql_plan_convert/output_schema/columns.rs new file mode 100644 index 000000000..29d257291 --- /dev/null +++ b/nodedb/src/control/planner/sql_plan_convert/output_schema/columns.rs @@ -0,0 +1,270 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! Column-level derivation shared by every output-schema rule. +//! +//! One projection entry, one GROUP BY key, or one collection's declared column +//! list maps to [`OutputColumn`]s here. Bare columns carry their real catalog +//! type; computed expressions are typed conservatively via +//! [`output_schema_types`](crate::control::planner::sql_plan_convert::output_schema_types). +//! A wrong non-TEXT OID makes clients fail to parse the text value, so every +//! uncertain case falls back to `DdlColType::Text`, the safe default. + +use std::collections::HashMap; + +use nodedb_sql::catalog::SqlCatalog; +use nodedb_sql::types::query::Projection; +use nodedb_sql::types_expr::SqlExpr; + +use crate::control::planner::sql_plan_convert::group_key_name::computed_group_key_name; +use crate::control::planner::sql_plan_convert::output_schema_types::infer_computed_expr_type; +use crate::control::server::response_shape::schema::{ + OutputColumn, OutputSchema, sql_data_type_to_ddl_col_type_with_width, +}; +use crate::control::server::response_shape::types::DdlColType; + +/// Maps one `Projection` entry to an `OutputColumn`, given a map of bare +/// column name -> resolved wire type for the collection in scope. +/// +/// This is the authoritative derivation rule (see also [`schema_from_projection`]): +/// for a qualified `table.column` reference, `lookup_key` keeps the full +/// dot-joined form (the join executor prefixes every key with its source +/// collection name) while `display_name` is the last segment. For a bare +/// column both are identical. +/// +/// `Projection::Star` / `Projection::QualifiedStar` have no single concrete +/// column and return `None`; the caller sets `is_star` instead. +pub(super) fn projection_to_column( + p: &Projection, + types: &HashMap, +) -> Option { + match p { + Projection::Column(qname) => { + let display_name = qname + .rsplit('.') + .next() + .map(str::to_string) + .unwrap_or_else(|| qname.clone()); + let ty = types + .get(&display_name) + .copied() + .unwrap_or(DdlColType::Text); + Some(OutputColumn { + display_name, + lookup_key: qname.clone(), + ty, + }) + } + Projection::Computed { expr, alias } => { + // For an aliased column reference (`o.id AS oid`) the Data Plane + // keys the value by the underlying column, not the alias — so the + // lookup_key must be the qualified expression (matching the join + // executor's prefixed keys) while the alias is only the display + // name. A genuine computed expression (`price * qty AS total`) is + // emitted by the executor under its alias, so that stays the key. + let lookup_key = match expr { + SqlExpr::Column { + table: Some(t), + name, + } => format!("{t}.{name}"), + SqlExpr::Column { table: None, name } => name.clone(), + _ => alias.clone(), + }; + Some(OutputColumn { + display_name: alias.clone(), + lookup_key, + ty: infer_computed_expr_type(expr, types), + }) + } + Projection::Star | Projection::QualifiedStar(_) => None, + } +} + +/// Builds a `HashMap` of bare column name -> resolved wire type for +/// `collection`, via a best-effort catalog lookup. Returns an empty map +/// (never an error) when the lookup fails or the collection is unknown — +/// callers fall back to `DdlColType::Text` for every column in that case. +pub(super) fn column_types_for( + catalog: &C, + database_id: nodedb_types::DatabaseId, + collection: &str, +) -> HashMap { + match catalog.get_collection(database_id, collection) { + Ok(Some(info)) => info + .columns + .iter() + .map(|c| { + ( + c.name.clone(), + sql_data_type_to_ddl_col_type_with_width( + &c.data_type, + c.int_width, + c.float_width, + ), + ) + }) + .collect(), + _ => HashMap::new(), + } +} + +/// Derives an `OutputColumn` for one GROUP BY key expression. +/// +/// The `display_name` is the SELECT-list output name: the explicit alias when +/// the projection aliased the key (`SELECT k AS label ... GROUP BY k` yields +/// output column `label`, matching Postgres), otherwise the key's own column +/// name. The `lookup_key` always stays the raw grouped column name (the key +/// the aggregate executor emits the value under), so `project_row` still finds +/// the value. +/// +/// The output type is the grouped column's catalog type when the key is a bare +/// column (resolved from `types`, default `Text`); a computed-expression key is +/// typed conservatively via [`infer_computed_expr_type`], defaulting to `Text`. +/// +/// A non-`Column` GROUP BY key (a computed expression) derives its `lookup_key` +/// from the shared index-based `computed_group_key_name` rule — the exact name +/// the aggregate spec emits the evaluated value under, so the two can never +/// diverge. Its `display_name` is the SELECT-list alias when present +/// (`UPPER(label) AS u` shows column `u`), else the same placeholder. +pub(super) fn group_by_key_column( + expr: &SqlExpr, + index: usize, + alias: Option<&str>, + types: &HashMap, +) -> OutputColumn { + match expr { + SqlExpr::Column { table, name } => { + let lookup_key = match table { + Some(t) => format!("{t}.{name}"), + None => name.clone(), + }; + let display_name = alias.map(str::to_string).unwrap_or_else(|| name.clone()); + let ty = types.get(name).copied().unwrap_or(DdlColType::Text); + OutputColumn { + display_name, + lookup_key, + ty, + } + } + _ => { + // The executor emits the evaluated value under the shared + // index-based name (see `group_by_to_specs`), so `lookup_key` MUST + // equal it. `display_name` is the SELECT-list alias when present + // (`UPPER(label) AS u` shows column `u`), else the same placeholder. + let lookup_key = computed_group_key_name(index); + let display_name = alias + .map(str::to_string) + .unwrap_or_else(|| lookup_key.clone()); + OutputColumn { + display_name, + lookup_key, + ty: infer_computed_expr_type(expr, types), + } + } + } +} + +/// Returns the collection's columns in declared catalog order, mapped to +/// `OutputColumn`s (`display_name` = `lookup_key` = column name). Returns an +/// empty `Vec` when the catalog/collection lookup fails or the collection has +/// no declared columns (e.g. a schemaless collection) — so a schemaless +/// `SELECT *` still yields empty columns, deriving its shape from the rows. +pub(super) fn ordered_columns_for( + catalog: &C, + database_id: nodedb_types::DatabaseId, + collection: &str, +) -> Vec { + match catalog.get_collection(database_id, collection) { + Ok(Some(info)) => info + .columns + .iter() + .map(|c| OutputColumn { + display_name: c.name.clone(), + lookup_key: c.name.clone(), + ty: sql_data_type_to_ddl_col_type_with_width( + &c.data_type, + c.int_width, + c.float_width, + ), + }) + .collect(), + _ => Vec::new(), + } +} + +/// Maps a projection list to an `OutputSchema` fragment using `types`. +/// +/// A `Star` / `QualifiedStar` in the projection sets `is_star` and expands +/// into `ordered_cols` (the collection's catalog columns in declared order), +/// appending only entries not already produced by a named projection. When +/// the projection has no star, `ordered_cols` is ignored and behavior is the +/// named-columns-only, `is_star=false` case. +pub(super) fn schema_from_projection( + projection: &[Projection], + types: &HashMap, + ordered_cols: &[OutputColumn], +) -> OutputSchema { + let mut columns = Vec::with_capacity(projection.len()); + let mut is_star = false; + for p in projection { + match projection_to_column(p, types) { + Some(col) => columns.push(col), + None => { + is_star = true; + for oc in ordered_cols { + if !columns.iter().any(|c| c.lookup_key == oc.lookup_key) { + columns.push(oc.clone()); + } + } + } + } + } + OutputSchema { columns, is_star } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn bare_column_uses_matching_type_from_map() { + let mut types = HashMap::new(); + types.insert("foo".to_string(), DdlColType::Int8); + let p = Projection::Column("foo".to_string()); + let col = projection_to_column(&p, &types).expect("Some for Column"); + assert_eq!(col.lookup_key, "foo"); + assert_eq!(col.display_name, "foo"); + assert_eq!(col.ty, DdlColType::Int8); + } + + #[test] + fn qualified_column_display_is_last_segment() { + let types = HashMap::new(); + let p = Projection::Column("t.bar".to_string()); + let col = projection_to_column(&p, &types).expect("Some for Column"); + assert_eq!(col.lookup_key, "t.bar"); + assert_eq!(col.display_name, "bar"); + assert_eq!(col.ty, DdlColType::Text); + } + + #[test] + fn computed_uses_alias_for_both_and_defaults_to_text() { + let types = HashMap::new(); + let p = Projection::Computed { + expr: nodedb_sql::types_expr::SqlExpr::Wildcard, + alias: "total".to_string(), + }; + let col = projection_to_column(&p, &types).expect("Some for Computed"); + assert_eq!(col.lookup_key, "total"); + assert_eq!(col.display_name, "total"); + assert_eq!(col.ty, DdlColType::Text); + } + + #[test] + fn star_returns_none() { + let types = HashMap::new(); + assert!(projection_to_column(&Projection::Star, &types).is_none()); + assert!( + projection_to_column(&Projection::QualifiedStar("t".to_string()), &types).is_none() + ); + } +} diff --git a/nodedb/src/control/planner/sql_plan_convert/output_schema/mod.rs b/nodedb/src/control/planner/sql_plan_convert/output_schema/mod.rs new file mode 100644 index 000000000..341537672 --- /dev/null +++ b/nodedb/src/control/planner/sql_plan_convert/output_schema/mod.rs @@ -0,0 +1,8 @@ +// SPDX-License-Identifier: BUSL-1.1 + +pub mod build; +pub mod columns; +pub mod returning; + +pub use build::build_output_schema; +pub use returning::build_returning_schema; diff --git a/nodedb/src/control/planner/sql_plan_convert/output_schema/returning.rs b/nodedb/src/control/planner/sql_plan_convert/output_schema/returning.rs new file mode 100644 index 000000000..6f95a8fdb --- /dev/null +++ b/nodedb/src/control/planner/sql_plan_convert/output_schema/returning.rs @@ -0,0 +1,217 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! Derives the announced [`OutputSchema`] of a DML `RETURNING` clause. +//! +//! A `RETURNING` clause is a projection over the target collection, so it is +//! typed by the same rule a `SELECT` projection is: the clause's column list is +//! mapped to [`Projection`] entries and handed to +//! [`schema_from_projection`](super::columns::schema_from_projection), against +//! the same catalog column types and declared order. There is one derivation, +//! so a write and a read of the same column can never announce different types. +//! +//! `RETURNING *` sets `is_star`, exactly as `SELECT *` does. The concrete +//! column list of a star is only knowable from the returned rows — a +//! schemaless row carries fields no catalog column declares — so the shaper +//! keeps the row-derived list and renders those cells as text, the same answer +//! `SELECT *` gives for the same row. + +use nodedb_physical::physical_plan::{ReturningColumns, ReturningSpec}; +use nodedb_sql::catalog::SqlCatalog; +use nodedb_sql::types::query::Projection; +use nodedb_sql::types_expr::SqlExpr; + +use crate::control::server::response_shape::schema::OutputSchema; + +use super::columns::{column_types_for, ordered_columns_for, schema_from_projection}; + +/// The output schema a write announces for `returning` against `collection`. +/// +/// `None` — the statement carries no `RETURNING` clause — announces nothing, +/// which is what a write with no result set must say. +pub fn build_returning_schema( + returning: Option<&ReturningSpec>, + collection: &str, + catalog: &C, + database_id: nodedb_types::DatabaseId, +) -> OutputSchema { + let Some(spec) = returning else { + return OutputSchema::default(); + }; + let projection = returning_projection(spec); + let types = column_types_for(catalog, database_id, collection); + let ordered_cols = ordered_columns_for(catalog, database_id, collection); + schema_from_projection(&projection, &types, &ordered_cols) +} + +/// Maps a `RETURNING` column list to the projection entries the shared +/// derivation reads. +/// +/// The clause's grammar admits a bare column name and an optional alias, and +/// nothing else: `parse_returning_columns` rejects every expression form with a +/// typed error before a spec exists. So an aliased item maps to a +/// `Projection::Computed` wrapping the column reference — the form that keeps +/// the alias as the display name while the value is still looked up under the +/// source column — and a bare item maps to `Projection::Column`. +fn returning_projection(spec: &ReturningSpec) -> Vec { + match &spec.columns { + ReturningColumns::Star => vec![Projection::Star], + ReturningColumns::Named(items) => items + .iter() + .map(|item| match &item.alias { + Some(alias) => Projection::Computed { + expr: SqlExpr::Column { + table: None, + name: item.name.clone(), + }, + alias: alias.clone(), + }, + None => Projection::Column(item.name.clone()), + }) + .collect(), + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::control::server::response_shape::types::DdlColType; + use nodedb_physical::physical_plan::ReturningItem; + + /// Catalog exposing one `points` collection: a `TIMESTAMP` time key, a + /// `TEXT` tag, and a `FLOAT` measurement — the shape a timeseries + /// collection declares. + struct PointsCatalog; + + impl SqlCatalog for PointsCatalog { + fn get_collection( + &self, + _database_id: nodedb_types::DatabaseId, + name: &str, + ) -> Result, nodedb_sql::catalog::SqlCatalogError> + { + use nodedb_sql::types::collection::ColumnInfo; + use nodedb_sql::types::query::EngineType; + use nodedb_sql::types_expr::SqlDataType; + + if name != "points" { + return Ok(None); + } + let col = |n: &str, t: SqlDataType| ColumnInfo { + name: n.to_string(), + data_type: t, + nullable: true, + is_primary_key: false, + default: None, + raw_type: None, + int_width: None, + float_width: None, + }; + Ok(Some(nodedb_sql::types::CollectionInfo { + name: "points".to_string(), + engine: EngineType::Timeseries, + columns: vec![ + col("ts", SqlDataType::Timestamp), + col("host", SqlDataType::String), + col("v", SqlDataType::Float64), + ], + primary_key: None, + has_auto_tier: false, + indexes: Vec::new(), + bitemporal: false, + primary: nodedb_types::PrimaryEngine::Document, + vector_primary: None, + partition_strategy: nodedb_types::PartitionStrategy::CollectionHomed, + open_schema: nodedb_sql::types::CollectionInfo::open_schema_for( + EngineType::Timeseries, + ), + })) + } + } + + fn named(items: &[(&str, Option<&str>)]) -> ReturningSpec { + ReturningSpec { + columns: ReturningColumns::Named( + items + .iter() + .map(|(name, alias)| ReturningItem { + name: (*name).to_string(), + alias: alias.map(str::to_string), + }) + .collect(), + ), + } + } + + fn schema(spec: Option<&ReturningSpec>) -> OutputSchema { + let database_id = nodedb_types::DatabaseId::DEFAULT; + build_returning_schema(spec, "points", &PointsCatalog, database_id) + } + + /// Named columns carry the declared catalog type, in clause order — the + /// same types `SELECT ts, host, v` announces for the same row. + #[test] + fn named_columns_carry_their_declared_types() { + let spec = named(&[("ts", None), ("host", None), ("v", None)]); + let out = schema(Some(&spec)); + assert!(!out.is_star); + let got: Vec<(&str, DdlColType)> = out + .columns + .iter() + .map(|c| (c.display_name.as_str(), c.ty)) + .collect(); + assert_eq!( + got, + vec![ + ("ts", DdlColType::Timestamp), + ("host", DdlColType::Text), + ("v", DdlColType::Float8), + ] + ); + } + + /// An alias names the output column while the value is still looked up + /// under the source column, and it keeps the source column's type. + #[test] + fn an_alias_renames_the_column_and_keeps_its_type() { + let spec = named(&[("v", Some("reading"))]); + let out = schema(Some(&spec)); + assert_eq!(out.columns.len(), 1); + assert_eq!(out.columns[0].display_name, "reading"); + assert_eq!(out.columns[0].lookup_key, "v"); + assert_eq!(out.columns[0].ty, DdlColType::Float8); + } + + /// A column the catalog does not declare falls back to `Text`, the safe + /// default for a schemaless field. + #[test] + fn an_undeclared_column_falls_back_to_text() { + let spec = named(&[("undeclared", None)]); + let out = schema(Some(&spec)); + assert_eq!(out.columns[0].ty, DdlColType::Text); + } + + /// `RETURNING *` sets `is_star`, so the shaper keeps the row-derived + /// column list — the same answer `SELECT *` gives. + #[test] + fn a_star_is_marked_as_one() { + let spec = ReturningSpec { + columns: ReturningColumns::Star, + }; + let out = schema(Some(&spec)); + assert!(out.is_star); + let names: Vec<&str> = out + .columns + .iter() + .map(|c| c.display_name.as_str()) + .collect(); + assert_eq!(names, vec!["ts", "host", "v"]); + } + + /// A write with no `RETURNING` clause announces nothing. + #[test] + fn no_clause_announces_nothing() { + let out = schema(None); + assert!(out.columns.is_empty()); + assert!(!out.is_star); + } +} diff --git a/nodedb/src/control/scatter_gather/hop.rs b/nodedb/src/control/scatter_gather/hop.rs index 4cb4af739..cd436aa37 100644 --- a/nodedb/src/control/scatter_gather/hop.rs +++ b/nodedb/src/control/scatter_gather/hop.rs @@ -192,7 +192,7 @@ pub async fn coordinate_cross_shard_hop( crate::types::TenantId::new(tenant_id_u64), database_id, &security.context(shared), - false, + None, ) .await { diff --git a/nodedb/src/control/server/pgwire/handler/cursor_query.rs b/nodedb/src/control/server/pgwire/handler/cursor_query.rs index 9cae1f414..7d0310141 100644 --- a/nodedb/src/control/server/pgwire/handler/cursor_query.rs +++ b/nodedb/src/control/server/pgwire/handler/cursor_query.rs @@ -68,7 +68,7 @@ impl NodeDbPgHandler { permission_cache: Some(&*perm_cache), }; let (tasks, _output_schema, versions, _) = query_ctx - .plan_sql_with_rls_and_versions(sql, tenant_id, database_id, &sec, false) + .plan_sql_with_rls_and_versions(sql, tenant_id, database_id, &sec, None) .await .map_err(StatementSetupError::from)?; drop(perm_cache); diff --git a/nodedb/src/control/server/pgwire/handler/prepared/parser.rs b/nodedb/src/control/server/pgwire/handler/prepared/parser.rs index 524fbb171..dd86e7569 100644 --- a/nodedb/src/control/server/pgwire/handler/prepared/parser.rs +++ b/nodedb/src/control/server/pgwire/handler/prepared/parser.rs @@ -304,8 +304,8 @@ impl NodeDbQueryParser { // When the original SQL had a RETURNING clause on a DML statement, // build result fields from the collection schema and the RETURNING spec. - if let Some(spec) = returning_spec - && let Some(fields) = result_fields_for_returning(&spec, plans.first(), catalog) + if let Some(ref spec) = returning_spec + && let Some(fields) = result_fields_for_returning(spec, plans.first(), catalog) { return (param_types, fields); } @@ -321,6 +321,7 @@ impl NodeDbQueryParser { &plans, catalog, database_id, + returning_spec.as_ref(), ); let result_fields: Vec = output_schema .columns diff --git a/nodedb/src/control/server/pgwire/handler/routing/calvin_dispatch.rs b/nodedb/src/control/server/pgwire/handler/routing/calvin_dispatch.rs index 2a00170c7..a9208b00e 100644 --- a/nodedb/src/control/server/pgwire/handler/routing/calvin_dispatch.rs +++ b/nodedb/src/control/server/pgwire/handler/routing/calvin_dispatch.rs @@ -53,14 +53,16 @@ fn meter_calvin_task( /// Who issued the statement and how its rows must be encoded back. /// -/// Bundled because these four are the connection's identity, not parameters of -/// the dispatch: they are looked up together at the call site and travel -/// unchanged through every branch below. +/// Bundled because these are the connection's identity and result contract, +/// not parameters of the dispatch: they are looked up together at the call +/// site and travel unchanged through every branch below. pub(super) struct CalvinDispatchSession<'a> { pub identity: &'a AuthenticatedIdentity, pub session_id: SessionId, pub result_formats: &'a [pgwire::api::results::FieldFormat], pub auth: &'a crate::control::security::auth_context::AuthContext, + /// The statement's announced output columns, when it announced any. + pub projection: Option<&'a crate::control::server::response_shape::schema::OutputSchema>, } impl NodeDbPgHandler { @@ -81,6 +83,7 @@ impl NodeDbPgHandler { session_id, result_formats, auth, + projection, } = session; let cross_shard_mode = self.sessions.cross_shard_txn_mode(session_id); let tx_state = self.sessions.transaction_state(session_id); @@ -180,6 +183,7 @@ impl NodeDbPgHandler { task, apply_resp.as_ref(), CalvinResponseCtx { + projection, state: &self.state, tenant_id, database_id, @@ -259,6 +263,7 @@ impl NodeDbPgHandler { task, outcome.apply_result.as_ref(), CalvinResponseCtx { + projection, state: &self.state, tenant_id, database_id, diff --git a/nodedb/src/control/server/pgwire/handler/routing/calvin_response.rs b/nodedb/src/control/server/pgwire/handler/routing/calvin_response.rs index a73d059e2..dd305ec63 100644 --- a/nodedb/src/control/server/pgwire/handler/routing/calvin_response.rs +++ b/nodedb/src/control/server/pgwire/handler/routing/calvin_response.rs @@ -16,6 +16,11 @@ use nodedb_physical::physical_task::PhysicalTask; /// Shared inputs for shaping one task of a completed Calvin batch. pub(super) struct CalvinResponseCtx<'a> { + /// The statement's announced output columns, when it announced any. A + /// `RETURNING` write is held to them here exactly as the single-shard + /// dispatch loop holds it, so the same statement renders the same row + /// whichever route it took. + pub(super) projection: Option<&'a crate::control::server::response_shape::schema::OutputSchema>, pub(super) state: &'a crate::control::state::SharedState, pub(super) tenant_id: TenantId, pub(super) database_id: crate::types::DatabaseId, @@ -55,6 +60,7 @@ pub(super) fn calvin_execution_response( use crate::control::server::response_shape::types::{PlanKind, describe_plan}; let CalvinResponseCtx { + projection, state, tenant_id, database_id, @@ -70,7 +76,7 @@ pub(super) fn calvin_execution_response( payload: resp.payload.as_bytes(), plan: &task.plan, plan_kind: PlanKind::ReturningRows, - projection: None, + projection, state, database_id, tenant_id, diff --git a/nodedb/src/control/server/pgwire/handler/routing/execute.rs b/nodedb/src/control/server/pgwire/handler/routing/execute.rs index 81a737d1b..26538d414 100644 --- a/nodedb/src/control/server/pgwire/handler/routing/execute.rs +++ b/nodedb/src/control/server/pgwire/handler/routing/execute.rs @@ -144,7 +144,10 @@ impl NodeDbPgHandler { tenant_id, identity, session_id, - shaping.formats, + ResultShaping { + projection: effective_schema, + formats: shaping.formats, + }, &auth_ctx, ) .await? @@ -214,6 +217,7 @@ impl NodeDbPgHandler { session_id, result_formats: shaping.formats, auth: &auth_ctx, + projection: effective_schema, }, &sum_target_reads, ) diff --git a/nodedb/src/control/server/pgwire/handler/routing/planning.rs b/nodedb/src/control/server/pgwire/handler/routing/planning.rs index 41f600a6b..194f2d032 100644 --- a/nodedb/src/control/server/pgwire/handler/routing/planning.rs +++ b/nodedb/src/control/server/pgwire/handler/routing/planning.rs @@ -156,13 +156,12 @@ impl NodeDbPgHandler { // Request-admission already ran once in `execute_single_sql` — must not admit again. // Per-query ON DENY always wins over the session-level override in `scope`. - let (clean_sql, scope) = + let (statement_sql, scope) = crate::control::server::session_auth::apply_per_query_on_deny(sql, scope); // Strip RETURNING clause before DataFusion planning. let (clean_sql, returning_spec) = - returning::strip_returning(&clean_sql).map_err(StatementSetupError::from)?; - let has_returning = returning_spec.is_some(); + returning::strip_returning(&statement_sql).map_err(StatementSetupError::from)?; // Forwards per-session planning GUCs into the shared query context, protocol-neutral // so pgwire and native honor them identically; flags drive the cache bypass below. @@ -215,9 +214,13 @@ impl NodeDbPgHandler { let state = Arc::clone(&self.state); let tenant = tenant_id.as_u64(); let db = database_id; + // Keyed on the statement INCLUDING its `RETURNING` clause: the + // clause is stripped before planning, so two statements that + // differ only in what they project share the stripped text — and + // the cached entry carries the announced output columns. self.sessions.get_cached_plan( session_id, - &clean_sql, + &statement_sql, move |id| current_descriptor_version(&state, tenant, db, id), current_permission_tree_version, current_rls_version, @@ -243,6 +246,7 @@ impl NodeDbPgHandler { tenant_id, database_id, &sec, + returning_spec.as_ref(), ) .await .map_err(StatementSetupError::from)?; @@ -275,7 +279,7 @@ impl NodeDbPgHandler { tenant_id, database_id, &sec, - has_returning, + returning_spec.as_ref(), ) .await .map_err(StatementSetupError::from)? @@ -286,7 +290,7 @@ impl NodeDbPgHandler { if !bypass_cache && cache_eligibility.is_cacheable() { self.sessions.put_cached_plan( session_id, - &clean_sql, + &statement_sql, planned.clone(), versions.clone(), output_schema.clone(), diff --git a/nodedb/src/control/server/pgwire/handler/routing/pre_dispatch.rs b/nodedb/src/control/server/pgwire/handler/routing/pre_dispatch.rs index 90af0937a..a6978eabf 100644 --- a/nodedb/src/control/server/pgwire/handler/routing/pre_dispatch.rs +++ b/nodedb/src/control/server/pgwire/handler/routing/pre_dispatch.rs @@ -2,7 +2,7 @@ //! Pre-dispatch routing gates for pgwire planned task sets. -use pgwire::api::results::{FieldFormat, Response}; +use pgwire::api::results::Response; use pgwire::error::{ErrorInfo, PgWireError, PgWireResult}; use nodedb_physical::physical_task::PhysicalTask; @@ -90,9 +90,13 @@ impl NodeDbPgHandler { tenant_id: TenantId, identity: &AuthenticatedIdentity, session_id: SessionId, - result_formats: &[FieldFormat], + shaping: ResultShaping<'_>, auth: &crate::control::security::auth_context::AuthContext, ) -> PgWireResult>> { + let ResultShaping { + projection, + formats: result_formats, + } = shaping; let tx_state = self.sessions.transaction_state(session_id); if tx_state == crate::control::server::shared::session::TransactionState::InBlock || self.state.calvin_completion_registry.get().is_none() @@ -121,6 +125,7 @@ impl NodeDbPgHandler { session_id, result_formats, auth, + projection, }, // No settled image read to carry — this fires before materialized-sum settlement. &[], diff --git a/nodedb/src/control/server/response_shape/returning.rs b/nodedb/src/control/server/response_shape/returning.rs index f2ef2c848..82af689ec 100644 --- a/nodedb/src/control/server/response_shape/returning.rs +++ b/nodedb/src/control/server/response_shape/returning.rs @@ -24,11 +24,15 @@ //! construction* rather than by coincidence, and every cell stays under the //! name it was stored with — no padding, no truncation, no re-alignment. //! -//! The simple-query protocol passes no projection (a DML plan's `OutputSchema` -//! is empty) and keeps the row-derived list. That divergence is correct: it -//! emits the RowDescription and the DataRows together out of this one -//! `ShapedRows`, so there is no earlier announcement to honour, and a -//! schemaless row's undeclared fields stay visible. +//! The simple-query protocol announces the same list: the planner derives a +//! DML plan's `OutputSchema` from its `RETURNING` column list, so a named +//! clause is held to those columns and their catalog types, and a returned +//! cell renders exactly as the same column renders under `SELECT`. +//! +//! `RETURNING *` announces `is_star` instead, and a star keeps the row-derived +//! list. The concrete columns of a star are only knowable once the rows exist — +//! a schemaless row carries fields no catalog column declares — which is the +//! same answer `SELECT *` gives for the same row. use serde_json::{Map, Value as JsonValue}; diff --git a/nodedb/src/control/server/shared/ddl/neutral/collection/dml/parse/dispatch.rs b/nodedb/src/control/server/shared/ddl/neutral/collection/dml/parse/dispatch.rs index a4c49ab5d..8370b4aee 100644 --- a/nodedb/src/control/server/shared/ddl/neutral/collection/dml/parse/dispatch.rs +++ b/nodedb/src/control/server/shared/ddl/neutral/collection/dml/parse/dispatch.rs @@ -134,7 +134,7 @@ pub(in crate::control::server::shared::ddl::neutral::collection) async fn plan_a // Injection happens HERE, before the task set is consumed: implicit-edge // extraction, authorization, staging, and dispatch all read `tasks` after // this point, and injecting later would hand them un-injected copies. - let (mut tasks, versions) = { + let (mut tasks, output_schema, versions) = { let scope = RequestAuthScope::for_database(identity, state.auth_stores(), database_id); let permission_cache = state.permission_cache.read().await; let sec = PlanSecurityContext { @@ -147,14 +147,20 @@ pub(in crate::control::server::shared::ddl::neutral::collection) async fn plan_a permission_cache: Some(&*permission_cache), }; let query_ctx = crate::control::planner::context::QueryContext::for_state(state); - let (tasks, _output_schema, versions, _) = query_ctx - .plan_sql_with_rls_and_versions(sql, tenant_id, database_id, &sec, false) + let (tasks, output_schema, versions, _) = query_ctx + .plan_sql_with_rls_and_versions( + sql, + tenant_id, + database_id, + &sec, + returning_spec.as_ref(), + ) .await .map_err(|error| { let (_, sqlstate, message) = error_to_sqlstate(&error); ddl_err(sqlstate, message) })?; - (tasks, versions) + (tasks, output_schema, versions) }; // Attach the projection to every planned write, refusing any insert shape @@ -414,7 +420,9 @@ pub(in crate::control::server::shared::ddl::neutral::collection) async fn plan_a payload: response.payload.as_bytes(), plan: &task.plan, plan_kind: PlanKind::ReturningRows, - projection: None, + // The statement's announced `RETURNING` columns, so this + // transport renders a returned cell exactly as pgwire does. + projection: Some(&output_schema), state, database_id, tenant_id, diff --git a/nodedb/src/control/server/shared/ddl/neutral/planning.rs b/nodedb/src/control/server/shared/ddl/neutral/planning.rs index 7c29e27dc..acf185a58 100644 --- a/nodedb/src/control/server/shared/ddl/neutral/planning.rs +++ b/nodedb/src/control/server/shared/ddl/neutral/planning.rs @@ -48,7 +48,7 @@ pub async fn plan_authorized_sql( }; let query_ctx = QueryContext::for_state(state); let (tasks, output_schema, versions, _) = query_ctx - .plan_sql_with_rls_and_versions(sql, identity.tenant_id, database_id, &sec, false) + .plan_sql_with_rls_and_versions(sql, identity.tenant_id, database_id, &sec, None) .await .map_err(|error| DdlError::new("42601", format!("query planning failed: {error}")))?; diff --git a/nodedb/src/control/server/shared/plan_admission.rs b/nodedb/src/control/server/shared/plan_admission.rs index 3402f4f15..98f3a2c22 100644 --- a/nodedb/src/control/server/shared/plan_admission.rs +++ b/nodedb/src/control/server/shared/plan_admission.rs @@ -106,7 +106,7 @@ async fn plan_authorize_and_admit_once( permission_cache: Some(&*permission_cache), }; let (tasks, output_schema, versions, _cache_eligibility) = query_ctx - .plan_sql_with_rls_and_versions(sql, tenant_id, database_id, &security, false) + .plan_sql_with_rls_and_versions(sql, tenant_id, database_id, &security, None) .await?; (tasks, output_schema, versions) }; From 960dee06196f36b82c313fdcf91080a12ec7ffe9 Mon Sep 17 00:00:00 2001 From: Farhan Syah Date: Thu, 10 Sep 2026 05:04:28 +0800 Subject: [PATCH 12/23] fix(timeseries): read declared TIMESTAMP columns as epoch microseconds The memtable stores every timeseries instant in epoch milliseconds, but a client reads a declared TIMESTAMP/TIMESTAMPTZ cell as epoch microseconds. Raw scans and RETURNING rows from ingest handed back the raw millisecond count unscaled, understating the stored instant by three orders of magnitude on read. Add CoreLoop::ts_instant_columns to list a collection's declared instant columns and scale_instant_cells to rescale those columns' values from milliseconds to microseconds once rows leave a scan or ingest. A BIGINT TIME_KEY shares the same storage column but is not a declared instant, so it keeps the integer that was inserted. --- .../executor/core_loop/ts_declared_schema.rs | 63 +++++++++ .../executor/handlers/timeseries/ingest.rs | 9 +- .../handlers/timeseries/raw_scan/mod.rs | 2 +- .../handlers/timeseries/raw_scan/row_emit.rs | 125 ++++++++++++++++++ .../handlers/timeseries/raw_scan/scan.rs | 9 ++ .../cases/timeseries_declared_time_key.rs | 28 ++-- 6 files changed, 226 insertions(+), 10 deletions(-) diff --git a/nodedb/src/data/executor/core_loop/ts_declared_schema.rs b/nodedb/src/data/executor/core_loop/ts_declared_schema.rs index 1f6279ee0..8e4ff48ab 100644 --- a/nodedb/src/data/executor/core_loop/ts_declared_schema.rs +++ b/nodedb/src/data/executor/core_loop/ts_declared_schema.rs @@ -116,6 +116,54 @@ impl CoreLoop { timestamp_idx, }) } + + /// Declared columns of a timeseries collection that carry an instant. + /// + /// A column declared `TIMESTAMP` or `TIMESTAMPTZ` is one. The memtable + /// keeps every timestamp column in epoch milliseconds, while a client + /// reads a `TIMESTAMP` cell as epoch microseconds, so row emission scales + /// exactly these columns. + /// + /// A `BIGINT TIME_KEY` shares the same millisecond column and is absent + /// from this list: its declared type is an integer, so it hands back the + /// number that was inserted. + /// + /// An undeclared measurement (raw ILP protocol ingest) has no entry and + /// yields an empty list — the planner types its columns as text, so no + /// cell of it is read as an instant. + pub(in crate::data::executor) fn ts_instant_columns( + &self, + database_id: DatabaseId, + tid: TenantId, + collection: &str, + ) -> Vec { + let Some(declared) = self.declared_timeseries(database_id, tid, collection) else { + return Vec::new(); + }; + declared + .columns + .iter() + .filter(|(_, type_str)| declared_type_is_instant(type_str)) + .map(|(name, _)| name.clone()) + .collect() + } +} + +/// Whether a declared DDL type makes a column an instant on the wire. +/// +/// Mirrors the two spellings the planner resolves to `SqlDataType::Timestamp` +/// in `control::planner::catalog_adapter::type_convert::parse_type_str`. The +/// two must name the same set: the planner decides how a cell is READ, this +/// decides the unit it is WRITTEN in. +/// +/// `SYSTEM_TIMESTAMP` is deliberately absent — the planner types it as text. +fn declared_type_is_instant(declared_type: &str) -> bool { + let bare = declared_type.split_whitespace().next().unwrap_or(""); + matches!( + bare.parse::(), + Ok(nodedb_types::columnar::ColumnType::Timestamp) + | Ok(nodedb_types::columnar::ColumnType::Timestamptz) + ) } /// Map a declared SQL type onto the memtable's storage type. @@ -189,6 +237,21 @@ mod tests { ); } + /// The declared types the planner reads back as instants are exactly the + /// ones emission scales. A `BIGINT` time key is stored in the same + /// millisecond column and must NOT be scaled. + #[test] + fn only_declared_timestamp_types_are_instants() { + assert!(declared_type_is_instant("TIMESTAMP TIME_KEY")); + assert!(declared_type_is_instant("TIMESTAMPTZ")); + assert!(declared_type_is_instant("timestamp")); + assert!(!declared_type_is_instant("BIGINT TIME_KEY")); + assert!(!declared_type_is_instant("INT")); + assert!(!declared_type_is_instant("TEXT")); + assert!(!declared_type_is_instant("SYSTEM_TIMESTAMP")); + assert!(!declared_type_is_instant("")); + } + #[test] fn unknown_types_fall_back_to_symbol() { assert_eq!( diff --git a/nodedb/src/data/executor/handlers/timeseries/ingest.rs b/nodedb/src/data/executor/handlers/timeseries/ingest.rs index 977c2c761..1135c32a6 100644 --- a/nodedb/src/data/executor/handlers/timeseries/ingest.rs +++ b/nodedb/src/data/executor/handlers/timeseries/ingest.rs @@ -327,7 +327,7 @@ impl CoreLoop { // missing float field is stored as NaN and both paths render it as SQL // NULL, which a hand-written projection over the ingest values would // have printed as "NaN". - let returned_rows: Vec = match returning { + let mut returned_rows: Vec = match returning { Some(_) => match self.columnar_memtables.get(&key) { Some(mt) => { super::raw_scan::emit_memtable_rows_at(mt, &outcome.accepted_row_indices) @@ -336,6 +336,13 @@ impl CoreLoop { }, None => Vec::new(), }; + // Same scan-unit rule `SELECT` applies: a declared `TIMESTAMP` cell + // leaves the engine as epoch microseconds, not the milliseconds + // storage holds. + let instant_columns = self.ts_instant_columns(task.request.database_id, tid, collection); + if let Err(e) = super::raw_scan::scale_instant_cells(&mut returned_rows, &instant_columns) { + return self.response_error(task, e); + } if accepted > 0 && let Some(lsn) = wal_lsn diff --git a/nodedb/src/data/executor/handlers/timeseries/raw_scan/mod.rs b/nodedb/src/data/executor/handlers/timeseries/raw_scan/mod.rs index 081da03b5..9b346ff12 100644 --- a/nodedb/src/data/executor/handlers/timeseries/raw_scan/mod.rs +++ b/nodedb/src/data/executor/handlers/timeseries/raw_scan/mod.rs @@ -8,5 +8,5 @@ pub mod partition_scan; pub mod row_emit; pub mod scan; -pub(in crate::data::executor) use row_emit::emit_memtable_rows_at; +pub(in crate::data::executor) use row_emit::{emit_memtable_rows_at, scale_instant_cells}; pub(in crate::data::executor) use scan::RawScanParams; diff --git a/nodedb/src/data/executor/handlers/timeseries/raw_scan/row_emit.rs b/nodedb/src/data/executor/handlers/timeseries/raw_scan/row_emit.rs index 1e50d4018..63180c16b 100644 --- a/nodedb/src/data/executor/handlers/timeseries/raw_scan/row_emit.rs +++ b/nodedb/src/data/executor/handlers/timeseries/raw_scan/row_emit.rs @@ -210,3 +210,128 @@ pub(super) fn nodedb_value_to_rmpv(v: &nodedb_types::Value) -> rmpv::Value { _ => rmpv::Value::Nil, } } + +/// Rescale every declared-instant cell of `rows` from the milliseconds the +/// memtable and the partitions store to the epoch microseconds a `TIMESTAMP` +/// cell carries on the wire. +/// +/// The engine's own unit stays milliseconds: partition ranges, retention, +/// `time_bucket` and every scan predicate read it. The scale therefore runs +/// once, as rows leave the scan — after filtering, sorting and computed +/// columns — so nothing inside the engine sees the wire unit. +/// +/// `instant_columns` comes from `CoreLoop::ts_instant_columns`, which lists +/// the columns declared `TIMESTAMP` or `TIMESTAMPTZ`. A `BIGINT TIME_KEY` +/// lives in the same millisecond column and is not in that list, so it keeps +/// the integer the client inserted. +/// +/// SQL NULL cells pass through untouched. A stored value that cannot be +/// expressed in microseconds fails the read rather than wrapping. +pub(in crate::data::executor) fn scale_instant_cells( + rows: &mut [rmpv::Value], + instant_columns: &[String], +) -> crate::Result<()> { + if instant_columns.is_empty() { + return Ok(()); + } + for row in rows.iter_mut() { + let rmpv::Value::Map(fields) = row else { + continue; + }; + for (key, value) in fields.iter_mut() { + let Some(name) = key.as_str() else { continue }; + if !instant_columns.iter().any(|c| c == name) { + continue; + } + let rmpv::Value::Integer(stored) = value else { + continue; + }; + let millis = stored.as_i64().ok_or_else(|| crate::Error::Internal { + detail: format!( + "timeseries column {name} holds {stored}, which is not a millisecond \ + count an instant can be read from" + ), + })?; + let micros = nodedb_types::NdbDateTime::from_millis(millis) + .map_err(|e| crate::Error::Internal { + detail: format!("timeseries column {name} at {millis} ms: {e}"), + })? + .micros; + *value = rmpv::Value::Integer(micros.into()); + } + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::scale_instant_cells; + + fn row(cells: &[(&str, i64)]) -> rmpv::Value { + rmpv::Value::Map( + cells + .iter() + .map(|(k, v)| { + ( + rmpv::Value::String((*k).into()), + rmpv::Value::Integer((*v).into()), + ) + }) + .collect(), + ) + } + + fn cell(row: &rmpv::Value, name: &str) -> Option { + let rmpv::Value::Map(fields) = row else { + return None; + }; + fields + .iter() + .find(|(k, _)| k.as_str() == Some(name)) + .and_then(|(_, v)| v.as_i64()) + } + + /// A declared `TIMESTAMP` column is read as epoch microseconds, so the + /// millisecond value storage holds is scaled on the way out. 2020-03-05 + /// stays 2020-03-05 instead of landing 50 years earlier. + #[test] + fn a_declared_instant_column_leaves_the_scan_in_microseconds() { + let mut rows = vec![row(&[("captured_at", 1_583_402_400_000)])]; + scale_instant_cells(&mut rows, &["captured_at".to_string()]).expect("scale"); + assert_eq!(cell(&rows[0], "captured_at"), Some(1_583_402_400_000_000)); + } + + /// A `BIGINT TIME_KEY` shares the millisecond storage column but is not a + /// declared instant, so its value is handed back exactly as inserted. + #[test] + fn a_column_that_is_not_a_declared_instant_keeps_its_value() { + let mut rows = vec![row(&[("ts", 1000), ("n", 7)])]; + scale_instant_cells(&mut rows, &["other".to_string()]).expect("scale"); + assert_eq!(cell(&rows[0], "ts"), Some(1000)); + assert_eq!(cell(&rows[0], "n"), Some(7)); + } + + /// A NULL instant cell stays NULL — there is no instant to scale. + #[test] + fn a_null_instant_cell_passes_through() { + let mut rows = vec![rmpv::Value::Map(vec![( + rmpv::Value::String("captured_at".into()), + rmpv::Value::Nil, + )])]; + scale_instant_cells(&mut rows, &["captured_at".to_string()]).expect("scale"); + assert_eq!(cell(&rows[0], "captured_at"), None); + } + + /// A stored millisecond count past the microsecond range fails the read. + /// Wrapping it would hand back an instant that is not the stored one. + #[test] + fn a_millisecond_value_beyond_the_microsecond_range_fails_the_read() { + let mut rows = vec![row(&[("captured_at", i64::MAX)])]; + let err = scale_instant_cells(&mut rows, &["captured_at".to_string()]) + .expect_err("i64::MAX ms cannot be expressed in microseconds"); + assert!( + err.to_string().contains("captured_at"), + "the error must name the column: {err}" + ); + } +} diff --git a/nodedb/src/data/executor/handlers/timeseries/raw_scan/scan.rs b/nodedb/src/data/executor/handlers/timeseries/raw_scan/scan.rs index 335d4f1a5..a51828b8f 100644 --- a/nodedb/src/data/executor/handlers/timeseries/raw_scan/scan.rs +++ b/nodedb/src/data/executor/handlers/timeseries/raw_scan/scan.rs @@ -271,6 +271,15 @@ impl CoreLoop { } results.truncate(limit); + // The engine stores its timestamp columns in milliseconds; a client + // reads a `TIMESTAMP` cell as epoch microseconds. Scale here, once + // every predicate, sort and computed column has run against the + // engine's own unit. + let instant_columns = self.ts_instant_columns(task.request.database_id, tid, collection); + if let Err(e) = super::row_emit::scale_instant_cells(&mut results, &instant_columns) { + return self.response_error(task, e); + } + let array = rmpv::Value::Array(results); let mut buf = Vec::new(); rmpv::encode::write_value(&mut buf, &array).unwrap_or(()); diff --git a/nodedb/tests/wire/cases/timeseries_declared_time_key.rs b/nodedb/tests/wire/cases/timeseries_declared_time_key.rs index 34a79507f..464ecd0f6 100644 --- a/nodedb/tests/wire/cases/timeseries_declared_time_key.rs +++ b/nodedb/tests/wire/cases/timeseries_declared_time_key.rs @@ -17,7 +17,11 @@ //! `BIGINT` time keys. A name the engine happens to recognise internally //! (`ts`, `timestamp`, `time`) is not special: it is the user's column. //! -//! Time-key values read back as the epoch milliseconds the engine stores. +//! A declared `TIMESTAMP` time key reads back as the instant that was +//! inserted: the engine stores epoch milliseconds and a `TIMESTAMP` cell is +//! read as epoch microseconds, so the two units meet as the row leaves the +//! scan. A `BIGINT` time key shares that storage column and is not a +//! timestamp, so it reads back as the number that was inserted. use crate::harness::TestServer; @@ -29,9 +33,14 @@ const LATE: &str = "2020-03-05 13:00:00"; /// Upper bound that both event times fall below but no ingest-assigned /// wall-clock time ever will. const AFTER_BOTH: &str = "2021-01-01 00:00:00"; -/// `EARLY` as the epoch milliseconds the timeseries engine stores and reads -/// back for a time-key column. -const EARLY_MS: &str = "1583402400000"; +/// `EARLY` as a declared `TIMESTAMP` column renders it. The engine stores +/// 1583402400000 epoch milliseconds; a `TIMESTAMP` cell carries epoch +/// microseconds, which the pgwire encoder writes as ISO-8601 UTC. +const EARLY_ISO: &str = "2020-03-05T10:00:00.000000Z"; +/// `EARLY` as `SELECT *` renders it: a star projection announces no catalog +/// type, so its cells stay the raw stored number — here the same instant in +/// epoch microseconds. +const EARLY_MICROS: &str = "1583402400000000"; #[tokio::test] async fn time_key_named_ts_round_trips() { @@ -58,7 +67,10 @@ async fn time_key_named_ts_round_trips() { !ts.is_empty(), "declared TIME_KEY column `ts` must not read back NULL: {rows:?}" ); - assert_eq!(ts, EARLY_MS, "`ts` must round-trip the inserted event time"); + assert_eq!( + ts, EARLY_ISO, + "`ts` must round-trip the inserted event time" + ); } #[tokio::test] @@ -114,7 +126,7 @@ async fn select_star_projects_declared_columns_only() { ); assert_eq!( rows[0].get("ts").map(String::as_str), - Some(EARLY_MS), + Some(EARLY_MICROS), "`SELECT *` must carry the inserted event time under `ts`: {rows:?}" ); } @@ -141,7 +153,7 @@ async fn custom_named_time_key_round_trips() { assert_eq!(rows.len(), 1, "one inserted row must read back: {rows:?}"); assert_eq!( rows[0].get("captured_at").map(String::as_str), - Some(EARLY_MS), + Some(EARLY_ISO), "`captured_at` must round-trip the inserted event time: {rows:?}" ); } @@ -289,7 +301,7 @@ async fn non_time_key_column_named_timestamp_keeps_its_value() { ); assert_eq!( rows[0].get("captured_at").map(String::as_str), - Some(EARLY_MS), + Some(EARLY_ISO), "the declared time key must still round-trip: {rows:?}" ); } From 99a40d4ada08864c9650f664e429cf0a72a81c4d Mon Sep 17 00:00:00 2001 From: Farhan Syah Date: Thu, 10 Sep 2026 06:19:27 +0800 Subject: [PATCH 13/23] refactor(sql): derive RETURNING result fields from OutputSchema Drop the separate result_fields_for_returning path and its private SqlDataType-to-pg mapping. Describe now infers RETURNING columns through the same build_output_schema call used for SELECT projections, so the extended-query and simple-query paths can never disagree on a column's type. Update the wire test to read numeric RETURNING columns by their announced OID (int2/int4/int8, float4/float8) instead of assuming int8/float8, since OutputSchema reports the narrower declared width. --- .../server/pgwire/handler/prepared/parser.rs | 35 +++--- .../pgwire/handler/prepared/parser_schema.rs | 105 +----------------- .../tests/wire/cases/pgwire_returning_dml.rs | 10 ++ 3 files changed, 29 insertions(+), 121 deletions(-) diff --git a/nodedb/src/control/server/pgwire/handler/prepared/parser.rs b/nodedb/src/control/server/pgwire/handler/prepared/parser.rs index dd86e7569..1ae47ff18 100644 --- a/nodedb/src/control/server/pgwire/handler/prepared/parser.rs +++ b/nodedb/src/control/server/pgwire/handler/prepared/parser.rs @@ -23,19 +23,19 @@ use crate::control::state::SharedState; use super::super::auth::{pgwire_authorization_error, resolve_session_identity}; use super::statement::ParsedStatement; -use parser_schema::{ - count_placeholders, is_dsl_statement, result_fields_for_returning, - substitute_placeholders_with_null, -}; +use parser_schema::{count_placeholders, is_dsl_statement, substitute_placeholders_with_null}; #[path = "parser_schema.rs"] mod parser_schema; -/// Maps the response shaper's protocol-neutral wire type to a pgwire -/// `Type` for RowDescription. Mirrors the (now-deleted) SQL-reparse path's -/// `sql_data_type_to_pg`; variants with no dedicated wire type still fall -/// back to `Type::TEXT` where that was the prior fallback, matching -/// today's behavior. +/// Maps the response shaper's protocol-neutral wire type to a pgwire `Type` +/// for RowDescription. +/// +/// This is the only mapping the extended-query path uses. Every result column +/// — a `SELECT` projection and a `RETURNING` clause alike — reaches it through +/// `build_output_schema`, so Describe and the simple-query path answer one +/// question with one rule. A `DdlColType` with no dedicated wire type resolves +/// to `Type::TEXT`, the safe default a client can always parse. fn ddl_col_type_to_pg(ty: &DdlColType) -> Type { match ty { DdlColType::Int8 => Type::INT8, @@ -302,20 +302,15 @@ impl NodeDbQueryParser { Err(_) => return (param_types, Vec::new()), }; - // When the original SQL had a RETURNING clause on a DML statement, - // build result fields from the collection schema and the RETURNING spec. - if let Some(ref spec) = returning_spec - && let Some(fields) = result_fields_for_returning(spec, plans.first(), catalog) - { - return (param_types, fields); - } - // Infer result fields from the planner's authoritative output // schema — the same derivation used to shape response rows, so // Describe's RowDescription always matches what Execute returns. - // Empty `plans` (already handled above) or a plan variant with no - // resolvable projection yields an empty `OutputSchema`, matching - // today's `Vec::new()` fallback for DSL/non-SELECT statements. + // A write plan announces what its `RETURNING` spec projects, through + // that same derivation, so Describe and the simple-query path can + // never disagree on a column's type. Empty `plans` (already handled + // above) or a plan variant with no resolvable projection yields an + // empty `OutputSchema`, matching the `Vec::new()` fallback for + // DSL/non-SELECT statements. let output_schema = crate::control::planner::sql_plan_convert::output_schema::build_output_schema( &plans, diff --git a/nodedb/src/control/server/pgwire/handler/prepared/parser_schema.rs b/nodedb/src/control/server/pgwire/handler/prepared/parser_schema.rs index f812c04de..4457aa8d4 100644 --- a/nodedb/src/control/server/pgwire/handler/prepared/parser_schema.rs +++ b/nodedb/src/control/server/pgwire/handler/prepared/parser_schema.rs @@ -1,14 +1,10 @@ // SPDX-License-Identifier: BUSL-1.1 -//! Schema-inference utilities for `NodeDbQueryParser`. +//! SQL-text utilities for `NodeDbQueryParser`. //! -//! These free functions are called by `parser.rs` during Parse-message -//! handling to infer parameter and result-field types from SQL text and -//! catalog metadata. - -use nodedb_types::DatabaseId; -use pgwire::api::Type; -use pgwire::api::results::FieldInfo; +//! `parser.rs` calls these during Parse-message handling to classify a +//! statement and to count and neutralise its `$N` placeholders. Result-column +//! types come from the planner's `OutputSchema`, never from here. /// Return true if `sql` starts with a DSL or DDL keyword that `plan_sql` /// cannot parse and must be routed through `execute_sql` at Execute time. @@ -123,99 +119,6 @@ pub(super) fn count_placeholders(sql: &str) -> usize { max_idx } -/// Build result `FieldInfo`s for a DML statement with a RETURNING clause. -/// -/// Resolves the target collection from the DML plan, looks up its schema, and -/// projects the RETURNING spec onto it. Returns `None` if the plan isn't a -/// recognized DML type or the collection schema cannot be found. -pub(super) fn result_fields_for_returning( - spec: &nodedb_physical::physical_plan::ReturningSpec, - plan: Option<&nodedb_sql::SqlPlan>, - catalog: &dyn nodedb_sql::SqlCatalog, -) -> Option> { - use nodedb_physical::physical_plan::{ReturningColumns, ReturningItem}; - use nodedb_sql::types::SqlDataType; - use pgwire::api::results::FieldFormat; - - // Local, self-contained mapping from the planner's `SqlDataType` to a - // pgwire `Type`; kept private to this function since RETURNING is the - // only remaining caller after the DESCRIBE path moved to the planner's - // authoritative `OutputSchema` (see `ddl_col_type_to_pg` in parser.rs). - fn returning_col_type_to_pg(dt: &SqlDataType) -> Type { - match dt { - SqlDataType::Int64 => Type::INT8, - SqlDataType::Float64 => Type::FLOAT8, - SqlDataType::String => Type::TEXT, - SqlDataType::Bool => Type::BOOL, - SqlDataType::Bytes => Type::BYTEA, - SqlDataType::Timestamp => Type::TIMESTAMP, - SqlDataType::Timestamptz => Type::TIMESTAMPTZ, - SqlDataType::Decimal => Type::NUMERIC, - SqlDataType::Uuid => Type::TEXT, - SqlDataType::Vector(_) => Type::BYTEA, - SqlDataType::Geometry => Type::BYTEA, - SqlDataType::Unknown => Type::TEXT, - } - } - - // Every write that can carry a RETURNING clause resolves its target here. - // A write whose target is missed announces NO result columns while its - // response still ships one field per stored column, which the client - // cannot read against the RowDescription it was given — so the list is - // the write plans, not just the two that first needed it. - let collection = match plan? { - nodedb_sql::SqlPlan::Update { collection, .. } - | nodedb_sql::SqlPlan::UpdateFrom { collection, .. } - | nodedb_sql::SqlPlan::Delete { collection, .. } - | nodedb_sql::SqlPlan::Insert { collection, .. } - | nodedb_sql::SqlPlan::KvInsert { collection, .. } - | nodedb_sql::SqlPlan::Upsert { collection, .. } - | nodedb_sql::SqlPlan::TimeseriesIngest { collection, .. } - | nodedb_sql::SqlPlan::VectorPrimaryInsert { collection, .. } => collection.as_str(), - nodedb_sql::SqlPlan::Merge { target, .. } - | nodedb_sql::SqlPlan::InsertSelect { target, .. } => target.as_str(), - _ => return None, - }; - - let info = catalog - .get_collection(DatabaseId::DEFAULT, collection) - .ok() - .flatten()?; - - let columns_to_field_info = |columns: &[nodedb_sql::ColumnInfo]| -> Vec { - columns - .iter() - .map(|c| { - FieldInfo::new( - c.name.clone(), - None, - None, - returning_col_type_to_pg(&c.data_type), - FieldFormat::Text, - ) - }) - .collect() - }; - - let fields = match &spec.columns { - ReturningColumns::Star => columns_to_field_info(&info.columns), - ReturningColumns::Named(items) => items - .iter() - .map(|item: &ReturningItem| { - let display_name = item.alias.clone().unwrap_or_else(|| item.name.clone()); - let pg_type = info - .columns - .iter() - .find(|c| c.name == item.name) - .map(|c| returning_col_type_to_pg(&c.data_type)) - .unwrap_or(Type::TEXT); - FieldInfo::new(display_name, None, None, pg_type, FieldFormat::Text) - }) - .collect(), - }; - Some(fields) -} - #[cfg(test)] mod tests { use super::*; diff --git a/nodedb/tests/wire/cases/pgwire_returning_dml.rs b/nodedb/tests/wire/cases/pgwire_returning_dml.rs index dedfb9bb2..a8ba13163 100644 --- a/nodedb/tests/wire/cases/pgwire_returning_dml.rs +++ b/nodedb/tests/wire/cases/pgwire_returning_dml.rs @@ -482,10 +482,20 @@ async fn extended_query_returning_star_matches_the_announced_row_description() { // RowDescription announced for it. for (i, column) in row.columns().iter().enumerate() { let ty = column.type_(); + // One arm per numeric OID the RowDescription can carry. `score` is + // declared `INT`, so it is announced as int4 and must be read as + // `i32`: reading it as `String` is a client-side type error, not a + // fallback. The narrower widths are listed for the same reason. let value = if *ty == Type::INT8 { row.get::<_, i64>(i).to_string() + } else if *ty == Type::INT4 { + row.get::<_, i32>(i).to_string() + } else if *ty == Type::INT2 { + row.get::<_, i16>(i).to_string() } else if *ty == Type::FLOAT8 { row.get::<_, f64>(i).to_string() + } else if *ty == Type::FLOAT4 { + row.get::<_, f32>(i).to_string() } else { row.get::<_, String>(i) }; From 1386fc36e4b776b9a45448017398f9e9fad99005 Mon Sep 17 00:00:00 2001 From: Farhan Syah Date: Thu, 10 Sep 2026 08:08:54 +0800 Subject: [PATCH 14/23] feat(sql): resolve real column types for joins and grouped timeseries scans An output schema for a JOIN previously defaulted every projected column to Text. It now resolves each column against the catalog of the side it came from, keyed on the qualified name the join executor emits, falling back to Text only for a bare name two sides declare with different types. A grouped, non-bucketed timeseries scan now announces its GROUP BY keys with their catalog types instead of leaving the whole schema untyped, matching the shape its aggregate encoder emits. --- .../sql_plan_convert/output_schema/build.rs | 62 +++- .../sql_plan_convert/output_schema/columns.rs | 7 +- .../output_schema/join_types.rs | 276 ++++++++++++++++++ .../sql_plan_convert/output_schema/mod.rs | 1 + .../sql_plan_convert/output_schema_types.rs | 21 +- 5 files changed, 354 insertions(+), 13 deletions(-) create mode 100644 nodedb/src/control/planner/sql_plan_convert/output_schema/join_types.rs diff --git a/nodedb/src/control/planner/sql_plan_convert/output_schema/build.rs b/nodedb/src/control/planner/sql_plan_convert/output_schema/build.rs index b0dbd54dd..f7b9fa374 100644 --- a/nodedb/src/control/planner/sql_plan_convert/output_schema/build.rs +++ b/nodedb/src/control/planner/sql_plan_convert/output_schema/build.rs @@ -11,10 +11,12 @@ use std::collections::HashMap; use nodedb_physical::physical_plan::ReturningSpec; +use nodedb_query::agg_key::canonical_agg_key; use nodedb_sql::catalog::SqlCatalog; use nodedb_sql::types::SqlPlan; use nodedb_sql::types::query::AggOutputSlot; +use crate::control::planner::sql_plan_convert::aggregate::agg_expr_to_pair; use crate::control::planner::sql_plan_convert::lateral::collection_name_from_plan; use crate::control::planner::sql_plan_convert::output_schema_types::infer_aggregate_type; use crate::control::server::response_shape::schema::{OutputColumn, OutputSchema}; @@ -46,6 +48,47 @@ pub fn build_output_schema( }; match plan { + // A grouped timeseries scan announces the columns its aggregate + // encoder emits, in that encoder's order: each GROUP BY key, then + // each aggregate. A GROUP BY key carries its own catalog type, so one + // stored instant renders the same grouped as it does through a plain + // `SELECT`. An aggregate result stays `Text`: the timeseries plan + // carries no SELECT-list alias for it, so its type cannot be resolved + // with certainty. + // + // A `time_bucket` query is excluded: its encoder prepends a `bucket` + // boundary column that the plan's GROUP BY list does not name, so the + // announced shape would not be the emitted one. + SqlPlan::TimeseriesScan { + collection, + group_by, + aggregates, + bucket_interval_ms, + .. + } if !group_by.is_empty() && *bucket_interval_ms == 0 => { + let types = column_types_for(catalog, database_id, collection); + let mut columns = Vec::with_capacity(group_by.len() + aggregates.len()); + for key in group_by { + columns.push(OutputColumn { + display_name: key.clone(), + lookup_key: key.clone(), + ty: types.get(key).copied().unwrap_or(DdlColType::Text), + }); + } + for agg in aggregates { + let (function, field) = agg_expr_to_pair(agg); + let key = canonical_agg_key(&function, &field); + columns.push(OutputColumn { + display_name: key.clone(), + lookup_key: key, + ty: DdlColType::Text, + }); + } + OutputSchema { + columns, + is_star: false, + } + } SqlPlan::Scan { collection, projection, @@ -115,12 +158,19 @@ pub fn build_output_schema( let ordered_cols = ordered_columns_for(catalog, database_id, collection); schema_from_projection(projection, &types, &ordered_cols) } - SqlPlan::Join { projection, .. } => { - // A join has no single source collection; column types default - // to `Text` for every projected field rather than picking one - // side arbitrarily. A star here has no single catalog to expand - // against, so no ordered columns are supplied. - let types = HashMap::new(); + SqlPlan::Join { + left, + right, + projection, + .. + } => { + // Each projected column resolves against the catalog of the side + // it came from, keyed on the qualified name the join executor + // emits (`orders.ts`). A bare name two sides declare differently + // cannot be attributed and stays `Text`. A star here has no + // single catalog to expand against, so no ordered columns are + // supplied. + let types = super::join_types::join_column_types(left, right, catalog, database_id); schema_from_projection(projection, &types, &[]) } SqlPlan::ConstantResult { columns, .. } => OutputSchema { diff --git a/nodedb/src/control/planner/sql_plan_convert/output_schema/columns.rs b/nodedb/src/control/planner/sql_plan_convert/output_schema/columns.rs index 29d257291..78cea8a45 100644 --- a/nodedb/src/control/planner/sql_plan_convert/output_schema/columns.rs +++ b/nodedb/src/control/planner/sql_plan_convert/output_schema/columns.rs @@ -44,8 +44,13 @@ pub(super) fn projection_to_column( .next() .map(str::to_string) .unwrap_or_else(|| qname.clone()); + // A join's map is keyed on the qualified name, because two + // sides can carry the same bare column with different types. A + // single-collection map holds bare names only, so the qualified + // lookup misses there and the bare name answers. let ty = types - .get(&display_name) + .get(qname) + .or_else(|| types.get(&display_name)) .copied() .unwrap_or(DdlColType::Text); Some(OutputColumn { diff --git a/nodedb/src/control/planner/sql_plan_convert/output_schema/join_types.rs b/nodedb/src/control/planner/sql_plan_convert/output_schema/join_types.rs new file mode 100644 index 000000000..3fed56b73 --- /dev/null +++ b/nodedb/src/control/planner/sql_plan_convert/output_schema/join_types.rs @@ -0,0 +1,276 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! Per-side column types for a join's output schema. +//! +//! A join has no single source collection, so each projected column resolves +//! against the catalog of the side it came from. Two sides can carry the same +//! bare column name with different types, so the map is keyed on the +//! qualified name the executor emits (`orders.ts`). The bare name is added +//! too, and only while exactly one side claims it: an ambiguous bare name +//! stays `Text`, which every client parses. + +use std::collections::HashMap; + +use nodedb_sql::catalog::SqlCatalog; +use nodedb_sql::types::SqlPlan; + +use crate::control::server::response_shape::types::DdlColType; + +use super::columns::column_types_for; + +/// One join side: the collection it reads and the alias qualifying its +/// columns in the merged row. `None` means the columns are qualified by the +/// collection name. +struct JoinSide { + collection: String, + alias: Option, +} + +/// Column types of every side of a join, keyed by qualified name. +/// +/// Each side contributes `.` and `.` — a +/// projection can spell either — plus the bare `` when no other side +/// declares that name with a different type. +pub(super) fn join_column_types( + left: &SqlPlan, + right: &SqlPlan, + catalog: &C, + database_id: nodedb_types::DatabaseId, +) -> HashMap { + let mut sides = Vec::new(); + collect_sides(left, &mut sides); + collect_sides(right, &mut sides); + + let mut qualified: HashMap = HashMap::new(); + // Bare names seen so far, and whether one side still owns the name. A + // second side declaring the same name with a different type marks it + // ambiguous, and an ambiguous bare name resolves to `Text`. + let mut bare: HashMap = HashMap::new(); + let mut ambiguous: Vec = Vec::new(); + + for side in &sides { + let types = column_types_for(catalog, database_id, &side.collection); + for (column, ty) in &types { + qualified.insert(format!("{}.{column}", side.collection), *ty); + if let Some(alias) = &side.alias { + qualified.insert(format!("{alias}.{column}"), *ty); + } + // `copied` ends the borrow on `bare` before the arms write to it. + match bare.get(column).copied() { + Some(seen) if seen == *ty => {} + Some(_) => { + if !ambiguous.iter().any(|c| c == column) { + ambiguous.push(column.clone()); + } + } + None => { + bare.insert(column.clone(), *ty); + } + } + } + } + + for column in &ambiguous { + bare.insert(column.clone(), DdlColType::Text); + } + for (column, ty) in bare { + qualified.entry(column).or_insert(ty); + } + qualified +} + +/// Collect the scan-like leaves of one join side. +/// +/// A nested join contributes both of its own sides. A plan shape with no +/// single source collection contributes none, so every column it projects +/// stays `Text`. +fn collect_sides(plan: &SqlPlan, out: &mut Vec) { + match plan { + SqlPlan::Join { left, right, .. } => { + collect_sides(left, out); + collect_sides(right, out); + } + // The three read shapes that carry a FROM-clause alias. + SqlPlan::Scan { + collection, alias, .. + } + | SqlPlan::PointGet { + collection, alias, .. + } + | SqlPlan::DocumentIndexLookup { + collection, alias, .. + } => out.push(JoinSide { + collection: collection.clone(), + alias: alias.clone(), + }), + // Reads over one collection that carry no alias slot: the merged row + // qualifies their columns by the collection name. + SqlPlan::RangeScan { collection, .. } + | SqlPlan::TimeseriesScan { collection, .. } + | SqlPlan::SpatialScan { collection, .. } + | SqlPlan::VectorSearch { collection, .. } + | SqlPlan::MultiVectorSearch { collection, .. } + | SqlPlan::SparseSearch { collection, .. } + | SqlPlan::TextSearch { collection, .. } + | SqlPlan::HybridSearch { collection, .. } + | SqlPlan::HybridSearchTriple { collection, .. } + | SqlPlan::RecursiveScan { collection, .. } => out.push(JoinSide { + collection: collection.clone(), + alias: None, + }), + // No single source collection to attribute a column to: set + // operations, aggregates, CTE/subquery bodies, lateral shapes, + // constant and recursive-value rows, every write, and the whole + // array and index DDL family. + SqlPlan::ConstantResult { .. } + | SqlPlan::Insert { .. } + | SqlPlan::KvInsert { .. } + | SqlPlan::Upsert { .. } + | SqlPlan::InsertSelect { .. } + | SqlPlan::Update { .. } + | SqlPlan::UpdateFrom { .. } + | SqlPlan::Delete { .. } + | SqlPlan::Truncate { .. } + | SqlPlan::Aggregate { .. } + | SqlPlan::TimeseriesIngest { .. } + | SqlPlan::Union { .. } + | SqlPlan::Intersect { .. } + | SqlPlan::Except { .. } + | SqlPlan::RecursiveValue { .. } + | SqlPlan::Cte { .. } + | SqlPlan::Subquery { .. } + | SqlPlan::CreateArray { .. } + | SqlPlan::DropArray { .. } + | SqlPlan::AlterArray { .. } + | SqlPlan::InsertArray { .. } + | SqlPlan::DeleteArray { .. } + | SqlPlan::ArraySlice { .. } + | SqlPlan::ArrayProject { .. } + | SqlPlan::ArrayAgg { .. } + | SqlPlan::ArrayElementwise { .. } + | SqlPlan::ArrayFlush { .. } + | SqlPlan::ArrayCompact { .. } + | SqlPlan::Merge { .. } + | SqlPlan::LateralTopK { .. } + | SqlPlan::LateralLoop { .. } + | SqlPlan::VectorPrimaryInsert { .. } + | SqlPlan::CreateIndex { .. } + | SqlPlan::DropIndex { .. } => {} + } +} + +#[cfg(test)] +mod tests { + use super::*; + use nodedb_sql::types::collection::ColumnInfo; + use nodedb_sql::types::query::EngineType; + use nodedb_sql::types_expr::SqlDataType; + + /// Two collections whose `id` columns differ in type, plus a `ts` + /// declared only on `events`. + struct TwoSideCatalog; + + fn column(name: &str, ty: SqlDataType) -> ColumnInfo { + ColumnInfo { + name: name.to_string(), + data_type: ty, + nullable: true, + is_primary_key: false, + default: None, + raw_type: None, + int_width: None, + float_width: None, + } + } + + impl SqlCatalog for TwoSideCatalog { + fn get_collection( + &self, + _database_id: nodedb_types::DatabaseId, + name: &str, + ) -> Result, nodedb_sql::catalog::SqlCatalogError> + { + let columns = match name { + "events" => vec![ + column("ts", SqlDataType::Timestamp), + column("id", SqlDataType::Int64), + ], + "hosts" => vec![column("id", SqlDataType::String)], + _ => return Ok(None), + }; + Ok(Some(nodedb_sql::types::CollectionInfo { + name: name.to_string(), + engine: EngineType::DocumentStrict, + columns, + primary_key: None, + has_auto_tier: false, + indexes: Vec::new(), + bitemporal: false, + primary: nodedb_types::PrimaryEngine::Document, + vector_primary: None, + partition_strategy: nodedb_types::PartitionStrategy::CollectionHomed, + open_schema: nodedb_sql::types::CollectionInfo::open_schema_for( + EngineType::DocumentStrict, + ), + })) + } + } + + fn scan(collection: &str, alias: Option<&str>) -> SqlPlan { + SqlPlan::Scan { + collection: collection.to_string(), + alias: alias.map(str::to_string), + engine: EngineType::DocumentStrict, + filters: Vec::new(), + projection: Vec::new(), + sort_keys: Vec::new(), + limit: None, + offset: 0, + distinct: false, + window_functions: Vec::new(), + temporal: nodedb_sql::temporal::TemporalScope::default(), + } + } + + /// A column declared on one side only keeps that side's catalog type, + /// under both its qualified and its bare name. + #[test] + fn a_column_unique_to_one_side_keeps_its_type() { + let types = join_column_types( + &scan("events", None), + &scan("hosts", None), + &TwoSideCatalog, + nodedb_types::DatabaseId::DEFAULT, + ); + assert_eq!(types.get("events.ts"), Some(&DdlColType::Timestamp)); + assert_eq!(types.get("ts"), Some(&DdlColType::Timestamp)); + } + + /// An alias qualifies the same columns the collection name does. + #[test] + fn an_alias_names_the_same_side() { + let types = join_column_types( + &scan("events", Some("e")), + &scan("hosts", None), + &TwoSideCatalog, + nodedb_types::DatabaseId::DEFAULT, + ); + assert_eq!(types.get("e.ts"), Some(&DdlColType::Timestamp)); + assert_eq!(types.get("events.ts"), Some(&DdlColType::Timestamp)); + } + + /// A bare name two sides declare with different types cannot be + /// attributed, so it stays `Text` while the qualified names keep theirs. + #[test] + fn an_ambiguous_bare_name_stays_text() { + let types = join_column_types( + &scan("events", None), + &scan("hosts", None), + &TwoSideCatalog, + nodedb_types::DatabaseId::DEFAULT, + ); + assert_eq!(types.get("id"), Some(&DdlColType::Text)); + assert_eq!(types.get("events.id"), Some(&DdlColType::Int8)); + assert_eq!(types.get("hosts.id"), Some(&DdlColType::Text)); + } +} diff --git a/nodedb/src/control/planner/sql_plan_convert/output_schema/mod.rs b/nodedb/src/control/planner/sql_plan_convert/output_schema/mod.rs index 341537672..7c219e6e8 100644 --- a/nodedb/src/control/planner/sql_plan_convert/output_schema/mod.rs +++ b/nodedb/src/control/planner/sql_plan_convert/output_schema/mod.rs @@ -2,6 +2,7 @@ pub mod build; pub mod columns; +pub mod join_types; pub mod returning; pub use build::build_output_schema; diff --git a/nodedb/src/control/planner/sql_plan_convert/output_schema_types.rs b/nodedb/src/control/planner/sql_plan_convert/output_schema_types.rs index 69d9cad86..f6c0f623f 100644 --- a/nodedb/src/control/planner/sql_plan_convert/output_schema_types.rs +++ b/nodedb/src/control/planner/sql_plan_convert/output_schema_types.rs @@ -21,14 +21,23 @@ use nodedb_sql::types_expr::{BinaryOp, SqlExpr, SqlValue, UnaryOp}; use crate::control::server::response_shape::types::DdlColType; -/// Resolves a bare (unqualified or table-qualified) column reference to its -/// catalog type from `types`. Returns `None` for any non-column expression, or -/// for a column absent from the map. `types` is keyed by bare column name (the -/// last dotted segment), matching how the catalog map is built for the -/// collection in scope. +/// Resolves a column reference to its catalog type from `types`. Returns +/// `None` for any non-column expression, or for a column absent from the map. +/// +/// A join's map is keyed on the qualified name, because two sides can carry +/// the same bare column with different types, so a table-qualified reference +/// tries `.` first. A single-collection map holds bare names +/// only, where that lookup misses and the bare name answers. fn bare_column_type(expr: &SqlExpr, types: &HashMap) -> Option { match expr { - SqlExpr::Column { name, .. } => types.get(name).copied(), + SqlExpr::Column { + table: Some(table), + name, + } => types + .get(&format!("{table}.{name}")) + .or_else(|| types.get(name)) + .copied(), + SqlExpr::Column { table: None, name } => types.get(name).copied(), _ => None, } } From 299e698dd319efb4acd471b2794a39753efc10db Mon Sep 17 00:00:00 2001 From: Farhan Syah Date: Thu, 10 Sep 2026 08:09:04 +0800 Subject: [PATCH 15/23] fix(join): rescale declared-instant columns a join scans locally A join reads a timeseries collection through scan_collection, which hands back its stored epoch-millisecond value, while a client reads a TIMESTAMP cell as epoch microseconds. Hash, nested-loop, and sort-merge joins now rescale the declared-instant cells they read from their own local scans immediately before emission, after every predicate has run so the join itself still compares milliseconds against milliseconds. A side supplied by a sub-plan is excluded: its rows were already rescaled by the handler that produced them, so rescaling again would double the correction. --- nodedb/src/data/executor/dispatch/query.rs | 121 ++++++++++++++---- .../executor/handlers/join/instant_scale.rs | 97 ++++++++++++++ nodedb/src/data/executor/handlers/join/mod.rs | 1 + .../executor/handlers/join/nested_loop.rs | 13 ++ .../src/data/executor/handlers/join/params.rs | 18 +++ .../executor/handlers/join/shuffle_join.rs | 1 + .../data/executor/handlers/join/sort_merge.rs | 13 ++ 7 files changed, 239 insertions(+), 25 deletions(-) create mode 100644 nodedb/src/data/executor/handlers/join/instant_scale.rs diff --git a/nodedb/src/data/executor/dispatch/query.rs b/nodedb/src/data/executor/dispatch/query.rs index 180a112d0..cfd90c5ea 100644 --- a/nodedb/src/data/executor/dispatch/query.rs +++ b/nodedb/src/data/executor/dispatch/query.rs @@ -8,6 +8,7 @@ use nodedb_physical::physical_plan::{GroupKeySpec, QueryOp}; use crate::data::executor::core_loop::CoreLoop; use crate::data::executor::handlers::join::{ HashJoinParams, JoinParams, NestedLoopJoinParams, ShuffleJoinInputs, SortMergeJoinParams, + instant_scale::JoinInstantSide, lateral::{LateralLoopParams, LateralTopKParams}, }; use crate::data::executor::task::ExecutionTask; @@ -110,7 +111,35 @@ impl CoreLoop { left_scan_filters, right_scan_filters, .. - } => self.execute_hash_join(HashJoinParams { + } => { + // Only a side this join scans itself carries stored + // milliseconds. A side supplied by a sub-plan was already + // rescaled by the handler that read it, so listing it here + // would scale one instant twice. + let mut local_sides = Vec::with_capacity(2); + if left_input.is_none() { + local_sides.push(JoinInstantSide { + collection: left_collection.as_str(), + qualifier: left_alias + .as_deref() + .unwrap_or_else(|| left_collection.as_str()), + }); + } + if right_input.is_none() { + local_sides.push(JoinInstantSide { + collection: right_collection.as_str(), + qualifier: right_alias + .as_deref() + .unwrap_or_else(|| right_collection.as_str()), + }); + } + let instant_columns = self.join_instant_columns( + task.request.database_id, + crate::types::TenantId::new(tid), + &local_sides, + projection, + ); + self.execute_hash_join(HashJoinParams { join: JoinParams { task, on, @@ -120,6 +149,7 @@ impl CoreLoop { computed_projection_bytes: computed_projection, join_filter_bytes: join_filters, post_filter_bytes: post_filters, + instant_columns: &instant_columns, }, tid, left_collection: left_collection.as_str(), @@ -134,7 +164,8 @@ impl CoreLoop { right_rls_filters, left_scan_filters, right_scan_filters, - }), + }) + } QueryOp::ShuffleJoinConsume { build_path, @@ -161,6 +192,10 @@ impl CoreLoop { computed_projection_bytes: &[], join_filter_bytes: &[], post_filter_bytes: &[], + // A shuffle consumer's rows are gathered by the + // coordinator, not sent to a client, and its staged + // frames were produced by the sides' own scans. + instant_columns: &[], }; let inputs = ShuffleJoinInputs { build_path: std::path::PathBuf::from(build_path), @@ -182,17 +217,35 @@ impl CoreLoop { limit, left_rls_filters, right_rls_filters, - } => self.execute_nested_loop_join(NestedLoopJoinParams { - task, - tid, - left_collection: left_collection.as_str(), - right_collection: right_collection.as_str(), - condition, - join_type, - limit: *limit, - left_rls_filters, - right_rls_filters, - }), + } => { + let instant_columns = self.join_instant_columns( + task.request.database_id, + crate::types::TenantId::new(tid), + &[ + JoinInstantSide { + collection: left_collection.as_str(), + qualifier: left_collection.as_str(), + }, + JoinInstantSide { + collection: right_collection.as_str(), + qualifier: right_collection.as_str(), + }, + ], + &[], + ); + self.execute_nested_loop_join(NestedLoopJoinParams { + task, + tid, + left_collection: left_collection.as_str(), + right_collection: right_collection.as_str(), + condition, + join_type, + limit: *limit, + left_rls_filters, + right_rls_filters, + instant_columns: &instant_columns, + }) + } QueryOp::SortMergeJoin { left_collection, @@ -203,18 +256,36 @@ impl CoreLoop { pre_sorted, left_rls_filters, right_rls_filters, - } => self.execute_sort_merge_join(SortMergeJoinParams { - task, - tid, - left_collection: left_collection.as_str(), - right_collection: right_collection.as_str(), - on, - join_type, - limit: *limit, - pre_sorted: *pre_sorted, - left_rls_filters, - right_rls_filters, - }), + } => { + let instant_columns = self.join_instant_columns( + task.request.database_id, + crate::types::TenantId::new(tid), + &[ + JoinInstantSide { + collection: left_collection.as_str(), + qualifier: left_collection.as_str(), + }, + JoinInstantSide { + collection: right_collection.as_str(), + qualifier: right_collection.as_str(), + }, + ], + &[], + ); + self.execute_sort_merge_join(SortMergeJoinParams { + task, + tid, + left_collection: left_collection.as_str(), + right_collection: right_collection.as_str(), + on, + join_type, + limit: *limit, + pre_sorted: *pre_sorted, + left_rls_filters, + right_rls_filters, + instant_columns: &instant_columns, + }) + } QueryOp::RecursiveScan { collection, diff --git a/nodedb/src/data/executor/handlers/join/instant_scale.rs b/nodedb/src/data/executor/handlers/join/instant_scale.rs new file mode 100644 index 000000000..ada4e4abc --- /dev/null +++ b/nodedb/src/data/executor/handlers/join/instant_scale.rs @@ -0,0 +1,97 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! Rescale the declared-instant cells a join materialized itself. +//! +//! A timeseries collection stores every timestamp column in epoch +//! milliseconds, and a client reads a `TIMESTAMP` cell as epoch microseconds. +//! A join scans its local sides through `scan_collection`, which hands back +//! the stored millisecond value, so the join rescales those cells as it emits +//! them — after every predicate has run, so the join itself still compares +//! milliseconds against milliseconds. +//! +//! Exactly once: a handler rescales ONLY the cells it read from a local +//! collection scan. Rows that arrive from a sub-plan response were already +//! rescaled by the handler that read them, and are never touched again. + +use nodedb_physical::physical_plan::JoinProjection; + +use crate::data::executor::core_loop::CoreLoop; +use crate::data::executor::handlers::timeseries::raw_scan::scale_instant_cells; +use crate::types::{DatabaseId, TenantId}; + +/// One locally-scanned join side: the collection its rows come from, and the +/// qualifier the merged row prefixes that side's keys with. +pub(in crate::data::executor) struct JoinInstantSide<'a> { + pub(in crate::data::executor) collection: &'a str, + pub(in crate::data::executor) qualifier: &'a str, +} + +impl CoreLoop { + /// The emitted names of the declared-instant cells `sides` contribute. + /// + /// A merged join row keys every cell `.`, and a + /// projection renames it to its output name. The returned names are what + /// the emitted row actually carries, so the caller matches on them + /// directly. A side that is not a timeseries collection contributes none, + /// so a join over ordinary collections returns an empty list and pays + /// nothing. + pub(in crate::data::executor) fn join_instant_columns( + &self, + database_id: DatabaseId, + tid: TenantId, + sides: &[JoinInstantSide<'_>], + projection: &[JoinProjection], + ) -> Vec { + let mut emitted = Vec::new(); + for side in sides { + for column in self.ts_instant_columns(database_id, tid, side.collection) { + let qualified = format!("{}.{column}", side.qualifier); + if projection.is_empty() { + emitted.push(qualified); + continue; + } + // Mirrors `binary_row_project`: a projection entry names + // either the qualified key or its bare last segment. + for entry in projection { + if entry.source == qualified || entry.source == column { + emitted.push(entry.output.clone()); + } + } + } + } + emitted + } +} + +/// Rescale the named cells of already-projected join rows, in place. +/// +/// Each row is a msgpack map. An empty `instant_columns` is a no-op, so a +/// join that touches no timeseries collection never decodes a row. +pub(in crate::data::executor) fn scale_join_instant_rows( + rows: &mut [Vec], + instant_columns: &[String], +) -> crate::Result<()> { + if instant_columns.is_empty() { + return Ok(()); + } + for row in rows.iter_mut() { + let mut decoded = [ + crate::util::bounded_msgpack::read_value(row.as_slice()).map_err(|e| { + crate::Error::Serialization { + format: "msgpack".into(), + detail: format!("decode join row for instant rescale: {e}"), + } + })?, + ]; + scale_instant_cells(&mut decoded, instant_columns)?; + let mut buf = Vec::with_capacity(row.len()); + rmpv::encode::write_value(&mut buf, &decoded[0]).map_err(|e| { + crate::Error::Serialization { + format: "msgpack".into(), + detail: format!("encode join row after instant rescale: {e}"), + } + })?; + *row = buf; + } + Ok(()) +} diff --git a/nodedb/src/data/executor/handlers/join/mod.rs b/nodedb/src/data/executor/handlers/join/mod.rs index f9d4f8ad7..a6dc9bff6 100644 --- a/nodedb/src/data/executor/handlers/join/mod.rs +++ b/nodedb/src/data/executor/handlers/join/mod.rs @@ -10,6 +10,7 @@ mod grace_repartition; mod grace_spill; pub mod hash; mod hash_handlers; +pub(in crate::data::executor) mod instant_scale; pub mod lateral; pub mod nested_loop; pub mod params; diff --git a/nodedb/src/data/executor/handlers/join/nested_loop.rs b/nodedb/src/data/executor/handlers/join/nested_loop.rs index e0e6db89a..d961bbc6a 100644 --- a/nodedb/src/data/executor/handlers/join/nested_loop.rs +++ b/nodedb/src/data/executor/handlers/join/nested_loop.rs @@ -30,6 +30,7 @@ impl CoreLoop { limit, left_rls_filters, right_rls_filters, + instant_columns, } = p; debug!( core = self.core_id, @@ -224,6 +225,18 @@ impl CoreLoop { return self.response_error(task, ErrorCode::ResourcesExhausted); } + // Last step before emission: the scans above compared the + // milliseconds storage holds, and the client reads microseconds. + if let Err(e) = super::instant_scale::scale_join_instant_rows(&mut results, instant_columns) + { + return self.response_error( + task, + ErrorCode::Internal { + detail: e.to_string(), + }, + ); + } + let payload = super::super::super::response_codec::encode_binary_rows(&results); self.response_with_payload(task, payload) } diff --git a/nodedb/src/data/executor/handlers/join/params.rs b/nodedb/src/data/executor/handlers/join/params.rs index c5620115e..3de0b4e1b 100644 --- a/nodedb/src/data/executor/handlers/join/params.rs +++ b/nodedb/src/data/executor/handlers/join/params.rs @@ -17,6 +17,11 @@ pub(crate) struct JoinParams<'a> { pub computed_projection_bytes: &'a [u8], pub join_filter_bytes: &'a [u8], pub post_filter_bytes: &'a [u8], + /// Emitted names of the declared-instant cells this join reads from its + /// own local scans. Empty when no timeseries collection is scanned + /// locally — including for every side supplied by a sub-plan, whose rows + /// were already rescaled by the handler that read them. + pub instant_columns: &'a [String], } /// Hash join: scans both sides from storage or executes resolved child sub-plans. @@ -76,6 +81,9 @@ pub(crate) struct NestedLoopJoinParams<'a> { pub left_rls_filters: &'a [u8], /// Row-level-security filters for the locally-scanned right side. pub right_rls_filters: &'a [u8], + /// Emitted names of the declared-instant cells the two local scans + /// contribute. Empty when neither side is a timeseries collection. + pub instant_columns: &'a [String], } /// Sort-merge join: O((N+M)·log N) equi-join with optional pre-sorted inputs. @@ -95,6 +103,9 @@ pub(crate) struct SortMergeJoinParams<'a> { pub left_rls_filters: &'a [u8], /// Row-level-security filters for the locally-scanned right side. pub right_rls_filters: &'a [u8], + /// Emitted names of the declared-instant cells the two local scans + /// contribute. Empty when neither side is a timeseries collection. + pub instant_columns: &'a [String], } // ── Test helpers ───────────────────────────────────────────────────────────── @@ -212,6 +223,10 @@ impl JoinParams<'_> { } } + // Last step before emission: every predicate above compared the + // milliseconds storage holds, and the client reads microseconds. + super::instant_scale::scale_join_instant_rows(results, self.instant_columns)?; + Ok(()) } } @@ -258,6 +273,7 @@ mod tests { computed_projection_bytes: &[], join_filter_bytes: &[], post_filter_bytes: &[], + instant_columns: &[], }; let mut results = vec![vec![1u8, 2, 3], vec![4u8, 5, 6]]; assert!(params.filter_and_project(&mut results).is_ok()); @@ -280,6 +296,7 @@ mod tests { computed_projection_bytes: &[], join_filter_bytes: &[], post_filter_bytes: corrupt, + instant_columns: &[], }; let mut results = vec![vec![0u8; 8]]; // would be "leaked" under the old code let err = params.filter_and_project(&mut results); @@ -310,6 +327,7 @@ mod tests { computed_projection_bytes: &[], join_filter_bytes: &[], post_filter_bytes: &filter_bytes, + instant_columns: &[], }; let mut results = vec![ row_with_score(99), // should be kept diff --git a/nodedb/src/data/executor/handlers/join/shuffle_join.rs b/nodedb/src/data/executor/handlers/join/shuffle_join.rs index d1fceefad..2cbe7abfd 100644 --- a/nodedb/src/data/executor/handlers/join/shuffle_join.rs +++ b/nodedb/src/data/executor/handlers/join/shuffle_join.rs @@ -439,6 +439,7 @@ mod tests { computed_projection_bytes: &[], join_filter_bytes: &[], post_filter_bytes: &[], + instant_columns: &[], }; let inputs = ShuffleJoinInputs { build_path, diff --git a/nodedb/src/data/executor/handlers/join/sort_merge.rs b/nodedb/src/data/executor/handlers/join/sort_merge.rs index 286de26f6..c4803a8a5 100644 --- a/nodedb/src/data/executor/handlers/join/sort_merge.rs +++ b/nodedb/src/data/executor/handlers/join/sort_merge.rs @@ -51,6 +51,7 @@ impl CoreLoop { pre_sorted, left_rls_filters, right_rls_filters, + instant_columns, } = p; debug!( core = self.core_id, @@ -299,6 +300,18 @@ impl CoreLoop { return self.response_error(task, ErrorCode::ResourcesExhausted); } + // Last step before emission: the scans above compared the + // milliseconds storage holds, and the client reads microseconds. + if let Err(e) = super::instant_scale::scale_join_instant_rows(&mut results, instant_columns) + { + return self.response_error( + task, + ErrorCode::Internal { + detail: e.to_string(), + }, + ); + } + let payload = super::super::super::response_codec::encode_binary_rows(&results); self.response_with_payload(task, payload) } From 7da56d6c28ff85579531262814770f0e35b58369 Mon Sep 17 00:00:00 2001 From: Farhan Syah Date: Thu, 10 Sep 2026 08:09:15 +0800 Subject: [PATCH 16/23] fix(timeseries): render GROUP BY keys with their declared column type MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A grouped timeseries scan reduces every key to a string, and the aggregate encoder rendered every one of them back as text regardless of the column's real type. It now resolves each GROUP BY column's storage kind — declared instant, integer, float, or text — from the collection's declared schema or its resident memtable schema, and renders the key with that type. A declared TIMESTAMP key converts its stored milliseconds to the microseconds a client expects, matching how a direct SELECT of the same column renders it. --- nodedb/src/data/executor/core_loop/mod.rs | 1 + .../executor/core_loop/ts_declared_schema.rs | 83 +++++++++++++++++++ .../executor/handlers/timeseries/aggregate.rs | 5 ++ .../executor/handlers/timeseries/encode.rs | 60 ++++++++++++-- 4 files changed, 140 insertions(+), 9 deletions(-) diff --git a/nodedb/src/data/executor/core_loop/mod.rs b/nodedb/src/data/executor/core_loop/mod.rs index 0398e78e3..55e9e6211 100644 --- a/nodedb/src/data/executor/core_loop/mod.rs +++ b/nodedb/src/data/executor/core_loop/mod.rs @@ -30,6 +30,7 @@ pub use doc_config_seed::DocConfigSeedEntry; pub(in crate::data::executor) use segment_keks::SegmentKeks; pub use state::CoreLoop; pub use test_governor::test_governor; +pub(in crate::data::executor) use ts_declared_schema::TsGroupKeyKind; /// Shared test fixtures (`make_core_with_dir`, `make_default_task`), kept /// alongside the write-version-index tests that exercise the same `CoreLoop` /// apply chokepoints. Re-exported here so external test modules keep using diff --git a/nodedb/src/data/executor/core_loop/ts_declared_schema.rs b/nodedb/src/data/executor/core_loop/ts_declared_schema.rs index 8e4ff48ab..c29312625 100644 --- a/nodedb/src/data/executor/core_loop/ts_declared_schema.rs +++ b/nodedb/src/data/executor/core_loop/ts_declared_schema.rs @@ -117,6 +117,58 @@ impl CoreLoop { }) } + /// How each named GROUP BY column of a timeseries collection renders. + /// + /// A grouped column must carry the type it carries ungrouped, so this + /// resolves the same declared shape row emission reads. A declared + /// collection answers from its DDL; a measurement ingested over the raw + /// ILP protocol has no DDL, so its resident memtable schema answers. + /// A column present in neither renders as text. + pub(in crate::data::executor) fn ts_group_key_kinds( + &self, + database_id: DatabaseId, + tid: TenantId, + collection: &str, + group_by: &[String], + ) -> Vec { + if let Some(declared) = self.declared_timeseries(database_id, tid, collection) { + let time_key_index = declared.time_key_index(); + return group_by + .iter() + .map(|name| { + let Some(index) = declared.columns.iter().position(|(c, _)| c == name) else { + return TsGroupKeyKind::Text; + }; + let declared_type = declared.columns[index].1.as_str(); + if declared_type_is_instant(declared_type) { + return TsGroupKeyKind::Instant; + } + kind_of_storage(memtable_column_type( + declared_type, + Some(index) == time_key_index, + )) + }) + .collect(); + } + + let key = (database_id, tid, collection.to_string()); + let Some(memtable) = self.columnar_memtables.get(&key) else { + return vec![TsGroupKeyKind::Text; group_by.len()]; + }; + let schema = memtable.schema(); + group_by + .iter() + .map(|name| { + schema + .columns + .iter() + .find(|(c, _)| c == name) + .map(|(_, ty)| kind_of_storage(*ty)) + .unwrap_or(TsGroupKeyKind::Text) + }) + .collect() + } + /// Declared columns of a timeseries collection that carry an instant. /// /// A column declared `TIMESTAMP` or `TIMESTAMPTZ` is one. The memtable @@ -149,6 +201,37 @@ impl CoreLoop { } } +/// How one grouped timeseries column renders in an aggregate result row. +/// +/// The grouped scan reduces every key to a string, so emission has to put the +/// column's own type back. The variants name the four shapes a stored +/// timeseries column can take on the wire. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(in crate::data::executor) enum TsGroupKeyKind { + /// A declared `TIMESTAMP` / `TIMESTAMPTZ` column: epoch microseconds. + Instant, + /// An integer column, including a `BIGINT TIME_KEY`, in its stored unit. + Integer, + /// A floating-point column. + Float, + /// A dictionary symbol, or a column with no resolvable storage type. + Text, +} + +/// The wire shape a memtable storage type renders as. +/// +/// `Timestamp` maps to `Integer` here: the instant case is decided from the +/// declared DDL type before this runs, so what reaches it is a `BIGINT` +/// time key or a system-time column, both of which render as the number +/// storage holds. +fn kind_of_storage(storage: ColumnType) -> TsGroupKeyKind { + match storage { + ColumnType::Int64 | ColumnType::Timestamp => TsGroupKeyKind::Integer, + ColumnType::Float64 => TsGroupKeyKind::Float, + ColumnType::Symbol => TsGroupKeyKind::Text, + } +} + /// Whether a declared DDL type makes a column an instant on the wire. /// /// Mirrors the two spellings the planner resolves to `SqlDataType::Timestamp` diff --git a/nodedb/src/data/executor/handlers/timeseries/aggregate.rs b/nodedb/src/data/executor/handlers/timeseries/aggregate.rs index 72b1c0453..7ee65f367 100644 --- a/nodedb/src/data/executor/handlers/timeseries/aggregate.rs +++ b/nodedb/src/data/executor/handlers/timeseries/aggregate.rs @@ -184,6 +184,10 @@ impl CoreLoop { }; // Phase 4: Encode response (MessagePack, no serde_json intermediate). + // A grouped column carries the type it carries ungrouped, so the + // encoder is told each key column's declared shape. + let group_key_kinds = + self.ts_group_key_kinds(task.request.database_id, tid, collection, group_by); let payload = match super::encode::encode_grouped_results( &merged, group_by, @@ -191,6 +195,7 @@ impl CoreLoop { limit, bucket_interval_ms, sort_keys, + &group_key_kinds, ) { Ok(p) => p, Err(e) => return self.response_error(task, e), diff --git a/nodedb/src/data/executor/handlers/timeseries/encode.rs b/nodedb/src/data/executor/handlers/timeseries/encode.rs index e28342bdb..922b4dfdc 100644 --- a/nodedb/src/data/executor/handlers/timeseries/encode.rs +++ b/nodedb/src/data/executor/handlers/timeseries/encode.rs @@ -4,6 +4,47 @@ use nodedb_query::agg_key::canonical_agg_key; +use crate::data::executor::core_loop::TsGroupKeyKind; + +/// Render one GROUP BY key part with the type its column carries ungrouped. +/// +/// The grouped scan reduces every key to a string, so the column's own type +/// is put back here. An empty part is SQL NULL. A declared instant is stored +/// in milliseconds and read in microseconds, exactly as row emission reads +/// it, so the two routes to one stored instant render it identically. +/// +/// A part that does not parse as its column's type falls back to the text it +/// holds: the key is data the scan produced, and dropping the group would +/// lose a row. +fn group_key_value(part: Option<&&str>, kind: TsGroupKeyKind) -> crate::Result { + let Some(text) = part.filter(|s| !s.is_empty()) else { + return Ok(rmpv::Value::Nil); + }; + let value = match kind { + TsGroupKeyKind::Instant => match text.parse::() { + Ok(millis) => { + let micros = nodedb_types::NdbDateTime::from_millis(millis) + .map_err(|e| crate::Error::Internal { + detail: format!("grouped timeseries key at {millis} ms: {e}"), + })? + .micros; + rmpv::Value::Integer(micros.into()) + } + Err(_) => rmpv::Value::String((*text).into()), + }, + TsGroupKeyKind::Integer => match text.parse::() { + Ok(n) => rmpv::Value::Integer(n.into()), + Err(_) => rmpv::Value::String((*text).into()), + }, + TsGroupKeyKind::Float => match text.parse::() { + Ok(f) => rmpv::Value::F64(f), + Err(_) => rmpv::Value::String((*text).into()), + }, + TsGroupKeyKind::Text => rmpv::Value::String((*text).into()), + }; + Ok(value) +} + /// Serialize GroupedAggResult directly to MessagePack bytes. /// /// Avoids building `Vec` (2M allocations for 2M groups). @@ -20,6 +61,7 @@ pub(in crate::data::executor) fn encode_grouped_results( limit: usize, bucket_interval_ms: i64, sort_keys: &[nodedb_physical::physical_plan::SortKeySpec], + group_key_kinds: &[TsGroupKeyKind], ) -> crate::Result> { let has_bucket = bucket_interval_ms > 0; // An ordered query has to see every group before cutting to `limit`: @@ -62,20 +104,20 @@ pub(in crate::data::executor) fn encode_grouped_results( )); for (i, field) in group_by.iter().enumerate() { - let val = parts - .get(i + 1) - .filter(|s| !s.is_empty()) - .map(|s| rmpv::Value::String((*s).into())) - .unwrap_or(rmpv::Value::Nil); + let kind = group_key_kinds + .get(i) + .copied() + .unwrap_or(TsGroupKeyKind::Text); + let val = group_key_value(parts.get(i + 1), kind)?; fields.push((rmpv::Value::String(field.as_str().into()), val)); } } else { for (i, field) in group_by.iter().enumerate() { - let val = parts + let kind = group_key_kinds .get(i) - .filter(|s| !s.is_empty()) - .map(|s| rmpv::Value::String((*s).into())) - .unwrap_or(rmpv::Value::Nil); + .copied() + .unwrap_or(TsGroupKeyKind::Text); + let val = group_key_value(parts.get(i), kind)?; fields.push((rmpv::Value::String(field.as_str().into()), val)); } } From 640cce8513578d49459016a924196cf10dfa4f29 Mon Sep 17 00:00:00 2001 From: Farhan Syah Date: Thu, 10 Sep 2026 08:09:24 +0800 Subject: [PATCH 17/23] test(timeseries): cover a declared time key rendered via join and GROUP BY A stored TIMESTAMP time key can be read directly, projected through a JOIN, or used as a GROUP BY key, each through its own scan and encoder. Cover that all three render one stored instant identically, anchored against the instant the INSERT actually supplied. --- nodedb/tests/wire/cases/mod.rs | 1 + .../cases/timeseries_join_time_rendering.rs | 157 ++++++++++++++++++ 2 files changed, 158 insertions(+) create mode 100644 nodedb/tests/wire/cases/timeseries_join_time_rendering.rs diff --git a/nodedb/tests/wire/cases/mod.rs b/nodedb/tests/wire/cases/mod.rs index e51303c21..8b585c811 100644 --- a/nodedb/tests/wire/cases/mod.rs +++ b/nodedb/tests/wire/cases/mod.rs @@ -264,6 +264,7 @@ mod strict_bitemporal_audit_query; mod strict_bitemporal_select_star; mod strict_schema_restart; mod timeseries_declared_time_key; +mod timeseries_join_time_rendering; mod timeseries_write_row_level_security; mod transactional_ddl_atomicity; mod transactional_ddl_compensation; diff --git a/nodedb/tests/wire/cases/timeseries_join_time_rendering.rs b/nodedb/tests/wire/cases/timeseries_join_time_rendering.rs new file mode 100644 index 000000000..35cdb9545 --- /dev/null +++ b/nodedb/tests/wire/cases/timeseries_join_time_rendering.rs @@ -0,0 +1,157 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! A stored instant renders the same however a query reaches it. +//! +//! A timeseries `TIMESTAMP` time key can be read three ways: a direct +//! `SELECT` of the column, the same column projected through a JOIN, and the +//! same column used as a `GROUP BY` key. Each route runs through its own +//! scan and its own encoder. All three name one stored instant, so all three +//! must render it identically. +//! +//! Every assertion here compares two reads of the SAME row against each +//! other. A hardcoded rendering would pin whichever route was written down +//! first and call the other one wrong, so the comparison states the +//! invariant instead. One absolute check anchors the pair, so the two routes +//! cannot agree on a value that denotes the wrong instant. + +use crate::harness::TestServer; + +/// One event time, years in the past, so a value the engine substitutes at +/// ingest (wall-clock "now") is separable from the value the INSERT supplied. +const EARLY: &str = "2020-03-05 10:00:00"; +/// `EARLY` as a declared `TIMESTAMP` column renders it. The engine stores +/// 1583402400000 epoch milliseconds; a `TIMESTAMP` cell carries epoch +/// microseconds, which the pgwire encoder writes as ISO-8601 UTC. +const EARLY_ISO: &str = "2020-03-05T10:00:00.000000Z"; +/// `EARLY` as epoch microseconds — 1583402400000 milliseconds times 1000. +/// A projection that announces no catalog type leaves its cells this number. +const EARLY_MICROS: &str = "1583402400000000"; + +/// Create a timeseries collection and a document collection that join on the +/// event's `host`, then insert exactly one row into each so the join yields +/// one row. +async fn setup(server: &TestServer, events: &str, hosts: &str) { + server + .exec(&format!( + "CREATE COLLECTION {events} \ + (captured_at TIMESTAMP TIME_KEY, host TEXT, v FLOAT) \ + WITH (engine='timeseries')" + )) + .await + .unwrap_or_else(|e| panic!("create {events}: {e}")); + server + .exec(&format!( + "CREATE COLLECTION {hosts} (id TEXT PRIMARY KEY, region TEXT) \ + WITH (engine='document_strict')" + )) + .await + .unwrap_or_else(|e| panic!("create {hosts}: {e}")); + + server + .exec(&format!( + "INSERT INTO {events} (captured_at, host, v) VALUES ('{EARLY}', 'h1', 1.5)" + )) + .await + .unwrap_or_else(|e| panic!("insert into {events}: {e}")); + server + .exec(&format!( + "INSERT INTO {hosts} (id, region) VALUES ('h1', 'eu')" + )) + .await + .unwrap_or_else(|e| panic!("insert into {hosts}: {e}")); +} + +/// A time key projected through a JOIN renders as the direct `SELECT` renders +/// it. Both reads name one stored row, so a divergence is a rendering defect. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn a_time_key_renders_the_same_through_a_join_as_through_a_select() { + let server = TestServer::start().await; + setup(&server, "tsj_join_events", "tsj_join_hosts").await; + + let direct = server + .query_text("SELECT captured_at FROM tsj_join_events") + .await + .expect("direct SELECT of the declared time key must succeed"); + assert_eq!(direct.len(), 1, "one stored point: {direct:?}"); + + let joined = server + .query_text( + "SELECT tsj_join_events.captured_at FROM tsj_join_events \ + INNER JOIN tsj_join_hosts ON tsj_join_events.host = tsj_join_hosts.id", + ) + .await + .expect("the same column projected through a JOIN must succeed"); + assert_eq!( + joined.len(), + 1, + "one event matches one host, so the join yields one row: {joined:?}" + ); + + assert_eq!( + joined[0], direct[0], + "the time key must render the same through a JOIN as through a SELECT: \ + joined={joined:?} direct={direct:?}" + ); +} + +/// A time key used as a `GROUP BY` key renders as the direct `SELECT` renders +/// it. The aggregate encoder writes the group key itself, so it is a third +/// route to the same stored instant. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn a_time_key_renders_the_same_through_an_aggregate_as_through_a_select() { + let server = TestServer::start().await; + setup(&server, "tsj_agg_events", "tsj_agg_hosts").await; + + let direct = server + .query_text("SELECT captured_at FROM tsj_agg_events") + .await + .expect("direct SELECT of the declared time key must succeed"); + assert_eq!(direct.len(), 1, "one stored point: {direct:?}"); + + let grouped = server + .query_text("SELECT captured_at, COUNT(*) FROM tsj_agg_events GROUP BY captured_at") + .await + .expect("GROUP BY on the declared time key must succeed"); + assert_eq!( + grouped.len(), + 1, + "one stored point falls in one group: {grouped:?}" + ); + + assert_eq!( + grouped[0], direct[0], + "the time key must render the same as a GROUP BY key as through a SELECT: \ + grouped={grouped:?} direct={direct:?}" + ); +} + +/// A time key read through a JOIN denotes the instant the INSERT supplied. +/// This anchors the comparisons above, which two wrong routes can otherwise +/// pass by agreeing with each other. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn a_joined_time_key_denotes_the_stored_instant() { + let server = TestServer::start().await; + setup(&server, "tsj_abs_events", "tsj_abs_hosts").await; + + let joined = server + .query_text( + "SELECT tsj_abs_events.captured_at FROM tsj_abs_events \ + INNER JOIN tsj_abs_hosts ON tsj_abs_events.host = tsj_abs_hosts.id", + ) + .await + .expect("the time key projected through a JOIN must succeed"); + assert_eq!(joined.len(), 1, "the join yields one row: {joined:?}"); + + // Two renderings denote 2020-03-05T10:00:00Z, and the expected value is + // whichever one the join announces a type for. EARLY_ISO is that instant + // written as ISO-8601 UTC, which a cell typed TIMESTAMP produces. + // EARLY_MICROS is the same instant in epoch microseconds — the unit a + // TIMESTAMP cell carries — which an untyped cell leaves as a number. + // Epoch MILLISECONDS denote 1970-01-19 read either way, so a millisecond + // value fails both arms. + assert!( + joined[0] == EARLY_ISO || joined[0] == EARLY_MICROS, + "a joined time key must denote {EARLY}: expected {EARLY_ISO} \ + or {EARLY_MICROS}, got {joined:?}" + ); +} From 7927b166a1bd1c884e9678091a852a99c85dc3e9 Mon Sep 17 00:00:00 2001 From: Farhan Syah Date: Thu, 10 Sep 2026 09:53:25 +0800 Subject: [PATCH 18/23] refactor(sql): move write-route resolution into EngineRules Each engine's EngineRules now picks the WriteRoute (Document or ColumnarFamily) when it builds an Insert or Upsert plan, carrying it on the SqlPlan variant instead of re-deriving it later from the engine type. The conversion layer reads the route directly, removing the separate route-resolution module. --- nodedb-sql/src/engine_rules/columnar.rs | 2 + .../src/engine_rules/document_schemaless.rs | 2 + .../src/engine_rules/document_strict.rs | 2 + nodedb-sql/src/engine_rules/spatial.rs | 2 + nodedb-sql/src/types/mod.rs | 2 +- nodedb-sql/src/types/plan/mod.rs | 2 +- nodedb-sql/src/types/plan/row_types.rs | 15 +++++ nodedb-sql/src/types/plan/variants.rs | 7 ++- nodedb-sql/src/visitor/plan_visitor/args.rs | 6 +- .../src/visitor/plan_visitor/dispatch.rs | 4 ++ .../planner/sql_plan_convert/dml/insert.rs | 15 ++--- .../planner/sql_plan_convert/dml/mod.rs | 1 - .../planner/sql_plan_convert/dml/route.rs | 55 ------------------- .../planner/sql_plan_convert/dml/upsert.rs | 11 ++-- .../sql_plan_convert/visitor/arms_dml.rs | 10 ++-- 15 files changed, 56 insertions(+), 80 deletions(-) delete mode 100644 nodedb/src/control/planner/sql_plan_convert/dml/route.rs diff --git a/nodedb-sql/src/engine_rules/columnar.rs b/nodedb-sql/src/engine_rules/columnar.rs index 366b86840..7decfbd69 100644 --- a/nodedb-sql/src/engine_rules/columnar.rs +++ b/nodedb-sql/src/engine_rules/columnar.rs @@ -13,6 +13,7 @@ impl EngineRules for ColumnarRules { Ok(vec![SqlPlan::Insert { collection: p.collection, engine: EngineType::Columnar, + route: WriteRoute::ColumnarFamily, rows: p.rows, column_defaults: p.column_defaults, if_absent: p.if_absent, @@ -29,6 +30,7 @@ impl EngineRules for ColumnarRules { Ok(vec![SqlPlan::Upsert { collection: p.collection, engine: EngineType::Columnar, + route: WriteRoute::ColumnarFamily, rows: p.rows, column_defaults: p.column_defaults, on_conflict_updates: p.on_conflict_updates, diff --git a/nodedb-sql/src/engine_rules/document_schemaless.rs b/nodedb-sql/src/engine_rules/document_schemaless.rs index 7b1af4059..17937962f 100644 --- a/nodedb-sql/src/engine_rules/document_schemaless.rs +++ b/nodedb-sql/src/engine_rules/document_schemaless.rs @@ -13,6 +13,7 @@ impl EngineRules for SchemalessRules { Ok(vec![SqlPlan::Insert { collection: p.collection, engine: EngineType::DocumentSchemaless, + route: WriteRoute::Document, rows: p.rows, column_defaults: p.column_defaults, if_absent: p.if_absent, @@ -25,6 +26,7 @@ impl EngineRules for SchemalessRules { Ok(vec![SqlPlan::Upsert { collection: p.collection, engine: EngineType::DocumentSchemaless, + route: WriteRoute::Document, rows: p.rows, column_defaults: p.column_defaults, on_conflict_updates: p.on_conflict_updates, diff --git a/nodedb-sql/src/engine_rules/document_strict.rs b/nodedb-sql/src/engine_rules/document_strict.rs index 1bc621cbc..0aaca86ac 100644 --- a/nodedb-sql/src/engine_rules/document_strict.rs +++ b/nodedb-sql/src/engine_rules/document_strict.rs @@ -13,6 +13,7 @@ impl EngineRules for StrictRules { Ok(vec![SqlPlan::Insert { collection: p.collection, engine: EngineType::DocumentStrict, + route: WriteRoute::Document, rows: p.rows, column_defaults: p.column_defaults, if_absent: p.if_absent, @@ -25,6 +26,7 @@ impl EngineRules for StrictRules { Ok(vec![SqlPlan::Upsert { collection: p.collection, engine: EngineType::DocumentStrict, + route: WriteRoute::Document, rows: p.rows, column_defaults: p.column_defaults, on_conflict_updates: p.on_conflict_updates, diff --git a/nodedb-sql/src/engine_rules/spatial.rs b/nodedb-sql/src/engine_rules/spatial.rs index 23396d985..afe708e52 100644 --- a/nodedb-sql/src/engine_rules/spatial.rs +++ b/nodedb-sql/src/engine_rules/spatial.rs @@ -13,6 +13,7 @@ impl EngineRules for SpatialRules { Ok(vec![SqlPlan::Insert { collection: p.collection, engine: EngineType::Spatial, + route: WriteRoute::ColumnarFamily, rows: p.rows, column_defaults: p.column_defaults, if_absent: p.if_absent, @@ -28,6 +29,7 @@ impl EngineRules for SpatialRules { Ok(vec![SqlPlan::Upsert { collection: p.collection, engine: EngineType::Spatial, + route: WriteRoute::ColumnarFamily, rows: p.rows, column_defaults: p.column_defaults, on_conflict_updates: p.on_conflict_updates, diff --git a/nodedb-sql/src/types/mod.rs b/nodedb-sql/src/types/mod.rs index 167995ebe..064e722b9 100644 --- a/nodedb-sql/src/types/mod.rs +++ b/nodedb-sql/src/types/mod.rs @@ -15,7 +15,7 @@ pub use filter::{CompareOp, Filter, FilterExpr}; pub use plan::{ ArrayPrefilter, DistanceMetric, KvInsertIntent, MergeClauseKind, MergePlanAction, MergePlanClause, PlanCacheEligibility, SqlPlan, VectorAnnOptions, VectorPrimaryRow, - VectorQuantization, + VectorQuantization, WriteRoute, }; pub use query::{ AggOutputSlot, AggregateExpr, EngineType, JoinType, Projection, SortKey, SpatialPredicate, diff --git a/nodedb-sql/src/types/plan/mod.rs b/nodedb-sql/src/types/plan/mod.rs index b1d21ba1c..ad6020b61 100644 --- a/nodedb-sql/src/types/plan/mod.rs +++ b/nodedb-sql/src/types/plan/mod.rs @@ -12,7 +12,7 @@ mod volatility_scan; pub use cacheability::PlanCacheEligibility; pub use merge_types::{MergeClauseKind, MergePlanAction, MergePlanClause}; -pub use row_types::{KvInsertIntent, VectorPrimaryRow}; +pub use row_types::{KvInsertIntent, VectorPrimaryRow, WriteRoute}; pub use variants::{DistanceMetric, SqlPlan}; pub use vector_opts::{ArrayPrefilter, VectorAnnOptions, VectorQuantization}; pub use volatility_scan::{default_expr_is_volatile, defaults_are_volatile, expr_is_volatile}; diff --git a/nodedb-sql/src/types/plan/row_types.rs b/nodedb-sql/src/types/plan/row_types.rs index 3e24c9e92..b002fc657 100644 --- a/nodedb-sql/src/types/plan/row_types.rs +++ b/nodedb-sql/src/types/plan/row_types.rs @@ -35,3 +35,18 @@ pub enum KvInsertIntent { /// duplicate key overwrites. Also the shape used by the RESP SET path. Put, } + +/// The lowering a row-shaped write takes, carried on `SqlPlan::Insert` and +/// `SqlPlan::Upsert`. +/// +/// The engine's `EngineRules` picks it while it builds the variant, so the +/// conversion layer reads the route instead of re-deciding from `EngineType`. +/// An engine with no row-shaped lowering builds another variant entirely and +/// never names a route. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum WriteRoute { + /// One task per row, carrying a `DocumentOp` or a `CrdtOp`. + Document, + /// One batched columnar task for the whole statement. + ColumnarFamily, +} diff --git a/nodedb-sql/src/types/plan/variants.rs b/nodedb-sql/src/types/plan/variants.rs index 7e11a6d5b..f5ad2b736 100644 --- a/nodedb-sql/src/types/plan/variants.rs +++ b/nodedb-sql/src/types/plan/variants.rs @@ -15,7 +15,7 @@ use crate::types::query::{ }; use super::merge_types::MergePlanClause; -use super::row_types::{KvInsertIntent, VectorPrimaryRow}; +use super::row_types::{KvInsertIntent, VectorPrimaryRow, WriteRoute}; use super::vector_opts::{ArrayPrefilter, VectorAnnOptions}; /// The top-level plan produced by the SQL planner. @@ -104,6 +104,9 @@ pub enum SqlPlan { Insert { collection: String, engine: EngineType, + /// The lowering these rows take, chosen by the engine's `EngineRules`. + /// The conversion layer reads it instead of re-deciding from `engine`. + route: WriteRoute, rows: Vec>, /// Column defaults from schema: `(column_name, default_expr)`. /// Used to auto-generate values for missing columns (e.g. `id` with `UUID_V7`). @@ -152,6 +155,8 @@ pub enum SqlPlan { Upsert { collection: String, engine: EngineType, + /// The lowering these rows take. Mirrors `Insert::route`. + route: WriteRoute, rows: Vec>, column_defaults: Vec<(String, String)>, /// `ON CONFLICT (...) DO UPDATE SET field = expr` assignments. diff --git a/nodedb-sql/src/visitor/plan_visitor/args.rs b/nodedb-sql/src/visitor/plan_visitor/args.rs index 206457f44..81207a39a 100644 --- a/nodedb-sql/src/visitor/plan_visitor/args.rs +++ b/nodedb-sql/src/visitor/plan_visitor/args.rs @@ -9,7 +9,7 @@ use crate::temporal::TemporalScope; use crate::types::SqlPlan; use crate::types::filter::Filter; -use crate::types::plan::{ArrayPrefilter, MergePlanClause, VectorAnnOptions}; +use crate::types::plan::{ArrayPrefilter, MergePlanClause, VectorAnnOptions, WriteRoute}; use crate::types::query::{ AggregateExpr, EngineType, JoinType, Projection, SortKey, SpatialPredicate, WindowSpec, }; @@ -65,6 +65,8 @@ pub struct DocumentIndexLookupVisitArgs<'a> { pub struct InsertVisitArgs<'a> { pub collection: &'a str, pub engine: EngineType, + /// The lowering these rows take, decided by the engine's `EngineRules`. + pub route: WriteRoute, pub rows: &'a [Vec<(String, SqlValue)>], pub column_defaults: &'a [(String, String)], pub if_absent: bool, @@ -76,6 +78,8 @@ pub struct InsertVisitArgs<'a> { pub struct UpsertVisitArgs<'a> { pub collection: &'a str, pub engine: EngineType, + /// The lowering these rows take, decided by the engine's `EngineRules`. + pub route: WriteRoute, pub rows: &'a [Vec<(String, SqlValue)>], pub column_defaults: &'a [(String, String)], pub on_conflict_updates: &'a [(String, SqlExpr)], diff --git a/nodedb-sql/src/visitor/plan_visitor/dispatch.rs b/nodedb-sql/src/visitor/plan_visitor/dispatch.rs index fe4bff77c..dc8c98930 100644 --- a/nodedb-sql/src/visitor/plan_visitor/dispatch.rs +++ b/nodedb-sql/src/visitor/plan_visitor/dispatch.rs @@ -95,6 +95,7 @@ pub fn dispatch(visitor: &mut V, plan: &SqlPlan) -> Result(visitor: &mut V, plan: &SqlPlan) -> Result visitor.insert(InsertVisitArgs { collection, engine: *engine, + route: *route, rows, column_defaults, if_absent: *if_absent, @@ -120,6 +122,7 @@ pub fn dispatch(visitor: &mut V, plan: &SqlPlan) -> Result(visitor: &mut V, plan: &SqlPlan) -> Result visitor.upsert(UpsertVisitArgs { collection, engine: *engine, + route: *route, rows, column_defaults, on_conflict_updates, diff --git a/nodedb/src/control/planner/sql_plan_convert/dml/insert.rs b/nodedb/src/control/planner/sql_plan_convert/dml/insert.rs index 693b323cb..8bd48d4fa 100644 --- a/nodedb/src/control/planner/sql_plan_convert/dml/insert.rs +++ b/nodedb/src/control/planner/sql_plan_convert/dml/insert.rs @@ -1,6 +1,6 @@ // SPDX-License-Identifier: BUSL-1.1 -use nodedb_sql::types::{EngineType, SqlValue}; +use nodedb_sql::types::{SqlValue, WriteRoute}; use nodedb_types::Surrogate; use nodedb_types::columnar::{ColumnDef, ColumnType, ColumnarSchema}; @@ -13,7 +13,6 @@ use super::super::convert::ConvertContext; use super::super::value::{ expand_row_defaults, row_to_msgpack, rows_to_msgpack_array, sql_value_to_string, }; -use super::route::{WriteRoute, insert_route}; use nodedb_physical::physical_task::{PhysicalTask, PostSetOp}; /// Build a `ColumnarSchema` from raw catalog column-type strings. @@ -219,7 +218,8 @@ pub(super) fn columnar_row_surrogates( /// Bundled arguments for [`convert_insert`]. pub(in super::super) struct ConvertInsertArgs<'a> { pub collection: &'a str, - pub engine: &'a EngineType, + /// The lowering these rows take, decided by `nodedb-sql`. + pub route: WriteRoute, pub rows: &'a [Vec<(String, SqlValue)>], pub column_defaults: &'a [(String, String)], pub column_schema: &'a [(String, String)], @@ -234,7 +234,7 @@ pub(in super::super) fn convert_insert( ) -> crate::Result> { let ConvertInsertArgs { collection, - engine, + route, rows, column_defaults, column_schema, @@ -249,9 +249,6 @@ pub(in super::super) fn convert_insert( let vshard = VShardId::from_collection_in_database(ctx.database_id, collection); let mut tasks = Vec::new(); let mut columnar_rows: Vec<&Vec<(String, SqlValue)>> = Vec::new(); - // Resolved once per statement, before any DEFAULT is materialized: an - // engine with no INSERT lowering here must not burn a sequence value. - let route = insert_route(engine, collection)?; // Both INSERT routing gates, read from the catalog once for the whole // statement (never re-hit per row). @@ -476,7 +473,7 @@ mod tests { let rows = vec![crdt_row("k1")]; let tasks = convert_insert(ConvertInsertArgs { collection: "crdt_coll", - engine: &EngineType::DocumentSchemaless, + route: WriteRoute::Document, rows: &rows, column_defaults: &[], column_schema: &[], @@ -508,7 +505,7 @@ mod tests { let rows = vec![crdt_row("k1")]; let tasks = convert_insert(ConvertInsertArgs { collection: "plain", - engine: &EngineType::DocumentSchemaless, + route: WriteRoute::Document, rows: &rows, column_defaults: &[], column_schema: &[], diff --git a/nodedb/src/control/planner/sql_plan_convert/dml/mod.rs b/nodedb/src/control/planner/sql_plan_convert/dml/mod.rs index f2271a9c9..d87cf86b8 100644 --- a/nodedb/src/control/planner/sql_plan_convert/dml/mod.rs +++ b/nodedb/src/control/planner/sql_plan_convert/dml/mod.rs @@ -5,7 +5,6 @@ mod crdt_gate; mod insert; mod kv_and_vector; mod merge; -mod route; mod update_delete; mod upsert; diff --git a/nodedb/src/control/planner/sql_plan_convert/dml/route.rs b/nodedb/src/control/planner/sql_plan_convert/dml/route.rs deleted file mode 100644 index af87a7006..000000000 --- a/nodedb/src/control/planner/sql_plan_convert/dml/route.rs +++ /dev/null @@ -1,55 +0,0 @@ -// SPDX-License-Identifier: BUSL-1.1 - -//! Engine routing for the row-shaped write converters. - -use nodedb_sql::types::EngineType; - -/// The lowering a statement's rows take. -pub(super) enum WriteRoute { - /// One task per row, carrying a `DocumentOp` or a `CrdtOp`. - Document, - /// One batched `ColumnarOp` task for the whole statement. - ColumnarFamily, -} - -/// Resolve an INSERT's route, refusing engines that lower elsewhere. -/// -/// Routing runs once per statement, ahead of DEFAULT materialization, so a -/// refused engine never allocates a sequence value it discards. -pub(super) fn insert_route(engine: &EngineType, collection: &str) -> crate::Result { - match engine { - EngineType::DocumentSchemaless | EngineType::DocumentStrict => Ok(WriteRoute::Document), - EngineType::Columnar | EngineType::Spatial => Ok(WriteRoute::ColumnarFamily), - EngineType::KeyValue => Err(crate::Error::PlanError { - detail: "KV INSERT must use SqlPlan::KvInsert path".into(), - }), - EngineType::Timeseries => Err(crate::Error::PlanError { - detail: format!( - "INSERT into '{collection}': timeseries collections use TimeseriesIngest, not Insert" - ), - }), - EngineType::Array => Err(crate::Error::PlanError { - detail: format!( - "INSERT into '{collection}': array engine uses INSERT INTO ARRAY syntax" - ), - }), - } -} - -/// Resolve an UPSERT's route, refusing engines with no upsert lowering. -/// -/// Runs ahead of DEFAULT materialization for the same reason as -/// [`insert_route`]. -pub(super) fn upsert_route(engine: &EngineType, collection: &str) -> crate::Result { - match engine { - EngineType::DocumentSchemaless | EngineType::DocumentStrict => Ok(WriteRoute::Document), - EngineType::Columnar | EngineType::Spatial => Ok(WriteRoute::ColumnarFamily), - EngineType::Timeseries | EngineType::KeyValue | EngineType::Array => { - Err(crate::Error::PlanError { - detail: format!( - "UPSERT into '{collection}': engine type {engine:?} does not support upsert" - ), - }) - } - } -} diff --git a/nodedb/src/control/planner/sql_plan_convert/dml/upsert.rs b/nodedb/src/control/planner/sql_plan_convert/dml/upsert.rs index 7586b436e..9813c9305 100644 --- a/nodedb/src/control/planner/sql_plan_convert/dml/upsert.rs +++ b/nodedb/src/control/planner/sql_plan_convert/dml/upsert.rs @@ -6,7 +6,7 @@ //! identity helper there (`resolve_doc_identity`) so a row's surrogate is //! derived identically whichever statement wrote it. -use nodedb_sql::types::{EngineType, SqlExpr, SqlValue}; +use nodedb_sql::types::{SqlExpr, SqlValue, WriteRoute}; use crate::bridge::envelope::PhysicalPlan; use crate::types::{TenantId, VShardId}; @@ -18,13 +18,13 @@ use super::super::value::{ assignments_to_update_values, expand_row_defaults, row_to_msgpack, rows_to_msgpack_array, }; use super::insert::{build_schema_bytes, columnar_row_surrogates, resolve_doc_identity}; -use super::route::{WriteRoute, upsert_route}; use nodedb_physical::physical_task::{PhysicalTask, PostSetOp}; /// Bundled arguments for [`convert_upsert`]. pub(in super::super) struct ConvertUpsertArgs<'a> { pub collection: &'a str, - pub engine: &'a EngineType, + /// The lowering these rows take, decided by `nodedb-sql`. + pub route: WriteRoute, pub rows: &'a [Vec<(String, SqlValue)>], pub column_defaults: &'a [(String, String)], pub column_schema: &'a [(String, String)], @@ -39,7 +39,7 @@ pub(in super::super) fn convert_upsert( ) -> crate::Result> { let ConvertUpsertArgs { collection, - engine, + route, rows, column_defaults, column_schema, @@ -53,9 +53,6 @@ pub(in super::super) fn convert_upsert( let collection = coll_qualified.as_str(); let vshard = VShardId::from_collection_in_database(ctx.database_id, collection); let mut tasks = Vec::new(); - // Resolved once per statement, before any DEFAULT is materialized: an - // engine with no UPSERT lowering must not burn a sequence value. - let route = upsert_route(engine, collection)?; // Detect CRDT document collections once. An explicit `ON CONFLICT DO UPDATE // SET ...` cannot be honored: CRDT conflict resolution IS the LWW diff --git a/nodedb/src/control/planner/sql_plan_convert/visitor/arms_dml.rs b/nodedb/src/control/planner/sql_plan_convert/visitor/arms_dml.rs index 93f937107..90dffec3d 100644 --- a/nodedb/src/control/planner/sql_plan_convert/visitor/arms_dml.rs +++ b/nodedb/src/control/planner/sql_plan_convert/visitor/arms_dml.rs @@ -10,7 +10,8 @@ macro_rules! impl_dml_arms_for_convert_visitor { ) -> crate::Result> { let nodedb_sql::InsertVisitArgs { collection, - engine, + engine: _engine, + route, rows, column_defaults, if_absent, @@ -19,7 +20,7 @@ macro_rules! impl_dml_arms_for_convert_visitor { } = args; super::super::dml::convert_insert(super::super::dml::ConvertInsertArgs { collection, - engine: &engine, + route, rows, column_defaults, column_schema, @@ -36,7 +37,8 @@ macro_rules! impl_dml_arms_for_convert_visitor { ) -> crate::Result> { let nodedb_sql::UpsertVisitArgs { collection, - engine, + engine: _engine, + route, rows, column_defaults, on_conflict_updates, @@ -45,7 +47,7 @@ macro_rules! impl_dml_arms_for_convert_visitor { } = args; super::super::dml::convert_upsert(super::super::dml::ConvertUpsertArgs { collection, - engine: &engine, + route, rows, column_defaults, column_schema, From 4be0b774450508a5e74bcda22fa54efebd40a88a Mon Sep 17 00:00:00 2001 From: Farhan Syah Date: Thu, 10 Sep 2026 09:53:35 +0800 Subject: [PATCH 19/23] fix(join): rescale instant cells before projection renames them A projection that renames a joined declared-instant cell (an alias or a computed expression) hid it from the millisecond-to-microsecond rescale, which matched on the emitted output name. Rescale now runs on the merged row against the keys the join itself wrote, before any projection or computed column touches it, so a rename or computation downstream cannot bypass the scale. --- nodedb/src/data/executor/dispatch/query.rs | 3 - .../executor/handlers/join/instant_scale.rs | 39 +++++------- .../src/data/executor/handlers/join/params.rs | 59 ++++++++++++++++--- .../cases/timeseries_join_time_rendering.rs | 59 +++++++++++++++++++ 4 files changed, 125 insertions(+), 35 deletions(-) diff --git a/nodedb/src/data/executor/dispatch/query.rs b/nodedb/src/data/executor/dispatch/query.rs index cfd90c5ea..cbe383a73 100644 --- a/nodedb/src/data/executor/dispatch/query.rs +++ b/nodedb/src/data/executor/dispatch/query.rs @@ -137,7 +137,6 @@ impl CoreLoop { task.request.database_id, crate::types::TenantId::new(tid), &local_sides, - projection, ); self.execute_hash_join(HashJoinParams { join: JoinParams { @@ -231,7 +230,6 @@ impl CoreLoop { qualifier: right_collection.as_str(), }, ], - &[], ); self.execute_nested_loop_join(NestedLoopJoinParams { task, @@ -270,7 +268,6 @@ impl CoreLoop { qualifier: right_collection.as_str(), }, ], - &[], ); self.execute_sort_merge_join(SortMergeJoinParams { task, diff --git a/nodedb/src/data/executor/handlers/join/instant_scale.rs b/nodedb/src/data/executor/handlers/join/instant_scale.rs index ada4e4abc..4f2e41620 100644 --- a/nodedb/src/data/executor/handlers/join/instant_scale.rs +++ b/nodedb/src/data/executor/handlers/join/instant_scale.rs @@ -12,8 +12,11 @@ //! Exactly once: a handler rescales ONLY the cells it read from a local //! collection scan. Rows that arrive from a sub-plan response were already //! rescaled by the handler that read them, and are never touched again. - -use nodedb_physical::physical_plan::JoinProjection; +//! +//! The rescale runs on the merged row, before any projection. A cell is named +//! by the key the join itself wrote, so a rename cannot hide it: the rename +//! happens strictly downstream, and a projected cell copies bytes the rescale +//! has already corrected. use crate::data::executor::core_loop::CoreLoop; use crate::data::executor::handlers::timeseries::raw_scan::scale_instant_cells; @@ -27,43 +30,29 @@ pub(in crate::data::executor) struct JoinInstantSide<'a> { } impl CoreLoop { - /// The emitted names of the declared-instant cells `sides` contribute. + /// The merged-row keys of the declared-instant cells `sides` contribute. /// - /// A merged join row keys every cell `.`, and a - /// projection renames it to its output name. The returned names are what - /// the emitted row actually carries, so the caller matches on them - /// directly. A side that is not a timeseries collection contributes none, - /// so a join over ordinary collections returns an empty list and pays - /// nothing. + /// A merged join row keys every cell `.`, so the + /// returned names are what the row carries before any projection runs. A + /// side that is not a timeseries collection contributes none, so a join + /// over ordinary collections returns an empty list and pays nothing. pub(in crate::data::executor) fn join_instant_columns( &self, database_id: DatabaseId, tid: TenantId, sides: &[JoinInstantSide<'_>], - projection: &[JoinProjection], ) -> Vec { - let mut emitted = Vec::new(); + let mut merged_keys = Vec::new(); for side in sides { for column in self.ts_instant_columns(database_id, tid, side.collection) { - let qualified = format!("{}.{column}", side.qualifier); - if projection.is_empty() { - emitted.push(qualified); - continue; - } - // Mirrors `binary_row_project`: a projection entry names - // either the qualified key or its bare last segment. - for entry in projection { - if entry.source == qualified || entry.source == column { - emitted.push(entry.output.clone()); - } - } + merged_keys.push(format!("{}.{column}", side.qualifier)); } } - emitted + merged_keys } } -/// Rescale the named cells of already-projected join rows, in place. +/// Rescale the named cells of merged join rows, in place. /// /// Each row is a msgpack map. An empty `instant_columns` is a no-op, so a /// join that touches no timeseries collection never decodes a row. diff --git a/nodedb/src/data/executor/handlers/join/params.rs b/nodedb/src/data/executor/handlers/join/params.rs index 3de0b4e1b..8b14df27f 100644 --- a/nodedb/src/data/executor/handlers/join/params.rs +++ b/nodedb/src/data/executor/handlers/join/params.rs @@ -17,7 +17,7 @@ pub(crate) struct JoinParams<'a> { pub computed_projection_bytes: &'a [u8], pub join_filter_bytes: &'a [u8], pub post_filter_bytes: &'a [u8], - /// Emitted names of the declared-instant cells this join reads from its + /// Merged-row keys of the declared-instant cells this join reads from its /// own local scans. Empty when no timeseries collection is scanned /// locally — including for every side supplied by a sub-plan, whose rows /// were already rescaled by the handler that read them. @@ -81,7 +81,7 @@ pub(crate) struct NestedLoopJoinParams<'a> { pub left_rls_filters: &'a [u8], /// Row-level-security filters for the locally-scanned right side. pub right_rls_filters: &'a [u8], - /// Emitted names of the declared-instant cells the two local scans + /// Merged-row keys of the declared-instant cells the two local scans /// contribute. Empty when neither side is a timeseries collection. pub instant_columns: &'a [String], } @@ -103,7 +103,7 @@ pub(crate) struct SortMergeJoinParams<'a> { pub left_rls_filters: &'a [u8], /// Row-level-security filters for the locally-scanned right side. pub right_rls_filters: &'a [u8], - /// Emitted names of the declared-instant cells the two local scans + /// Merged-row keys of the declared-instant cells the two local scans /// contribute. Empty when neither side is a timeseries collection. pub instant_columns: &'a [String], } @@ -202,6 +202,11 @@ impl JoinParams<'_> { } } + // Every predicate above compared the milliseconds storage holds, and + // the client reads microseconds. Scale here, on the keys the join + // wrote, so a projection that renames a cell copies a corrected value. + super::instant_scale::scale_join_instant_rows(results, self.instant_columns)?; + if !self.computed_projection_bytes.is_empty() { let computed: Vec = zerompk::from_msgpack(self.computed_projection_bytes).map_err(|e| { @@ -223,10 +228,6 @@ impl JoinParams<'_> { } } - // Last step before emission: every predicate above compared the - // milliseconds storage holds, and the client reads microseconds. - super::instant_scale::scale_join_instant_rows(results, self.instant_columns)?; - Ok(()) } } @@ -312,6 +313,50 @@ mod tests { ); } + /// A renaming projection cannot hide an instant cell from the rescale. + /// + /// The rescale names the merged key the join wrote, and the projection + /// renames the cell afterwards, so the emitted cell carries microseconds + /// under its output name. + #[test] + fn projection_rename_still_rescales_the_instant_cell() { + let task = make_dummy_task(); + let projection = vec![JoinProjection { + source: "e.captured_at".into(), + output: "ts".into(), + }]; + let params = JoinParams { + task: &task, + on: &[], + join_type: "inner", + limit: usize::MAX, + projection: &projection, + computed_projection_bytes: &[], + join_filter_bytes: &[], + post_filter_bytes: &[], + instant_columns: &["e.captured_at".to_string()], + }; + let mut results = vec![ + nodedb_types::json_to_msgpack( + &serde_json::json!({"e.captured_at": 1_583_402_400_000i64}), + ) + .expect("encode test row"), + ]; + params + .filter_and_project(&mut results) + .expect("filter_and_project"); + + let decoded = nodedb_types::value_from_msgpack(&results[0]).expect("decode emitted row"); + let nodedb_types::Value::Object(fields) = decoded else { + panic!("emitted row must be a map, got {decoded:?}"); + }; + assert_eq!( + fields.get("ts"), + Some(&nodedb_types::Value::Integer(1_583_402_400_000_000)), + "the renamed cell must carry epoch microseconds: {fields:?}" + ); + } + /// Valid encoded filters → matching rows retained, non-matching rows dropped /// (happy path unchanged). #[test] diff --git a/nodedb/tests/wire/cases/timeseries_join_time_rendering.rs b/nodedb/tests/wire/cases/timeseries_join_time_rendering.rs index 35cdb9545..ecc4d4e71 100644 --- a/nodedb/tests/wire/cases/timeseries_join_time_rendering.rs +++ b/nodedb/tests/wire/cases/timeseries_join_time_rendering.rs @@ -155,3 +155,62 @@ async fn a_joined_time_key_denotes_the_stored_instant() { or {EARLY_MICROS}, got {joined:?}" ); } + +/// A computed projection of a time key renders it as a direct read does. +/// `COALESCE` returns the stored cell untouched, so both reads name one instant. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn a_computed_projection_of_a_time_key_matches_a_direct_read() { + let server = TestServer::start().await; + setup(&server, "tsj_comp_events", "tsj_comp_hosts").await; + + let direct = server + .query_text("SELECT COALESCE(captured_at, captured_at) AS captured_at FROM tsj_comp_events") + .await + .expect("a computed projection of the time key must succeed"); + assert_eq!(direct.len(), 1, "one stored point: {direct:?}"); + + let joined = server + .query_text( + "SELECT COALESCE(tsj_comp_events.captured_at, tsj_comp_events.captured_at) \ + AS captured_at FROM tsj_comp_events \ + INNER JOIN tsj_comp_hosts ON tsj_comp_events.host = tsj_comp_hosts.id", + ) + .await + .expect("the same computed projection over a JOIN must succeed"); + assert_eq!(joined.len(), 1, "the join yields one row: {joined:?}"); + + assert_eq!( + joined[0], direct[0], + "a computed projection of the time key must render the same through a JOIN \ + as through a SELECT: joined={joined:?} direct={direct:?}" + ); +} + +/// An aliased joined time key denotes the instant the INSERT supplied. +/// The alias renames the cell, so the emitted name must still carry the instant. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn an_aliased_joined_time_key_denotes_the_stored_instant() { + let server = TestServer::start().await; + setup(&server, "tsj_alias_events", "tsj_alias_hosts").await; + + let joined = server + .query_text( + "SELECT tsj_alias_events.captured_at AS ts FROM tsj_alias_events \ + INNER JOIN tsj_alias_hosts ON tsj_alias_events.host = tsj_alias_hosts.id", + ) + .await + .expect("an aliased time key projected through a JOIN must succeed"); + assert_eq!(joined.len(), 1, "the join yields one row: {joined:?}"); + + // The expected value is EARLY, the inserted instant, in whichever unit the + // alias announces a type for. EARLY_ISO is 2020-03-05T10:00:00Z as a cell + // typed TIMESTAMP renders it; EARLY_MICROS is the same instant in the epoch + // microseconds a TIMESTAMP cell carries, which an untyped cell leaves as a + // number. The stored 1583402400000 milliseconds denote 1970-01-19 read + // either way, so a millisecond value fails both arms. + assert!( + joined[0] == EARLY_ISO || joined[0] == EARLY_MICROS, + "an aliased joined time key must denote {EARLY}: expected {EARLY_ISO} \ + or {EARLY_MICROS}, got {joined:?}" + ); +} From d530c30b23d3856e962f6b1c01d6140904728cef Mon Sep 17 00:00:00 2001 From: Farhan Syah Date: Thu, 10 Sep 2026 11:35:14 +0800 Subject: [PATCH 20/23] refactor(types): unify declared-type resolution and instant detection ColumnType::from_declared_type resolves a catalog fields entry's raw DDL text (e.g. "INT DEFAULT 5") to its ColumnType through one shared classifier, and ColumnType::is_instant answers whether a column carries an instant. The Control Plane's parse_type_str and the Data Plane's declared_type_is_instant / memtable_column_type now both defer to these instead of keeping their own copies of the integer/float keyword lists and timestamp-variant matches, so the two planes cannot drift on which columns are instants or which spellings resolve to which type. DECLARED_INT_KEYWORDS and DECLARED_FLOAT_KEYWORDS name the accepted PostgreSQL wire-width spellings once, shared with the corresponding IntWidth/FloatWidth classifiers. --- nodedb-types/src/columnar/column_parse.rs | 201 ++++++++++++++++-- nodedb-types/src/columnar/column_type.rs | 13 ++ nodedb-types/src/columnar/mod.rs | 2 +- .../planner/catalog_adapter/type_convert.rs | 106 ++++++--- .../executor/core_loop/ts_declared_schema.rs | 40 ++-- 5 files changed, 284 insertions(+), 78 deletions(-) diff --git a/nodedb-types/src/columnar/column_parse.rs b/nodedb-types/src/columnar/column_parse.rs index 880ac3d8c..af1cf3095 100644 --- a/nodedb-types/src/columnar/column_parse.rs +++ b/nodedb-types/src/columnar/column_parse.rs @@ -8,6 +8,36 @@ use std::str::FromStr; use super::column_type::ColumnType; +/// Every declared spelling that resolves to [`ColumnType::Int64`]. +/// +/// nodedb stores every integer as a full `i64`, so all of these collapse to +/// one storage variant. The declared width travels separately as an +/// [`IntWidth`](super::IntWidth), which narrows the advertised wire OID and +/// bounds writes. [`IntWidth::from_declared_type`](super::IntWidth::from_declared_type) +/// must recognize every spelling listed here — a spelling it does not know +/// advertises OID 20 (`bigint`) whatever the author declared. +pub const DECLARED_INT_KEYWORDS: [&str; 8] = [ + "BIGINT", "INT64", "INTEGER", "INT", "INT4", "INT8", "SMALLINT", "INT2", +]; + +/// Every declared spelling that resolves to [`ColumnType::Float64`]. +/// +/// The float counterpart of [`DECLARED_INT_KEYWORDS`]: nodedb stores every +/// float as a full `f64`, and +/// [`FloatWidth::from_declared_type`](super::FloatWidth::from_declared_type) +/// carries the declared width that narrows the advertised wire OID. It must +/// recognize every spelling listed here. +pub const DECLARED_FLOAT_KEYWORDS: [&str; 8] = [ + "FLOAT64", + "DOUBLE", + "DOUBLE PRECISION", + "FLOAT8", + "REAL", + "FLOAT4", + "FLOAT32", + "FLOAT", +]; + /// Error from parsing a column type string. #[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)] #[non_exhaustive] @@ -121,28 +151,12 @@ impl FromStr for ColumnType { } match upper.as_str() { - // `INT4`/`INT8`/`SMALLINT`/`INT2` are PostgreSQL wire-width integer - // keywords: strict/kv `CREATE COLLECTION` must accept - // them as valid aliases, not reject them as unknown types. They - // all collapse to the same `Int64` storage variant as - // `BIGINT`/`INTEGER`/`INT` — nodedb always stores integers as a - // full i64. The declared width is carried separately as an - // [`super::IntWidth`], which is what bounds writes and narrows the - // advertised wire OID; it is deliberately not a storage variant. - "BIGINT" | "INT64" | "INTEGER" | "INT" | "INT4" | "INT8" | "SMALLINT" | "INT2" => { - Ok(Self::Int64) - } - // `FLOAT4`/`FLOAT8`/`FLOAT32`/`DOUBLE PRECISION` are PostgreSQL - // wire-width float keywords, rejected as unknown types here for - // the same reason `INT4` was. They all collapse to - // the same `Float64` storage variant as `DOUBLE`/`REAL`/`FLOAT` — - // nodedb always stores floats as a full f64. The declared width is - // carried separately as a [`super::FloatWidth`], which narrows the - // advertised wire OID; it is deliberately not a storage variant. - // Unlike integers it bounds no writes: narrowing a float rounds - // rather than wraps, and PostgreSQL accepts-and-rounds too. - "FLOAT64" | "DOUBLE" | "DOUBLE PRECISION" | "FLOAT8" | "REAL" | "FLOAT4" - | "FLOAT32" | "FLOAT" => Ok(Self::Float64), + // Every PostgreSQL wire-width integer keyword collapses to the + // one `Int64` storage variant; `DECLARED_INT_KEYWORDS` lists them + // and carries the width contract. + keyword if DECLARED_INT_KEYWORDS.contains(&keyword) => Ok(Self::Int64), + // The float counterpart, listed by `DECLARED_FLOAT_KEYWORDS`. + keyword if DECLARED_FLOAT_KEYWORDS.contains(&keyword) => Ok(Self::Float64), "TEXT" | "STRING" | "VARCHAR" => Ok(Self::String), "BOOL" | "BOOLEAN" => Ok(Self::Bool), "BYTES" | "BYTEA" | "BLOB" => Ok(Self::Bytes), @@ -169,3 +183,146 @@ impl FromStr for ColumnType { } } } + +impl ColumnType { + /// Resolve a declared DDL type string to a column type. + /// + /// This is the single answer to "which [`ColumnType`] does this declared + /// string denote", shared by both planes. The catalog records the raw DDL + /// text that followed the column name, so an entry reads `INT DEFAULT 5` + /// or `DECIMAL(10, 2) NOT NULL`, not `INT`. This resolves the leading type + /// token, so a trailing modifier never changes the answer. + /// + /// `None` means the token names no known type. A caller that needs a + /// fallback picks its own — this reports the absence rather than guessing. + /// + /// A multi-word spelling resolves from its first word: `DOUBLE PRECISION` + /// is [`ColumnType::Float64`], and `TIMESTAMP WITH TIME ZONE` reaching + /// here as catalog text resolves to [`ColumnType::Timestamp`]. Pass such a + /// spelling to [`str::parse`] instead to resolve it whole. + pub fn from_declared_type(declared: &str) -> Option { + bare_declared_token(declared).parse().ok() + } +} + +/// The leading type token of a declared DDL type string. +/// +/// The cut is the first whitespace outside parentheses, so a parameter list +/// keeps its internal spaces and `DECIMAL(10, 2) NOT NULL` yields +/// `DECIMAL(10, 2)`. A trailing comma left by a column-list split is dropped. +fn bare_declared_token(declared: &str) -> &str { + let trimmed = declared.trim_start(); + let mut depth = 0usize; + let mut end = trimmed.len(); + for (index, ch) in trimmed.char_indices() { + match ch { + '(' => depth += 1, + ')' => depth = depth.saturating_sub(1), + _ if depth == 0 && ch.is_whitespace() => { + end = index; + break; + } + _ => {} + } + } + trimmed.get(..end).unwrap_or(trimmed).trim_end_matches(',') +} + +#[cfg(test)] +mod tests { + use super::super::{FloatWidth, IntWidth}; + use super::*; + + /// Every spelling the parser resolves to `Int64` must also resolve to a + /// declared [`IntWidth`]. A spelling one side knows and the other does not + /// advertises the wrong `RowDescription` OID for the column. + /// + /// The list is the one the parser itself matches on, so adding a spelling + /// there extends this test rather than leaving it behind. + #[test] + fn every_declared_int_keyword_resolves_to_int64_and_a_width() { + for keyword in DECLARED_INT_KEYWORDS { + assert_eq!( + keyword.parse::(), + Ok(ColumnType::Int64), + "{keyword} must resolve to Int64" + ); + assert!( + IntWidth::from_declared_type(keyword).is_some(), + "{keyword} must resolve to a declared IntWidth" + ); + } + } + + /// The float counterpart of + /// [`every_declared_int_keyword_resolves_to_int64_and_a_width`]. + #[test] + fn every_declared_float_keyword_resolves_to_float64_and_a_width() { + for keyword in DECLARED_FLOAT_KEYWORDS { + assert_eq!( + keyword.parse::(), + Ok(ColumnType::Float64), + "{keyword} must resolve to Float64" + ); + assert!( + FloatWidth::from_declared_type(keyword).is_some(), + "{keyword} must resolve to a declared FloatWidth" + ); + } + } + + #[test] + fn declared_type_ignores_trailing_modifiers() { + assert_eq!( + ColumnType::from_declared_type("INT DEFAULT 5"), + Some(ColumnType::Int64) + ); + assert_eq!( + ColumnType::from_declared_type("TEXT NOT NULL PRIMARY KEY"), + Some(ColumnType::String) + ); + assert_eq!( + ColumnType::from_declared_type("timestamp time_key"), + Some(ColumnType::Timestamp) + ); + assert_eq!( + ColumnType::from_declared_type("BIGINT,"), + Some(ColumnType::Int64) + ); + } + + /// A parameter list keeps its internal spaces: cutting at the first + /// whitespace would leave `DECIMAL(10,` and lose the type. + #[test] + fn declared_type_keeps_a_spaced_parameter_list() { + assert_eq!( + ColumnType::from_declared_type("DECIMAL(10, 2) NOT NULL"), + Some(ColumnType::Decimal { + precision: 10, + scale: 2 + }) + ); + assert_eq!( + ColumnType::from_declared_type("VECTOR(768)"), + Some(ColumnType::Vector(768)) + ); + } + + #[test] + fn declared_type_reports_an_unknown_token_as_none() { + assert_eq!(ColumnType::from_declared_type("SOMETHING_ELSE"), None); + assert_eq!(ColumnType::from_declared_type(""), None); + assert_eq!(ColumnType::from_declared_type(" "), None); + } + + /// `Timestamp` and `Timestamptz` are instants; `SystemTimestamp` is + /// engine-assigned and is not. + #[test] + fn only_timestamp_types_are_instants() { + assert!(ColumnType::Timestamp.is_instant()); + assert!(ColumnType::Timestamptz.is_instant()); + assert!(!ColumnType::SystemTimestamp.is_instant()); + assert!(!ColumnType::Int64.is_instant()); + assert!(!ColumnType::String.is_instant()); + } +} diff --git a/nodedb-types/src/columnar/column_type.rs b/nodedb-types/src/columnar/column_type.rs index ec369fc6f..262ef622a 100644 --- a/nodedb-types/src/columnar/column_type.rs +++ b/nodedb-types/src/columnar/column_type.rs @@ -108,6 +108,19 @@ impl ColumnType { self.fixed_size().is_none() } + /// Whether a column of this type carries an instant. + /// + /// This is the single answer to that question for both planes: the + /// Control Plane types such a column as a timestamp on the wire, and the + /// Data Plane scales exactly these columns from the epoch milliseconds + /// storage holds to the epoch microseconds a client reads. + /// + /// `SystemTimestamp` is not an instant. It is engine-assigned from HLC at + /// commit and the planner types it as text. + pub const fn is_instant(&self) -> bool { + matches!(self, Self::Timestamp | Self::Timestamptz) + } + /// Return the canonical PostgreSQL type OID for this column type. /// /// This is the single authoritative mapping between NodeDB `ColumnType` diff --git a/nodedb-types/src/columnar/mod.rs b/nodedb-types/src/columnar/mod.rs index 2d55ef73c..b7741b8c4 100644 --- a/nodedb-types/src/columnar/mod.rs +++ b/nodedb-types/src/columnar/mod.rs @@ -13,7 +13,7 @@ pub mod schema; pub mod wal_record; pub use column_def::{ColumnDef, ColumnModifier}; -pub use column_parse::ColumnTypeParseError; +pub use column_parse::{ColumnTypeParseError, DECLARED_FLOAT_KEYWORDS, DECLARED_INT_KEYWORDS}; pub use column_type::ColumnType; pub use declared_type_keyword::declared_type_matches; pub use dml_wal_record::ColumnarDmlWalRecord; diff --git a/nodedb/src/control/planner/catalog_adapter/type_convert.rs b/nodedb/src/control/planner/catalog_adapter/type_convert.rs index c0bfcdd90..8aa2a1dc8 100644 --- a/nodedb/src/control/planner/catalog_adapter/type_convert.rs +++ b/nodedb/src/control/planner/catalog_adapter/type_convert.rs @@ -272,40 +272,47 @@ fn convert_column_type(ct: &nodedb_types::columnar::ColumnType) -> SqlDataType { /// Resolve the declared SQL type of a catalog `fields` entry. /// /// The catalog records the raw DDL text that followed the column name, so an -/// entry reads `INT DEFAULT 5` or `INT NOT NULL`, not `INT`. The bare type -/// token comes from `parse_column_type_str_full`, the same splitter -/// `declared_default` uses and the same boundary `IntWidth::from_declared_type` -/// and `FloatWidth::from_declared_type` respect. A trailing modifier therefore -/// never changes the resolved type. +/// entry reads `INT DEFAULT 5` or `INT NOT NULL`, not `INT`. +/// [`nodedb_types::columnar::ColumnType::from_declared_type`] is the single +/// classifier both planes resolve that text through, so a trailing modifier +/// never changes the resolved type and the Data Plane cannot disagree about +/// which columns carry an instant. +/// +/// A token that names no known type resolves to `SqlDataType::String`, the +/// widest rendering a catalog column can fall back to. fn parse_type_str(s: &str) -> SqlDataType { - let (bare, _, _, _) = nodedb_sql::ddl_ast::collection_type::parse_column_type_str_full(s); - let upper = bare.to_uppercase(); - // Handle DECIMAL/NUMERIC with optional (p,s) params. - if upper.starts_with("DECIMAL") || upper.starts_with("NUMERIC") { - return SqlDataType::Decimal; + match nodedb_types::columnar::ColumnType::from_declared_type(s) { + Some(declared) => declared_column_type_to_sql(declared), + None => SqlDataType::String, } - match upper.as_str() { - // Every spelling `IntWidth::from_declared_type` recognizes must appear - // here too, or the column resolves to the `_ => String` default and - // advertises OID 25 (text) — the exact failure that made `SMALLINT` - // columns unreadable. `parse_type_str` decides *whether* - // the column is an integer; `IntWidth` decides *how wide*. - "INT" | "INTEGER" | "INT4" | "INT8" | "INT64" | "BIGINT" | "SMALLINT" | "INT2" => { - SqlDataType::Int64 - } - // Same contract as the integer arm above, for the float family: every - // spelling `FloatWidth::from_declared_type` recognizes must appear - // here, or the column falls through to `_ => String` and advertises - // OID 25 (text) no matter what width was declared. `DOUBLE PRECISION` - // arrives as the bare token `DOUBLE`, matching how - // `FloatWidth::from_declared_type` recognizes it. - "FLOAT" | "FLOAT4" | "FLOAT8" | "FLOAT32" | "FLOAT64" | "DOUBLE" | "REAL" => { - SqlDataType::Float64 - } - "BOOL" | "BOOLEAN" => SqlDataType::Bool, - "BYTES" | "BYTEA" | "BLOB" => SqlDataType::Bytes, - "TIMESTAMP" | "TIMESTAMPTZ" => SqlDataType::Timestamp, - _ => SqlDataType::String, +} + +/// Map a declared column type onto the SQL type a `fields` column advertises. +/// +/// Every type whose declared spelling and whose resolved strict/kv schema +/// type advertise the same SQL type defers to [`convert_column_type`], so the +/// two mappings cannot drift. The arms above that tail are the exceptions: a +/// schemaless or columnar-family column stores these as the text the client +/// wrote, not in the strict engine's binary encoding, so it renders as text. +fn declared_column_type_to_sql(declared: nodedb_types::columnar::ColumnType) -> SqlDataType { + use nodedb_types::columnar::ColumnType; + match declared { + // A declared TIMESTAMPTZ column reads back in the naive timestamp + // shape these engines store, so it advertises OID 1114. + ColumnType::Timestamptz => SqlDataType::Timestamp, + // Bitemporal system time is engine-assigned and renders as text. + ColumnType::SystemTimestamp => SqlDataType::String, + // Stored as the client's own text: WKT geometry, JSON text, a vector + // or duration literal, and the collection literals. + ColumnType::Geometry + | ColumnType::Json + | ColumnType::Vector(_) + | ColumnType::Duration + | ColumnType::Array + | ColumnType::Set + | ColumnType::Range + | ColumnType::Record => SqlDataType::String, + other => convert_column_type(&other), } } @@ -316,6 +323,41 @@ mod tests { use super::{SqlDataType, convert_collection_type, parse_type_str}; use crate::control::security::catalog::StoredCollection; + /// The planner reads a cell as an instant for exactly the declared types + /// `ColumnType::is_instant` names, which is the same predicate the Data + /// Plane scales emission by. Pinning the equivalence here is what a + /// comment could not do: a spelling added to one side and not the other + /// fails this test instead of shipping a millisecond value labelled as + /// microseconds. + #[test] + fn parse_type_str_reads_exactly_the_instant_declared_types_as_timestamps() { + use nodedb_types::columnar::ColumnType; + for declared in [ + "TIMESTAMP", + "TIMESTAMPTZ", + "timestamp", + "TIMESTAMP TIME_KEY", + "TIMESTAMPTZ NOT NULL", + "SYSTEM_TIMESTAMP", + "BIGINT TIME_KEY", + "INT", + "TEXT", + "GEOMETRY", + "DECIMAL(10, 2)", + "VECTOR(768)", + "SOMETHING_ELSE", + "", + ] { + let is_instant = + ColumnType::from_declared_type(declared).is_some_and(|ty| ty.is_instant()); + assert_eq!( + is_instant, + parse_type_str(declared) == SqlDataType::Timestamp, + "{declared}: the instant predicate and the planner type must agree" + ); + } + } + /// `SMALLINT`/`INT2` are valid PostgreSQL wire-width integer keywords /// that must resolve to the same `SqlDataType::Int64` arm as /// `INT`/`INTEGER`/`INT4`/`INT8`/`BIGINT` — previously they were unlisted diff --git a/nodedb/src/data/executor/core_loop/ts_declared_schema.rs b/nodedb/src/data/executor/core_loop/ts_declared_schema.rs index c29312625..acad6049d 100644 --- a/nodedb/src/data/executor/core_loop/ts_declared_schema.rs +++ b/nodedb/src/data/executor/core_loop/ts_declared_schema.rs @@ -234,19 +234,12 @@ fn kind_of_storage(storage: ColumnType) -> TsGroupKeyKind { /// Whether a declared DDL type makes a column an instant on the wire. /// -/// Mirrors the two spellings the planner resolves to `SqlDataType::Timestamp` -/// in `control::planner::catalog_adapter::type_convert::parse_type_str`. The -/// two must name the same set: the planner decides how a cell is READ, this -/// decides the unit it is WRITTEN in. -/// -/// `SYSTEM_TIMESTAMP` is deliberately absent — the planner types it as text. +/// Both planes answer this from `nodedb_types::columnar::ColumnType`: the +/// Control Plane decides how a cell is READ, this decides the unit it is +/// WRITTEN in, and one classifier keeps them from naming different sets. fn declared_type_is_instant(declared_type: &str) -> bool { - let bare = declared_type.split_whitespace().next().unwrap_or(""); - matches!( - bare.parse::(), - Ok(nodedb_types::columnar::ColumnType::Timestamp) - | Ok(nodedb_types::columnar::ColumnType::Timestamptz) - ) + nodedb_types::columnar::ColumnType::from_declared_type(declared_type) + .is_some_and(|declared| declared.is_instant()) } /// Map a declared SQL type onto the memtable's storage type. @@ -255,20 +248,21 @@ fn declared_type_is_instant(declared_type: &str) -> bool { /// regardless of how it was spelled — `TIMESTAMP`, `TIMESTAMPTZ`, and /// `BIGINT` time keys all store epoch milliseconds. fn memtable_column_type(declared_type: &str, is_time_key: bool) -> ColumnType { + use nodedb_types::columnar::ColumnType as DeclaredType; + if is_time_key { return ColumnType::Timestamp; } - let bare = declared_type.split_whitespace().next().unwrap_or(""); - match bare.parse::() { - Ok(nodedb_types::columnar::ColumnType::Timestamp) - | Ok(nodedb_types::columnar::ColumnType::Timestamptz) - | Ok(nodedb_types::columnar::ColumnType::SystemTimestamp) => ColumnType::Timestamp, - Ok(nodedb_types::columnar::ColumnType::Int64) => ColumnType::Int64, - // The memtable has no boolean column; ILP ingest has always widened - // booleans to f64, so a declared BOOLEAN lands in the same place. - Ok(nodedb_types::columnar::ColumnType::Float64) - | Ok(nodedb_types::columnar::ColumnType::Bool) - | Ok(nodedb_types::columnar::ColumnType::Decimal { .. }) => ColumnType::Float64, + match DeclaredType::from_declared_type(declared_type) { + Some( + DeclaredType::Timestamp | DeclaredType::Timestamptz | DeclaredType::SystemTimestamp, + ) => ColumnType::Timestamp, + Some(DeclaredType::Int64) => ColumnType::Int64, + // The memtable has no boolean column; ILP ingest widens booleans to + // f64, so a declared BOOLEAN lands in the same place. + Some(DeclaredType::Float64 | DeclaredType::Bool | DeclaredType::Decimal { .. }) => { + ColumnType::Float64 + } // Everything else — TEXT, UUID, JSON, and any type the memtable // cannot represent natively — is stored as a dictionary symbol. _ => ColumnType::Symbol, From caa424b50536fa9f8554af49d353ddb633566d0c Mon Sep 17 00:00:00 2001 From: Farhan Syah Date: Thu, 10 Sep 2026 11:35:27 +0800 Subject: [PATCH 21/23] refactor(sql): compile column DEFAULTs once per statement Split planner::defaults into a module with a ColumnDefaults / CompiledDefault pair: declaration text is classified and parsed once into a DefaultKind (generator, literal, or parsed expression), and each row then evaluates the compiled form without re-parsing. The KV insert path, declared-defaults materialization, and the columnar/ document row-expansion path (expand_row_defaults) all build one ColumnDefaults outside their row loops instead of re-invoking evaluate_default_expr per row per column. Drop Volatility::Stable, which named a per-statement reuse boundary nothing in the planner ever used; Volatility now distinguishes only Immutable (foldable at plan time) from Volatile (fresh per call). --- nodedb-sql/src/planner/defaults.rs | 284 -------------- nodedb-sql/src/planner/defaults/compiled.rs | 357 ++++++++++++++++++ nodedb-sql/src/planner/defaults/convert.rs | 60 +++ nodedb-sql/src/planner/defaults/kind.rs | 138 +++++++ nodedb-sql/src/planner/defaults/mod.rs | 30 ++ .../planner/dml_helpers/declared_defaults.rs | 46 +-- .../src/planner/dml_helpers/kv_insert.rs | 7 +- nodedb-sql/src/types/plan/volatility_scan.rs | 2 +- nodedb-types/src/volatility.rs | 7 +- .../sql_plan_convert/value/defaults.rs | 10 +- 10 files changed, 617 insertions(+), 324 deletions(-) delete mode 100644 nodedb-sql/src/planner/defaults.rs create mode 100644 nodedb-sql/src/planner/defaults/compiled.rs create mode 100644 nodedb-sql/src/planner/defaults/convert.rs create mode 100644 nodedb-sql/src/planner/defaults/kind.rs create mode 100644 nodedb-sql/src/planner/defaults/mod.rs diff --git a/nodedb-sql/src/planner/defaults.rs b/nodedb-sql/src/planner/defaults.rs deleted file mode 100644 index 7799354f2..000000000 --- a/nodedb-sql/src/planner/defaults.rs +++ /dev/null @@ -1,284 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 - -//! Column DEFAULT expression evaluation at insert time. -//! -//! Supports ID generation functions (UUIDv4/v7, ULID, CUID2, NANOID), `NOW()`, -//! sequence accessors (`nextval`, `currval`), and literal values. Anything -//! else routes through the plan-time const-folder. -//! -//! A DEFAULT that cannot be evaluated raises [`SqlError::UnevaluableDefault`]. -//! The column is never omitted: an omitted column stores NULL where the -//! declaration promised a value, and nothing reports it. -//! -//! Lives in the SQL crate rather than beside one engine's converter because -//! every engine that materializes a DEFAULT has to produce the SAME value for -//! the same expression — a `DEFAULT now()` that means one thing on a document -//! collection and another on a key-value one would be a difference nobody -//! declared. The key-value planner also needs it BEFORE its declared-type -//! coercion and range checks run, so a materialized default is validated -//! exactly like a supplied one. - -use crate::catalog::SqlCatalog; -use crate::error::SqlError; -use crate::types::{SqlExpr, SqlValue}; - -/// Evaluate `expr`, the DEFAULT declared on `column`, to one value. -/// -/// `catalog` resolves the sequence accessors `nextval` and `currval`. It is -/// required rather than optional: a catalog-free evaluator silently dropped -/// every sequence-backed DEFAULT. -/// -/// The absence of a DEFAULT is the caller's `ColumnInfo::default` being `None` -/// and never reaches here. Every call therefore either yields a value or -/// raises. -pub fn evaluate_default_expr( - expr: &str, - column: &str, - catalog: &dyn SqlCatalog, -) -> crate::Result { - let upper = expr.trim().to_uppercase(); - if let Some(value) = eval_keyword_default(&upper) { - return Ok(value); - } - if let Some(value) = eval_parametric_or_literal(expr, &upper)? { - return Ok(value); - } - evaluate_parsed_default(expr, column, catalog) -} - -/// Check that `expr`, the DEFAULT declared on `column`, can be evaluated. -/// -/// DDL calls this to refuse an unevaluable DEFAULT at declaration time. It -/// classifies the expression through the same arms `evaluate_default_expr` -/// uses, then parses anything left over. Parsing runs the resolver's -/// `FunctionRegistry` gate, so an unregistered function name raises -/// [`SqlError::UndefinedFunction`]. -/// -/// A sequence accessor is parsed, never called, so declaring a column must -/// never advance a sequence. -pub fn validate_default_expr(expr: &str, column: &str) -> crate::Result<()> { - let upper = expr.trim().to_uppercase(); - if eval_keyword_default(&upper).is_some() { - return Ok(()); - } - if eval_parametric_or_literal(expr, &upper)?.is_some() { - return Ok(()); - } - let sql_expr = crate::parse_expr_string(expr)?; - reject_setval_default(&sql_expr, column) -} - -/// Evaluate the keyword-spelled defaults: the ID generators and `NOW()`. -/// -/// Returns `None` for every other expression. This is the one list of -/// keyword forms; the DDL gate classifies through it rather than repeating it. -fn eval_keyword_default(upper: &str) -> Option { - let value = match upper { - "UUID_V7" | "UUIDV7" | "GEN_UUID_V7()" | "UUID_V7()" => { - nodedb_types::Value::String(nodedb_types::id_gen::uuid_v7()) - } - "UUID_V4" | "UUIDV4" | "UUID" | "GEN_UUID_V4()" | "UUID_V4()" => { - nodedb_types::Value::String(nodedb_types::id_gen::uuid_v4()) - } - "ULID" | "GEN_ULID()" | "ULID()" => { - nodedb_types::Value::String(nodedb_types::id_gen::ulid()) - } - "CUID2" | "CUID2()" => nodedb_types::Value::String(nodedb_types::id_gen::cuid2()), - "NANOID" | "NANOID()" => nodedb_types::Value::String(nodedb_types::id_gen::nanoid()), - "NOW()" => { - let now = std::time::SystemTime::now() - .duration_since(std::time::UNIX_EPOCH) - .unwrap_or_default(); - nodedb_types::Value::String( - chrono::DateTime::from_timestamp_millis(now.as_millis() as i64) - .map(|dt| dt.to_rfc3339()) - .unwrap_or_else(|| now.as_millis().to_string()), - ) - } - _ => return None, - }; - Some(value) -} - -/// Evaluate the parametric ID generators and the bare literals. -/// -/// Returns `Ok(None)` when `expr` is none of them, leaving it to the parser. -/// This is the one list of literal forms; the DDL gate reuses it. -fn eval_parametric_or_literal( - expr: &str, - upper: &str, -) -> crate::Result> { - // NANOID(N) — custom length. - if upper.starts_with("NANOID(") && upper.ends_with(')') { - let len_str = &upper[7..upper.len() - 1]; - if let Ok(len) = len_str.parse::() { - return Ok(Some(nodedb_types::Value::String( - nodedb_types::id_gen::nanoid_with_length(len), - ))); - } - } - // CUID2(N) — custom length; validates length range and surfaces planning errors. - if upper.starts_with("CUID2(") && upper.ends_with(')') { - let len_str = &upper[6..upper.len() - 1]; - if let Ok(len) = len_str.parse::() { - let id = nodedb_types::id_gen::cuid2_with_length(len).map_err(|e| SqlError::Parse { - detail: format!("CUID2({len}) default expression is invalid: {e}"), - })?; - return Ok(Some(nodedb_types::Value::String(id))); - } - } - // Numeric literal. - if let Ok(i) = expr.trim().parse::() { - return Ok(Some(nodedb_types::Value::Integer(i))); - } - if let Ok(f) = expr.trim().parse::() { - return Ok(Some(nodedb_types::Value::Float(f))); - } - // Quoted string literal. - let trimmed = expr.trim(); - if (trimmed.starts_with('\'') && trimmed.ends_with('\'')) - || (trimmed.starts_with('"') && trimmed.ends_with('"')) - { - return Ok(Some(nodedb_types::Value::String( - trimmed[1..trimmed.len() - 1].to_string(), - ))); - } - - Ok(None) -} - -/// Parse the DEFAULT as SQL, then resolve it against the catalog or the folder. -fn evaluate_parsed_default( - expr: &str, - column: &str, - catalog: &dyn SqlCatalog, -) -> crate::Result { - let sql_expr = crate::parse_expr_string(expr).map_err(|_| unevaluable(column, expr))?; - if let Some(value) = evaluate_sequence_default(&sql_expr, column, catalog)? { - return Ok(sql_value_to_ndb(value)); - } - // `Once`: a materialized DEFAULT serves this insert only, and an INSERT - // plan carrying a volatile DEFAULT is never admitted to the plan cache. - let folded = crate::planner::const_fold::fold_constant_scoped( - &sql_expr, - crate::planner::const_fold::default_registry(), - crate::planner::const_fold::FoldScope::Once, - ) - .map_err(|_| unevaluable(column, expr))? - .ok_or_else(|| unevaluable(column, expr))?; - Ok(sql_value_to_ndb(folded)) -} - -/// Resolve `nextval` / `currval` through the catalog; refuse `setval`. -/// -/// Returns `Ok(None)` for every other expression, leaving it to the folder. -fn evaluate_sequence_default( - expr: &SqlExpr, - column: &str, - catalog: &dyn SqlCatalog, -) -> crate::Result> { - reject_setval_default(expr, column)?; - super::catalog_expr_fold::eval_sequence_accessor(expr, catalog) -} - -/// Refuse `setval` as a column DEFAULT. -/// -/// `setval` moves a sequence rather than reading one, so a column cannot take -/// its result as a value. Both the evaluator and the DDL gate call this. -fn reject_setval_default(expr: &SqlExpr, column: &str) -> crate::Result<()> { - if let SqlExpr::Function { name, .. } = expr - && name.eq_ignore_ascii_case("setval") - { - return Err(SqlError::SetvalInColumnDefault { - column: column.to_string(), - }); - } - Ok(()) -} - -fn unevaluable(column: &str, expr: &str) -> SqlError { - SqlError::UnevaluableDefault { - column: column.to_string(), - expr: expr.to_string(), - } -} - -fn sql_value_to_ndb(v: SqlValue) -> nodedb_types::Value { - match v { - SqlValue::Null => nodedb_types::Value::Null, - SqlValue::Bool(b) => nodedb_types::Value::Bool(b), - SqlValue::Int(i) => nodedb_types::Value::Integer(i), - SqlValue::Float(f) => nodedb_types::Value::Float(f), - SqlValue::Decimal(d) => nodedb_types::Value::Decimal(d), - SqlValue::String(s) => nodedb_types::Value::String(s), - SqlValue::Bytes(b) => nodedb_types::Value::Bytes(b), - SqlValue::Array(a) => { - nodedb_types::Value::Array(a.into_iter().map(sql_value_to_ndb).collect()) - } - SqlValue::Timestamp(dt) => nodedb_types::Value::NaiveDateTime(dt), - SqlValue::Timestamptz(dt) => nodedb_types::Value::DateTime(dt), - } -} - -/// Convert an evaluated default back into the planner's literal type. -/// -/// The inverse of `sql_value_to_ndb` above, which is the only producer of -/// these values — so every shape the evaluator can emit has an exact -/// counterpart here. Anything else raises rather than rendering through -/// `Debug`: a `DEFAULT` that stored `Uuid("…")` as its own debug text is the -/// same class of defect as dropping it, and harder to notice because the -/// column looks populated. -pub fn default_value_to_sql(column: &str, value: nodedb_types::Value) -> crate::Result { - Ok(match value { - nodedb_types::Value::Null => SqlValue::Null, - nodedb_types::Value::Bool(b) => SqlValue::Bool(b), - nodedb_types::Value::Integer(i) => SqlValue::Int(i), - nodedb_types::Value::Float(f) => SqlValue::Float(f), - nodedb_types::Value::Decimal(d) => SqlValue::Decimal(d), - nodedb_types::Value::String(s) => SqlValue::String(s), - nodedb_types::Value::Bytes(b) => SqlValue::Bytes(b), - nodedb_types::Value::NaiveDateTime(dt) => SqlValue::Timestamp(dt), - nodedb_types::Value::DateTime(dt) => SqlValue::Timestamptz(dt), - nodedb_types::Value::Array(items) => SqlValue::Array( - items - .into_iter() - .map(|item| default_value_to_sql(column, item)) - .collect::>>()?, - ), - other => { - return Err(SqlError::Unsupported { - detail: format!( - "default for column '{column}' evaluates to a value with no SQL literal \ - form: {other:?}" - ), - }); - } - }) -} - -/// Fill in every column of `row` that declares a DEFAULT and the statement omitted. -/// -/// `column_defaults` is the catalog's `(column_name, default_expr)` list. -/// Each entry evaluates at most once per row, so a `nextval` DEFAULT allocates -/// exactly one value per row. -/// -/// A column the statement supplied stays untouched, an explicit `NULL` -/// included: `NULL` is a value the author chose, and overwriting it with the -/// default makes storing one impossible. -/// -/// A DEFAULT the evaluator cannot resolve raises -/// [`SqlError::UnevaluableDefault`] rather than leaving the column out. -pub fn materialize_row_defaults( - row: &mut Vec<(String, SqlValue)>, - column_defaults: &[(String, String)], - catalog: &dyn SqlCatalog, -) -> crate::Result<()> { - for (column, default_expr) in column_defaults { - if row.iter().any(|(name, _)| name == column) { - continue; - } - let evaluated = evaluate_default_expr(default_expr, column, catalog)?; - row.push((column.clone(), default_value_to_sql(column, evaluated)?)); - } - Ok(()) -} diff --git a/nodedb-sql/src/planner/defaults/compiled.rs b/nodedb-sql/src/planner/defaults/compiled.rs new file mode 100644 index 000000000..9905e3e23 --- /dev/null +++ b/nodedb-sql/src/planner/defaults/compiled.rs @@ -0,0 +1,357 @@ +// SPDX-License-Identifier: Apache-2.0 + +//! Column DEFAULTs compiled once per statement and evaluated per row. + +use super::convert::{default_value_to_sql, sql_value_to_ndb}; +use super::kind::{DefaultKind, keyword_generator, parametric_or_literal}; +use crate::catalog::SqlCatalog; +use crate::error::SqlError; +use crate::types::{ColumnInfo, SqlExpr, SqlValue}; + +/// One column DEFAULT, classified and parsed. +/// +/// Compilation is the only step that reads the declaration text, so a caller +/// holding a `CompiledDefault` cannot parse the expression a second time. +#[derive(Debug, Clone)] +pub struct CompiledDefault { + column: String, + /// The declaration text, kept for the `UnevaluableDefault` message only. + text: String, + kind: DefaultKind, + volatile: bool, +} + +impl CompiledDefault { + /// Compile the DEFAULT declared on `column` without evaluating it. + /// + /// A parse error propagates verbatim, so the DDL gate can map an + /// unregistered function name onto SQLSTATE `42883`. + /// + /// A sequence accessor is parsed, never called, so declaring a column must + /// never advance a sequence. + pub fn declare(column: &str, expr: &str) -> crate::Result { + let kind = classify(column, expr)?; + let volatile = match &kind { + DefaultKind::Generator(_) => true, + DefaultKind::Literal(_) => false, + DefaultKind::Expr(parsed) => crate::types::plan::expr_is_volatile(parsed), + }; + Ok(Self { + column: column.to_string(), + text: expr.to_string(), + kind, + volatile, + }) + } + + /// Compile the DEFAULT declared on `column` for row materialization. + /// + /// An expression the parser refuses raises [`SqlError::UnevaluableDefault`] + /// rather than a parse error: DDL already refused those at declaration, so + /// a statement reaching here found one the catalog cannot resolve. + pub fn compile(column: &str, expr: &str) -> crate::Result { + Self::declare(column, expr).map_err(|error| match error { + SqlError::Parse { .. } | SqlError::UndefinedFunction { .. } => { + unevaluable(column, expr) + } + other => other, + }) + } + + /// The column this DEFAULT fills. + pub fn column(&self) -> &str { + &self.column + } + + /// Whether this DEFAULT produces a fresh value on every evaluation. + /// + /// A plan carrying one is never admitted to the plan cache, or the cache + /// replays one execution's value into every later one. + pub fn is_volatile(&self) -> bool { + self.volatile + } + + /// Evaluate to one value. `catalog` resolves `nextval` and `currval`. + /// + /// Takes no expression text, so the per-row path cannot re-parse. + pub fn evaluate(&self, catalog: &dyn SqlCatalog) -> crate::Result { + match &self.kind { + DefaultKind::Generator(generator) => generator.generate(), + DefaultKind::Literal(value) => Ok(value.clone()), + DefaultKind::Expr(parsed) => self.evaluate_expr(parsed, catalog), + } + } + + /// Resolve a parsed DEFAULT through the catalog, then the const-folder. + fn evaluate_expr( + &self, + parsed: &SqlExpr, + catalog: &dyn SqlCatalog, + ) -> crate::Result { + if let Some(value) = + crate::planner::catalog_expr_fold::eval_sequence_accessor(parsed, catalog)? + { + return Ok(sql_value_to_ndb(value)); + } + // `Once`: a materialized DEFAULT serves this insert only, and an INSERT + // plan carrying a volatile DEFAULT is never admitted to the plan cache. + let folded = crate::planner::const_fold::fold_constant_scoped( + parsed, + crate::planner::const_fold::default_registry(), + crate::planner::const_fold::FoldScope::Once, + ) + .map_err(|_| unevaluable(&self.column, &self.text))? + .ok_or_else(|| unevaluable(&self.column, &self.text))?; + Ok(sql_value_to_ndb(folded)) + } +} + +/// Every declared DEFAULT of one collection, compiled once for a statement. +/// +/// Build this OUTSIDE the row loop. `materialize_row` then fills each row from +/// the parsed forms, so a multi-row `VALUES` clause parses each declaration +/// exactly once however many rows it carries. +#[derive(Debug, Clone, Default)] +pub struct ColumnDefaults { + defaults: Vec, +} + +impl ColumnDefaults { + /// Compile the catalog's `(column_name, default_expr)` list. + pub fn compile_pairs(pairs: &[(String, String)]) -> crate::Result { + let defaults = pairs + .iter() + .map(|(column, expr)| CompiledDefault::compile(column, expr)) + .collect::>>()?; + Ok(Self { defaults }) + } + + /// Compile every declared column that carries a DEFAULT. + pub fn compile_columns(columns: &[ColumnInfo]) -> crate::Result { + let defaults = columns + .iter() + .filter_map(|column| { + column + .default + .as_deref() + .map(|expr| CompiledDefault::compile(&column.name, expr)) + }) + .collect::>>()?; + Ok(Self { defaults }) + } + + /// Whether no column declares a DEFAULT. + pub fn is_empty(&self) -> bool { + self.defaults.is_empty() + } + + /// Fill in every column of `row` that declares a DEFAULT and the statement + /// omitted. + /// + /// Each entry evaluates at most once per row, so a `nextval` DEFAULT + /// allocates exactly one value per row. + /// + /// A column the statement supplied stays untouched, an explicit `NULL` + /// included: `NULL` is a value the author chose, and overwriting it with + /// the default makes storing one impossible. + /// + /// Returns whether any default it materialized was volatile, so the caller + /// can keep the plan out of the plan cache. + pub fn materialize_row( + &self, + row: &mut Vec<(String, SqlValue)>, + catalog: &dyn SqlCatalog, + ) -> crate::Result { + let mut volatile = false; + for default in &self.defaults { + if row.iter().any(|(name, _)| name == default.column()) { + continue; + } + let evaluated = default.evaluate(catalog)?; + let value = default_value_to_sql(default.column(), evaluated)?; + volatile |= default.is_volatile(); + row.push((default.column().to_string(), value)); + } + Ok(volatile) + } +} + +/// Check that `expr`, the DEFAULT declared on `column`, can be evaluated. +/// +/// DDL calls this to refuse an unevaluable DEFAULT at declaration time. It +/// classifies and parses the expression, and evaluates nothing. Parsing runs +/// the resolver's `FunctionRegistry` gate, so an unregistered function name +/// raises [`SqlError::UndefinedFunction`]. +pub fn validate_default_expr(expr: &str, column: &str) -> crate::Result<()> { + CompiledDefault::declare(column, expr).map(|_| ()) +} + +/// Classify a DEFAULT into its compiled form. +fn classify(column: &str, expr: &str) -> crate::Result { + let upper = expr.trim().to_uppercase(); + if let Some(generator) = keyword_generator(&upper) { + return Ok(DefaultKind::Generator(generator)); + } + if let Some(kind) = parametric_or_literal(expr, &upper)? { + return Ok(kind); + } + let parsed = crate::parse_expr_string(expr)?; + reject_setval_default(&parsed, column)?; + Ok(DefaultKind::Expr(parsed)) +} + +/// Refuse `setval` as a column DEFAULT. +/// +/// `setval` moves a sequence rather than reading one, so a column cannot take +/// its result as a value. +fn reject_setval_default(expr: &SqlExpr, column: &str) -> crate::Result<()> { + if let SqlExpr::Function { name, .. } = expr + && name.eq_ignore_ascii_case("setval") + { + return Err(SqlError::SetvalInColumnDefault { + column: column.to_string(), + }); + } + Ok(()) +} + +fn unevaluable(column: &str, expr: &str) -> SqlError { + SqlError::UnevaluableDefault { + column: column.to_string(), + expr: expr.to_string(), + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::catalog::SqlCatalogError; + use std::cell::Cell; + + /// A catalog whose only state is a sequence counter, so a test can count + /// exactly how many times a DEFAULT reached `nextval`. + #[derive(Default)] + struct CountingCatalog { + nextval_calls: Cell, + } + + impl SqlCatalog for CountingCatalog { + fn get_collection( + &self, + _database_id: nodedb_types::DatabaseId, + _name: &str, + ) -> std::result::Result, SqlCatalogError> { + Ok(None) + } + + fn sequence_nextval( + &self, + _database_id: nodedb_types::DatabaseId, + _tenant_id: u64, + _name: &str, + ) -> crate::Result { + let next = self.nextval_calls.get() + 1; + self.nextval_calls.set(next); + Ok(next) + } + } + + fn pairs(column: &str, expr: &str) -> Vec<(String, String)> { + vec![(column.to_string(), expr.to_string())] + } + + #[test] + fn one_compilation_still_generates_a_fresh_value_per_row() { + let catalog = CountingCatalog::default(); + let compiled = ColumnDefaults::compile_pairs(&pairs("u", "UUID_V7()")).expect("compiles"); + let mut first = Vec::new(); + let mut second = Vec::new(); + compiled.materialize_row(&mut first, &catalog).expect("row"); + compiled + .materialize_row(&mut second, &catalog) + .expect("row"); + assert_ne!(first[0].1, second[0].1, "each row needs its own UUID"); + } + + #[test] + fn nextval_allocates_exactly_one_value_per_row() { + let catalog = CountingCatalog::default(); + let compiled = + ColumnDefaults::compile_pairs(&pairs("id", "nextval('s')")).expect("compiles"); + let mut first = Vec::new(); + let mut second = Vec::new(); + compiled.materialize_row(&mut first, &catalog).expect("row"); + compiled + .materialize_row(&mut second, &catalog) + .expect("row"); + assert_eq!(first[0].1, SqlValue::Int(1)); + assert_eq!(second[0].1, SqlValue::Int(2)); + assert_eq!(catalog.nextval_calls.get(), 2); + } + + #[test] + fn declaring_a_sequence_default_never_advances_it() { + let catalog = CountingCatalog::default(); + validate_default_expr("nextval('s')", "id").expect("declaration is valid"); + assert_eq!( + catalog.nextval_calls.get(), + 0, + "the DDL gate must parse, never call" + ); + } + + #[test] + fn a_supplied_column_keeps_its_value() { + let catalog = CountingCatalog::default(); + let compiled = + ColumnDefaults::compile_pairs(&pairs("id", "nextval('s')")).expect("compiles"); + let mut row = vec![("id".to_string(), SqlValue::Int(7))]; + compiled.materialize_row(&mut row, &catalog).expect("row"); + assert_eq!(row, vec![("id".to_string(), SqlValue::Int(7))]); + assert_eq!(catalog.nextval_calls.get(), 0); + } + + #[test] + fn volatility_is_decided_once_at_compile_time() { + assert!( + CompiledDefault::declare("u", "UUID_V7()") + .expect("compiles") + .is_volatile() + ); + assert!( + CompiledDefault::declare("id", "nextval('s')") + .expect("compiles") + .is_volatile() + ); + assert!( + !CompiledDefault::declare("s", "'active'") + .expect("compiles") + .is_volatile() + ); + assert!( + !CompiledDefault::declare("n", "1 + 2") + .expect("compiles") + .is_volatile() + ); + } + + #[test] + fn setval_is_refused_as_a_default() { + let error = CompiledDefault::declare("id", "setval('s', 10)").expect_err("setval refused"); + assert!(matches!(error, SqlError::SetvalInColumnDefault { .. })); + } + + #[test] + fn an_unregistered_function_is_refused_at_declaration() { + let error = CompiledDefault::declare("a", "no_such_function_here('x')") + .expect_err("unknown function refused"); + assert!(matches!(error, SqlError::UndefinedFunction { .. })); + } + + #[test] + fn an_unregistered_function_is_unevaluable_at_insert() { + let error = CompiledDefault::compile("a", "no_such_function_here('x')") + .expect_err("unknown function refused"); + assert!(matches!(error, SqlError::UnevaluableDefault { .. })); + } +} diff --git a/nodedb-sql/src/planner/defaults/convert.rs b/nodedb-sql/src/planner/defaults/convert.rs new file mode 100644 index 000000000..6817a5cca --- /dev/null +++ b/nodedb-sql/src/planner/defaults/convert.rs @@ -0,0 +1,60 @@ +// SPDX-License-Identifier: Apache-2.0 + +//! Conversion between the evaluator's value type and the planner's literal type. + +use crate::error::SqlError; +use crate::types::SqlValue; + +/// Convert an evaluated SQL literal into the engine-facing value type. +pub(super) fn sql_value_to_ndb(v: SqlValue) -> nodedb_types::Value { + match v { + SqlValue::Null => nodedb_types::Value::Null, + SqlValue::Bool(b) => nodedb_types::Value::Bool(b), + SqlValue::Int(i) => nodedb_types::Value::Integer(i), + SqlValue::Float(f) => nodedb_types::Value::Float(f), + SqlValue::Decimal(d) => nodedb_types::Value::Decimal(d), + SqlValue::String(s) => nodedb_types::Value::String(s), + SqlValue::Bytes(b) => nodedb_types::Value::Bytes(b), + SqlValue::Array(a) => { + nodedb_types::Value::Array(a.into_iter().map(sql_value_to_ndb).collect()) + } + SqlValue::Timestamp(dt) => nodedb_types::Value::NaiveDateTime(dt), + SqlValue::Timestamptz(dt) => nodedb_types::Value::DateTime(dt), + } +} + +/// Convert an evaluated default back into the planner's literal type. +/// +/// The inverse of `sql_value_to_ndb` above, which is the only producer of +/// these values — so every shape the evaluator can emit has an exact +/// counterpart here. Anything else raises rather than rendering through +/// `Debug`: a `DEFAULT` that stored `Uuid("…")` as its own debug text is the +/// same class of defect as dropping it, and harder to notice because the +/// column looks populated. +pub fn default_value_to_sql(column: &str, value: nodedb_types::Value) -> crate::Result { + Ok(match value { + nodedb_types::Value::Null => SqlValue::Null, + nodedb_types::Value::Bool(b) => SqlValue::Bool(b), + nodedb_types::Value::Integer(i) => SqlValue::Int(i), + nodedb_types::Value::Float(f) => SqlValue::Float(f), + nodedb_types::Value::Decimal(d) => SqlValue::Decimal(d), + nodedb_types::Value::String(s) => SqlValue::String(s), + nodedb_types::Value::Bytes(b) => SqlValue::Bytes(b), + nodedb_types::Value::NaiveDateTime(dt) => SqlValue::Timestamp(dt), + nodedb_types::Value::DateTime(dt) => SqlValue::Timestamptz(dt), + nodedb_types::Value::Array(items) => SqlValue::Array( + items + .into_iter() + .map(|item| default_value_to_sql(column, item)) + .collect::>>()?, + ), + other => { + return Err(SqlError::Unsupported { + detail: format!( + "default for column '{column}' evaluates to a value with no SQL literal \ + form: {other:?}" + ), + }); + } + }) +} diff --git a/nodedb-sql/src/planner/defaults/kind.rs b/nodedb-sql/src/planner/defaults/kind.rs new file mode 100644 index 000000000..2e1b2deab --- /dev/null +++ b/nodedb-sql/src/planner/defaults/kind.rs @@ -0,0 +1,138 @@ +// SPDX-License-Identifier: Apache-2.0 + +//! The compiled forms a column DEFAULT classifies into. + +use crate::error::SqlError; +use crate::types::SqlExpr; + +/// What a DEFAULT expression compiles to. +/// +/// Classification happens once per statement. Nothing here holds the original +/// text, so evaluation cannot fall back to parsing it again. +#[derive(Debug, Clone)] +pub(super) enum DefaultKind { + /// A generator that produces a fresh value on every call. + Generator(Generator), + /// A constant the declaration spells out. + Literal(nodedb_types::Value), + /// A parsed expression: a sequence accessor, or a const-folder target. + Expr(SqlExpr), +} + +/// A DEFAULT spelled as a value generator rather than a registered call. +/// +/// The catalog accepts both the bare form (`UUID_V7`) and the call form +/// (`UUID_V7()`), so both spellings classify to the same variant. +#[derive(Debug, Clone, Copy)] +pub(super) enum Generator { + UuidV7, + UuidV4, + Ulid, + Cuid2, + Cuid2Len(usize), + Nanoid, + NanoidLen(usize), + Now, +} + +impl Generator { + /// Produce one fresh value. + pub(super) fn generate(self) -> crate::Result { + let value = match self { + Self::UuidV7 => nodedb_types::Value::String(nodedb_types::id_gen::uuid_v7()), + Self::UuidV4 => nodedb_types::Value::String(nodedb_types::id_gen::uuid_v4()), + Self::Ulid => nodedb_types::Value::String(nodedb_types::id_gen::ulid()), + Self::Cuid2 => nodedb_types::Value::String(nodedb_types::id_gen::cuid2()), + Self::Cuid2Len(len) => nodedb_types::Value::String(cuid2_with_length(len)?), + Self::Nanoid => nodedb_types::Value::String(nodedb_types::id_gen::nanoid()), + Self::NanoidLen(len) => { + nodedb_types::Value::String(nodedb_types::id_gen::nanoid_with_length(len)) + } + Self::Now => nodedb_types::Value::String(now_rfc3339()), + }; + Ok(value) + } +} + +/// Generate a CUID2 of `len` characters, mapping a rejected length to a +/// planning error. +pub(super) fn cuid2_with_length(len: usize) -> crate::Result { + nodedb_types::id_gen::cuid2_with_length(len).map_err(|e| SqlError::Parse { + detail: format!("CUID2({len}) default expression is invalid: {e}"), + }) +} + +/// Render the current wall-clock instant the way `DEFAULT NOW()` stores it. +fn now_rfc3339() -> String { + let now = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap_or_default(); + chrono::DateTime::from_timestamp_millis(now.as_millis() as i64) + .map(|dt| dt.to_rfc3339()) + .unwrap_or_else(|| now.as_millis().to_string()) +} + +/// Classify the keyword-spelled defaults: the ID generators and `NOW()`. +/// +/// Returns `None` for every other expression. This is the one list of keyword +/// forms; the DDL gate classifies through it rather than repeating it. +pub(super) fn keyword_generator(upper: &str) -> Option { + let generator = match upper { + "UUID_V7" | "UUIDV7" | "GEN_UUID_V7()" | "UUID_V7()" => Generator::UuidV7, + "UUID_V4" | "UUIDV4" | "UUID" | "GEN_UUID_V4()" | "UUID_V4()" => Generator::UuidV4, + "ULID" | "GEN_ULID()" | "ULID()" => Generator::Ulid, + "CUID2" | "CUID2()" => Generator::Cuid2, + "NANOID" | "NANOID()" => Generator::Nanoid, + "NOW()" => Generator::Now, + _ => return None, + }; + Some(generator) +} + +/// Classify the parametric ID generators and the bare literals. +/// +/// Returns `Ok(None)` when `expr` is none of them, leaving it to the parser. +/// This is the one list of literal forms; the DDL gate reuses it. +/// +/// `CUID2(N)` validates its length here, so a rejected length raises at +/// declaration rather than at the first insert. +pub(super) fn parametric_or_literal(expr: &str, upper: &str) -> crate::Result> { + // NANOID(N) — custom length. + if upper.starts_with("NANOID(") && upper.ends_with(')') { + let len_str = &upper[7..upper.len() - 1]; + if let Ok(len) = len_str.parse::() { + return Ok(Some(DefaultKind::Generator(Generator::NanoidLen(len)))); + } + } + // CUID2(N) — custom length; validates length range and surfaces planning errors. + if upper.starts_with("CUID2(") && upper.ends_with(')') { + let len_str = &upper[6..upper.len() - 1]; + if let Ok(len) = len_str.parse::() { + // One generation checks the length range against the one authority + // for it. The value is discarded; each row generates its own. + cuid2_with_length(len)?; + return Ok(Some(DefaultKind::Generator(Generator::Cuid2Len(len)))); + } + } + // Numeric literal. + if let Ok(i) = expr.trim().parse::() { + return Ok(Some(DefaultKind::Literal(nodedb_types::Value::Integer(i)))); + } + if let Ok(f) = expr.trim().parse::() { + return Ok(Some(DefaultKind::Literal(nodedb_types::Value::Float(f)))); + } + // Quoted string literal. + // A length of two is the shortest quoted literal, the empty string. One + // lone quote character opens a literal nothing closes, so it is not one. + let trimmed = expr.trim(); + if trimmed.len() >= 2 + && ((trimmed.starts_with('\'') && trimmed.ends_with('\'')) + || (trimmed.starts_with('"') && trimmed.ends_with('"'))) + { + return Ok(Some(DefaultKind::Literal(nodedb_types::Value::String( + trimmed[1..trimmed.len() - 1].to_string(), + )))); + } + + Ok(None) +} diff --git a/nodedb-sql/src/planner/defaults/mod.rs b/nodedb-sql/src/planner/defaults/mod.rs new file mode 100644 index 000000000..98b05a51b --- /dev/null +++ b/nodedb-sql/src/planner/defaults/mod.rs @@ -0,0 +1,30 @@ +// SPDX-License-Identifier: Apache-2.0 + +//! Column DEFAULT expression compilation and evaluation at insert time. +//! +//! A declared DEFAULT is stored in the catalog as text. [`ColumnDefaults`] +//! compiles that text ONCE per statement; the per-row path evaluates the +//! compiled form and never sees a string, so it cannot re-parse. +//! +//! Supported forms: ID generation functions (UUIDv4/v7, ULID, CUID2, NANOID), +//! `NOW()`, sequence accessors (`nextval`, `currval`), literals, and any other +//! expression the plan-time const-folder can resolve. +//! +//! A DEFAULT that cannot be evaluated raises [`crate::SqlError::UnevaluableDefault`]. +//! The column is never omitted: an omitted column stores NULL where the +//! declaration promised a value, and nothing reports it. +//! +//! Lives in the SQL crate rather than beside one engine's converter because +//! every engine that materializes a DEFAULT has to produce the SAME value for +//! the same expression — a `DEFAULT now()` that means one thing on a document +//! collection and another on a key-value one would be a difference nobody +//! declared. The key-value planner also needs it BEFORE its declared-type +//! coercion and range checks run, so a materialized default is validated +//! exactly like a supplied one. + +mod compiled; +mod convert; +mod kind; + +pub use compiled::{ColumnDefaults, CompiledDefault, validate_default_expr}; +pub use convert::default_value_to_sql; diff --git a/nodedb-sql/src/planner/dml_helpers/declared_defaults.rs b/nodedb-sql/src/planner/dml_helpers/declared_defaults.rs index 0fc5125f8..5c5e6e5ab 100644 --- a/nodedb-sql/src/planner/dml_helpers/declared_defaults.rs +++ b/nodedb-sql/src/planner/dml_helpers/declared_defaults.rs @@ -1,19 +1,19 @@ // SPDX-License-Identifier: Apache-2.0 -//! Materialization of declared column DEFAULTs into a parsed `VALUES` row. +//! Materialization of declared column DEFAULTs into a parsed `VALUES` row set. use crate::catalog::SqlCatalog; use crate::error::Result; +use crate::planner::defaults::ColumnDefaults; use crate::types::*; -/// Fill in every declared column the statement omitted that carries a DEFAULT. +/// Materialize declared DEFAULTs across a whole `VALUES` row set. /// /// The key-value and vector-primary engines store the values they are handed /// and have no typed write path, so a DEFAULT that is not materialized HERE is /// materialized nowhere: the catalog would keep the declaration and every read /// return nothing for it. Documents and columnar rows expand theirs through the -/// same `evaluate_default_expr`, so one expression yields one value on every -/// engine. +/// same [`ColumnDefaults`], so one expression yields one value on every engine. /// /// Two rules the ordering encodes: /// @@ -26,44 +26,26 @@ use crate::types::*; /// on a `SMALLINT` column a way to store a value the same literal is /// rejected for. /// +/// Every declaration compiles once, before the row loop, so a multi-row +/// `VALUES` clause parses each DEFAULT expression exactly once. +/// /// A DEFAULT the evaluator cannot resolve raises `SqlError::UnevaluableDefault` /// rather than leaving the column out. `catalog` resolves `nextval` / `currval`. /// -/// Returns whether any materialized default came from a `Volatile` -/// expression, so the caller can keep the plan out of the plan cache. -pub(crate) fn materialize_declared_defaults( - declared_columns: &[ColumnInfo], - row: &mut Vec<(String, SqlValue)>, - catalog: &dyn SqlCatalog, -) -> Result { - let mut volatile = false; - for column in declared_columns { - let Some(default_expr) = column.default.as_deref() else { - continue; - }; - if row.iter().any(|(name, _)| name == &column.name) { - continue; - } - let evaluated = - crate::planner::defaults::evaluate_default_expr(default_expr, &column.name, catalog)?; - let value = crate::planner::defaults::default_value_to_sql(&column.name, evaluated)?; - volatile |= crate::types::plan::default_expr_is_volatile(default_expr); - row.push((column.name.clone(), value)); - } - Ok(volatile) -} - -/// Materialize declared DEFAULTs across a whole `VALUES` row set. -/// -/// Returns whether any materialized default was volatile. +/// Returns whether any materialized default was volatile, so the caller can +/// keep the plan out of the plan cache. pub(crate) fn materialize_defaults_in_rows( declared_columns: &[ColumnInfo], rows: &mut [Vec<(String, SqlValue)>], catalog: &dyn SqlCatalog, ) -> Result { + let compiled = ColumnDefaults::compile_columns(declared_columns)?; + if compiled.is_empty() { + return Ok(false); + } let mut volatile = false; for row in rows.iter_mut() { - volatile |= materialize_declared_defaults(declared_columns, row, catalog)?; + volatile |= compiled.materialize_row(row, catalog)?; } Ok(volatile) } diff --git a/nodedb-sql/src/planner/dml_helpers/kv_insert.rs b/nodedb-sql/src/planner/dml_helpers/kv_insert.rs index 0c486e5b1..68525971e 100644 --- a/nodedb-sql/src/planner/dml_helpers/kv_insert.rs +++ b/nodedb-sql/src/planner/dml_helpers/kv_insert.rs @@ -3,7 +3,6 @@ //! Plan construction for the KV engine's `VALUES`-clause insert paths //! (plain `INSERT`, `UPSERT`, and `INSERT ... ON CONFLICT DO UPDATE`). -use super::declared_defaults::materialize_declared_defaults; use super::params::KvInsertParams; use super::range_check::{ check_declared_float_ranges, check_declared_float_ranges_in_assignments, @@ -14,6 +13,7 @@ use crate::error::{Result, SqlError}; use crate::planner::declared_type_coerce::{ coerce_assignments_to_declared_types, coerce_row_to_declared_types, }; +use crate::planner::defaults::ColumnDefaults; use crate::types::*; /// Build a `SqlPlan::KvInsert` from a VALUES clause. Shared by plain INSERT, @@ -72,6 +72,9 @@ pub(crate) fn build_kv_insert_plan(params: KvInsertParams<'_>) -> Result> = Vec::with_capacity(rows_ast.len()); let mut volatile_defaults = false; for row_exprs in rows_ast { @@ -80,7 +83,7 @@ pub(crate) fn build_kv_insert_plan(params: KvInsertParams<'_>) -> Result bool { /// Column DEFAULT spellings that generate a fresh value per row but name no /// registered function. Kept in step with the generator arms of -/// `crate::planner::defaults::evaluate_default_expr`. +/// `crate::planner::defaults`. const VOLATILE_DEFAULT_ALIASES: &[&str] = &["uuidv7", "uuidv4", "gen_uuid_v7", "gen_uuid_v4", "gen_ulid"]; diff --git a/nodedb-types/src/volatility.rs b/nodedb-types/src/volatility.rs index 8dfad6180..b1fb7b497 100644 --- a/nodedb-types/src/volatility.rs +++ b/nodedb-types/src/volatility.rs @@ -6,13 +6,16 @@ //! `Scalar` and `Volatile` at the same time. /// How far a function call's result can be reused. +/// +/// Two states, because the engine has two reuse boundaries: a value folded +/// into a cached plan, and a value produced fresh for one execution. A +/// per-statement memoization boundary would need a third state; no such +/// boundary exists, so no third state does either. #[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] pub enum Volatility { /// Same arguments always give the same result. Foldable at plan time. #[default] Immutable, - /// Constant within one statement, not across statements. - Stable, /// Can change per call, or has side effects. Volatile, } diff --git a/nodedb/src/control/planner/sql_plan_convert/value/defaults.rs b/nodedb/src/control/planner/sql_plan_convert/value/defaults.rs index 43dfaf105..cb6e4aba2 100644 --- a/nodedb/src/control/planner/sql_plan_convert/value/defaults.rs +++ b/nodedb/src/control/planner/sql_plan_convert/value/defaults.rs @@ -15,8 +15,9 @@ use crate::types::TenantId; /// row. A DEFAULT materialized per engine after that gate refuses a key the /// declaration supplies. /// -/// Each DEFAULT evaluates once per row, so a `nextval` DEFAULT allocates -/// exactly one value per row of a multi-row VALUES clause. +/// Each DEFAULT compiles once per statement and evaluates once per row, so a +/// `nextval` DEFAULT allocates exactly one value per row of a multi-row VALUES +/// clause and the expression is parsed once however many rows it fills. /// /// `column_defaults` empty means nothing to expand: the rows pass through and /// the catalog is never read. @@ -30,10 +31,13 @@ pub(in super::super) fn expand_row_defaults( return Ok(rows.to_vec()); } let catalog = ctx.sql_catalog()?; + let compiled = nodedb_sql::planner::defaults::ColumnDefaults::compile_pairs(column_defaults) + .map_err(|e| map_plan_error(e, tenant_id))?; let mut expanded = Vec::with_capacity(rows.len()); for row in rows { let mut row = row.clone(); - nodedb_sql::planner::defaults::materialize_row_defaults(&mut row, column_defaults, catalog) + compiled + .materialize_row(&mut row, catalog) .map_err(|e| map_plan_error(e, tenant_id))?; expanded.push(row); } From 3cbbb1968bc4b17f899a986283e437439f4ee473 Mon Sep 17 00:00:00 2001 From: Farhan Syah Date: Thu, 10 Sep 2026 14:47:20 +0800 Subject: [PATCH 22/23] feat(sql): unify declared-type parsing and gate value-producing clauses Route the TYPEGUARD type-expression parser through the same ColumnType resolver CONVERT and CREATE use, so one declared spelling (VARCHAR(n), TIMESTAMP WITH/WITHOUT TIME ZONE, SYSTEM_TIMESTAMP, DECIMAL(p, s), and every registered numeric alias) resolves to the same type everywhere, and a trailing word after a keyword is rejected instead of silently dropped. Generalize the CREATE-time column DEFAULT gate into a shared column_default module and reuse it from CONVERT's column list and from a new typeguard DEFAULT/VALUE gate, so a declaration naming an unregistered or non-deterministic function is refused at DDL time across all three surfaces instead of failing at first write. Split the CONVERT COLLECTION handler out of its single file into a directory of driver, column-defs, type-map, and typeguard-columns modules to hold the added validation without growing an already large file. --- nodedb-sql/src/parser/type_expr.rs | 381 +++++++++---- nodedb-types/src/columnar/column_parse.rs | 56 +- .../ddl/neutral/collection/create/build.rs | 2 +- .../create/build_column_defaults.rs | 49 -- .../ddl/neutral/collection/create/mod.rs | 5 +- .../shared/ddl/neutral/column_default.rs | 70 +++ .../server/shared/ddl/neutral/convert.rs | 515 ------------------ .../shared/ddl/neutral/convert/column_defs.rs | 373 +++++++++++++ .../shared/ddl/neutral/convert/driver.rs | 155 ++++++ .../server/shared/ddl/neutral/convert/mod.rs | 16 + .../shared/ddl/neutral/convert/support.rs | 13 + .../shared/ddl/neutral/convert/type_map.rs | 397 ++++++++++++++ .../ddl/neutral/convert/typeguard_columns.rs | 123 +++++ .../control/server/shared/ddl/neutral/mod.rs | 1 + .../ddl/neutral/typeguard/injected_expr.rs | 110 ++++ .../shared/ddl/neutral/typeguard/mod.rs | 1 + .../shared/ddl/neutral/typeguard/parse.rs | 59 ++ .../inproc/cases/system_task_call_sites.rs | 2 +- nodedb/tests/wire/cases/mod.rs | 2 + .../wire/cases/sql_convert_column_defs.rs | 88 +++ .../wire/cases/sql_typeguard_default_gate.rs | 100 ++++ .../wire/cases/sql_typeguard_defaults.rs | 34 ++ 22 files changed, 1868 insertions(+), 684 deletions(-) delete mode 100644 nodedb/src/control/server/shared/ddl/neutral/collection/create/build_column_defaults.rs create mode 100644 nodedb/src/control/server/shared/ddl/neutral/column_default.rs delete mode 100644 nodedb/src/control/server/shared/ddl/neutral/convert.rs create mode 100644 nodedb/src/control/server/shared/ddl/neutral/convert/column_defs.rs create mode 100644 nodedb/src/control/server/shared/ddl/neutral/convert/driver.rs create mode 100644 nodedb/src/control/server/shared/ddl/neutral/convert/mod.rs create mode 100644 nodedb/src/control/server/shared/ddl/neutral/convert/support.rs create mode 100644 nodedb/src/control/server/shared/ddl/neutral/convert/type_map.rs create mode 100644 nodedb/src/control/server/shared/ddl/neutral/convert/typeguard_columns.rs create mode 100644 nodedb/src/control/server/shared/ddl/neutral/typeguard/injected_expr.rs create mode 100644 nodedb/tests/wire/cases/sql_convert_column_defs.rs create mode 100644 nodedb/tests/wire/cases/sql_typeguard_default_gate.rs diff --git a/nodedb-sql/src/parser/type_expr.rs b/nodedb-sql/src/parser/type_expr.rs index d372658c5..7bcba19dc 100644 --- a/nodedb-sql/src/parser/type_expr.rs +++ b/nodedb-sql/src/parser/type_expr.rs @@ -7,6 +7,7 @@ //! at write time. use nodedb_types::Value; +use nodedb_types::columnar::ColumnType; use crate::error::SqlError; @@ -26,6 +27,10 @@ pub enum TypeExpr { } /// Leaf type variants that map to a single Value discriminant. +/// +/// Every variant except [`SimpleType::Object`] names one +/// [`ColumnType`](nodedb_types::columnar::ColumnType), so a spelling resolves +/// to the same type here and in the CONVERT path. #[derive(Debug, Clone, PartialEq)] pub enum SimpleType { Int, @@ -35,13 +40,23 @@ pub enum SimpleType { Bytes, Timestamp, Timestamptz, - Decimal, + /// Engine-assigned instant. Matching mirrors + /// `ColumnType::SystemTimestamp`: a client writes an instant, never text. + SystemTimestamp, + /// Declared precision and scale carry the spelling the author wrote. + /// Matching is variant-level, as it is for [`SimpleType::Vector`]. + Decimal { + precision: u8, + scale: u8, + }, Uuid, Ulid, Geometry, Duration, /// Untyped array (any element type). Array, + /// Typeguard-only leaf: a field holding a nested map. No declared DDL + /// spelling names it, and CONVERT maps such a field to `ColumnType::Json`. Object, Json, /// Untyped set (any element type). @@ -60,6 +75,12 @@ pub enum SimpleType { /// Parse a type expression string into a [`TypeExpr`]. /// +/// Each leaf spelling resolves through +/// [`ColumnType::from_str`](std::str::FromStr), the parser the CONVERT path +/// resolves the same typeguard text through. A leaf the shared parser does not +/// know is an error, so a trailing word such as `TIMESTAMP GARBAGE` is +/// rejected rather than read as `TIMESTAMP`. +/// /// # Examples /// /// ``` @@ -80,7 +101,15 @@ pub fn parse_type_expr(s: &str) -> Result { } let mut pos = 0usize; let chars: Vec = s.chars().collect(); - parse_union(&chars, &mut pos, false) + let expr = parse_union(&chars, &mut pos, false)?; + skip_ws(&chars, &mut pos); + if pos < chars.len() { + let rest: String = chars[pos..].iter().collect(); + return Err(SqlError::Parse { + detail: format!("unexpected trailing input in type expression: '{rest}'"), + }); + } + Ok(expr) } /// Parse `single_type ('|' single_type)*`. @@ -89,7 +118,7 @@ pub fn parse_type_expr(s: &str) -> Result { /// when parsing inside `ARRAY<...>` / `SET<...>`). fn parse_union(chars: &[char], pos: &mut usize, stop_at_gt: bool) -> Result { let mut variants: Vec = Vec::new(); - variants.push(parse_single(chars, pos, stop_at_gt)?); + variants.push(parse_single(chars, pos)?); loop { skip_ws(chars, pos); @@ -104,7 +133,7 @@ fn parse_union(chars: &[char], pos: &mut usize, stop_at_gt: bool) -> Result Result, SET<...>, VECTOR(N)). -fn parse_single(chars: &[char], pos: &mut usize, stop_at_gt: bool) -> Result { +/// Parse a single type token: a keyword, `ARRAY<...>`, or `SET<...>`. +fn parse_single(chars: &[char], pos: &mut usize) -> Result { skip_ws(chars, pos); let keyword = read_keyword(chars, pos); if keyword.is_empty() { @@ -128,122 +157,155 @@ fn parse_single(chars: &[char], pos: &mut usize, stop_at_gt: bool) -> Result Ok(TypeExpr::Null), - "INT" | "INTEGER" | "BIGINT" | "INT64" => Ok(TypeExpr::Simple(SimpleType::Int)), - "FLOAT" | "DOUBLE" | "REAL" | "FLOAT64" => Ok(TypeExpr::Simple(SimpleType::Float)), - "STRING" | "TEXT" | "VARCHAR" => Ok(TypeExpr::Simple(SimpleType::String)), - "BOOL" | "BOOLEAN" => Ok(TypeExpr::Simple(SimpleType::Bool)), - "BYTES" | "BYTEA" | "BLOB" => Ok(TypeExpr::Simple(SimpleType::Bytes)), - "TIMESTAMP" => Ok(TypeExpr::Simple(SimpleType::Timestamp)), - "TIMESTAMPTZ" => Ok(TypeExpr::Simple(SimpleType::Timestamptz)), - "DECIMAL" | "NUMERIC" => Ok(TypeExpr::Simple(SimpleType::Decimal)), - "UUID" => Ok(TypeExpr::Simple(SimpleType::Uuid)), - "ULID" => Ok(TypeExpr::Simple(SimpleType::Ulid)), - "GEOMETRY" => Ok(TypeExpr::Simple(SimpleType::Geometry)), - "DURATION" => Ok(TypeExpr::Simple(SimpleType::Duration)), + // Typeguard-only keyword: a field holding a nested map. The shared + // declared-type parser has no `OBJECT` spelling, and the CONVERT path + // maps such a field to `ColumnType::Json`. "OBJECT" => Ok(TypeExpr::Simple(SimpleType::Object)), - "JSON" => Ok(TypeExpr::Simple(SimpleType::Json)), - "REGEX" => Ok(TypeExpr::Simple(SimpleType::Regex)), - "RANGE" => Ok(TypeExpr::Simple(SimpleType::Range)), - "RECORD" => Ok(TypeExpr::Simple(SimpleType::Record)), - - // Dimensionless — matched before the parameterized "VECTOR" block. - // `read_keyword` reads the whole "SPARSEVECTOR" token, so this exact - // arm claims it and the "VECTOR" `(N)` parser never sees it. - "SPARSEVECTOR" => Ok(TypeExpr::Simple(SimpleType::SparseVector)), - - "VECTOR" => { - // Expect '(' digits ')'. - skip_ws(chars, pos); - if *pos >= chars.len() || chars[*pos] != '(' { - return Err(SqlError::Parse { - detail: format!("expected '(' after VECTOR at position {pos}"), - }); - } - *pos += 1; // consume '(' - skip_ws(chars, pos); - let digits = read_digits(chars, pos); - if digits.is_empty() { - return Err(SqlError::Parse { - detail: "expected dimension digits inside VECTOR(...)".to_string(), - }); - } - let dim: u32 = digits.parse().map_err(|_| SqlError::Parse { - detail: format!("invalid VECTOR dimension: '{digits}'"), - })?; - if dim == 0 { - return Err(SqlError::Parse { - detail: "VECTOR dimension must be > 0".to_string(), - }); - } - skip_ws(chars, pos); - if *pos >= chars.len() || chars[*pos] != ')' { - return Err(SqlError::Parse { - detail: format!("expected ')' to close VECTOR({dim} at position {pos}"), - }); - } - *pos += 1; // consume ')' - Ok(TypeExpr::Simple(SimpleType::Vector(dim))) - } - "ARRAY" => { - // Optional typed variant: ARRAY + // Typed generics: the typeguard grammar types the element, which no + // declared DDL spelling does. Bare `ARRAY` / `SET` resolve as leaves. + "ARRAY" | "SET" => { skip_ws(chars, pos); if *pos < chars.len() && chars[*pos] == '<' { *pos += 1; // consume '<' skip_ws(chars, pos); - let inner = parse_union(chars, pos, true)?; + let inner = Box::new(parse_union(chars, pos, true)?); skip_ws(chars, pos); if *pos >= chars.len() || chars[*pos] != '>' { return Err(SqlError::Parse { - detail: format!("expected '>' to close ARRAY<...> at position {pos}"), + detail: format!("expected '>' to close {keyword}<...> at position {pos}"), }); } *pos += 1; // consume '>' - Ok(TypeExpr::TypedArray(Box::new(inner))) - } else { - Ok(TypeExpr::Simple(SimpleType::Array)) + return Ok(if keyword == "ARRAY" { + TypeExpr::TypedArray(inner) + } else { + TypeExpr::TypedSet(inner) + }); } + parse_leaf(chars, pos, &keyword) } - "SET" => { - // Optional typed variant: SET - skip_ws(chars, pos); - if *pos < chars.len() && chars[*pos] == '<' && !stop_at_gt { - *pos += 1; // consume '<' - skip_ws(chars, pos); - let inner = parse_union(chars, pos, true)?; - skip_ws(chars, pos); - if *pos >= chars.len() || chars[*pos] != '>' { - return Err(SqlError::Parse { - detail: format!("expected '>' to close SET<...> at position {pos}"), - }); - } - *pos += 1; // consume '>' - Ok(TypeExpr::TypedSet(Box::new(inner))) - } else if *pos < chars.len() && chars[*pos] == '<' { - // Inside a nested context — consume the '<' and inner normally. - *pos += 1; - skip_ws(chars, pos); - let inner = parse_union(chars, pos, true)?; - skip_ws(chars, pos); - if *pos >= chars.len() || chars[*pos] != '>' { - return Err(SqlError::Parse { - detail: format!("expected '>' to close SET<...> at position {pos}"), - }); + _ => parse_leaf(chars, pos, &keyword), + } +} + +/// Resolve one leaf spelling through the shared declared-type parser. +/// +/// `ColumnType` answers which type a declared spelling names for both planes, +/// so routing here is what keeps typeguard enforcement and typeguard CONVERT +/// from reading one spelling two ways. +fn parse_leaf(chars: &[char], pos: &mut usize, keyword: &str) -> Result { + let spelling = read_spelling(chars, pos, keyword)?; + let column_type: ColumnType = spelling.parse().map_err(|e| SqlError::Parse { + detail: format!("type '{spelling}': {e}"), + })?; + Ok(TypeExpr::Simple(simple_from_column_type( + column_type, + &spelling, + )?)) +} + +/// Read one declared type spelling: the leading keyword, the words that +/// continue it, and an attached parameter list. +/// +/// `TIMESTAMP WITH TIME ZONE` is one type name, so continuation words join +/// with single spaces and the shared parser resolves the whole spelling. A +/// word that continues no known spelling makes the spelling unknown, which is +/// how a trailing garbage word is rejected instead of ignored. +/// +/// A parameter list attaches to its keyword with no space, as +/// `DECIMAL(10, 2)` does. A `(` behind whitespace stays unread and the caller +/// reports it as trailing input. +fn read_spelling(chars: &[char], pos: &mut usize, keyword: &str) -> Result { + let mut spelling = keyword.to_string(); + loop { + if *pos < chars.len() && chars[*pos] == '(' { + spelling.push_str(&read_paren_group(chars, pos)?); + continue; + } + let mut probe = *pos; + skip_ws(chars, &mut probe); + if probe == *pos { + break; + } + let word = read_keyword(chars, &mut probe); + if word.is_empty() { + break; + } + spelling.push(' '); + spelling.push_str(&word); + *pos = probe; + } + Ok(spelling) +} + +/// Read a balanced parenthesized group, both parentheses included. +fn read_paren_group(chars: &[char], pos: &mut usize) -> Result { + let start = *pos; + let mut depth = 0usize; + let mut group = String::new(); + while *pos < chars.len() { + let ch = chars[*pos]; + group.push(ch); + *pos += 1; + match ch { + '(' => depth += 1, + ')' => { + depth = depth.saturating_sub(1); + if depth == 0 { + return Ok(group); } - *pos += 1; - Ok(TypeExpr::TypedSet(Box::new(inner))) - } else { - Ok(TypeExpr::Simple(SimpleType::Set)) } + _ => {} } - - other => Err(SqlError::Parse { - detail: format!("unknown type keyword: '{other}'"), - }), } + Err(SqlError::Parse { + detail: format!("unclosed '(' at position {start}"), + }) +} + +/// Map a resolved column type onto the typeguard leaf that validates it. +/// +/// `ColumnType` is `#[non_exhaustive]`: a variant added later reaches the +/// typed error arm until a leaf is written for it. +fn simple_from_column_type( + column_type: ColumnType, + spelling: &str, +) -> Result { + Ok(match column_type { + ColumnType::Int64 => SimpleType::Int, + ColumnType::Float64 => SimpleType::Float, + ColumnType::String => SimpleType::String, + ColumnType::Bool => SimpleType::Bool, + ColumnType::Bytes => SimpleType::Bytes, + ColumnType::Timestamp => SimpleType::Timestamp, + ColumnType::Timestamptz => SimpleType::Timestamptz, + ColumnType::SystemTimestamp => SimpleType::SystemTimestamp, + ColumnType::Decimal { precision, scale } => SimpleType::Decimal { precision, scale }, + ColumnType::Geometry => SimpleType::Geometry, + ColumnType::Vector(dim) => SimpleType::Vector(dim), + ColumnType::SparseVector => SimpleType::SparseVector, + ColumnType::Uuid => SimpleType::Uuid, + ColumnType::Json => SimpleType::Json, + ColumnType::Ulid => SimpleType::Ulid, + ColumnType::Duration => SimpleType::Duration, + ColumnType::Array => SimpleType::Array, + ColumnType::Set => SimpleType::Set, + ColumnType::Regex => SimpleType::Regex, + ColumnType::Range => SimpleType::Range, + ColumnType::Record => SimpleType::Record, + other => { + return Err(SqlError::Parse { + detail: format!( + "type '{spelling}' resolves to {other}, which no typeguard validates" + ), + }); + } + }) } fn skip_ws(chars: &[char], pos: &mut usize) { @@ -267,16 +329,6 @@ fn read_keyword(chars: &[char], pos: &mut usize) -> String { s } -/// Read contiguous ASCII digits. -fn read_digits(chars: &[char], pos: &mut usize) -> String { - let mut s = String::new(); - while *pos < chars.len() && chars[*pos].is_ascii_digit() { - s.push(chars[*pos]); - *pos += 1; - } - s -} - // ── Validator ──────────────────────────────────────────────────────────────── /// Check if a [`Value`] matches a [`TypeExpr`]. @@ -286,7 +338,8 @@ fn read_digits(chars: &[char], pos: &mut usize) -> String { /// - `Simple(Timestamp)` accepts `Value::DateTime`, `Value::Integer`, and /// `Value::String` (same rules as `ColumnType::Timestamp.accepts()`). /// - `Simple(Decimal)` accepts `Value::Decimal`, `Value::Float`, `Value::Integer`, -/// and `Value::String`. +/// and `Value::String`. Declared precision and scale do not narrow it. +/// - `Simple(SystemTimestamp)` accepts `Value::DateTime` and `Value::Integer`. /// - `Simple(Uuid)` accepts `Value::Uuid` and `Value::String`. /// - `Simple(Geometry)` accepts `Value::Geometry` and `Value::String`. /// - `TypedArray(inner)` matches `Value::Array` where every element matches `inner`. @@ -327,7 +380,10 @@ fn value_matches_simple(value: &Value, simple: &SimpleType) -> bool { value, Value::DateTime(_) | Value::Integer(_) | Value::String(_) ), - SimpleType::Decimal => matches!( + // Mirrors `ColumnType::SystemTimestamp.accepts`: an engine-assigned + // instant takes no text form. + SimpleType::SystemTimestamp => matches!(value, Value::DateTime(_) | Value::Integer(_)), + SimpleType::Decimal { .. } => matches!( value, Value::Decimal(_) | Value::Float(_) | Value::Integer(_) | Value::String(_) ), @@ -485,6 +541,105 @@ mod tests { ); } + /// Both zone spellings are one type name each, so the parser reads the + /// zone the author wrote instead of the leading word alone. + #[test] + fn parse_timestamp_zone_spellings() { + assert_eq!( + parse_type_expr("TIMESTAMP WITH TIME ZONE").unwrap(), + TypeExpr::Simple(SimpleType::Timestamptz) + ); + assert_eq!( + parse_type_expr("timestamp without time zone").unwrap(), + TypeExpr::Simple(SimpleType::Timestamp) + ); + } + + /// A word that continues no known spelling is an error. Reading it as the + /// leading keyword is what let one declaration mean two types. + #[test] + fn parse_error_trailing_words() { + assert!(parse_type_expr("TIMESTAMP GARBAGE WORDS").is_err()); + assert!(parse_type_expr("STRING FOO").is_err()); + assert!(parse_type_expr("TIMESTAMP WITH").is_err()); + assert!(parse_type_expr("INT )").is_err()); + assert!(parse_type_expr("VECTOR(3) EXTRA").is_err()); + } + + /// Every integer and float spelling the shared parser knows parses here. + /// A keyword added to either list extends this test, so a declaration the + /// rest of the database resolves cannot fail typeguard enforcement. + #[test] + fn parse_every_declared_numeric_keyword() { + use nodedb_types::columnar::{DECLARED_FLOAT_KEYWORDS, DECLARED_INT_KEYWORDS}; + for keyword in DECLARED_INT_KEYWORDS { + assert_eq!( + parse_type_expr(keyword).unwrap(), + TypeExpr::Simple(SimpleType::Int), + "{keyword} must parse as an integer leaf" + ); + } + for keyword in DECLARED_FLOAT_KEYWORDS { + assert_eq!( + parse_type_expr(keyword).unwrap(), + TypeExpr::Simple(SimpleType::Float), + "{keyword} must parse as a float leaf" + ); + } + } + + #[test] + fn parse_jsonb() { + assert_eq!( + parse_type_expr("JSONB").unwrap(), + TypeExpr::Simple(SimpleType::Json) + ); + } + + #[test] + fn parse_decimal_carries_declared_params() { + assert_eq!( + parse_type_expr("DECIMAL(10, 2)").unwrap(), + TypeExpr::Simple(SimpleType::Decimal { + precision: 10, + scale: 2 + }) + ); + assert_eq!( + parse_type_expr("NUMERIC").unwrap(), + TypeExpr::Simple(SimpleType::Decimal { + precision: 38, + scale: 10 + }) + ); + assert!(parse_type_expr("DECIMAL(0)").is_err()); + } + + /// A declared character length resolves to the same leaf bare `VARCHAR` + /// gives, and matching stays a string check. + #[test] + fn parse_varchar_length() { + let expr = parse_type_expr("VARCHAR(255)").unwrap(); + assert_eq!(expr, TypeExpr::Simple(SimpleType::String)); + assert!(value_matches_type(&Value::String("x".into()), &expr)); + assert!(parse_type_expr("VARCHAR(0)").is_err()); + } + + /// `SYSTEM_TIMESTAMP` is engine-assigned, so it takes an instant and no + /// text — the rule `ColumnType::SystemTimestamp` enforces. + #[test] + fn parse_and_match_system_timestamp() { + let expr = parse_type_expr("SYSTEM_TIMESTAMP").unwrap(); + assert_eq!(expr, TypeExpr::Simple(SimpleType::SystemTimestamp)); + let dt = nodedb_types::NdbDateTime::from_micros(1_700_000_000_000_000); + assert!(value_matches_type(&Value::DateTime(dt), &expr)); + assert!(value_matches_type(&Value::Integer(1_700_000_000), &expr)); + assert!(!value_matches_type( + &Value::String("2024-01-01".into()), + &expr + )); + } + #[test] fn parse_error_unknown_keyword() { assert!(parse_type_expr("FOOBAR").is_err()); diff --git a/nodedb-types/src/columnar/column_parse.rs b/nodedb-types/src/columnar/column_parse.rs index af1cf3095..b184352da 100644 --- a/nodedb-types/src/columnar/column_parse.rs +++ b/nodedb-types/src/columnar/column_parse.rs @@ -48,6 +48,8 @@ pub enum ColumnTypeParseError { UseTimestamp, #[error("invalid VECTOR dimension: '{0}' (must be a positive integer)")] InvalidVectorDim(String), + #[error("invalid VARCHAR length: '{0}' (must be a positive integer)")] + InvalidCharLength(String), #[error( "invalid DECIMAL/NUMERIC params: '{0}' (expected DECIMAL(precision, scale) with precision 1-38 and scale <= precision)" )] @@ -150,6 +152,24 @@ impl FromStr for ColumnType { return Ok(Self::Vector(dim)); } + // VARCHAR(n) special case. A declared character length is a wire + // contract, not a storage property — nodedb stores every string + // unbounded — so `VARCHAR(255)` resolves to the answer bare `VARCHAR` + // gives. The length still parses, so a malformed one errors here + // instead of resolving to a type the author did not write. + if let Some(rest) = upper.strip_prefix("VARCHAR") + && let Some(inner) = rest.strip_prefix('(').and_then(|r| r.strip_suffix(')')) + { + let length: u32 = inner + .trim() + .parse() + .map_err(|_| ColumnTypeParseError::InvalidCharLength(inner.trim().to_string()))?; + if length == 0 { + return Err(ColumnTypeParseError::InvalidCharLength("0".into())); + } + return Ok(Self::String); + } + match upper.as_str() { // Every PostgreSQL wire-width integer keyword collapses to the // one `Int64` storage variant; `DECLARED_INT_KEYWORDS` lists them @@ -160,7 +180,7 @@ impl FromStr for ColumnType { "TEXT" | "STRING" | "VARCHAR" => Ok(Self::String), "BOOL" | "BOOLEAN" => Ok(Self::Bool), "BYTES" | "BYTEA" | "BLOB" => Ok(Self::Bytes), - "TIMESTAMP" => Ok(Self::Timestamp), + "TIMESTAMP" | "TIMESTAMP WITHOUT TIME ZONE" => Ok(Self::Timestamp), "TIMESTAMPTZ" | "TIMESTAMP WITH TIME ZONE" => Ok(Self::Timestamptz), "SYSTEM_TIMESTAMP" | "SYSTEMTIMESTAMP" => Ok(Self::SystemTimestamp), "GEOMETRY" => Ok(Self::Geometry), @@ -308,6 +328,40 @@ mod tests { ); } + /// The two zone spellings resolve whole, so a parser that reads the full + /// string reads the zone the author wrote. + #[test] + fn both_timestamp_zone_spellings_resolve_whole() { + assert_eq!( + "TIMESTAMP WITH TIME ZONE".parse::(), + Ok(ColumnType::Timestamptz) + ); + assert_eq!( + "timestamp without time zone".parse::(), + Ok(ColumnType::Timestamp) + ); + } + + /// A declared character length resolves to `String` and a malformed one + /// errors rather than resolving to an unwritten type. + #[test] + fn varchar_length_resolves_to_string() { + assert_eq!("VARCHAR(255)".parse::(), Ok(ColumnType::String)); + assert_eq!("varchar(1)".parse::(), Ok(ColumnType::String)); + assert_eq!( + ColumnType::from_declared_type("VARCHAR(8) NOT NULL"), + Some(ColumnType::String) + ); + assert_eq!( + "VARCHAR(0)".parse::(), + Err(ColumnTypeParseError::InvalidCharLength("0".into())) + ); + assert_eq!( + "VARCHAR(abc)".parse::(), + Err(ColumnTypeParseError::InvalidCharLength("ABC".into())) + ); + } + #[test] fn declared_type_reports_an_unknown_token_as_none() { assert_eq!(ColumnType::from_declared_type("SOMETHING_ELSE"), None); diff --git a/nodedb/src/control/server/shared/ddl/neutral/collection/create/build.rs b/nodedb/src/control/server/shared/ddl/neutral/collection/create/build.rs index 2c81900df..a3ed56209 100644 --- a/nodedb/src/control/server/shared/ddl/neutral/collection/create/build.rs +++ b/nodedb/src/control/server/shared/ddl/neutral/collection/create/build.rs @@ -34,10 +34,10 @@ use super::super::enforcement::{ use super::engine_option::validate_engine_name; use super::request::CreateCollectionRequest; -use super::build_column_defaults::validate_column_defaults; use super::build_flags::{err, resolve_crdt_flag, validate_crdt_signing_storage, validate_name}; use super::build_post_create::{create_serial_sequences, log_vector_fields}; use super::build_primary_engine::resolve_primary_engine; +use crate::control::server::shared::ddl::neutral::column_default::validate_column_defaults; /// Per-surface configuration. The fields are the entire surface-level /// difference between `CREATE COLLECTION` and `CREATE TABLE`. diff --git a/nodedb/src/control/server/shared/ddl/neutral/collection/create/build_column_defaults.rs b/nodedb/src/control/server/shared/ddl/neutral/collection/create/build_column_defaults.rs deleted file mode 100644 index ae201f2d1..000000000 --- a/nodedb/src/control/server/shared/ddl/neutral/collection/create/build_column_defaults.rs +++ /dev/null @@ -1,49 +0,0 @@ -// SPDX-License-Identifier: BUSL-1.1 - -//! DDL-time gate for column `DEFAULT` expressions. -//! -//! A `DEFAULT` clause is stored as raw text on the column type string and is -//! evaluated only when an INSERT omits the column. Without this gate a -//! `CREATE` accepts a call to a function that does not exist, and the author -//! learns about it at the first insert instead of at the declaration. -//! -//! The check runs `nodedb_sql`'s DEFAULT classifier, which consults the same -//! `FunctionRegistry` the resolver's undefined-function gate consults. No -//! second registry and no second name list exist here. - -use nodedb_sql::SqlError; -use nodedb_sql::ddl_ast::collection_type::parse_column_type_str_full; -use nodedb_types::error::sqlstate; - -use super::super::super::super::result::DdlError; - -/// Refuse a declared column `DEFAULT` the server cannot evaluate. -/// -/// The expression is classified and parsed, never evaluated, so a -/// `DEFAULT nextval('s')` column never advances its sequence at `CREATE`. -/// -/// An unregistered function name raises SQLSTATE `42883`; every other -/// rejection raises SQLSTATE `42601`. -pub(super) fn validate_column_defaults(columns: &[(String, String)]) -> Result<(), DdlError> { - for (column, type_str) in columns { - let (_, _, _, default_expr) = parse_column_type_str_full(type_str); - let Some(expr) = default_expr else { - continue; - }; - nodedb_sql::planner::defaults::validate_default_expr(&expr, column) - .map_err(|error| default_error(column, &error))?; - } - Ok(()) -} - -/// Map a DEFAULT validation error onto its SQLSTATE. -fn default_error(column: &str, error: &SqlError) -> DdlError { - let sqlstate = match error { - SqlError::UndefinedFunction { .. } => sqlstate::UNDEFINED_FUNCTION, - _ => sqlstate::SYNTAX_ERROR, - }; - DdlError::new( - sqlstate, - format!("DEFAULT for column '{column}' is invalid: {error}"), - ) -} diff --git a/nodedb/src/control/server/shared/ddl/neutral/collection/create/mod.rs b/nodedb/src/control/server/shared/ddl/neutral/collection/create/mod.rs index ee032daf9..c59a645ad 100644 --- a/nodedb/src/control/server/shared/ddl/neutral/collection/create/mod.rs +++ b/nodedb/src/control/server/shared/ddl/neutral/collection/create/mod.rs @@ -4,7 +4,6 @@ //! //! Relocated from `pgwire::ddl::collection::create` (now deleted): //! - [`build`] — the shared `build_and_persist` body + `Variant` -//! - [`build_column_defaults`] — DDL-time column `DEFAULT` gate for `build` //! - [`build_flags`] — name / flag validation for `build` //! - [`build_primary_engine`] — vector-primary resolution for `build` //! - [`build_post_create`] — post-create side effects for `build` @@ -13,14 +12,12 @@ //! - [`table`] — the `create_table` entry point //! - [`request`] — `CreateCollectionRequest` //! -//! `build_column_defaults`, `build_flags`, `build_primary_engine`, and -//! `build_post_create` are +//! `build_flags`, `build_primary_engine`, and `build_post_create` are //! internal to [`build`] — declared here (siblings must be declared by the //! parent module, not by `build` itself) but scoped no wider than `build` //! needs them. pub mod build; -pub(in crate::control::server::shared::ddl::neutral::collection::create) mod build_column_defaults; pub(in crate::control::server::shared::ddl::neutral::collection::create) mod build_flags; pub(in crate::control::server::shared::ddl::neutral::collection::create) mod build_post_create; pub(in crate::control::server::shared::ddl::neutral::collection::create) mod build_primary_engine; diff --git a/nodedb/src/control/server/shared/ddl/neutral/column_default.rs b/nodedb/src/control/server/shared/ddl/neutral/column_default.rs new file mode 100644 index 000000000..390409a66 --- /dev/null +++ b/nodedb/src/control/server/shared/ddl/neutral/column_default.rs @@ -0,0 +1,70 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! DDL-time gate for declared value-producing clauses. +//! +//! A column `DEFAULT`, a typeguard `DEFAULT` and a typeguard `VALUE` all pass +//! through here, so one registry decides which function names a declaration +//! can name. +//! +//! A `DEFAULT` clause is stored as raw text on the column and is evaluated +//! only when an INSERT omits the column. Without this gate a `CREATE` or a +//! `CONVERT` accepts a call to a function that does not exist, and the author +//! learns about it at the first insert instead of at the declaration. +//! +//! The check runs `nodedb_sql`'s DEFAULT classifier, which consults the same +//! `FunctionRegistry` the resolver's undefined-function gate consults. No +//! second registry and no second name list exist here. + +use nodedb_sql::SqlError; +use nodedb_sql::ddl_ast::collection_type::parse_column_type_str_full; +use nodedb_types::error::sqlstate; + +use super::super::result::DdlError; + +/// Refuse a declared column `DEFAULT` the server cannot evaluate. +/// +/// Each pair carries a column name and its declared type text, and the +/// `DEFAULT` clause is read out of that text. +pub(super) fn validate_column_defaults(columns: &[(String, String)]) -> Result<(), DdlError> { + for (column, type_str) in columns { + let (_, _, _, default_expr) = parse_column_type_str_full(type_str); + let Some(expr) = default_expr else { + continue; + }; + validate_column_default(column, &expr)?; + } + Ok(()) +} + +/// Refuse one declared column `DEFAULT` the server cannot evaluate. +pub(super) fn validate_column_default(column: &str, expr: &str) -> Result<(), DdlError> { + validate_clause_expr("DEFAULT", column, expr) +} + +/// Refuse one declared value-producing clause the server cannot evaluate. +/// +/// `clause` names the keyword the author wrote and `owner` the column or field +/// carrying it, so a typeguard `VALUE` reports itself rather than borrowing the +/// `DEFAULT` wording. +/// +/// The expression is classified and parsed, never evaluated, so a +/// `DEFAULT nextval('s')` column never advances its sequence at DDL time. +/// +/// An unregistered function name raises SQLSTATE `42883`; every other +/// rejection raises SQLSTATE `42601`. +pub(super) fn validate_clause_expr(clause: &str, owner: &str, expr: &str) -> Result<(), DdlError> { + nodedb_sql::planner::defaults::validate_default_expr(expr, owner) + .map_err(|error| clause_error(clause, owner, &error)) +} + +/// Map a clause validation error onto its SQLSTATE. +fn clause_error(clause: &str, owner: &str, error: &SqlError) -> DdlError { + let sqlstate = match error { + SqlError::UndefinedFunction { .. } => sqlstate::UNDEFINED_FUNCTION, + _ => sqlstate::SYNTAX_ERROR, + }; + DdlError::new( + sqlstate, + format!("{clause} for '{owner}' is invalid: {error}"), + ) +} diff --git a/nodedb/src/control/server/shared/ddl/neutral/convert.rs b/nodedb/src/control/server/shared/ddl/neutral/convert.rs deleted file mode 100644 index 46c218ba8..000000000 --- a/nodedb/src/control/server/shared/ddl/neutral/convert.rs +++ /dev/null @@ -1,515 +0,0 @@ -// SPDX-License-Identifier: BUSL-1.1 - -//! Protocol-neutral DDL handler for CONVERT COLLECTION. -//! -//! Syntax: -//! - `CONVERT COLLECTION TO document` -//! - `CONVERT COLLECTION TO strict (col1 TYPE, col2 TYPE, ...)` -//! - `CONVERT COLLECTION TO kv` -//! -//! Ported from the pgwire `ddl::convert` handler. The accepted-target -//! validation (document_schemaless / document_strict / kv — columnar / -//! timeseries / spatial rejected), the catalog read + write, the Data Plane -//! conversion dispatch, and the typeguard → CHECK-constraint carry-over are -//! preserved verbatim; only the result construction changed from pgwire -//! `Response` / `PgWireError` to the protocol-neutral [`DdlResult`] / -//! [`DdlError`]. - -use nodedb_sql::parser::preprocess::lex::{ - find_ascii_case_insensitive, find_ascii_case_insensitive_from, -}; -use nodedb_types::DatabaseId; -use std::time::Duration; - -use sonic_rs; - -use crate::bridge::envelope::PhysicalPlan; -use crate::control::catalog_entry::persist_collection_replicated; -use crate::control::security::identity::AuthenticatedIdentity; -use crate::control::server::shared::ddl::sql_parse::parse_ident_token; -use crate::control::server::shared::ddl::sync_dispatch::{ - SystemReason, SystemTask, dispatch_system, -}; -use crate::control::state::SharedState; -use nodedb_physical::physical_plan::MetaOp; - -use super::super::result::{DdlError, DdlResult}; - -fn err(sqlstate: &str, message: &str) -> DdlError { - DdlError::new(sqlstate, message) -} - -/// CONVERT COLLECTION TO [()] -pub async fn convert_collection( - state: &SharedState, - identity: &AuthenticatedIdentity, - database_id: DatabaseId, - sql: &str, -) -> Result, DdlError> { - let (collection, target_type, explicit_columns) = parse_convert_sql(sql)?; - let tenant_id = identity.tenant_id; - - // Validate collection exists. - let catalog = state.credentials.catalog(); - - let mut coll = catalog - .get_collection(database_id, tenant_id.as_u64(), &collection) - .map_err(|e| err("XX000", &e.to_string()))? - .ok_or_else(|| { - err( - "42P01", - &format!("collection '{collection}' does not exist"), - ) - })?; - - // Build columns before dispatch — needed for both Data Plane and catalog. - let columns: Option> = match target_type.as_str() { - "document_strict" | "kv" => { - let cols = if let Some(cols) = explicit_columns { - cols - } else if !coll.type_guards.is_empty() { - typeguards_to_column_defs(&coll.type_guards)? - } else { - return Err(err( - "42601", - "CONVERT TO strict requires column definitions or active typeguards", - )); - }; - Some(cols) - } - _ => None, - }; - - let schema_json_for_dp = if let Some(ref cols) = columns { - sonic_rs::to_string(cols) - .map_err(|e| err("XX000", &format!("schema serialization: {e}")))? - } else { - String::new() - }; - - // Dispatch to Data Plane: re-encode if needed (strict = Binary Tuple). - let plan = PhysicalPlan::Meta(MetaOp::ConvertCollection { - collection: nodedb_types::QualifiedCollection::new(database_id, &collection), - target_type: target_type.clone(), - schema_json: schema_json_for_dp, - }); - - dispatch_system( - state, - SystemTask::new( - SystemReason::DdlApply, - tenant_id, - database_id, - &collection, - plan, - ), - Duration::from_secs(60), - ) - .await - .map_err(|e| err("XX000", &format!("conversion failed: {e}")))?; - - // Update catalog collection type. - let new_type = match target_type.as_str() { - "document_schemaless" => nodedb_types::CollectionType::document(), - "document_strict" | "kv" => { - let columns = columns.expect( - "invariant: columns is Some for document_strict/kv targets, validated above", - ); - let schema = nodedb_types::columnar::StrictSchema { - columns, - version: 1, - dropped_columns: Vec::new(), - bitemporal: false, - }; - if target_type == "kv" { - nodedb_types::CollectionType::kv(schema) - } else { - nodedb_types::CollectionType::strict(schema) - } - } - _ => { - return Err(err( - "42601", - &format!("unsupported target type: {target_type}"), - )); - } - }; - - coll.collection_type = new_type; - - // CONVERT TO document_strict: if collection had typeguards, carry over CHECK constraints - // and drop typeguard definitions (strict schema subsumes type checking). - if target_type == "document_strict" && !coll.type_guards.is_empty() { - for guard in &coll.type_guards { - if let Some(ref check_expr) = guard.check_expr { - // Avoid duplicate names. - let name = format!("_guard_{}", guard.field); - if !coll.check_constraints.iter().any(|c| c.name == name) { - coll.check_constraints.push( - crate::control::security::catalog::types::CheckConstraintDef { - name, - check_sql: check_expr.clone(), - has_subquery: false, - }, - ); - } - } - } - coll.type_guards.clear(); - } - - persist_collection_replicated(state, database_id, &coll) - .map_err(|e| err("XX000", &e.to_string()))?; - - tracing::info!( - %collection, - target_type, - tenant = tenant_id.as_u64(), - "collection converted" - ); - - Ok(vec![DdlResult::Status { - command: "CONVERT COLLECTION".to_string(), - rows_affected: None, - }]) -} - -// ── SQL Parsing ────────────────────────────────────────────────────────── - -/// Parse CONVERT COLLECTION SQL. -/// -/// Returns `(collection_name, target_type, explicit_columns)`. -/// `explicit_columns` is `None` for `TO document` or `TO strict` without parens. -fn parse_convert_sql( - sql: &str, -) -> Result< - ( - String, - String, - Option>, - ), - DdlError, -> { - // Extract collection name: CONVERT COLLECTION TO ... - let coll_pos = find_ascii_case_insensitive(sql, "COLLECTION ") - .ok_or_else(|| err("42601", "expected COLLECTION keyword"))?; - let after_coll = sql[coll_pos + 11..].trim_start(); - let collection = parse_ident_token( - after_coll - .split_whitespace() - .next() - .ok_or_else(|| err("42601", "missing collection name"))?, - )?; - - // Extract target type: TO - let to_pos = find_ascii_case_insensitive_from(sql, " TO ", coll_pos + 11) - .ok_or_else(|| err("42601", "expected TO clause"))?; - let after_to = sql[to_pos + 4..].trim_start(); - let target_type = after_to - .split_whitespace() - .next() - .ok_or_else(|| err("42601", "missing target type after TO"))? - .to_lowercase() - .trim_matches('(') - .to_string(); - - match target_type.as_str() { - "document_schemaless" => Ok((collection, "document_schemaless".into(), None)), - "document_strict" | "kv" => { - if after_to.contains('(') { - let cols = parse_column_defs(after_to)?; - Ok((collection, target_type, Some(cols))) - } else { - Ok((collection, target_type, None)) - } - } - "document" | "doc" => Err(err( - "42601", - "deprecated target type 'document'; use 'document_schemaless'", - )), - "strict" => Err(err( - "42601", - "deprecated target type 'strict'; use 'document_strict'", - )), - "key_value" | "keyvalue" => Err(err("42601", "deprecated target type; use 'kv'")), - other => Err(err( - "42601", - &format!( - "unsupported target type: '{other}' \ - (expected document_schemaless, document_strict, kv)" - ), - )), - } -} - -/// Parse `(col1 TYPE, col2 TYPE, ...)` into `Vec`. -fn parse_column_defs(s: &str) -> Result, DdlError> { - use nodedb_types::columnar::ColumnDef; - - let open = s - .find('(') - .ok_or_else(|| err("42601", "expected (column definitions) after type"))?; - let close = s - .rfind(')') - .ok_or_else(|| err("42601", "missing closing parenthesis"))?; - if close <= open { - return Err(err("42601", "empty column definitions")); - } - - let inner = &s[open + 1..close]; - let mut columns: Vec = Vec::new(); - - for part in inner.split(',') { - let part = part.trim(); - if part.is_empty() { - continue; - } - let tokens: Vec<&str> = part.split_whitespace().collect(); - if tokens.len() < 2 { - return Err(err( - "42601", - &format!("expected 'name TYPE' in column def: {part}"), - )); - } - let col_name = parse_ident_token(tokens[0])?; - let col_type = tokens[1].to_uppercase(); - let nullable = !tokens - .windows(2) - .any(|w| w[0].eq_ignore_ascii_case("NOT") && w[1].eq_ignore_ascii_case("NULL")) - && !tokens.iter().any(|t| t.eq_ignore_ascii_case("NOTNULL")); - let primary_key = tokens - .windows(2) - .any(|w| w[0].eq_ignore_ascii_case("PRIMARY") && w[1].eq_ignore_ascii_case("KEY")); - - let ct = sql_type_to_column_type(&col_type); - let mut col = if nullable { - ColumnDef::nullable(col_name, ct) - } else { - ColumnDef::required(col_name, ct) - }; - if primary_key { - col = col.with_primary_key(); - } - columns.push(col); - } - - if columns.is_empty() { - return Err(err("42601", "at least one column required")); - } - - Ok(columns) -} - -/// Map SQL type names to ColumnType. -fn sql_type_to_column_type(sql_type: &str) -> nodedb_types::columnar::ColumnType { - use nodedb_types::columnar::ColumnType; - match sql_type { - "INT" | "INTEGER" | "INT8" | "BIGINT" | "INT4" | "INT2" | "SMALLINT" => ColumnType::Int64, - "FLOAT" | "FLOAT8" | "DOUBLE" | "REAL" => ColumnType::Float64, - "NUMERIC" | "DECIMAL" => ColumnType::Decimal { - precision: 38, - scale: 10, - }, - "BOOL" | "BOOLEAN" => ColumnType::Bool, - "TIMESTAMP" | "TIMESTAMP WITHOUT TIME ZONE" => ColumnType::Timestamp, - "TIMESTAMPTZ" | "TIMESTAMP WITH TIME ZONE" => ColumnType::Timestamptz, - "BLOB" | "BYTEA" | "BINARY" => ColumnType::Bytes, - "UUID" => ColumnType::Uuid, - "JSON" | "JSONB" => ColumnType::Json, - "SPARSEVECTOR" => ColumnType::SparseVector, - "GEOMETRY" => ColumnType::Geometry, - _ => ColumnType::String, // VARCHAR, TEXT, STRING, and anything else - } -} - -/// Convert typeguard field definitions to strict schema column definitions. -/// -/// Each typeguard field becomes a column. The type expression is mapped to ColumnType. -/// REQUIRED fields become NOT NULL. DEFAULT expressions carry over. -fn typeguards_to_column_defs( - guards: &[nodedb_types::TypeGuardFieldDef], -) -> Result, DdlError> { - use nodedb_types::columnar::{ColumnDef, ColumnType}; - - // Always include an `id` column as primary key. - let mut columns = vec![ColumnDef::required("id", ColumnType::String).with_primary_key()]; - - for guard in guards { - // Skip dot-path fields (nested) — strict schema is flat. - if guard.field.contains('.') { - continue; - } - // Skip if already added (e.g., "id"). - if guard.field == "id" || columns.iter().any(|c| c.name == guard.field) { - continue; - } - - let ct = typeguard_type_to_column_type(&guard.type_expr); - let mut col = if guard.required { - ColumnDef::required(guard.field.clone(), ct) - } else { - ColumnDef::nullable(guard.field.clone(), ct) - }; - col.default = guard.default_expr.clone().or(guard.value_expr.clone()); - columns.push(col); - } - - Ok(columns) -} - -/// Map a typeguard type expression string to a ColumnType. -fn typeguard_type_to_column_type(type_expr: &str) -> nodedb_types::columnar::ColumnType { - use nodedb_types::columnar::ColumnType; - - // Strip union types — take the first non-NULL type. - let base = type_expr - .split('|') - .map(|s| s.trim()) - .find(|s| !s.eq_ignore_ascii_case("NULL")) - .unwrap_or(type_expr) - .to_uppercase(); - - // Strip generic parameters: ARRAY → ARRAY, SET → SET. - let base = base.split('<').next().unwrap_or(&base); - - match base { - "INT" | "INTEGER" | "BIGINT" | "INT64" => ColumnType::Int64, - "FLOAT" | "DOUBLE" | "REAL" | "FLOAT64" => ColumnType::Float64, - "STRING" | "TEXT" | "VARCHAR" => ColumnType::String, - "BOOL" | "BOOLEAN" => ColumnType::Bool, - "BYTES" | "BYTEA" | "BLOB" => ColumnType::Bytes, - "TIMESTAMP" | "TIMESTAMP WITHOUT TIME ZONE" => ColumnType::Timestamp, - "TIMESTAMPTZ" | "TIMESTAMP WITH TIME ZONE" => ColumnType::Timestamptz, - "DECIMAL" | "NUMERIC" => ColumnType::Decimal { - precision: 38, - scale: 10, - }, - "UUID" => ColumnType::Uuid, - "GEOMETRY" => ColumnType::Geometry, - "JSON" | "JSONB" | "OBJECT" => ColumnType::Json, - "SPARSEVECTOR" => ColumnType::SparseVector, - _ => ColumnType::String, // fallback - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn parse_convert_to_document_schemaless() { - let (coll, target, cols) = - parse_convert_sql("CONVERT COLLECTION users TO document_schemaless").unwrap(); - assert_eq!(coll, "users"); - assert_eq!(target, "document_schemaless"); - assert!(cols.is_none()); - } - - #[test] - fn parse_convert_deprecated_document_rejected() { - assert!(parse_convert_sql("CONVERT COLLECTION users TO document").is_err()); - } - - #[test] - fn parse_convert_to_document_strict() { - let sql = - "CONVERT COLLECTION users TO document_strict (name VARCHAR, age INT, active BOOLEAN)"; - let (coll, target, cols) = parse_convert_sql(sql).unwrap(); - assert_eq!(coll, "users"); - assert_eq!(target, "document_strict"); - - let cols = cols.unwrap(); - assert_eq!(cols.len(), 3); - assert_eq!(cols[0].name, "name"); - assert!(matches!( - cols[0].column_type, - nodedb_types::columnar::ColumnType::String - )); - assert_eq!(cols[1].name, "age"); - assert!(matches!( - cols[1].column_type, - nodedb_types::columnar::ColumnType::Int64 - )); - assert_eq!(cols[2].name, "active"); - assert!(matches!( - cols[2].column_type, - nodedb_types::columnar::ColumnType::Bool - )); - } - - #[test] - fn parse_convert_deprecated_strict_rejected() { - let sql = "CONVERT COLLECTION users TO strict (name VARCHAR)"; - assert!(parse_convert_sql(sql).is_err()); - } - - #[test] - fn parse_convert_to_kv() { - let sql = "CONVERT COLLECTION cache TO kv (key VARCHAR, value BLOB)"; - let (coll, target, cols) = parse_convert_sql(sql).unwrap(); - assert_eq!(coll, "cache"); - assert_eq!(target, "kv"); - assert!(cols.is_some()); - } - - #[test] - fn parse_convert_not_null_constraint() { - let sql = "CONVERT COLLECTION users TO document_strict (id INT NOT NULL, name VARCHAR)"; - let (_, _, cols) = parse_convert_sql(sql).unwrap(); - let cols = cols.unwrap(); - assert!(!cols[0].nullable); - assert!(cols[1].nullable); - } - - #[test] - fn parse_convert_missing_to_errors() { - assert!(parse_convert_sql("CONVERT COLLECTION users").is_err()); - } - - #[test] - fn parse_convert_unknown_type_errors() { - assert!(parse_convert_sql("CONVERT COLLECTION users TO graph").is_err()); - } - - #[test] - fn sql_type_to_column_type_distinguishes_timestamp_from_timestamptz() { - use nodedb_types::columnar::ColumnType; - assert!(matches!( - sql_type_to_column_type("TIMESTAMP"), - ColumnType::Timestamp - )); - assert!(matches!( - sql_type_to_column_type("TIMESTAMP WITHOUT TIME ZONE"), - ColumnType::Timestamp - )); - assert!(matches!( - sql_type_to_column_type("TIMESTAMPTZ"), - ColumnType::Timestamptz - )); - assert!(matches!( - sql_type_to_column_type("TIMESTAMP WITH TIME ZONE"), - ColumnType::Timestamptz - )); - } - - #[test] - fn typeguard_type_to_column_type_distinguishes_timestamp_from_timestamptz() { - use nodedb_types::columnar::ColumnType; - assert!(matches!( - typeguard_type_to_column_type("TIMESTAMP"), - ColumnType::Timestamp - )); - assert!(matches!( - typeguard_type_to_column_type("TIMESTAMP WITHOUT TIME ZONE"), - ColumnType::Timestamp - )); - assert!(matches!( - typeguard_type_to_column_type("TIMESTAMPTZ"), - ColumnType::Timestamptz - )); - assert!(matches!( - typeguard_type_to_column_type("TIMESTAMP WITH TIME ZONE"), - ColumnType::Timestamptz - )); - } -} diff --git a/nodedb/src/control/server/shared/ddl/neutral/convert/column_defs.rs b/nodedb/src/control/server/shared/ddl/neutral/convert/column_defs.rs new file mode 100644 index 000000000..f72364ef9 --- /dev/null +++ b/nodedb/src/control/server/shared/ddl/neutral/convert/column_defs.rs @@ -0,0 +1,373 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! CONVERT COLLECTION statement text parsed into a target type and columns. +//! +//! A column definition is `name TYPE [NOT NULL] [PRIMARY KEY] [DEFAULT expr]`. +//! The `DEFAULT` clause is validated here and stored on the created column, so +//! a later insert that omits the column takes its value. + +use nodedb_sql::ddl_ast::collection_type::parse_column_type_str_full; +use nodedb_sql::parser::preprocess::lex::{ + find_ascii_case_insensitive, find_ascii_case_insensitive_from, +}; + +use crate::control::server::shared::ddl::sql_parse::{parse_ident_token, split_values}; + +use super::super::super::result::DdlError; +use super::super::column_default::validate_column_default; +use super::support::err; +use super::type_map::sql_type_to_column_type; + +/// Parse CONVERT COLLECTION SQL. +/// +/// Returns `(collection_name, target_type, explicit_columns)`. +/// `explicit_columns` is `None` for `TO document` or `TO strict` without parens. +pub(super) fn parse_convert_sql( + sql: &str, +) -> Result< + ( + String, + String, + Option>, + ), + DdlError, +> { + // Extract collection name: CONVERT COLLECTION TO ... + let coll_pos = find_ascii_case_insensitive(sql, "COLLECTION ") + .ok_or_else(|| err("42601", "expected COLLECTION keyword"))?; + let after_coll = sql[coll_pos + 11..].trim_start(); + let collection = parse_ident_token( + after_coll + .split_whitespace() + .next() + .ok_or_else(|| err("42601", "missing collection name"))?, + )?; + + // Extract target type: TO + let to_pos = find_ascii_case_insensitive_from(sql, " TO ", coll_pos + 11) + .ok_or_else(|| err("42601", "expected TO clause"))?; + let after_to = sql[to_pos + 4..].trim_start(); + let target_type = after_to + .split_whitespace() + .next() + .ok_or_else(|| err("42601", "missing target type after TO"))? + .to_lowercase() + .trim_matches('(') + .to_string(); + + match target_type.as_str() { + "document_schemaless" => Ok((collection, "document_schemaless".into(), None)), + "document_strict" | "kv" => { + if after_to.contains('(') { + let cols = parse_column_defs(after_to)?; + Ok((collection, target_type, Some(cols))) + } else { + Ok((collection, target_type, None)) + } + } + "document" | "doc" => Err(err( + "42601", + "deprecated target type 'document'; use 'document_schemaless'", + )), + "strict" => Err(err( + "42601", + "deprecated target type 'strict'; use 'document_strict'", + )), + "key_value" | "keyvalue" => Err(err("42601", "deprecated target type; use 'kv'")), + other => Err(err( + "42601", + format!( + "unsupported target type: '{other}' \ + (expected document_schemaless, document_strict, kv)" + ), + )), + } +} + +/// Words that end a type spelling in a CONVERT column definition. +const COLUMN_MODIFIER_KEYWORDS: [&str; 6] = ["NOT", "NULL", "NOTNULL", "PRIMARY", "KEY", "DEFAULT"]; + +/// Parse `(col1 TYPE, col2 TYPE, ...)` into `Vec`. +fn parse_column_defs(s: &str) -> Result, DdlError> { + use nodedb_types::columnar::ColumnDef; + + let open = s + .find('(') + .ok_or_else(|| err("42601", "expected (column definitions) after type"))?; + let close = s + .rfind(')') + .ok_or_else(|| err("42601", "missing closing parenthesis"))?; + if close <= open { + return Err(err("42601", "empty column definitions")); + } + + let inner = &s[open + 1..close]; + let mut columns: Vec = Vec::new(); + + // The list splits on top-level commas, so a parameter list keeps its own + // comma and `amount DECIMAL(10, 2)` stays one column definition. + for part in split_values(inner) { + let part = part.trim(); + if part.is_empty() { + continue; + } + let name_token = part.split_whitespace().next().ok_or_else(|| { + err( + "42601", + format!("expected 'name TYPE' in column def: {part}"), + ) + })?; + let (head, tail, default_expr) = split_column_default(part, name_token)?; + // The modifier scan reads the definition without its DEFAULT + // expression, so `NOT NULL` written after the clause still lands. + let declaration = format!("{head} {tail}"); + let tokens: Vec<&str> = declaration.split_whitespace().collect(); + if tokens.len() < 2 { + return Err(err( + "42601", + format!("expected 'name TYPE' in column def: {part}"), + )); + } + let col_name = parse_ident_token(tokens[0])?; + // The type spelling runs from the second token to the first modifier + // word, so `TIMESTAMP WITH TIME ZONE` resolves whole. + let type_end = tokens[1..] + .iter() + .position(|token| { + COLUMN_MODIFIER_KEYWORDS + .iter() + .any(|keyword| token.eq_ignore_ascii_case(keyword)) + }) + .map(|offset| offset + 1) + .unwrap_or(tokens.len()); + if type_end < 2 { + return Err(err( + "42601", + format!("expected 'name TYPE' in column def: {part}"), + )); + } + let col_type = tokens[1..type_end].join(" ").to_uppercase(); + let nullable = !tokens + .windows(2) + .any(|w| w[0].eq_ignore_ascii_case("NOT") && w[1].eq_ignore_ascii_case("NULL")) + && !tokens.iter().any(|t| t.eq_ignore_ascii_case("NOTNULL")); + let primary_key = tokens + .windows(2) + .any(|w| w[0].eq_ignore_ascii_case("PRIMARY") && w[1].eq_ignore_ascii_case("KEY")); + + let ct = sql_type_to_column_type(&col_type)?; + let mut col = if nullable { + ColumnDef::nullable(col_name, ct) + } else { + ColumnDef::required(col_name, ct) + }; + if primary_key { + col = col.with_primary_key(); + } + if let Some(expr) = default_expr { + validate_column_default(&col.name, &expr)?; + col = col.with_default(expr); + } + columns.push(col); + } + + if columns.is_empty() { + return Err(err("42601", "at least one column required")); + } + + Ok(columns) +} + +/// Split a column definition around its `DEFAULT` clause. +/// +/// Returns the text before the keyword, the text after the expression, and +/// the expression itself. The clause is read by +/// `parse_column_type_str_full`, the parser `CREATE COLLECTION` reads a +/// column `DEFAULT` with, so both statements accept one syntax. +fn split_column_default<'a>( + part: &'a str, + name_token: &str, +) -> Result<(&'a str, &'a str, Option), DdlError> { + let after_name = &part[name_token.len()..]; + let Some(offset) = find_ascii_case_insensitive(after_name, "DEFAULT") else { + return Ok((part, "", None)); + }; + let (_, _, _, default_expr) = parse_column_type_str_full(after_name); + let expr = default_expr.ok_or_else(|| { + err( + "42601", + format!("DEFAULT needs an expression in column def: {part}"), + ) + })?; + let after_keyword = after_name[offset + "DEFAULT".len()..].trim(); + let Some(tail) = after_keyword.strip_prefix(expr.as_str()) else { + return Err(err( + "42601", + format!("DEFAULT clause is malformed in column def: {part}"), + )); + }; + Ok((&part[..name_token.len() + offset], tail, Some(expr))) +} + +#[cfg(test)] +mod tests { + use super::*; + use nodedb_types::columnar::ColumnType; + + /// A CONVERT column definition resolves a multi-word type spelling whole, + /// so the declared zone survives. + #[test] + fn convert_column_defs_resolve_a_multi_word_type_spelling() { + let sql = "CONVERT COLLECTION events TO document_strict \ + (id TEXT PRIMARY KEY, at TIMESTAMP WITH TIME ZONE)"; + let (_, _, cols) = parse_convert_sql(sql).expect("multi-word column type must parse"); + let cols = cols.expect("explicit column defs must be present"); + assert_eq!(cols[1].name, "at"); + assert_eq!(cols[1].column_type, ColumnType::Timestamptz); + } + + /// A CONVERT column definition naming no known type stops the statement. + #[test] + fn convert_column_defs_refuse_an_unknown_type() { + let sql = "CONVERT COLLECTION events TO document_strict (id TEXT, payload WIDGET)"; + let error = parse_convert_sql(sql).expect_err("an unknown column type must refuse"); + assert_eq!(error.sqlstate, "42601"); + } + + /// A parameter list keeps its own comma, so the space after it changes + /// nothing about how the column list splits. + #[test] + fn convert_column_defs_split_around_a_parameter_list_comma() { + for sql in [ + "CONVERT COLLECTION sales TO document_strict (id TEXT, amount DECIMAL(10, 2))", + "CONVERT COLLECTION sales TO document_strict (id TEXT, amount DECIMAL(10,2))", + ] { + let (_, _, cols) = parse_convert_sql(sql).expect("a parameter list must parse"); + let cols = cols.expect("explicit column defs must be present"); + assert_eq!( + cols.iter().map(|c| c.name.as_str()).collect::>(), + ["id", "amount"], + "column list must split on the top-level comma only: {sql}" + ); + assert_eq!( + cols[1].column_type, + ColumnType::Decimal { + precision: 10, + scale: 2 + } + ); + } + } + + /// A `DEFAULT` clause reaches the created column instead of being dropped. + #[test] + fn convert_column_defs_carry_a_default_clause() { + let sql = "CONVERT COLLECTION orders TO document_strict \ + (id TEXT PRIMARY KEY, status TEXT NOT NULL DEFAULT 'pending')"; + let (_, _, cols) = parse_convert_sql(sql).expect("a DEFAULT clause must parse"); + let cols = cols.expect("explicit column defs must be present"); + assert_eq!(cols[1].name, "status"); + assert_eq!(cols[1].column_type, ColumnType::String); + assert!( + !cols[1].nullable, + "NOT NULL must survive the DEFAULT clause" + ); + assert_eq!(cols[1].default.as_deref(), Some("'pending'")); + } + + /// `NOT NULL` written after the `DEFAULT` clause still reaches the column. + #[test] + fn convert_column_defs_read_a_modifier_after_the_default_clause() { + let sql = "CONVERT COLLECTION orders TO document_strict \ + (id TEXT, status TEXT DEFAULT 'pending' NOT NULL)"; + let (_, _, cols) = parse_convert_sql(sql).expect("a trailing modifier must parse"); + let cols = cols.expect("explicit column defs must be present"); + assert_eq!(cols[1].column_type, ColumnType::String); + assert!(!cols[1].nullable, "NOT NULL after DEFAULT must land"); + assert_eq!(cols[1].default.as_deref(), Some("'pending'")); + } + + /// A `DEFAULT` naming a function the server cannot evaluate stops CONVERT + /// with the SQLSTATE `CREATE COLLECTION` raises for the same clause. + #[test] + fn convert_column_defs_refuse_an_unevaluable_default() { + let sql = "CONVERT COLLECTION orders TO document_strict \ + (id TEXT, status TEXT DEFAULT no_such_function())"; + let error = parse_convert_sql(sql).expect_err("an unevaluable DEFAULT must refuse"); + assert_eq!(error.sqlstate, "42883"); + } + + /// A `DEFAULT` with no expression is a syntax error, not a dropped clause. + #[test] + fn convert_column_defs_refuse_an_empty_default() { + let sql = "CONVERT COLLECTION orders TO document_strict (id TEXT, status TEXT DEFAULT)"; + let error = parse_convert_sql(sql).expect_err("an empty DEFAULT must refuse"); + assert_eq!(error.sqlstate, "42601"); + } + + #[test] + fn parse_convert_to_document_schemaless() { + let (coll, target, cols) = + parse_convert_sql("CONVERT COLLECTION users TO document_schemaless").unwrap(); + assert_eq!(coll, "users"); + assert_eq!(target, "document_schemaless"); + assert!(cols.is_none()); + } + + #[test] + fn parse_convert_deprecated_document_rejected() { + assert!(parse_convert_sql("CONVERT COLLECTION users TO document").is_err()); + } + + #[test] + fn parse_convert_to_document_strict() { + let sql = + "CONVERT COLLECTION users TO document_strict (name VARCHAR, age INT, active BOOLEAN)"; + let (coll, target, cols) = parse_convert_sql(sql).unwrap(); + assert_eq!(coll, "users"); + assert_eq!(target, "document_strict"); + + let cols = cols.unwrap(); + assert_eq!(cols.len(), 3); + assert_eq!(cols[0].name, "name"); + assert!(matches!(cols[0].column_type, ColumnType::String)); + assert_eq!(cols[1].name, "age"); + assert!(matches!(cols[1].column_type, ColumnType::Int64)); + assert_eq!(cols[2].name, "active"); + assert!(matches!(cols[2].column_type, ColumnType::Bool)); + } + + #[test] + fn parse_convert_deprecated_strict_rejected() { + let sql = "CONVERT COLLECTION users TO strict (name VARCHAR)"; + assert!(parse_convert_sql(sql).is_err()); + } + + #[test] + fn parse_convert_to_kv() { + let sql = "CONVERT COLLECTION cache TO kv (key VARCHAR, value BLOB)"; + let (coll, target, cols) = parse_convert_sql(sql).unwrap(); + assert_eq!(coll, "cache"); + assert_eq!(target, "kv"); + assert!(cols.is_some()); + } + + #[test] + fn parse_convert_not_null_constraint() { + let sql = "CONVERT COLLECTION users TO document_strict (id INT NOT NULL, name VARCHAR)"; + let (_, _, cols) = parse_convert_sql(sql).unwrap(); + let cols = cols.unwrap(); + assert!(!cols[0].nullable); + assert!(cols[1].nullable); + } + + #[test] + fn parse_convert_missing_to_errors() { + assert!(parse_convert_sql("CONVERT COLLECTION users").is_err()); + } + + #[test] + fn parse_convert_unknown_type_errors() { + assert!(parse_convert_sql("CONVERT COLLECTION users TO graph").is_err()); + } +} diff --git a/nodedb/src/control/server/shared/ddl/neutral/convert/driver.rs b/nodedb/src/control/server/shared/ddl/neutral/convert/driver.rs new file mode 100644 index 000000000..18ffe1ed8 --- /dev/null +++ b/nodedb/src/control/server/shared/ddl/neutral/convert/driver.rs @@ -0,0 +1,155 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! CONVERT COLLECTION execution: catalog read, Data Plane re-encode, catalog write. +//! +//! Accepted targets are `document_schemaless`, `document_strict` and `kv`. +//! The columnar, timeseries and spatial engines are creation-time choices and +//! are rejected here. + +use nodedb_types::DatabaseId; +use std::time::Duration; + +use sonic_rs; + +use crate::bridge::envelope::PhysicalPlan; +use crate::control::catalog_entry::persist_collection_replicated; +use crate::control::security::identity::AuthenticatedIdentity; +use crate::control::server::shared::ddl::sync_dispatch::{ + SystemReason, SystemTask, dispatch_system, +}; +use crate::control::state::SharedState; +use nodedb_physical::physical_plan::MetaOp; + +use super::super::super::result::{DdlError, DdlResult}; +use super::column_defs::parse_convert_sql; +use super::support::err; +use super::typeguard_columns::typeguards_to_column_defs; + +/// CONVERT COLLECTION TO [()] +pub async fn convert_collection( + state: &SharedState, + identity: &AuthenticatedIdentity, + database_id: DatabaseId, + sql: &str, +) -> Result, DdlError> { + let (collection, target_type, explicit_columns) = parse_convert_sql(sql)?; + let tenant_id = identity.tenant_id; + + // Validate collection exists. + let catalog = state.credentials.catalog(); + + let mut coll = catalog + .get_collection(database_id, tenant_id.as_u64(), &collection) + .map_err(|e| err("XX000", e.to_string()))? + .ok_or_else(|| err("42P01", format!("collection '{collection}' does not exist")))?; + + // Build columns before dispatch — needed for both Data Plane and catalog. + let columns: Option> = match target_type.as_str() { + "document_strict" | "kv" => { + let cols = if let Some(cols) = explicit_columns { + cols + } else if !coll.type_guards.is_empty() { + typeguards_to_column_defs(&coll.type_guards)? + } else { + return Err(err( + "42601", + "CONVERT TO strict requires column definitions or active typeguards", + )); + }; + Some(cols) + } + _ => None, + }; + + let schema_json_for_dp = if let Some(ref cols) = columns { + sonic_rs::to_string(cols).map_err(|e| err("XX000", format!("schema serialization: {e}")))? + } else { + String::new() + }; + + // Dispatch to Data Plane: re-encode if needed (strict = Binary Tuple). + let plan = PhysicalPlan::Meta(MetaOp::ConvertCollection { + collection: nodedb_types::QualifiedCollection::new(database_id, &collection), + target_type: target_type.clone(), + schema_json: schema_json_for_dp, + }); + + dispatch_system( + state, + SystemTask::new( + SystemReason::DdlApply, + tenant_id, + database_id, + &collection, + plan, + ), + Duration::from_secs(60), + ) + .await + .map_err(|e| err("XX000", format!("conversion failed: {e}")))?; + + // Update catalog collection type. + let new_type = match target_type.as_str() { + "document_schemaless" => nodedb_types::CollectionType::document(), + "document_strict" | "kv" => { + let columns = columns.expect( + "invariant: columns is Some for document_strict/kv targets, validated above", + ); + let schema = nodedb_types::columnar::StrictSchema { + columns, + version: 1, + dropped_columns: Vec::new(), + bitemporal: false, + }; + if target_type == "kv" { + nodedb_types::CollectionType::kv(schema) + } else { + nodedb_types::CollectionType::strict(schema) + } + } + _ => { + return Err(err( + "42601", + format!("unsupported target type: {target_type}"), + )); + } + }; + + coll.collection_type = new_type; + + // CONVERT TO document_strict: if collection had typeguards, carry over CHECK constraints + // and drop typeguard definitions (strict schema subsumes type checking). + if target_type == "document_strict" && !coll.type_guards.is_empty() { + for guard in &coll.type_guards { + if let Some(ref check_expr) = guard.check_expr { + // Avoid duplicate names. + let name = format!("_guard_{}", guard.field); + if !coll.check_constraints.iter().any(|c| c.name == name) { + coll.check_constraints.push( + crate::control::security::catalog::types::CheckConstraintDef { + name, + check_sql: check_expr.clone(), + has_subquery: false, + }, + ); + } + } + } + coll.type_guards.clear(); + } + + persist_collection_replicated(state, database_id, &coll) + .map_err(|e| err("XX000", e.to_string()))?; + + tracing::info!( + %collection, + target_type, + tenant = tenant_id.as_u64(), + "collection converted" + ); + + Ok(vec![DdlResult::Status { + command: "CONVERT COLLECTION".to_string(), + rows_affected: None, + }]) +} diff --git a/nodedb/src/control/server/shared/ddl/neutral/convert/mod.rs b/nodedb/src/control/server/shared/ddl/neutral/convert/mod.rs new file mode 100644 index 000000000..1a2704585 --- /dev/null +++ b/nodedb/src/control/server/shared/ddl/neutral/convert/mod.rs @@ -0,0 +1,16 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! Protocol-neutral DDL handler for CONVERT COLLECTION. +//! +//! Syntax: +//! - `CONVERT COLLECTION TO document_schemaless` +//! - `CONVERT COLLECTION TO document_strict (col1 TYPE, col2 TYPE, ...)` +//! - `CONVERT COLLECTION TO kv` + +pub mod column_defs; +pub mod driver; +mod support; +pub mod type_map; +pub mod typeguard_columns; + +pub use driver::convert_collection; diff --git a/nodedb/src/control/server/shared/ddl/neutral/convert/support.rs b/nodedb/src/control/server/shared/ddl/neutral/convert/support.rs new file mode 100644 index 000000000..816fe3b9e --- /dev/null +++ b/nodedb/src/control/server/shared/ddl/neutral/convert/support.rs @@ -0,0 +1,13 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! Shared helpers for the CONVERT COLLECTION family. + +use super::super::super::result::DdlError; + +/// Build a protocol-neutral [`DdlError`] with the given SQLSTATE and message. +/// +/// `message` takes anything convertible to `String`, so a `format!` result +/// moves straight in without a borrow and a second allocation. +pub(super) fn err(sqlstate: &str, message: impl Into) -> DdlError { + DdlError::new(sqlstate, message) +} diff --git a/nodedb/src/control/server/shared/ddl/neutral/convert/type_map.rs b/nodedb/src/control/server/shared/ddl/neutral/convert/type_map.rs new file mode 100644 index 000000000..f9c3eaf5d --- /dev/null +++ b/nodedb/src/control/server/shared/ddl/neutral/convert/type_map.rs @@ -0,0 +1,397 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! Declared type spellings resolved to a `ColumnType` for CONVERT COLLECTION. +//! +//! Two mappers share one resolver: the column-definition mapper reads the +//! spelling an author wrote in the statement, the typeguard mapper reads the +//! spelling a guard declared. Both answer through +//! `nodedb_sql::parser::type_expr::parse_type_expr`'s resolver, so one +//! spelling resolves to one type on every path. + +use super::super::super::result::DdlError; +use super::support::err; + +/// Resolve a declared type spelling to a `ColumnType`. +/// +/// The whole spelling resolves, so `TIMESTAMP WITH TIME ZONE` keeps the zone +/// the author wrote and `TIMESTAMP GARBAGE` names no type at all. +/// +/// This is the parser `nodedb_sql::parser::type_expr::parse_type_expr` +/// resolves the same text through, so a spelling added there reaches both +/// CONVERT mappers with no edit here. +fn resolve_declared_type( + declared: &str, +) -> Result { + declared.trim().parse() +} + +/// Map a declared SQL type spelling to a `ColumnType`. +/// +/// The spelling resolves through [`resolve_declared_type`]. +/// +/// `BINARY` resolves ahead of that call. It is a CONVERT-only spelling for +/// `ColumnType::Bytes` that the shared parser rejects. +/// +/// A spelling the shared parser rejects raises `42601`. The author names the +/// stored column type, so an unresolvable spelling must never become +/// `ColumnType::String`. +pub(super) fn sql_type_to_column_type( + sql_type: &str, +) -> Result { + use nodedb_types::columnar::ColumnType; + + let declared = sql_type.trim(); + if declared.eq_ignore_ascii_case("BINARY") { + return Ok(ColumnType::Bytes); + } + resolve_declared_type(declared) + .map_err(|e| err("42601", format!("column type '{declared}': {e}"))) +} + +/// Map a typeguard type expression string to a `ColumnType`. +/// +/// Union and generic wrappers reduce to their leaf spelling first. The leaf +/// then resolves through [`resolve_declared_type`], apart from the one +/// typeguard-only spelling named below. +/// +/// A leaf the shared parser rejects raises `42601`. The guard declares what +/// the column holds, so an unresolvable leaf must never become +/// `ColumnType::String`. +pub(super) fn typeguard_type_to_column_type( + type_expr: &str, +) -> Result { + use nodedb_types::columnar::ColumnType; + + // Strip union types — take the first non-NULL type. + let base = type_expr + .split('|') + .map(|s| s.trim()) + .find(|s| !s.eq_ignore_ascii_case("NULL")) + .unwrap_or(type_expr) + .to_uppercase(); + + // Strip generic parameters: ARRAY → ARRAY, SET → SET. + let base = base.split('<').next().unwrap_or(&base).trim(); + + // `OBJECT` is a typeguard-only keyword. It names a field holding a nested + // map, has no declared-DDL spelling, and stores as inline MessagePack. + if base.eq_ignore_ascii_case("OBJECT") { + return Ok(ColumnType::Json); + } + + // Every other spelling resolves through the one shared parser, so a + // spelling added there reaches typeguard conversion with no edit here. + resolve_declared_type(base).map_err(|e| err("42601", format!("type guard type '{base}': {e}"))) +} + +#[cfg(test)] +mod tests { + use super::*; + use nodedb_types::columnar::{ColumnType, DECLARED_FLOAT_KEYWORDS, DECLARED_INT_KEYWORDS}; + + /// Both CONVERT type mappers must answer with the shared parser for every + /// integer spelling it knows. A keyword added to `DECLARED_INT_KEYWORDS` + /// extends this test, so neither mapper can fall to its `String` default + /// for a spelling the rest of the database resolves to `Int64`. + #[test] + fn convert_mappers_resolve_every_declared_int_keyword() { + for keyword in DECLARED_INT_KEYWORDS { + assert_eq!( + sql_type_to_column_type(keyword) + .expect("the column def mapper resolves a declared int/float keyword"), + ColumnType::Int64, + "column def mapper must resolve {keyword}" + ); + assert_eq!( + typeguard_type_to_column_type(keyword) + .expect("the typeguard mapper resolves a declared int/float keyword"), + ColumnType::Int64, + "typeguard mapper must resolve {keyword}" + ); + } + } + + /// The float counterpart of + /// [`convert_mappers_resolve_every_declared_int_keyword`]. + #[test] + fn convert_mappers_resolve_every_declared_float_keyword() { + for keyword in DECLARED_FLOAT_KEYWORDS { + assert_eq!( + sql_type_to_column_type(keyword) + .expect("the column def mapper resolves a declared int/float keyword"), + ColumnType::Float64, + "column def mapper must resolve {keyword}" + ); + assert_eq!( + typeguard_type_to_column_type(keyword) + .expect("the typeguard mapper resolves a declared int/float keyword"), + ColumnType::Float64, + "typeguard mapper must resolve {keyword}" + ); + } + } + + /// Both mappers delegate to `ColumnType::from_declared_type` outside their + /// documented exceptions. This pins the delegation itself, so a mapper + /// cannot regrow a private keyword table that answers differently. + #[test] + fn convert_mappers_delegate_to_the_shared_parser() { + for declared in [ + "TEXT", + "VARCHAR", + "STRING", + "BOOL", + "BOOLEAN", + "BYTES", + "BYTEA", + "BLOB", + "TIMESTAMP", + "TIMESTAMPTZ", + "NUMERIC", + "DECIMAL", + "UUID", + "ULID", + "JSON", + "JSONB", + "GEOMETRY", + "SPARSEVECTOR", + "VECTOR(384)", + "DURATION", + "ARRAY", + "SET", + "REGEX", + "RANGE", + "RECORD", + "SYSTEM_TIMESTAMP", + ] { + let shared = ColumnType::from_declared_type(declared) + .expect("test input names a type the shared parser knows"); + assert_eq!( + sql_type_to_column_type(declared) + .expect("the column def mapper resolves a shared-parser spelling"), + shared, + "column def mapper must match the shared parser on {declared}" + ); + assert_eq!( + typeguard_type_to_column_type(declared) + .expect("the typeguard mapper resolves a shared-parser spelling"), + shared, + "typeguard mapper must match the shared parser on {declared}" + ); + } + } + + /// A spelling the shared parser rejects raises `42601` on both mappers. + /// + /// A silent `String` answer would store a column type the author never + /// declared, so the refusal is the whole contract here. `TIMESTAMP + /// GARBAGE` covers the trailing-word case, which resolving the leading + /// token alone would read as `TIMESTAMP`. + #[test] + fn convert_mappers_refuse_a_spelling_the_shared_parser_rejects() { + for declared in ["WIDGET", "TIMESTAMP GARBAGE"] { + let column_def = sql_type_to_column_type(declared) + .expect_err("the column def mapper must refuse an unresolvable spelling"); + assert_eq!( + column_def.sqlstate, "42601", + "column def mapper must refuse {declared} with 42601" + ); + let typeguard = typeguard_type_to_column_type(declared) + .expect_err("the typeguard mapper must refuse an unresolvable spelling"); + assert_eq!( + typeguard.sqlstate, "42601", + "typeguard mapper must refuse {declared} with 42601" + ); + } + } + + /// `BINARY` is a CONVERT-only spelling for `Bytes`; the shared parser + /// rejects it, so the column def mapper resolves it itself. + #[test] + fn binary_stays_a_convert_only_bytes_spelling() { + assert_eq!( + sql_type_to_column_type("BINARY").expect("BINARY resolves"), + ColumnType::Bytes + ); + assert_eq!(ColumnType::from_declared_type("BINARY"), None); + } + + /// `OBJECT` is a typeguard-only keyword the shared parser rejects, and a + /// multi-word timestamp spelling keeps the zone the author wrote. + #[test] + fn typeguard_only_spellings_keep_their_own_answer() { + assert_eq!( + typeguard_type_to_column_type("OBJECT").expect("OBJECT resolves"), + ColumnType::Json + ); + assert_eq!(ColumnType::from_declared_type("OBJECT"), None); + assert_eq!( + typeguard_type_to_column_type("TIMESTAMP WITH TIME ZONE") + .expect("the zone spelling resolves"), + ColumnType::Timestamptz + ); + assert_eq!( + typeguard_type_to_column_type("TIMESTAMP WITHOUT TIME ZONE") + .expect("the no-zone spelling resolves"), + ColumnType::Timestamp + ); + } + + /// Union and generic wrappers resolve from their inner leaf type. + #[test] + fn typeguard_unions_and_generics_resolve_to_the_leaf() { + assert_eq!( + typeguard_type_to_column_type("INT|NULL").expect("INT|NULL resolves"), + ColumnType::Int64 + ); + assert_eq!( + typeguard_type_to_column_type("NULL|STRING").expect("NULL|STRING resolves"), + ColumnType::String + ); + assert_eq!( + typeguard_type_to_column_type("ARRAY").expect("ARRAY resolves"), + ColumnType::Array + ); + assert_eq!( + typeguard_type_to_column_type("SET").expect("SET resolves"), + ColumnType::Set + ); + } + + /// One spelling, one answer. The typeguard ENFORCEMENT parser + /// (`nodedb_sql::parser::type_expr::parse_type_expr`, which decides + /// whether a written value is valid) and this CONVERT mapper (which + /// decides the column type the same guard becomes) must resolve every + /// declared spelling to the same type. + /// + /// A spelling one path reads as `Timestamp` and the other as + /// `Timestamptz` is a document validated under one rule and stored under + /// another. Adding a spelling to either parser extends this list, and a + /// leaf added to `SimpleType` fails the mapping below until it names its + /// column type. + #[test] + fn enforcement_and_convert_agree_on_every_typeguard_spelling() { + use nodedb_sql::parser::type_expr::{SimpleType, TypeExpr, parse_type_expr}; + + // The column type each typeguard leaf denotes. `Object` is the + // typeguard-only leaf; every other leaf names its own `ColumnType`. + fn leaf_column_type(leaf: &SimpleType) -> ColumnType { + match leaf { + SimpleType::Int => ColumnType::Int64, + SimpleType::Float => ColumnType::Float64, + SimpleType::String => ColumnType::String, + SimpleType::Bool => ColumnType::Bool, + SimpleType::Bytes => ColumnType::Bytes, + SimpleType::Timestamp => ColumnType::Timestamp, + SimpleType::Timestamptz => ColumnType::Timestamptz, + SimpleType::SystemTimestamp => ColumnType::SystemTimestamp, + SimpleType::Decimal { precision, scale } => ColumnType::Decimal { + precision: *precision, + scale: *scale, + }, + SimpleType::Uuid => ColumnType::Uuid, + SimpleType::Ulid => ColumnType::Ulid, + SimpleType::Geometry => ColumnType::Geometry, + SimpleType::Duration => ColumnType::Duration, + SimpleType::Array => ColumnType::Array, + SimpleType::Object => ColumnType::Json, + SimpleType::Json => ColumnType::Json, + SimpleType::Set => ColumnType::Set, + SimpleType::Regex => ColumnType::Regex, + SimpleType::Range => ColumnType::Range, + SimpleType::Record => ColumnType::Record, + SimpleType::Vector(dim) => ColumnType::Vector(*dim), + SimpleType::SparseVector => ColumnType::SparseVector, + } + } + + let mut spellings: Vec = [ + "TEXT", + "VARCHAR", + "VARCHAR(255)", + "STRING", + "BOOL", + "BOOLEAN", + "BYTES", + "BYTEA", + "BLOB", + "TIMESTAMP", + "TIMESTAMPTZ", + "TIMESTAMP WITH TIME ZONE", + "TIMESTAMP WITHOUT TIME ZONE", + "SYSTEM_TIMESTAMP", + "DECIMAL", + "NUMERIC", + "DECIMAL(10, 2)", + "UUID", + "ULID", + "GEOMETRY", + "DURATION", + "JSON", + "JSONB", + "OBJECT", + "ARRAY", + "SET", + "REGEX", + "RANGE", + "RECORD", + "SPARSEVECTOR", + "VECTOR(384)", + ] + .iter() + .map(|spelling| spelling.to_string()) + .collect(); + spellings.extend( + DECLARED_INT_KEYWORDS + .iter() + .chain(DECLARED_FLOAT_KEYWORDS.iter()) + .map(|keyword| keyword.to_string()), + ); + + for spelling in spellings { + let parsed = parse_type_expr(&spelling) + .unwrap_or_else(|e| panic!("enforcement must parse {spelling}: {e}")); + let TypeExpr::Simple(leaf) = parsed else { + panic!("{spelling} must parse to a leaf type, got {parsed:?}") + }; + assert_eq!( + leaf_column_type(&leaf), + typeguard_type_to_column_type(&spelling) + .unwrap_or_else(|e| panic!("CONVERT must resolve {spelling}: {}", e.message)), + "enforcement and CONVERT must agree on {spelling}" + ); + } + } + + #[test] + fn sql_type_to_column_type_distinguishes_timestamp_from_timestamptz() { + for (spelling, expected) in [ + ("TIMESTAMP", ColumnType::Timestamp), + ("TIMESTAMP WITHOUT TIME ZONE", ColumnType::Timestamp), + ("TIMESTAMPTZ", ColumnType::Timestamptz), + ("TIMESTAMP WITH TIME ZONE", ColumnType::Timestamptz), + ] { + assert_eq!( + sql_type_to_column_type(spelling).expect("the spelling resolves"), + expected, + "column def mapper on {spelling}" + ); + } + } + + #[test] + fn typeguard_type_to_column_type_distinguishes_timestamp_from_timestamptz() { + for (spelling, expected) in [ + ("TIMESTAMP", ColumnType::Timestamp), + ("TIMESTAMP WITHOUT TIME ZONE", ColumnType::Timestamp), + ("TIMESTAMPTZ", ColumnType::Timestamptz), + ("TIMESTAMP WITH TIME ZONE", ColumnType::Timestamptz), + ] { + assert_eq!( + typeguard_type_to_column_type(spelling).expect("the spelling resolves"), + expected, + "typeguard mapper on {spelling}" + ); + } + } +} diff --git a/nodedb/src/control/server/shared/ddl/neutral/convert/typeguard_columns.rs b/nodedb/src/control/server/shared/ddl/neutral/convert/typeguard_columns.rs new file mode 100644 index 000000000..0bc1749cc --- /dev/null +++ b/nodedb/src/control/server/shared/ddl/neutral/convert/typeguard_columns.rs @@ -0,0 +1,123 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! Typeguard field definitions turned into strict-schema columns. +//! +//! `CONVERT COLLECTION TO document_strict` with no explicit column list +//! reads the collection's active typeguards instead. + +use super::super::super::result::DdlError; +use super::super::column_default::validate_column_default; +use super::support::err; +use super::type_map::typeguard_type_to_column_type; + +/// Convert typeguard field definitions to strict schema column definitions. +/// +/// Each typeguard field becomes a column. The type expression is mapped to ColumnType. +/// REQUIRED fields become NOT NULL. DEFAULT expressions carry over. +/// +/// A carried-over expression becomes a column `DEFAULT`, so it passes the gate +/// every declared column `DEFAULT` passes. An unregistered function name +/// raises `42883`; every other rejection raises `42601`. +pub(super) fn typeguards_to_column_defs( + guards: &[nodedb_types::TypeGuardFieldDef], +) -> Result, DdlError> { + use nodedb_types::columnar::{ColumnDef, ColumnType}; + + // Always include an `id` column as primary key. + let mut columns = vec![ColumnDef::required("id", ColumnType::String).with_primary_key()]; + + for guard in guards { + // Skip dot-path fields (nested) — strict schema is flat. + if guard.field.contains('.') { + continue; + } + // Skip if already added (e.g., "id"). + if guard.field == "id" || columns.iter().any(|c| c.name == guard.field) { + continue; + } + + let ct = typeguard_type_to_column_type(&guard.type_expr).map_err(|e| { + err( + &e.sqlstate, + format!("field '{}': {}", guard.field, e.message), + ) + })?; + let mut col = if guard.required { + ColumnDef::required(guard.field.clone(), ct) + } else { + ColumnDef::nullable(guard.field.clone(), ct) + }; + // A guard carries either DEFAULT or VALUE, never both. Strict schema + // has one materialization slot, so both land on the column `DEFAULT`. + if let Some(expr) = guard.default_expr.clone().or(guard.value_expr.clone()) { + validate_column_default(&col.name, &expr)?; + col.default = Some(expr); + } + columns.push(col); + } + + Ok(columns) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn guard(field: &str, type_expr: &str) -> nodedb_types::TypeGuardFieldDef { + nodedb_types::TypeGuardFieldDef { + field: field.to_string(), + type_expr: type_expr.to_string(), + required: false, + check_expr: None, + default_expr: None, + value_expr: None, + } + } + + /// An unresolvable guard type stops CONVERT instead of storing a column + /// type the guard never declared. + #[test] + fn typeguards_to_column_defs_refuses_an_unresolvable_guard_type() { + let guards = vec![guard("gadget", "WIDGET")]; + let error = + typeguards_to_column_defs(&guards).expect_err("an unresolvable guard type must refuse"); + assert_eq!(error.sqlstate, "42601"); + assert!( + error.message.contains("gadget"), + "the error must name the field: {}", + error.message + ); + } + + /// A guard DEFAULT the column path can evaluate reaches the column. + #[test] + fn typeguards_to_column_defs_carry_a_literal_default() { + let mut status = guard("status", "STRING"); + status.default_expr = Some("'draft'".to_string()); + let columns = typeguards_to_column_defs(&[status]).expect("a literal DEFAULT must convert"); + assert_eq!(columns[1].name, "status"); + assert_eq!(columns[1].default.as_deref(), Some("'draft'")); + } + + /// A guard VALUE lands on the column `DEFAULT`, the one strict-schema slot + /// that materializes an omitted field. + #[test] + fn typeguards_to_column_defs_carry_a_value_expression() { + let mut computed = guard("computed", "STRING"); + computed.value_expr = Some("'server'".to_string()); + let columns = + typeguards_to_column_defs(&[computed]).expect("a VALUE expression must convert"); + assert_eq!(columns[1].default.as_deref(), Some("'server'")); + } + + /// A carried-over expression naming an unregistered function stops CONVERT + /// with the SQLSTATE every other DEFAULT source raises for it. + #[test] + fn typeguards_to_column_defs_refuse_an_unevaluable_default() { + let mut status = guard("status", "STRING"); + status.default_expr = Some("no_such_function()".to_string()); + let error = typeguards_to_column_defs(&[status]) + .expect_err("an unevaluable carried-over DEFAULT must refuse"); + assert_eq!(error.sqlstate, "42883"); + } +} diff --git a/nodedb/src/control/server/shared/ddl/neutral/mod.rs b/nodedb/src/control/server/shared/ddl/neutral/mod.rs index 5d0c9664e..7d6d82e97 100644 --- a/nodedb/src/control/server/shared/ddl/neutral/mod.rs +++ b/nodedb/src/control/server/shared/ddl/neutral/mod.rs @@ -18,6 +18,7 @@ pub mod change_stream; pub mod chunk_text; pub mod cluster; pub mod collection; +mod column_default; pub mod conflict_policy; pub mod constraint; pub mod consumer_group; diff --git a/nodedb/src/control/server/shared/ddl/neutral/typeguard/injected_expr.rs b/nodedb/src/control/server/shared/ddl/neutral/typeguard/injected_expr.rs new file mode 100644 index 000000000..1fb3afed4 --- /dev/null +++ b/nodedb/src/control/server/shared/ddl/neutral/typeguard/injected_expr.rs @@ -0,0 +1,110 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! DDL-time gate for a typeguard `DEFAULT` / `VALUE` expression. +//! +//! Both clauses store raw text and produce a field value on every write. The +//! Data Plane's `inject_defaults` reads that text with +//! `nodedb_query::expr_parse::parse_generated_expr` and evaluates it with +//! `SqlExpr::eval`, so a declaration the write path cannot evaluate is refused +//! here instead of at the first write. +//! +//! Two checks run, and neither evaluates the expression: +//! - The canonical declared-clause gate resolves every function name through +//! the `FunctionRegistry`. `nodedb_query::functions::eval_function` answers +//! `Null` for a name it does not know, so an unregistered name would store +//! `Null` on every write and report nothing. +//! - The write path's own parser refuses what it cannot evaluate, including +//! the non-deterministic functions a guard expression must not call. +//! +//! An unregistered function name raises SQLSTATE `42883`; every other +//! rejection raises SQLSTATE `42601`. `nextval` is refused outright, so +//! declaring a typeguard cannot advance a sequence. + +use super::super::super::result::DdlError; +use super::super::column_default::validate_clause_expr; +use super::parse::err; + +/// Refuse a typeguard `DEFAULT` or `VALUE` the engine cannot evaluate. +/// +/// `clause` names the keyword the author wrote, so the message points at it. +pub(super) fn validate_injected_expr( + field: &str, + clause: &str, + expr: &str, +) -> Result<(), DdlError> { + validate_clause_expr(clause, field, expr)?; + nodedb_query::expr_parse::parse_generated_expr(expr).map_err(|error| { + err( + "42601", + &format!("field '{field}': {clause} expression '{expr}' is invalid: {error}"), + ) + })?; + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + + /// Every clause form the write path evaluates stays accepted. + #[test] + fn evaluable_clause_expressions_are_accepted() { + for (clause, expr) in [ + ("DEFAULT", "'draft'"), + ("DEFAULT", "1"), + ("DEFAULT", "0"), + ("VALUE", "'server_computed'"), + ("VALUE", "name"), + ("VALUE", "LOWER(REPLACE(title, ' ', '-'))"), + ] { + validate_injected_expr("f", clause, expr) + .unwrap_or_else(|e| panic!("{clause} {expr} must be accepted: {}", e.message)); + } + } + + /// An unregistered function name raises `42883` rather than storing `Null` + /// on every write. + #[test] + fn an_unregistered_function_raises_undefined_function() { + let error = validate_injected_expr("status", "DEFAULT", "no_such_function()") + .expect_err("an unregistered function must refuse"); + assert_eq!(error.sqlstate, "42883"); + assert!( + error.message.contains("status"), + "the error must name the field: {}", + error.message + ); + } + + /// The same refusal covers a `VALUE` clause, and names that clause. + #[test] + fn an_unregistered_function_in_a_value_clause_names_the_clause() { + let error = validate_injected_expr("computed", "VALUE", "no_such_function()") + .expect_err("an unregistered function must refuse"); + assert_eq!(error.sqlstate, "42883"); + assert!( + error.message.contains("VALUE"), + "the error must name the clause: {}", + error.message + ); + } + + /// A function the write path refuses as non-deterministic is refused at + /// declaration with `42601`. + #[test] + fn a_non_deterministic_function_is_refused() { + for expr in ["now()", "uuid_v7()", "nextval('s')"] { + let error = validate_injected_expr("at", "DEFAULT", expr) + .expect_err("a non-deterministic function must refuse"); + assert_eq!(error.sqlstate, "42601", "on {expr}"); + } + } + + /// A malformed expression is a syntax error, not a stored clause. + #[test] + fn a_malformed_expression_is_refused() { + let error = validate_injected_expr("f", "DEFAULT", "'unterminated") + .expect_err("a malformed expression must refuse"); + assert_eq!(error.sqlstate, "42601"); + } +} diff --git a/nodedb/src/control/server/shared/ddl/neutral/typeguard/mod.rs b/nodedb/src/control/server/shared/ddl/neutral/typeguard/mod.rs index e1a509d6e..814d039f7 100644 --- a/nodedb/src/control/server/shared/ddl/neutral/typeguard/mod.rs +++ b/nodedb/src/control/server/shared/ddl/neutral/typeguard/mod.rs @@ -3,6 +3,7 @@ //! Protocol-neutral TYPEGUARD DDL family handlers. pub mod handlers; +mod injected_expr; pub mod parse; pub mod validate; diff --git a/nodedb/src/control/server/shared/ddl/neutral/typeguard/parse.rs b/nodedb/src/control/server/shared/ddl/neutral/typeguard/parse.rs index c7cdadfe0..f758ed234 100644 --- a/nodedb/src/control/server/shared/ddl/neutral/typeguard/parse.rs +++ b/nodedb/src/control/server/shared/ddl/neutral/typeguard/parse.rs @@ -9,9 +9,11 @@ use nodedb_sql::parser::preprocess::lex::{ find_ascii_case_insensitive, find_ascii_case_insensitive_from, }; +use nodedb_sql::parser::type_expr::parse_type_expr; use nodedb_types::TypeGuardFieldDef; use super::super::super::result::DdlError; +use super::injected_expr::validate_injected_expr; /// Extract collection name from `... TYPEGUARD [IF EXISTS] ON ...`. pub(super) fn extract_collection_name(sql: &str) -> Result { @@ -173,6 +175,11 @@ pub(super) fn parse_single_field(s: &str) -> Result )); } + // The declaration must name a type the engine resolves. `parse_type_expr` + // is the parser that enforces the guard on every write, so a spelling it + // rejects fails here instead of failing each later write. + parse_type_expr(&type_expr).map_err(|e| err("42601", &format!("field '{field}': {e}")))?; + // Extract DEFAULT expression if present. let default_expr = if let Some(def_pos) = find_word_boundary(rest, "DEFAULT") { let after_default = rest[def_pos + 7..].trim_start(); @@ -200,6 +207,15 @@ pub(super) fn parse_single_field(s: &str) -> Result )); } + // Both clauses produce a field value on every write, so a declaration the + // write path cannot evaluate is refused here. + if let Some(expr) = default_expr.as_deref() { + validate_injected_expr(&field, "DEFAULT", expr)?; + } + if let Some(expr) = value_expr.as_deref() { + validate_injected_expr(&field, "VALUE", expr)?; + } + Ok(TypeGuardFieldDef { field, type_expr, @@ -260,6 +276,49 @@ mod tests { assert_eq!(name, "metrics"); } + /// A declared type the engine cannot resolve stops the declaration. + /// + /// The guard would otherwise be stored and fail every later write. + /// `TIMESTAMP GARBAGE` covers the trailing-word case, which resolving the + /// leading token alone would read as `TIMESTAMP`. + #[test] + fn unresolvable_field_type_is_refused_at_declaration() { + for declaration in ["gadget WIDGET", "at TIMESTAMP GARBAGE"] { + let error = parse_single_field(declaration) + .expect_err("an unresolvable declared type must be refused"); + assert_eq!(error.sqlstate, "42601", "on {declaration}"); + } + } + + /// Every spelling the engine resolves stays accepted at declaration. + #[test] + fn every_resolvable_field_type_is_accepted_at_declaration() { + for declaration in [ + "a INT2", + "b INT4", + "c INT8", + "d SMALLINT", + "e FLOAT4", + "f FLOAT8", + "g FLOAT32", + "h DOUBLE PRECISION", + "i JSONB", + "j SYSTEM_TIMESTAMP", + "k VARCHAR(255)", + "l DECIMAL(10, 2)", + "m TIMESTAMP WITH TIME ZONE", + "n TIMESTAMP WITHOUT TIME ZONE", + "o OBJECT", + "p ARRAY", + "q STRING|NULL", + "r VECTOR(384)", + "s STRING REQUIRED CHECK (s <> '')", + ] { + parse_single_field(declaration) + .unwrap_or_else(|e| panic!("{declaration} must parse: {}", e.message)); + } + } + #[test] fn field_keywords_after_unicode_default_preserve_original_offsets() { let field = parse_single_field("label STRING DEFAULT 'ffff' CHECK (label <> '') REQUIRED") diff --git a/nodedb/tests/inproc/cases/system_task_call_sites.rs b/nodedb/tests/inproc/cases/system_task_call_sites.rs index 584f882d7..62a4779da 100644 --- a/nodedb/tests/inproc/cases/system_task_call_sites.rs +++ b/nodedb/tests/inproc/cases/system_task_call_sites.rs @@ -41,7 +41,7 @@ const ALLOWED: &[&str] = &[ "control/cluster/snapshot_applier.rs", // Committed DDL applied to engine state, and catalog maintenance. "control/server/shared/ddl/engine_apply.rs", - "control/server/shared/ddl/neutral/convert.rs", + "control/server/shared/ddl/neutral/convert/driver.rs", "control/server/shared/ddl/neutral/continuous_agg/create.rs", "control/server/shared/ddl/neutral/continuous_agg/drop.rs", "control/server/shared/ddl/neutral/continuous_agg/register.rs", diff --git a/nodedb/tests/wire/cases/mod.rs b/nodedb/tests/wire/cases/mod.rs index 8b585c811..49316db90 100644 --- a/nodedb/tests/wire/cases/mod.rs +++ b/nodedb/tests/wire/cases/mod.rs @@ -158,6 +158,7 @@ mod sql_bitemporal_document_visibility; mod sql_check_constraints; mod sql_collection_drop_index_cleanup; mod sql_conflict_policy; +mod sql_convert_column_defs; mod sql_copy_from; mod sql_copy_to; mod sql_cursors; @@ -246,6 +247,7 @@ mod sql_transactions_unique_violation; mod sql_transactions_upsert_overlay; mod sql_transactions_vector_overlay; mod sql_trigger_fuel; +mod sql_typeguard_default_gate; mod sql_typeguard_defaults; mod sql_undefined_column; mod sql_undefined_column_dml; diff --git a/nodedb/tests/wire/cases/sql_convert_column_defs.rs b/nodedb/tests/wire/cases/sql_convert_column_defs.rs new file mode 100644 index 000000000..9844b0c86 --- /dev/null +++ b/nodedb/tests/wire/cases/sql_convert_column_defs.rs @@ -0,0 +1,88 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! Integration tests for the `CONVERT COLLECTION` column list. +//! +//! Covers the two properties a column definition must hold: +//! - A parameter list carrying a comma stays one column definition. +//! - A `DEFAULT` clause reaches the created column and fills a later insert. +//! +//! A `DEFAULT` the server cannot evaluate is refused at CONVERT with SQLSTATE +//! `42883`, the SQLSTATE `CREATE COLLECTION` raises for the same clause. + +use crate::harness::TestServer; + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn convert_accepts_a_parameter_list_with_a_space_after_the_comma() { + let server = TestServer::start().await; + + server.exec("CREATE COLLECTION conv_decimal").await.unwrap(); + + server + .exec( + "CONVERT COLLECTION conv_decimal TO document_strict \ + (id TEXT PRIMARY KEY, amount DECIMAL(10, 2))", + ) + .await + .unwrap(); + + server + .exec("INSERT INTO conv_decimal (id, amount) VALUES ('d1', 12.50)") + .await + .unwrap(); + + let rows = server + .query_text_joined("SELECT amount FROM conv_decimal") + .await + .unwrap(); + assert!( + rows.iter().any(|row| row.contains("12.5")), + "the parameterized column must be declared and hold its value: {rows:?}" + ); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn convert_carries_a_column_default_into_a_later_insert() { + let server = TestServer::start().await; + + server.exec("CREATE COLLECTION conv_default").await.unwrap(); + + server + .exec( + "CONVERT COLLECTION conv_default TO document_strict \ + (id TEXT PRIMARY KEY, status TEXT DEFAULT 'pending')", + ) + .await + .unwrap(); + + server + .exec("INSERT INTO conv_default (id) VALUES ('c1')") + .await + .unwrap(); + + let rows = server + .query_text_joined("SELECT status FROM conv_default") + .await + .unwrap(); + assert!( + rows.iter().any(|row| row.contains("pending")), + "the converted column must take its DEFAULT: {rows:?}" + ); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn convert_refuses_a_default_naming_an_unknown_function() { + let server = TestServer::start().await; + + server + .exec("CREATE COLLECTION conv_bad_default") + .await + .unwrap(); + + server + .expect_error( + "CONVERT COLLECTION conv_bad_default TO document_strict \ + (id TEXT PRIMARY KEY, status TEXT DEFAULT no_such_function())", + "42883", + ) + .await; +} diff --git a/nodedb/tests/wire/cases/sql_typeguard_default_gate.rs b/nodedb/tests/wire/cases/sql_typeguard_default_gate.rs new file mode 100644 index 000000000..7f621680c --- /dev/null +++ b/nodedb/tests/wire/cases/sql_typeguard_default_gate.rs @@ -0,0 +1,100 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! A typeguard `DEFAULT` / `VALUE` the engine cannot evaluate is refused at +//! declaration. +//! +//! Both clauses produce a field value on every write. An unregistered function +//! name would otherwise store `NULL` on every write and report nothing, and a +//! non-deterministic call would fail every write instead of the declaration. + +use crate::harness::TestServer; + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn typeguard_unevaluable_default_is_refused_at_declaration() { + let server = TestServer::start().await; + + server.exec("CREATE COLLECTION tg_gate").await.unwrap(); + + // A function name no registry knows. + server + .expect_error( + "CREATE TYPEGUARD ON tg_gate (status STRING DEFAULT no_such_function())", + "42883", + ) + .await; + + // The same refusal covers the VALUE clause. + server + .expect_error( + "CREATE TYPEGUARD ON tg_gate (computed STRING VALUE no_such_function())", + "42883", + ) + .await; + + // A call the write path refuses as non-deterministic. + server + .expect_error( + "CREATE TYPEGUARD ON tg_gate (at STRING DEFAULT now())", + "42601", + ) + .await; + + // ALTER carries the same refusal. + server + .expect_error( + "ALTER TYPEGUARD ON tg_gate ADD status STRING DEFAULT no_such_function()", + "42883", + ) + .await; + + // A refused declaration reaches no storage. + let rows = server + .query_text("SHOW TYPEGUARD ON tg_gate") + .await + .unwrap(); + assert_eq!(rows.len(), 0, "refused guard must not be stored: {rows:?}"); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn typeguard_evaluable_defaults_stay_accepted() { + let server = TestServer::start().await; + + server.exec("CREATE COLLECTION tg_gate_ok").await.unwrap(); + + server + .exec( + "CREATE TYPEGUARD ON tg_gate_ok (\ + status STRING DEFAULT 'draft',\ + version INT REQUIRED DEFAULT 1\ + )", + ) + .await + .unwrap(); + + server + .exec("ALTER TYPEGUARD ON tg_gate_ok ADD slug STRING VALUE LOWER(status)") + .await + .unwrap(); + + let rows = server + .query_text("SHOW TYPEGUARD ON tg_gate_ok") + .await + .unwrap(); + assert_eq!(rows.len(), 3, "every accepted guard is stored: {rows:?}"); + + // The accepted DEFAULT still injects at write time. + server + .exec("INSERT INTO tg_gate_ok { id: 'g1', name: 'Alice' }") + .await + .unwrap(); + + let stored = server + .query_text_joined("SELECT * FROM tg_gate_ok WHERE id = 'g1'") + .await + .unwrap(); + assert_eq!(stored.len(), 1); + assert!( + stored[0].contains("draft"), + "DEFAULT must still inject: {stored:?}" + ); +} diff --git a/nodedb/tests/wire/cases/sql_typeguard_defaults.rs b/nodedb/tests/wire/cases/sql_typeguard_defaults.rs index 9c966f4fa..726716db5 100644 --- a/nodedb/tests/wire/cases/sql_typeguard_defaults.rs +++ b/nodedb/tests/wire/cases/sql_typeguard_defaults.rs @@ -422,3 +422,37 @@ async fn strict_default_now() { .unwrap(); assert_eq!(rows.len(), 1, "should have 1 row: {rows:?}"); } + +// ── Unresolvable declared type ── + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn typeguard_unresolvable_type_is_refused_at_declaration() { + let server = TestServer::start().await; + + server.exec("CREATE COLLECTION tg_bad_type").await.unwrap(); + + // A type name the engine resolves to nothing. + server + .expect_error("CREATE TYPEGUARD ON tg_bad_type (gadget WIDGET)", "42601") + .await; + + // A trailing word that reading the leading token alone would ignore. + server + .expect_error( + "CREATE TYPEGUARD ON tg_bad_type (at TIMESTAMP GARBAGE)", + "42601", + ) + .await; + + // ALTER carries the same refusal. + server + .expect_error("ALTER TYPEGUARD ON tg_bad_type ADD gadget WIDGET", "42601") + .await; + + // A refused declaration reaches no storage. + let rows = server + .query_text("SHOW TYPEGUARD ON tg_bad_type") + .await + .unwrap(); + assert_eq!(rows.len(), 0, "refused guard must not be stored: {rows:?}"); +} From 7464d2ce411747a07cdf201eafb9dc545a3a8129 Mon Sep 17 00:00:00 2001 From: Farhan Syah Date: Thu, 10 Sep 2026 16:49:29 +0800 Subject: [PATCH 23/23] fix(sql): refuse per-row sequence accessors instead of returning NULL nextval/currval/setval only evaluate through SqlCatalog's sequence state at plan time, in a FROM-less SELECT or a column DEFAULT. A call over a FROM relation instead reached the row evaluator, which holds no sequence state and silently returned NULL for every row. Name the three accessor names once in a shared sequence_accessor module and use it from both the constant folder and the new resolver gate. TableScope::is_row_scope and ColumnScope::is_row_scope report whether an expression sits behind a relation; convert_function_depth refuses a sequence accessor there with the new SqlError::SequencePerRowUnsupported. Wire the refusal through to SQLSTATE 0A000 (feature_not_supported) on both the plan-error and pgwire error-mapping paths via a new crate::Error::FeatureNotSupported, and through error_classify for the native surface. --- nodedb-sql/src/error.rs | 16 +++ nodedb-sql/src/functions/mod.rs | 1 + nodedb-sql/src/functions/sequence_accessor.rs | 14 ++ nodedb-sql/src/planner/catalog_expr_fold.rs | 2 +- nodedb-sql/src/resolver/columns.rs | 9 ++ nodedb-sql/src/resolver/expr/functions.rs | 47 +++++++ nodedb-sql/src/resolver/scope.rs | 11 ++ nodedb/src/control/planner/plan_error_map.rs | 7 + .../control/server/pgwire/types/error_map.rs | 3 + nodedb/src/error/types.rs | 8 ++ nodedb/src/error_classify.rs | 3 + nodedb/tests/wire/cases/sql_sequences.rs | 127 ++++++++++++++++++ 12 files changed, 247 insertions(+), 1 deletion(-) create mode 100644 nodedb-sql/src/functions/sequence_accessor.rs diff --git a/nodedb-sql/src/error.rs b/nodedb-sql/src/error.rs index 7cad3031a..ae2763b89 100644 --- a/nodedb-sql/src/error.rs +++ b/nodedb-sql/src/error.rs @@ -19,6 +19,22 @@ pub enum SqlError { #[error("function {name}(...) does not exist")] UndefinedFunction { name: String }, + /// A sequence accessor appeared where every output row needs its own + /// allocation, such as a SELECT list over a FROM clause. + /// + /// Rendered as SQLSTATE `0A000` (feature_not_supported). Constant + /// contexts evaluate the call for real: a FROM-less `SELECT`, a column + /// `DEFAULT`, a `VALUES` list. The refusal keeps a per-row call from + /// reaching the row evaluator, which has no sequence state and would + /// return `NULL` for every row. + #[error( + "{name}(...) is not supported in a per-row context; \ + a SELECT list, WHERE clause, or SET clause over a FROM relation \ + evaluates once per row. Call it in a FROM-less SELECT or a column \ + DEFAULT instead" + )] + SequencePerRowUnsupported { name: String }, + /// A statement names a database object that does not exist — a sequence, /// most commonly. Distinct from [`SqlError::UndefinedFunction`]: the /// function exists, the object it names does not. PostgreSQL rejects the diff --git a/nodedb-sql/src/functions/mod.rs b/nodedb-sql/src/functions/mod.rs index 0da11726c..1adc04c26 100644 --- a/nodedb-sql/src/functions/mod.rs +++ b/nodedb-sql/src/functions/mod.rs @@ -4,6 +4,7 @@ pub mod arg_types; pub mod builtins; pub mod fts_ops; pub mod registry; +pub mod sequence_accessor; pub use registry::{ ArgTypeSpec, FunctionCategory, FunctionMeta, FunctionRegistry, SearchTrigger, Version, diff --git a/nodedb-sql/src/functions/sequence_accessor.rs b/nodedb-sql/src/functions/sequence_accessor.rs new file mode 100644 index 000000000..74249dd61 --- /dev/null +++ b/nodedb-sql/src/functions/sequence_accessor.rs @@ -0,0 +1,14 @@ +// SPDX-License-Identifier: Apache-2.0 + +//! The sequence-accessor function names, named once for every gate that +//! treats them apart from other scalars. + +/// The accessors the planner routes to `SqlCatalog` sequence state. +pub const SEQUENCE_ACCESSORS: [&str; 3] = ["nextval", "currval", "setval"]; + +/// Whether `name` calls a sequence accessor. Comparison ignores ASCII case. +pub fn is_sequence_accessor(name: &str) -> bool { + SEQUENCE_ACCESSORS + .iter() + .any(|accessor| name.eq_ignore_ascii_case(accessor)) +} diff --git a/nodedb-sql/src/planner/catalog_expr_fold.rs b/nodedb-sql/src/planner/catalog_expr_fold.rs index 4c5562c52..cf2d75ed8 100644 --- a/nodedb-sql/src/planner/catalog_expr_fold.rs +++ b/nodedb-sql/src/planner/catalog_expr_fold.rs @@ -55,7 +55,7 @@ pub(super) fn eval_sequence_accessor( return Ok(None); }; let lowered = name.to_ascii_lowercase(); - if !matches!(lowered.as_str(), "nextval" | "currval" | "setval") { + if !crate::functions::sequence_accessor::is_sequence_accessor(&lowered) { return Ok(None); } let sequence = match args.first() { diff --git a/nodedb-sql/src/resolver/columns.rs b/nodedb-sql/src/resolver/columns.rs index 463859824..cc5d7f18b 100644 --- a/nodedb-sql/src/resolver/columns.rs +++ b/nodedb-sql/src/resolver/columns.rs @@ -123,6 +123,15 @@ impl TableScope { } } + /// Whether an expression here is evaluated once per row of some relation. + /// + /// A scope with no relation of its own and no enclosing query stands + /// behind a FROM-less `SELECT`, which produces exactly one row and + /// evaluates its projection at plan time. + pub fn is_row_scope(&self) -> bool { + !self.tables.is_empty() || self.outer.is_some() + } + /// A copy of this scope nested inside `outer`, for planning a correlated /// subquery body. pub fn nested_in(mut self, outer: TableScope) -> Self { diff --git a/nodedb-sql/src/resolver/expr/functions.rs b/nodedb-sql/src/resolver/expr/functions.rs index f7bdf1980..10f0f6b58 100644 --- a/nodedb-sql/src/resolver/expr/functions.rs +++ b/nodedb-sql/src/resolver/expr/functions.rs @@ -86,6 +86,16 @@ pub(super) fn convert_function_depth( return Err(SqlError::UndefinedFunction { name }); } + // Per-row gate: a sequence accessor is evaluated at plan time, by + // `planner::catalog_expr_fold` for a FROM-less SELECT and by + // `planner::defaults` for a column DEFAULT. A call over a FROM relation + // reaches the row evaluator instead, which holds no sequence state and + // answers `NULL` for every row. Refuse it here so the statement fails + // loudly at plan time. + if scope.is_row_scope() && crate::functions::sequence_accessor::is_sequence_accessor(&name) { + return Err(SqlError::SequencePerRowUnsupported { name }); + } + let args = collect_function_args(func, depth, scope)?; let distinct = match &func.args { @@ -297,4 +307,41 @@ mod tests { "quoted 'UPPER' must still resolve via case-insensitive registry lookup" ); } + + /// A FROM-less SELECT resolves against an empty relation set, where the + /// planner evaluates the accessor through the catalog. + #[test] + fn a_sequence_accessor_resolves_without_a_relation() { + let func = function_ast("SELECT nextval('s')"); + let mut depth = 0; + let scope = crate::resolver::columns::TableScope::new(); + let expr = convert_function_depth(&func, &mut depth, &ColumnScope::Relations(&scope)) + .expect("a FROM-less sequence accessor must resolve"); + match expr { + SqlExpr::Function { name, .. } => assert_eq!(name, "nextval"), + other => panic!("expected SqlExpr::Function, got {other:?}"), + } + } + + /// A relation in scope makes the call per-row, which the row evaluator + /// cannot serve, so the resolver refuses it. + #[test] + fn a_sequence_accessor_over_a_relation_is_refused() { + let mut depth = 0; + let outer = crate::resolver::columns::TableScope::new(); + let scope = crate::resolver::columns::TableScope::new().nested_in(outer); + for call in [ + "SELECT nextval('s')", + "SELECT currval('s')", + "SELECT setval('s', 1)", + ] { + let func = function_ast(call); + let err = convert_function_depth(&func, &mut depth, &ColumnScope::Relations(&scope)) + .unwrap_err(); + assert!( + matches!(err, SqlError::SequencePerRowUnsupported { .. }), + "expected SqlError::SequencePerRowUnsupported for {call}, got {err:?}" + ); + } + } } diff --git a/nodedb-sql/src/resolver/scope.rs b/nodedb-sql/src/resolver/scope.rs index 378bd6bc5..e829dd79d 100644 --- a/nodedb-sql/src/resolver/scope.rs +++ b/nodedb-sql/src/resolver/scope.rs @@ -31,4 +31,15 @@ impl ColumnScope<'_> { Self::Relations(scope) => scope.check_name(table_ref, column), } } + + /// Whether an expression here is evaluated once per row of some relation. + /// + /// `Unchecked` stands behind stored DEFAULTs, index predicates, and the + /// constant folders, none of which iterate rows. + pub fn is_row_scope(&self) -> bool { + match self { + Self::Unchecked => false, + Self::Relations(scope) => scope.is_row_scope(), + } + } } diff --git a/nodedb/src/control/planner/plan_error_map.rs b/nodedb/src/control/planner/plan_error_map.rs index dfd3f957d..1e9dd8dbc 100644 --- a/nodedb/src/control/planner/plan_error_map.rs +++ b/nodedb/src/control/planner/plan_error_map.rs @@ -35,6 +35,13 @@ pub(crate) fn map_plan_error( nodedb_sql::SqlError::UndefinedFunction { name } => { crate::Error::UndefinedFunction { name } } + // A per-row sequence accessor is a refusal, not a syntax error, so it + // keeps SQLSTATE `0A000` rather than the `42601` the fallback gives. + nodedb_sql::SqlError::SequencePerRowUnsupported { .. } => { + crate::Error::FeatureNotSupported { + detail: error.to_string(), + } + } nodedb_sql::SqlError::UndefinedObject { kind, name } => { crate::Error::UndefinedObject { kind, name } } diff --git a/nodedb/src/control/server/pgwire/types/error_map.rs b/nodedb/src/control/server/pgwire/types/error_map.rs index 3cd3fb262..976634640 100644 --- a/nodedb/src/control/server/pgwire/types/error_map.rs +++ b/nodedb/src/control/server/pgwire/types/error_map.rs @@ -45,6 +45,9 @@ pub fn error_to_sqlstate(err: &crate::Error) -> (&'static str, &'static str, Str it is hard-deleted" ), ), + crate::Error::FeatureNotSupported { detail } => { + ("ERROR", sqlstate::FEATURE_NOT_SUPPORTED, detail.clone()) + } crate::Error::UndefinedFunction { name } => ( "ERROR", sqlstate::UNDEFINED_FUNCTION, diff --git a/nodedb/src/error/types.rs b/nodedb/src/error/types.rs index e15239bb8..a4ed8d330 100644 --- a/nodedb/src/error/types.rs +++ b/nodedb/src/error/types.rs @@ -276,6 +276,14 @@ pub enum Error { #[error("query plan error: {detail}")] PlanError { detail: String }, + /// A statement asked for a SQL feature this server does not implement. + /// Propagated from a planner refusal such as + /// `SqlError::SequencePerRowUnsupported`; the pgwire layer renders this + /// as SQLSTATE `0A000` (feature_not_supported), so a client stops rather + /// than retries. + #[error("{detail}")] + FeatureNotSupported { detail: String }, + /// A function call in a query names no registered scalar, aggregate, or /// window function. Propagated from `SqlError::UndefinedFunction`; the /// pgwire layer renders this as SQLSTATE `42883` (undefined_function). diff --git a/nodedb/src/error_classify.rs b/nodedb/src/error_classify.rs index 9607ee789..aaf58663c 100644 --- a/nodedb/src/error_classify.rs +++ b/nodedb/src/error_classify.rs @@ -149,6 +149,9 @@ pub(crate) fn classify(e: &Error) -> NodeDbError { NodeDbError::quota_overcommit(field.clone(), detail) } Error::PlanError { detail } => NodeDbError::plan_error(detail), + // The native surface carries this as a plan error; the pgwire + // surface renders SQLSTATE `0A000`. + Error::FeatureNotSupported { detail } => NodeDbError::plan_error(detail), Error::UndefinedFunction { name } => NodeDbError::undefined_function(name.clone()), Error::UndefinedObject { kind, name } => { NodeDbError::undefined_object(format!("{kind} \"{name}\"")) diff --git a/nodedb/tests/wire/cases/sql_sequences.rs b/nodedb/tests/wire/cases/sql_sequences.rs index a86949309..a14daefa6 100644 --- a/nodedb/tests/wire/cases/sql_sequences.rs +++ b/nodedb/tests/wire/cases/sql_sequences.rs @@ -299,3 +299,130 @@ async fn describing_an_insert_leaves_the_sequence_untouched() { "describe must not allocate, got {first:?}" ); } + +/// Create a two-row KV collection and a sequence, both named after `label`. +/// +/// Returns `(sequence_name, collection_name)`. +async fn two_row_collection_with_sequence(server: &TestServer, label: &str) -> (String, String) { + let sequence = format!("seq_row_ctx_{label}"); + let collection = format!("row_ctx_{label}"); + + server + .exec(&format!("CREATE SEQUENCE {sequence}")) + .await + .unwrap(); + server + .exec(&format!( + "CREATE COLLECTION {collection} (id BIGINT PRIMARY KEY, v TEXT) \ + WITH (engine = 'kv')" + )) + .await + .unwrap(); + server + .exec(&format!("INSERT INTO {collection} (id, v) VALUES (1, 'a')")) + .await + .unwrap(); + server + .exec(&format!("INSERT INTO {collection} (id, v) VALUES (2, 'b')")) + .await + .unwrap(); + + (sequence, collection) +} + +/// `nextval` in a SELECT list over a FROM relation raises `0A000` +/// (feature_not_supported). Per-row allocation is not implemented, and a +/// silent NULL for every row is worse than the refusal. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn nextval_in_a_select_list_over_a_scan_is_refused() { + let server = TestServer::start().await; + let (sequence, collection) = two_row_collection_with_sequence(&server, "nextval").await; + + server + .expect_error( + &format!("SELECT nextval('{sequence}') FROM {collection}"), + "0A000", + ) + .await; +} + +/// `currval` in a SELECT list over a FROM relation raises `0A000`, the same +/// as `nextval`: both read session sequence state the row evaluator lacks. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn currval_in_a_select_list_over_a_scan_is_refused() { + let server = TestServer::start().await; + let (sequence, collection) = two_row_collection_with_sequence(&server, "currval").await; + + server + .query_text(&format!("SELECT nextval('{sequence}')")) + .await + .unwrap(); + server + .expect_error( + &format!("SELECT currval('{sequence}') FROM {collection}"), + "0A000", + ) + .await; +} + +/// A projection mixing a plain column with a sequence accessor is refused as +/// a whole. Returning the column and a NULL beside it would ship the silent +/// cell this refusal exists to prevent. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn a_projection_mixing_a_column_and_an_accessor_is_refused() { + let server = TestServer::start().await; + let (sequence, collection) = two_row_collection_with_sequence(&server, "mixed").await; + + server + .expect_error( + &format!("SELECT id, nextval('{sequence}') FROM {collection}"), + "0A000", + ) + .await; +} + +/// A sequence accessor in a WHERE clause over a FROM relation is refused for +/// the same reason as one in the SELECT list. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn a_sequence_accessor_in_a_where_clause_is_refused() { + let server = TestServer::start().await; + let (sequence, collection) = two_row_collection_with_sequence(&server, "where").await; + + server + .expect_error( + &format!("SELECT id FROM {collection} WHERE id = nextval('{sequence}')"), + "0A000", + ) + .await; +} + +/// The refusal yields no result set at all, so no row can carry an empty or +/// NULL cell, and the statement allocates nothing from the sequence. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn a_refused_accessor_produces_no_row_and_no_allocation() { + let server = TestServer::start().await; + let (sequence, collection) = two_row_collection_with_sequence(&server, "guard").await; + + let result = server + .query_text(&format!("SELECT nextval('{sequence}') FROM {collection}")) + .await; + let message = match result { + Ok(rows) => panic!("per-row nextval must not return rows, got {rows:?}"), + Err(message) => message, + }; + assert!( + message.contains("0A000"), + "refusal must carry SQLSTATE 0A000, got: {message}" + ); + + // A refused statement consumes nothing, so the first real allocation is 1. + let first = server + .query_text(&format!("SELECT nextval('{sequence}')")) + .await + .unwrap(); + assert_eq!( + first, + vec!["1".to_string()], + "the refused statement must not have allocated, got {first:?}" + ); +}