From 97dd3740bf2d43bfe42f0333fcb0d80da0b826bc Mon Sep 17 00:00:00 2001 From: kould Date: Tue, 6 Oct 2026 17:23:12 +0800 Subject: [PATCH 01/16] fix: intersect every OR branch when detaching AND ranges An AND over a union was merged like an OR: pieces disjoint from the other operand were kept, yet the predicate was reported fully consumed, so EliminateIndexFilter dropped the filter and index scans returned extra rows (e.g. id > 10 AND (id = 3 OR id = 21) returned 3). An empty piece in the union also hit an unreachable!() arm. Split range merging by operator: AND sweeps both sorted unions and keeps only the intersections of overlapping pieces, OR keeps the in-place union merge. --- src/expression/range_detacher.rs | 350 ++++++++++++++++++++----------- tests/slt/where_by_index.slt | 20 ++ 2 files changed, 242 insertions(+), 128 deletions(-) diff --git a/src/expression/range_detacher.rs b/src/expression/range_detacher.rs index 7f0ebbde..019ebe72 100644 --- a/src/expression/range_detacher.rs +++ b/src/expression/range_detacher.rs @@ -330,6 +330,12 @@ impl RangeColumnMatcher for IndexRangeColumn { } } +enum Placement { + Before, + After, + Overlap, +} + pub struct RangeDetacher< 'a, M: RangeColumnMatcher = IndexRangeColumn, @@ -739,9 +745,7 @@ impl<'a, M: RangeColumnMatcher, A: MetaArena + ?Sized> RangeDetacher<'a, M, A> { if bounds_have_parameter(&[&min, &max]) || ranges.iter().any(Range::has_parameter) { return Err((Range::Scope { min, max }, Range::SortedRanges(ranges))); } - let merged_ranges = - Self::extract_merge_ranges(op, Some(Range::Scope { min, max }), ranges, &mut 0); - Ok(Self::ranges2range(merged_ranges)) + Self::merge_ranges(op, Range::Scope { min, max }, ranges) } // e.g. (c1 = 1 or c1 = 2) ? c1 > 1 (Range::SortedRanges(ranges), Range::Scope { min, max }) => { @@ -779,9 +783,7 @@ impl<'a, M: RangeColumnMatcher, A: MetaArena + ?Sized> RangeDetacher<'a, M, A> { if eq.has_parameter() || ranges.iter().any(Range::has_parameter) { return Err((Range::Eq(eq), Range::SortedRanges(ranges))); } - let merged_ranges = - Self::extract_merge_ranges(op, Some(Range::Eq(eq)), ranges, &mut 0); - Ok(Self::ranges2range(merged_ranges)) + Self::merge_ranges(op, Range::Eq(eq), ranges) } // e.g. (c1 = 1 or c1 = 2) ? c1 = 1 (Range::SortedRanges(ranges), Range::Eq(eq)) => { @@ -789,7 +791,7 @@ impl<'a, M: RangeColumnMatcher, A: MetaArena + ?Sized> RangeDetacher<'a, M, A> { .map_err(|(right, left)| (left, right)) } // e.g. (c1 = 1 or c1 = 2) ? (c1 = 1 or c1 = 2) - (Range::SortedRanges(left_ranges), Range::SortedRanges(mut right_ranges)) => { + (Range::SortedRanges(left_ranges), Range::SortedRanges(right_ranges)) => { if left_ranges.iter().any(Range::has_parameter) || right_ranges.iter().any(Range::has_parameter) { @@ -798,12 +800,7 @@ impl<'a, M: RangeColumnMatcher, A: MetaArena + ?Sized> RangeDetacher<'a, M, A> { Range::SortedRanges(right_ranges), )); } - let mut idx = 0; - for left_range in left_ranges { - right_ranges = - Self::extract_merge_ranges(op, Some(left_range), right_ranges, &mut idx) - } - Ok(Self::ranges2range(right_ranges)) + Self::merge_ranges(op, Range::SortedRanges(left_ranges), right_ranges) } } } @@ -818,132 +815,177 @@ impl<'a, M: RangeColumnMatcher, A: MetaArena + ?Sized> RangeDetacher<'a, M, A> { } } - #[allow(unreachable_code)] - fn extract_merge_ranges( + #[allow(clippy::result_large_err)] + fn merge_ranges( op: BinaryOperator, - mut binary: Option, + left: Range, mut ranges: Vec, - idx: &mut usize, - ) -> Vec { - // FIXME: Lots of duplicate code - while *idx < ranges.len() { - match (&binary, &ranges[*idx]) { - ( - Some(Range::Scope { - min: l_min, - max: l_max, - }), - Range::Scope { - min: r_min, - max: r_max, - }, - ) => { - if let Some(true) = - Self::bound_compared(l_max, r_min, true, false).map(Ordering::is_lt) - { - ranges.insert(*idx, binary.unwrap()); - return ranges; - } else if let Some(true) = - Self::bound_compared(l_min, r_max, false, true).map(Ordering::is_gt) - { - *idx += 1; - continue; - } else { - binary = Self::merge_binary(op, binary.unwrap(), ranges.remove(*idx)).ok(); - } - } - ( - Some(Range::Scope { - min: l_min, - max: l_max, - }), - Range::Eq(r_val), - ) => { - let r_bound = Bound::Included(r_val); - - if let Some(true) = - Self::bound_compared(l_max, &r_bound, true, false).map(Ordering::is_lt) - { - ranges.insert(*idx, binary.unwrap()); - return ranges; - } else if Self::bound_compared(l_min, &r_bound, false, true) - .map(Ordering::is_gt) - .unwrap_or_else(|| op == BinaryOperator::Or) - { - *idx += 1; - continue; - } else if r_val.is_null() { - let _ = ranges.remove(*idx); - } else { - binary = Self::merge_binary(op, binary.unwrap(), ranges.remove(*idx)).ok(); + ) -> Result { + match op { + BinaryOperator::And => Self::intersect_ranges(left, ranges), + BinaryOperator::Or => { + Self::union_ranges(left, &mut ranges); + Ok(Self::ranges2range(ranges)) + } + _ => Err((left, Range::SortedRanges(ranges))), + } + } + + fn placement(op: BinaryOperator, left: &Range, right: &Range) -> Option { + let placement = match (left, right) { + ( + Range::Scope { + min: l_min, + max: l_max, + }, + Range::Scope { + min: r_min, + max: r_max, + }, + ) => Self::bound_compared(l_max, r_min, true, false) + .is_some_and(Ordering::is_lt) + .then_some(Placement::Before) + .or_else(|| { + Self::bound_compared(l_min, r_max, false, true) + .is_some_and(Ordering::is_gt) + .then_some(Placement::After) + }), + ( + Range::Scope { + min: l_min, + max: l_max, + }, + Range::Eq(r_val), + ) => { + let r_bound = Bound::Included(r_val); + Self::bound_compared(l_max, &r_bound, true, false) + .is_some_and(Ordering::is_lt) + .then_some(Placement::Before) + .or_else(|| { + Self::bound_compared(l_min, &r_bound, false, true) + .map(Ordering::is_gt) + .unwrap_or_else(|| op == BinaryOperator::Or) + .then_some(Placement::After) + }) + } + (Range::Eq(l_val), Range::Eq(r_val)) => match l_val.partial_cmp(r_val) { + Some(Ordering::Less) => Some(Placement::Before), + Some(Ordering::Greater) => Some(Placement::After), + _ => None, + }, + ( + Range::Eq(l_val), + Range::Scope { + min: r_min, + max: r_max, + }, + ) => { + let l_bound = Bound::Included(l_val); + Self::bound_compared(&l_bound, r_min, true, false) + .map(Ordering::is_lt) + .unwrap_or_else(|| op == BinaryOperator::Or) + .then_some(Placement::Before) + .or_else(|| { + Self::bound_compared(&l_bound, r_max, false, true) + .is_some_and(Ordering::is_gt) + .then_some(Placement::After) + }) + } + _ => return None, + }; + Some(placement.unwrap_or(Placement::Overlap)) + } + + #[allow(clippy::result_large_err)] + fn intersect_ranges(left: Range, mut ranges: Vec) -> Result { + let len = ranges.len(); + let l_ranges = match &left { + Range::SortedRanges(l_ranges) => l_ranges.as_slice(), + single => std::slice::from_ref(single), + }; + let mut start = 0; + for l_range in l_ranges { + let mut idx = start; + while idx < len { + let Some(placement) = Self::placement(BinaryOperator::And, l_range, &ranges[idx]) + else { + ranges.truncate(len); + return Err((left, Range::SortedRanges(ranges))); + }; + match placement { + Placement::After => { + idx += 1; + start = idx; } - } - (Some(Range::Eq(l_val)), Range::Eq(r_val)) => { - if let Some(true) = l_val.partial_cmp(r_val).map(Ordering::is_lt) { - ranges.insert(*idx, binary.unwrap()); - return ranges; - } else if let Some(true) = l_val.partial_cmp(r_val).map(Ordering::is_gt) { - *idx += 1; - continue; - } else { - binary = Self::merge_binary(op, binary.unwrap(), ranges.remove(*idx)).ok(); + Placement::Before => break, + Placement::Overlap => { + match Self::merge_binary( + BinaryOperator::And, + l_range.clone(), + ranges[idx].clone(), + ) { + Ok(Range::Dummy) => (), + Ok(range) => ranges.push(range), + Err(_) => { + ranges.truncate(len); + return Err((left, Range::SortedRanges(ranges))); + } + } + idx += 1; } } - ( - Some(Range::Eq(l_val)), - Range::Scope { - min: r_min, - max: r_max, - }, - ) => { - let l_bound = Bound::Included(l_val); - - if Self::bound_compared(&l_bound, r_min, true, false) - .map(Ordering::is_lt) - .unwrap_or_else(|| op == BinaryOperator::Or) - { - ranges.insert(*idx, binary.unwrap()); - return ranges; - } else if let Some(true) = - Self::bound_compared(&l_bound, r_max, false, true).map(Ordering::is_gt) - { - *idx += 1; - continue; - } else if l_val.is_null() { - binary = Some(ranges.remove(*idx)); - } else { - binary = Self::merge_binary(op, binary.unwrap(), ranges.remove(*idx)).ok(); - } + } + } + drop(ranges.drain(..len)); + Ok(Self::ranges2range(ranges)) + } + + fn union_ranges(binary: Range, ranges: &mut Vec) { + match binary { + Range::Dummy => (), + Range::SortedRanges(l_ranges) => { + for l_range in l_ranges { + Self::union_ranges(l_range, ranges); } - (Some(Range::Dummy), _) => { - binary = match op { - BinaryOperator::And => return vec![], - BinaryOperator::Or => Some(ranges.remove(*idx)), - _ => None, - }; + } + piece => Self::union_piece(piece, ranges), + } + } + + #[allow(unreachable_code)] + fn union_piece(mut binary: Range, ranges: &mut Vec) { + let mut idx = 0; + while idx < ranges.len() { + let Some(placement) = Self::placement(BinaryOperator::Or, &binary, &ranges[idx]) else { + #[cfg(debug_assertions)] + { + unreachable!(); } - (Some(Range::SortedRanges(l_ranges)), r_range) => { - return Self::extract_merge_ranges( - op, - Some(r_range.clone()), - l_ranges.clone(), - &mut 0, - ); + ranges.clear(); + return; + }; + match placement { + Placement::Before => { + ranges.insert(idx, binary); + return; } - (None, _) => break, - _ => { - #[cfg(debug_assertions)] - { - unreachable!(); + Placement::After => idx += 1, + Placement::Overlap => match (&binary, &ranges[idx]) { + (Range::Scope { .. }, Range::Eq(r_val)) if r_val.is_null() => { + let _ = ranges.remove(idx); } - return vec![]; - } + (Range::Eq(l_val), Range::Scope { .. }) if l_val.is_null() => { + binary = ranges.remove(idx); + } + _ => match Self::merge_binary(BinaryOperator::Or, binary, ranges.remove(idx)) { + Ok(range @ (Range::Scope { .. } | Range::Eq(_))) => binary = range, + Ok(range) => return Self::union_ranges(range, ranges), + Err(_) => return, + }, + }, } } - if let Some(range) = binary { - ranges.push(range); - } - ranges + ranges.push(binary); } fn or_scope_merge( @@ -2865,4 +2907,56 @@ mod test { .is_none()); assert!(!suffix.only_eq()); } + + /// AND over a union must intersect every piece instead of keeping the disjoint ones. + #[test] + fn test_detach_and_over_or() -> Result<(), DatabaseError> { + let table_state = build_t1_table()?; + let mut plan_arena = crate::planner::PlanArena::new(&table_state.table_arena); + let eq = |v| Range::Eq(DataValue::Int32(v)); + let cases = [ + ("c1 > 10 and (c1 = 1 or c1 = 20)", eq(20)), + // `c1 > 3` must still be applied after it has been intersected with `c1 = 5`. + ( + "c1 > 3 and (c1 = 5 or c1 = 7)", + Range::SortedRanges(vec![eq(5), eq(7)]), + ), + ("(c1 = 1 or c1 = 20) and (c1 = 5 or c1 = 20)", eq(20)), + // One `rj` overlapping several `li` must be intersected with each of them. + ( + "(c1 = 5 or c1 = 7) and (c1 < 10 or c1 > 20)", + Range::SortedRanges(vec![eq(5), eq(7)]), + ), + ( + "(c1 < 10 or c1 > 20) and (c1 = 5 or c1 = 7 or c1 = 15 or c1 = 25)", + Range::SortedRanges(vec![eq(5), eq(7), eq(25)]), + ), + ( + "c1 > 3 and (c1 < 2 or c1 > 30)", + Range::Scope { + min: Bound::Excluded(DataValue::Int32(30)), + max: Bound::Unbounded, + }, + ), + ("c1 = 1 and (c1 > 10 or c1 < 3)", eq(1)), + ( + "(c1 > 5 or c1 < 0) and (c1 = -1 or c1 = 3 or c1 = 7)", + Range::SortedRanges(vec![eq(-1), eq(7)]), + ), + ("(c1 = 1 or c1 = 2) and (c1 = 3 or c1 = 4)", Range::Dummy), + ("c1 = 3 and (c1 between 5 and 2 or c1 > 9)", Range::Dummy), + ]; + for (predicate, expected) in cases { + let sql = format!("select * from t1 where {predicate}"); + let plan = table_state.plan_with_arena(&sql, &mut plan_arena)?; + let op = plan_filter(plan, &mut plan_arena)?.unwrap(); + let detached = + RangeDetacher::new("t1", table_state.column_id_by_name("c1"), &mut plan_arena) + .detach(op.predicate)? + .expect("c1 predicate should be consumed"); + assert_eq!(detached.range, expected, "{predicate}"); + assert_eq!(detached.residual, None, "{predicate}"); + } + Ok(()) + } } diff --git a/tests/slt/where_by_index.slt b/tests/slt/where_by_index.slt index b87056cf..2378bbd6 100644 --- a/tests/slt/where_by_index.slt +++ b/tests/slt/where_by_index.slt @@ -46,6 +46,26 @@ query IIT select * from t1 where id = 0 and id != 0; ---- +# AND over an OR must intersect every OR branch, not union them +query IIT +select * from t1 where id > 10 and (id = 3 or id = 21); +---- +21 22 23 + +query IIT +select * from t1 where (id = 3 or id = 21) and (id = 21 or id = 30); +---- +21 22 23 + +query IIT +select * from t1 where (id = 3 or id = 21) and (id = 6 or id = 9); +---- + +query IIT +select * from t1 where id > 10 and id < 30 and (id < 5 or id > 25); +---- +27 28 29 + query IIT select * from t1 where id = 0 or id != 0 limit 10; ---- From 112238446eda3aa730e70b914f70c2307036d255 Mon Sep 17 00:00:00 2001 From: kould Date: Tue, 6 Oct 2026 17:59:31 +0800 Subject: [PATCH 02/16] fix: order NULL last when merging index ranges NULL index keys sort after every value, but range merging disagreed: - Eq x Eq placement used partial_cmp, so Eq(NULL) vs Eq(x) counted as overlapping; OR-merging them made no progress and recursed until the stack overflowed (e.g. c1 = 1 or c1 = 2 or c1 is null). - Scope OR Eq(NULL) put NULL first, and kept it only when the lower bound was excluded, so c1 >= 3 or c1 is null and c1 < 3 or c1 is null dropped the NULL rows. A scope only covers NULL when its upper bound is unbounded. Compare Eq x Eq with bound_compared, keep NULL as a trailing piece unless the scope already covers it, and drop the special NULL arms in union_piece that were masking the inconsistency. --- src/expression/range_detacher.rs | 28 ++++++++-------- tests/slt/where_by_index.slt | 56 ++++++++++++++++++++++++++++++++ 2 files changed, 70 insertions(+), 14 deletions(-) diff --git a/src/expression/range_detacher.rs b/src/expression/range_detacher.rs index 019ebe72..2829d0fa 100644 --- a/src/expression/range_detacher.rs +++ b/src/expression/range_detacher.rs @@ -693,11 +693,12 @@ impl<'a, M: RangeColumnMatcher, A: MetaArena + ?Sized> RangeDetacher<'a, M, A> { Ok(Range::Eq(unpack_bound(bound_eq))) } BinaryOperator::Or => { + // NULL sorts after every value, so only an unbounded `max` already covers it. if eq.is_null() { - return Ok(if matches!(min, Bound::Excluded(_)) { - Range::SortedRanges(vec![Range::Eq(eq), Range::Scope { min, max }]) - } else { + return Ok(if matches!(max, Bound::Unbounded) { Range::Scope { min, max } + } else { + Range::SortedRanges(vec![Range::Scope { min, max }, Range::Eq(eq)]) }); } let bound_eq = Bound::Excluded(eq); @@ -868,7 +869,12 @@ impl<'a, M: RangeColumnMatcher, A: MetaArena + ?Sized> RangeDetacher<'a, M, A> { .then_some(Placement::After) }) } - (Range::Eq(l_val), Range::Eq(r_val)) => match l_val.partial_cmp(r_val) { + (Range::Eq(l_val), Range::Eq(r_val)) => match Self::bound_compared( + &Bound::Included(l_val), + &Bound::Included(r_val), + false, + false, + ) { Some(Ordering::Less) => Some(Placement::Before), Some(Ordering::Greater) => Some(Placement::After), _ => None, @@ -970,19 +976,13 @@ impl<'a, M: RangeColumnMatcher, A: MetaArena + ?Sized> RangeDetacher<'a, M, A> { return; } Placement::After => idx += 1, - Placement::Overlap => match (&binary, &ranges[idx]) { - (Range::Scope { .. }, Range::Eq(r_val)) if r_val.is_null() => { - let _ = ranges.remove(idx); - } - (Range::Eq(l_val), Range::Scope { .. }) if l_val.is_null() => { - binary = ranges.remove(idx); - } - _ => match Self::merge_binary(BinaryOperator::Or, binary, ranges.remove(idx)) { + Placement::Overlap => { + match Self::merge_binary(BinaryOperator::Or, binary, ranges.remove(idx)) { Ok(range @ (Range::Scope { .. } | Range::Eq(_))) => binary = range, Ok(range) => return Self::union_ranges(range, ranges), Err(_) => return, - }, - }, + } + } } } ranges.push(binary); diff --git a/tests/slt/where_by_index.slt b/tests/slt/where_by_index.slt index 2378bbd6..e5a7bd94 100644 --- a/tests/slt/where_by_index.slt +++ b/tests/slt/where_by_index.slt @@ -508,3 +508,59 @@ select item,w,qty from stock_bounds where w=1 and qty<2 order by item; statement ok drop table stock_bounds; + +# OR / AND with IS NULL over an index: NULL sorts after every value +statement ok +create table t_null_or(id int primary key, c1 int); + +statement ok +create index t_null_or_c1 on t_null_or (c1); + +statement ok +insert into t_null_or values (0, null), (1, 1), (2, 2), (3, 3), (4, 4), (5, 5), (6, 6), (7, 7), (8, 8), (9, 9), (10, 0), (11, 1), (12, 2), (13, 3), (14, 4), (15, 5), (16, 6), (17, 7), (18, 8), (19, 9), (20, null), (21, 1), (22, 2), (23, 3), (24, 4), (25, 5), (26, 6), (27, 7), (28, 8), (29, 9), (30, 0), (31, 1), (32, 2), (33, 3), (34, 4), (35, 5), (36, 6), (37, 7), (38, 8), (39, 9), (40, null), (41, 1), (42, 2), (43, 3), (44, 4), (45, 5), (46, 6), (47, 7), (48, 8), (49, 9), (50, 0), (51, 1), (52, 2), (53, 3), (54, 4), (55, 5), (56, 6), (57, 7), (58, 8), (59, 9), (60, null), (61, 1), (62, 2), (63, 3), (64, 4), (65, 5), (66, 6), (67, 7), (68, 8), (69, 9), (70, 0), (71, 1), (72, 2), (73, 3), (74, 4), (75, 5), (76, 6), (77, 7), (78, 8), (79, 9), (80, null), (81, 1), (82, 2), (83, 3), (84, 4), (85, 5), (86, 6), (87, 7), (88, 8), (89, 9), (90, 0), (91, 1), (92, 2), (93, 3), (94, 4), (95, 5), (96, 6), (97, 7), (98, 8), (99, 9), (100, null), (101, 1), (102, 2), (103, 3), (104, 4), (105, 5), (106, 6), (107, 7), (108, 8), (109, 9), (110, 0), (111, 1), (112, 2), (113, 3), (114, 4), (115, 5), (116, 6), (117, 7), (118, 8), (119, 9), (120, null), (121, 1), (122, 2), (123, 3), (124, 4), (125, 5), (126, 6), (127, 7), (128, 8), (129, 9), (130, 0), (131, 1), (132, 2), (133, 3), (134, 4), (135, 5), (136, 6), (137, 7), (138, 8), (139, 9), (140, null), (141, 1), (142, 2), (143, 3), (144, 4), (145, 5), (146, 6), (147, 7), (148, 8), (149, 9), (150, 0), (151, 1), (152, 2), (153, 3), (154, 4), (155, 5), (156, 6), (157, 7), (158, 8), (159, 9), (160, null), (161, 1), (162, 2), (163, 3), (164, 4), (165, 5), (166, 6), (167, 7), (168, 8), (169, 9), (170, 0), (171, 1), (172, 2), (173, 3), (174, 4), (175, 5), (176, 6), (177, 7), (178, 8), (179, 9), (180, null), (181, 1), (182, 2), (183, 3), (184, 4), (185, 5), (186, 6), (187, 7), (188, 8), (189, 9), (190, 0), (191, 1), (192, 2), (193, 3), (194, 4), (195, 5), (196, 6), (197, 7), (198, 8), (199, 9); + +statement ok +analyze table t_null_or; + +query I +select count(*) from t_null_or where c1 = 1 or c1 = 2 or c1 is null; +---- +50 + +query I +select count(*) from t_null_or where c1 > 3 or c1 is null; +---- +130 + +query I +select count(*) from t_null_or where c1 >= 3 or c1 is null; +---- +150 + +query I +select count(*) from t_null_or where c1 < 3 or c1 is null; +---- +60 + +query I +select count(*) from t_null_or where c1 is not null or c1 is null; +---- +200 + +query I +select count(*) from t_null_or where c1 = 1 or c1 > 3 or c1 is null; +---- +150 + +query I +select count(*) from t_null_or where c1 = 5 and (c1 is null or c1 > 3); +---- +20 + +query I +select count(*) from t_null_or where (c1 = 1 or c1 is null) and (c1 is null or c1 > 3); +---- +10 + +statement ok +drop table t_null_or; From ad29084e03453b0029847a84feccbf67ab3393d9 Mon Sep 17 00:00:00 2001 From: kould Date: Tue, 6 Oct 2026 17:59:31 +0800 Subject: [PATCH 03/16] feat: display NULL values as uppercase NULL Match SQLite, PostgreSQL and MySQL. Also show a missing column default as NULL in describe, and update slt expectations (two rowsort blocks reorder because NULL now sorts before lowercase text). --- src/execution/dql/describe.rs | 2 +- src/types/value.rs | 4 +- tests/slt/aggregation.slt | 6 +- tests/slt/alter_table.slt | 24 +- tests/slt/basic_test.slt | 2 +- tests/slt/change_column.slt | 50 ++-- tests/slt/char.slt | 4 +- tests/slt/crdb/and_or.slt | 20 +- tests/slt/crdb/condition.slt | 8 +- tests/slt/crdb/join.slt | 304 ++++++++++----------- tests/slt/crdb/natural_join.slt | 6 +- tests/slt/crdb/order_by.slt | 94 +++---- tests/slt/crdb/select.slt | 22 +- tests/slt/crdb/where.slt | 4 +- tests/slt/describe.slt | 10 +- tests/slt/distinct.slt | 4 +- tests/slt/dummy.slt | 20 +- tests/slt/filter_null.slt | 2 +- tests/slt/insert.slt | 10 +- tests/slt/insert_multiple_primary_keys.slt | 6 +- tests/slt/join.slt | 32 +-- tests/slt/join_left_inner.slt | 8 +- tests/slt/order_by.slt | 12 +- tests/slt/set_operation.slt | 18 +- tests/slt/sql_2016/E091_01.slt | 2 +- tests/slt/sql_2016/E091_03.slt | 2 +- tests/slt/sql_2016/E091_04.slt | 2 +- tests/slt/sql_2016/E091_05.slt | 2 +- tests/slt/sql_2016/E091_06.slt | 8 +- tests/slt/sql_2016/E091_07.slt | 8 +- tests/slt/sql_2016/E131.slt | 2 +- tests/slt/sql_2016/F131_03.slt | 2 +- tests/slt/sql_2016/F261_01.slt | 16 +- tests/slt/sql_2016/F261_02.slt | 8 +- tests/slt/sql_2016/F261_03.slt | 2 +- tests/slt/string_length.slt | 2 +- tests/slt/subquery.slt | 10 +- tests/slt/substring.slt | 6 +- tests/slt/values.slt | 2 +- tests/slt/where.slt | 12 +- tests/slt/where_by_index.slt | 2 +- tests/slt/where_by_index_explain.slt | 2 +- 42 files changed, 381 insertions(+), 381 deletions(-) diff --git a/src/execution/dql/describe.rs b/src/execution/dql/describe.rs index 98316a26..2c64137c 100644 --- a/src/execution/dql/describe.rs +++ b/src/execution/dql/describe.rs @@ -115,7 +115,7 @@ fn describe_default(column: &ColumnCatalog, arena: &(dyn MetaArena + '_)) -> Str .default .as_ref() .map(|expr| expr.output_name(arena)) - .unwrap_or_else(|| "null".to_string()) + .unwrap_or_else(|| "NULL".to_string()) } fn fill_describe_row(values: &mut Vec, column: &ColumnCatalog, default: String) { diff --git a/src/types/value.rs b/src/types/value.rs index a2bdc311..7564783c 100644 --- a/src/types/value.rs +++ b/src/types/value.rs @@ -1801,7 +1801,7 @@ impl fmt::Display for DataValue { DataValue::UInt32(e) => write!(f, "{e}")?, DataValue::UInt64(e) => write!(f, "{e}")?, DataValue::Utf8 { value: e, .. } => write!(f, "{e}")?, - DataValue::Null => write!(f, "null")?, + DataValue::Null => write!(f, "NULL")?, DataValue::Date32(e) => { #[cfg(feature = "time")] write!(f, "{}", DataValue::date_format(*e).unwrap())?; @@ -2361,7 +2361,7 @@ mod test { (DataValue::UInt32(3), "3", "UInt32(3)"), (DataValue::UInt64(4), "4", "UInt64(4)"), (utf8("kite"), "kite", "Utf8(\"kite\")"), - (DataValue::Null, "null", "null"), + (DataValue::Null, "NULL", "null"), #[cfg(feature = "decimal")] ( DataValue::Decimal(Decimal::new(123, 2)), diff --git a/tests/slt/aggregation.slt b/tests/slt/aggregation.slt index 6afd2efb..0d6b0b1c 100644 --- a/tests/slt/aggregation.slt +++ b/tests/slt/aggregation.slt @@ -86,17 +86,17 @@ insert into sn values (1, 1), (2, 2); query I select sum(null) from sn; ---- -null +NULL query I select sum(distinct null) from sn; ---- -null +NULL query I select (select sum(null) from sn) from sn where id = 1; ---- -null +NULL statement ok drop table sn diff --git a/tests/slt/alter_table.slt b/tests/slt/alter_table.slt index 0b87fadf..289a6ef0 100644 --- a/tests/slt/alter_table.slt +++ b/tests/slt/alter_table.slt @@ -19,10 +19,10 @@ alter table t1 add column v4 int not null query IIII rowsort select * from t1 ---- -1 1 null 0 -2 2 null 0 -3 3 null 0 -4 4 null 0 +1 1 NULL 0 +2 2 NULL 0 +3 3 NULL 0 +4 4 NULL 0 statement ok alter table t1 drop column if exists v1 @@ -38,10 +38,10 @@ alter table t1 drop column id query IIII rowsort select * from t1 ---- -1 null 0 -2 null 0 -3 null 0 -4 null 0 +1 NULL 0 +2 NULL 0 +3 NULL 0 +4 NULL 0 statement ok create table t2(id int primary key, v1 int) @@ -72,8 +72,8 @@ insert into t3 values (1, 10), (2, 20) query TTTTTT describe t3 ---- -id Integer 4 false PRIMARY null -v Integer 4 true EMPTY null +id Integer 4 false PRIMARY NULL +v Integer 4 true EMPTY NULL statement ok insert into t3 values (3, 10) @@ -107,8 +107,8 @@ select * from t3 query TTTTTT describe t3 ---- -id Integer 4 false PRIMARY null -v Integer 4 true EMPTY null +id Integer 4 false PRIMARY NULL +v Integer 4 true EMPTY NULL statement error alter table t3 modify column v int first diff --git a/tests/slt/basic_test.slt b/tests/slt/basic_test.slt index f05fd01d..6a8e6c72 100644 --- a/tests/slt/basic_test.slt +++ b/tests/slt/basic_test.slt @@ -54,7 +54,7 @@ insert into t values(0, null) query T select * from t ---- -0 null +0 NULL statement ok drop table t diff --git a/tests/slt/change_column.slt b/tests/slt/change_column.slt index c25c441a..aad116af 100644 --- a/tests/slt/change_column.slt +++ b/tests/slt/change_column.slt @@ -25,9 +25,9 @@ alter table alter_users alter column v2 type bigint query TTTTTT describe alter_users ---- -id Integer 4 false PRIMARY null -value1 Integer 4 true UNIQUE null -v2 Bigint 8 true EMPTY null +id Integer 4 false PRIMARY NULL +value1 Integer 4 true UNIQUE NULL +v2 Bigint 8 true EMPTY NULL query I select v2 from alter_users order by id @@ -41,9 +41,9 @@ alter table alter_users change column v2 value2 bigint query TTTTTT describe alter_users ---- -id Integer 4 false PRIMARY null -value1 Integer 4 true UNIQUE null -value2 Bigint 8 true EMPTY null +id Integer 4 false PRIMARY NULL +value1 Integer 4 true UNIQUE NULL +value2 Bigint 8 true EMPTY NULL statement ok alter table alter_users alter column value2 set default 999 @@ -65,7 +65,7 @@ insert into alter_users (id, value1) values (4, 44) query T select value2 from alter_users where id = 4 ---- -null +NULL statement error alter table alter_users alter column value2 set not null @@ -79,9 +79,9 @@ alter table alter_users alter column value2 set not null query TTTTTT describe alter_users ---- -id Integer 4 false PRIMARY null -value1 Integer 4 true UNIQUE null -value2 Bigint 8 false EMPTY null +id Integer 4 false PRIMARY NULL +value1 Integer 4 true UNIQUE NULL +value2 Bigint 8 false EMPTY NULL statement error insert into alter_users (id, value1) values (5, 55) @@ -95,7 +95,7 @@ insert into alter_users (id, value1) values (6, 66) query T select value2 from alter_users where id = 6 ---- -null +NULL statement ok alter table alter_users modify column value2 int @@ -103,9 +103,9 @@ alter table alter_users modify column value2 int query TTTTTT describe alter_users ---- -id Integer 4 false PRIMARY null -value1 Integer 4 true UNIQUE null -value2 Integer 4 true EMPTY null +id Integer 4 false PRIMARY NULL +value1 Integer 4 true UNIQUE NULL +value2 Integer 4 true EMPTY NULL query I select value2 from alter_users where value2 is not null order by id @@ -124,9 +124,9 @@ alter table alter_users change column value2 value2 int not null query TTTTTT describe alter_users ---- -id Integer 4 false PRIMARY null -value1 Integer 4 true UNIQUE null -value2 Integer 4 false EMPTY null +id Integer 4 false PRIMARY NULL +value1 Integer 4 true UNIQUE NULL +value2 Integer 4 false EMPTY NULL statement error insert into alter_users (id, value1) values (7, 77) @@ -145,8 +145,8 @@ select value2 from alter_users where id = 7 query TTTTTT describe alter_users ---- -id Integer 4 false PRIMARY null -value1 Integer 4 true UNIQUE null +id Integer 4 false PRIMARY NULL +value1 Integer 4 true UNIQUE NULL value2 Integer 4 false EMPTY 707 statement ok @@ -163,8 +163,8 @@ select value2 from alter_users where id = 8 query TTTTTT describe alter_users ---- -id Integer 4 false PRIMARY null -value1 Integer 4 true UNIQUE null +id Integer 4 false PRIMARY NULL +value1 Integer 4 true UNIQUE NULL value2 Integer 4 true EMPTY 707 statement ok @@ -173,8 +173,8 @@ alter table alter_users modify column value2 int not null query TTTTTT describe alter_users ---- -id Integer 4 false PRIMARY null -value1 Integer 4 true UNIQUE null +id Integer 4 false PRIMARY NULL +value1 Integer 4 true UNIQUE NULL value2 Integer 4 false EMPTY 707 statement ok @@ -191,8 +191,8 @@ select value2 from alter_users where id = 9 query TTTTTT describe alter_users ---- -id Integer 4 false PRIMARY null -value1 Integer 4 true UNIQUE null +id Integer 4 false PRIMARY NULL +value1 Integer 4 true UNIQUE NULL value2 Integer 4 false EMPTY 808 statement ok diff --git a/tests/slt/char.slt b/tests/slt/char.slt index e054634e..75938baf 100644 --- a/tests/slt/char.slt +++ b/tests/slt/char.slt @@ -17,7 +17,7 @@ query ITT select * from t1; ---- 0 🪁 🪁🪁🪁🪁🪁 🪁 -1 null null null +1 NULL NULL NULL statement ok create table t2(id int primary key, v1 varchar, v2 varchar(5 characters), v3 varchar(5 octets)) @@ -38,7 +38,7 @@ query ITT select * from t2; ---- 0 🪁 🪁🪁🪁🪁🪁 🪁 -1 null null null +1 NULL NULL NULL statement ok drop table t1; diff --git a/tests/slt/crdb/and_or.slt b/tests/slt/crdb/and_or.slt index e418691f..a54e57da 100644 --- a/tests/slt/crdb/and_or.slt +++ b/tests/slt/crdb/and_or.slt @@ -10,9 +10,9 @@ INSERT INTO t VALUES (1, NULL, NULL), (2, NULL, 1), (3, 1, NULL), (4, 2, 0), (5, query T SELECT a <> 2 AND 3 / b = 1 FROM t ORDER BY k ---- -null +NULL false -null +NULL false true @@ -24,9 +24,9 @@ SELECT a FROM t WHERE a <> 2 AND 3 / b = 1 ORDER BY k query T SELECT a = 2 OR 3 / b = 1 FROM t ORDER BY k ---- -null -null -null +NULL +NULL +NULL true true @@ -45,9 +45,9 @@ INSERT INTO t VALUES (1, NULL, NULL), (2, NULL, 1), (3, 1, NULL), (4, 2, 1), (5, query T SELECT a <> 2 AND 3 / b = 1 FROM t ORDER BY k ---- -null +NULL false -null +NULL false true @@ -59,9 +59,9 @@ SELECT a FROM t WHERE a <> 2 AND 3 / b = 1 ORDER BY k query T SELECT a = 2 OR 3 / b = 1 FROM t ORDER BY k ---- -null -null -null +NULL +NULL +NULL true true diff --git a/tests/slt/crdb/condition.slt b/tests/slt/crdb/condition.slt index 4b104c91..c23e4e15 100644 --- a/tests/slt/crdb/condition.slt +++ b/tests/slt/crdb/condition.slt @@ -1,12 +1,12 @@ query IT SELECT IF(1 = 2, NULL, 1), IF(2 = 2, NULL, 2) ---- -1 null +1 NULL query ITT SELECT NULLIF(1, 2), NULLIF(2, 2), NULLIF(NULL, NULL) ---- -1 null null +1 NULL NULL query IIII SELECT IFNULL(1, 2), IFNULL(NULL, 2), COALESCE(1, 2), COALESCE(NULL, 2) @@ -40,13 +40,13 @@ query III SELECT a, NULLIF(a, 2), IF(a = 2, NULL, a) FROM t ORDER BY a ---- 1 1 1 -2 null null +2 NULL NULL 3 3 3 query TTTT SELECT CASE WHEN false THEN 'one' WHEN true THEN 'two' ELSE 'three' END, CASE 1 WHEN 2 THEN 'two' WHEN 1 THEN 'one' ELSE 'three' END, CASE WHEN false THEN 'one' ELSE 'three' END, CASE WHEN false THEN 'one' END ---- -two one three null +two one three NULL query TTTTT SELECT CASE WHEN 1 = 1 THEN 'one' END, CASE false WHEN 0 = 1 THEN 'one' END, CASE 1 WHEN 2 THEN 'one' ELSE 'three' END, CASE NULL WHEN true THEN 'one' WHEN false THEN 'two' WHEN NULL THEN 'three' ELSE 'four' END, CASE WHEN false THEN 'one' WHEN true THEN 'two' END diff --git a/tests/slt/crdb/join.slt b/tests/slt/crdb/join.slt index 62debc74..3b89057a 100644 --- a/tests/slt/crdb/join.slt +++ b/tests/slt/crdb/join.slt @@ -11,14 +11,14 @@ query II SELECT * FROM onecolumn AS a(aid, x) CROSS JOIN onecolumn AS b(bid, y) order by x ---- 2 42 0 44 -2 42 1 null +2 42 1 NULL 2 42 2 42 0 44 0 44 -0 44 1 null +0 44 1 NULL 0 44 2 42 -1 null 0 44 -1 null 1 null -1 null 2 42 +1 NULL 0 44 +1 NULL 1 NULL +1 NULL 2 42 statement error SELECT x FROM onecolumn AS a, onecolumn AS b; @@ -46,14 +46,14 @@ SELECT * FROM onecolumn AS a(aid, x) LEFT OUTER JOIN onecolumn AS b(bid, y) ON a ---- 2 42 2 42 0 44 0 44 -1 null null null +1 NULL NULL NULL query I SELECT * FROM onecolumn AS a LEFT OUTER JOIN onecolumn AS b USING(x) ORDER BY x ---- 2 42 2 0 44 0 -1 null null +1 NULL NULL statement error SELECT * FROM onecolumn AS a, onecolumn AS b ORDER BY x @@ -63,28 +63,28 @@ SELECT * FROM onecolumn AS a NATURAL LEFT OUTER JOIN onecolumn AS b order by a.x ---- 2 42 0 44 -1 null +1 NULL query II SELECT * FROM onecolumn AS a(aid, x) RIGHT OUTER JOIN onecolumn AS b(bid, y) ON a.x = b.y order by x ---- 2 42 2 42 0 44 0 44 -null null 1 null +NULL NULL 1 NULL query I SELECT * FROM onecolumn AS a RIGHT OUTER JOIN onecolumn AS b USING(x) ORDER BY x ---- 2 2 42 0 0 44 -null 1 null +NULL 1 NULL query I SELECT * FROM onecolumn AS a NATURAL RIGHT OUTER JOIN onecolumn AS b order by x ---- 2 42 0 44 -1 null +1 NULL statement ok drop table if exists onecolumn_w @@ -100,8 +100,8 @@ SELECT * FROM onecolumn AS a NATURAL JOIN onecolumn_w as b ---- 0 44 0 42 0 44 1 43 -1 null 0 42 -1 null 1 43 +1 NULL 0 42 +1 NULL 1 43 2 42 0 42 2 42 1 43 @@ -118,38 +118,38 @@ query II SELECT * FROM onecolumn AS a FULL OUTER JOIN othercolumn AS b ON a.x = b.x ORDER BY a.x,b.x ---- 2 42 1 42 -0 44 null null -null null 2 16 -null null 0 43 -1 null null null +0 44 NULL NULL +NULL NULL 2 16 +NULL NULL 0 43 +1 NULL NULL NULL query II SELECT * FROM onecolumn AS a full OUTER JOIN othercolumn AS b ON a.x = b.x and a.x > 16 order by a.x ---- 2 42 1 42 -0 44 null null -null null 0 43 -null null 2 16 -1 null null null +0 44 NULL NULL +NULL NULL 0 43 +NULL NULL 2 16 +1 NULL NULL NULL query II SELECT * FROM onecolumn AS a full OUTER JOIN othercolumn AS b ON a.x = b.x and b.x > 16 order by b.x,a.x ---- -null null 2 16 +NULL NULL 2 16 2 42 1 42 -null null 0 43 -0 44 null null -1 null null null +NULL NULL 0 43 +0 44 NULL NULL +1 NULL NULL NULL query II SELECT a.x, b.x FROM onecolumn AS a full OUTER JOIN othercolumn AS b ON false order by a.x, b.x ---- -42 null -44 null -null 16 -null 42 -null 43 -null null +42 NULL +44 NULL +NULL 16 +NULL 42 +NULL 43 +NULL NULL query II SELECT a.x, b.x FROM onecolumn AS a full OUTER JOIN othercolumn AS b ON true order by a.x, b.x @@ -160,9 +160,9 @@ SELECT a.x, b.x FROM onecolumn AS a full OUTER JOIN othercolumn AS b ON true ord 44 16 44 42 44 43 -null 16 -null 42 -null 43 +NULL 16 +NULL 42 +NULL 43 # Tips: This case will make x take the value of both sides # query @@ -212,16 +212,16 @@ SELECT * FROM empty AS a JOIN onecolumn AS b USING(x) query IT SELECT * FROM onecolumn AS a(aid, x) LEFT OUTER JOIN empty AS b(bid, y) ON a.x = b.y ORDER BY a.x ---- -2 42 null null -0 44 null null -1 null null null +2 42 NULL NULL +0 44 NULL NULL +1 NULL NULL NULL query I rowsort SELECT * FROM onecolumn AS a LEFT OUTER JOIN empty AS b USING(x) ORDER BY x ---- -0 44 null -1 null null -2 42 null +0 44 NULL +1 NULL NULL +2 42 NULL query I SELECT * FROM empty AS a(aid, x) LEFT OUTER JOIN onecolumn AS b(bid, y) ON a.x = b.y @@ -242,9 +242,9 @@ SELECT * FROM onecolumn AS a RIGHT OUTER JOIN empty AS b USING(x) query II SELECT * FROM empty AS a(aid, x) FULL OUTER JOIN onecolumn AS b(bid, y) ON a.x = b.y ORDER BY b.y ---- -null null 2 42 -null null 0 44 -null null 1 null +NULL NULL 2 42 +NULL NULL 0 44 +NULL NULL 1 NULL statement ok SELECT * FROM empty AS a FULL OUTER JOIN onecolumn AS b USING(x) ORDER BY x @@ -252,23 +252,23 @@ SELECT * FROM empty AS a FULL OUTER JOIN onecolumn AS b USING(x) ORDER BY x query IIII SELECT * FROM onecolumn AS a(aid, x) FULL OUTER JOIN empty AS b(bid, y) ON a.x = b.y ORDER BY a.x ---- -2 42 null null -0 44 null null -1 null null null +2 42 NULL NULL +0 44 NULL NULL +1 NULL NULL NULL query III rowsort SELECT * FROM onecolumn AS a FULL OUTER JOIN empty AS b USING(x) ORDER BY x ---- -0 44 null -1 null null -2 42 null +0 44 NULL +1 NULL NULL +2 42 NULL query II SELECT * FROM empty AS a(aid, x) FULL OUTER JOIN onecolumn AS b(bid, y) ON a.x = b.y ORDER BY b.y ---- -null null 2 42 -null null 0 44 -null null 1 null +NULL NULL 2 42 +NULL NULL 0 44 +NULL NULL 1 NULL # query # SELECT * FROM empty AS a FULL OUTER JOIN onecolumn AS b USING(x) ORDER BY x @@ -292,7 +292,7 @@ query IIII SELECT * FROM twocolumn AS a JOIN twocolumn AS b ON a.x = a.y order by a.x ---- 3 45 45 0 44 51 -3 45 45 1 null 52 +3 45 45 1 NULL 52 3 45 45 2 42 53 3 45 45 3 45 45 @@ -307,22 +307,22 @@ query IT SELECT o.x, t.y FROM onecolumn o LEFT OUTER JOIN twocolumn t ON (o.x=t.x AND t.y=53) order by o.x ---- 42 53 -44 null -null null +44 NULL +NULL NULL query II SELECT o.x, t.y FROM onecolumn o LEFT OUTER JOIN twocolumn t ON (o.x=t.x AND o.x=44) order by o.x ---- -42 null +42 NULL 44 51 -null null +NULL NULL query II SELECT o.x, t.y FROM onecolumn o LEFT OUTER JOIN twocolumn t ON (o.x=t.x AND t.x=44) order by o.x ---- -42 null +42 NULL 44 51 -null null +NULL NULL # query # SELECT * FROM (SELECT x, 2 two FROM onecolumn) NATURAL FULL JOIN (SELECT x, y+1 plus1 FROM twocolumn) @@ -356,31 +356,31 @@ SELECT * FROM a LEFT OUTER JOIN b ON a.i = b.i ---- 1 2 0 2 true 2 3 1 3 true -0 1 null null null +0 1 NULL NULL NULL query III SELECT * FROM a RIGHT OUTER JOIN b ON a.i = b.i order by b ---- -null null 2 4 false +NULL NULL 2 4 false 1 2 0 2 true 2 3 1 3 true query III SELECT * FROM a FULL OUTER JOIN b ON a.i = b.i order by b ---- -null null 2 4 false +NULL NULL 2 4 false 1 2 0 2 true 2 3 1 3 true -0 1 null null null +0 1 NULL NULL NULL query III SELECT * FROM a FULL OUTER JOIN b ON (a.i = b.i and a.i>2) ORDER BY a.i, b.i ---- -0 1 null null null -1 2 null null null +0 1 NULL NULL NULL +1 2 NULL NULL NULL 2 3 1 3 true -null null 0 2 true -null null 2 4 false +NULL NULL 0 2 true +NULL NULL 2 4 false statement ok INSERT INTO b VALUES (3, 3, false) @@ -391,7 +391,7 @@ SELECT * FROM a RIGHT OUTER JOIN b ON a.i=b.i ORDER BY b.i, b.b 1 2 0 2 true 2 3 3 3 false 2 3 1 3 true -null null 2 4 false +NULL NULL 2 4 false query III SELECT * FROM a FULL OUTER JOIN b ON a.i=b.i ORDER BY b.i, b.b @@ -399,8 +399,8 @@ SELECT * FROM a FULL OUTER JOIN b ON a.i=b.i ORDER BY b.i, b.b 1 2 0 2 true 2 3 3 3 false 2 3 1 3 true -null null 2 4 false -0 1 null null null +NULL NULL 2 4 false +0 1 NULL NULL NULL # TODO # query IIIIII @@ -541,25 +541,25 @@ SELECT * FROM pairs, square WHERE pairs.a + pairs.b = square.sq query III SELECT a, b, n, sq FROM pairs FULL OUTER JOIN square ON pairs.a + pairs.b = square.sq order by a ---- -1 1 null null -1 2 null null +1 1 NULL NULL +1 2 NULL NULL 1 3 2 4 -1 4 null null -1 5 null null -1 6 null null -2 3 null null -2 4 null null -2 5 null null -2 6 null null -3 4 null null -3 5 null null +1 4 NULL NULL +1 5 NULL NULL +1 6 NULL NULL +2 3 NULL NULL +2 4 NULL NULL +2 5 NULL NULL +2 6 NULL NULL +3 4 NULL NULL +3 5 NULL NULL 3 6 3 9 4 5 3 9 -4 6 null null -null null 1 1 -null null 4 16 -null null 5 25 -null null 6 36 +4 6 NULL NULL +NULL NULL 1 1 +NULL NULL 4 16 +NULL NULL 5 25 +NULL NULL 6 36 query IIII SELECT pairs.a, pairs.b, square.* FROM pairs FULL OUTER JOIN square ON pairs.a + pairs.b = square.sq WHERE pairs.b%2 <> square.sq%2 order by a @@ -570,29 +570,29 @@ SELECT pairs.a, pairs.b, square.* FROM pairs FULL OUTER JOIN square ON pairs.a + query IITT rowsort SELECT * FROM (SELECT * FROM pairs LEFT JOIN square ON b = sq AND a > 1 AND n < 6) WHERE b > 1 AND (n IS NULL OR n > 1) AND (n IS NULL OR a < sq) ---- -1 1 2 null null +1 1 2 NULL NULL 10 3 4 2 4 -11 3 5 null null -12 3 6 null null -13 4 5 null null -14 4 6 null null -2 1 3 null null -3 1 4 null null -4 1 5 null null -5 1 6 null null -6 2 3 null null +11 3 5 NULL NULL +12 3 6 NULL NULL +13 4 5 NULL NULL +14 4 6 NULL NULL +2 1 3 NULL NULL +3 1 4 NULL NULL +4 1 5 NULL NULL +5 1 6 NULL NULL +6 2 3 NULL NULL 7 2 4 2 4 -8 2 5 null null -9 2 6 null null +8 2 5 NULL NULL +9 2 6 NULL NULL query IIII SELECT * FROM (SELECT * FROM pairs RIGHT JOIN square ON b = sq AND a > 1 AND n < 6) WHERE (a IS NULL OR a > 2) AND n > 1 AND (a IS NULL OR a < sq) order by n ---- 10 3 4 2 4 -null null null 3 9 -null null null 4 16 -null null null 5 25 -null null null 6 36 +NULL NULL NULL 3 9 +NULL NULL NULL 4 16 +NULL NULL NULL 5 25 +NULL NULL NULL 6 36 statement ok drop table if exists t1 @@ -697,25 +697,25 @@ query TTT SELECT s, str1.s, str2.s FROM str1 LEFT OUTER JOIN str2 USING(s) order by str1.s ---- A A A -D D null -a a null -c c null +D D NULL +a a NULL +c c NULL query TTT SELECT s, str1.s, str2.s FROM str1 RIGHT OUTER JOIN str2 USING(s) order by str2.s ---- A A A -B null B -C null C -E null E +B NULL B +C NULL C +E NULL E query ITIT SELECT * FROM str1 LEFT OUTER JOIN str2 ON str1.s = str2.s order by str1.a ---- -0 1 a null null null +0 1 a NULL NULL NULL 1 2 A 0 1 A -2 3 c null null null -3 4 D null null null +2 3 c NULL NULL NULL +3 4 D NULL NULL NULL statement ok INSERT INTO str1 VALUES (4, 1, 'a' ), (5, 2, 'A'), (6, 3, 'c'), (7, 4, 'D') @@ -725,17 +725,17 @@ select * from str1 right join str2 on str1.s = str2.s order by str2.a ---- 1 2 A 0 1 A 5 2 A 0 1 A -null null null 1 2 B -null null null 2 3 C -null null null 3 4 E +NULL NULL NULL 1 2 B +NULL NULL NULL 2 3 C +NULL NULL NULL 3 4 E query ITIT select * from str1 right join str2 on false order by str2.a ---- -null null null 0 1 A -null null null 1 2 B -null null null 2 3 C -null null null 3 4 E +NULL NULL NULL 0 1 A +NULL NULL NULL 1 2 B +NULL NULL NULL 2 3 C +NULL NULL NULL 3 4 E # query # SELECT s, str1.s, str2.s FROM str1 FULL OUTER JOIN str2 USING(s) @@ -767,8 +767,8 @@ query IIII SELECT * FROM xyu LEFT OUTER JOIN xyv USING(x, y) WHERE x > 2 order by xyu.uid ---- 2 3 1 31 7 31 -3 3 2 32 null null -4 4 4 44 null null +3 3 2 32 NULL NULL +4 4 4 44 NULL NULL # TODO: x > 2 means x on both sides # query IIII @@ -780,10 +780,10 @@ query IIIIII rowsort SELECT * FROM xyu FULL OUTER JOIN xyv USING(x, y) WHERE x > 2 ---- 2 3 1 31 7 31 -3 3 2 32 null null -4 4 4 44 null null -null 3 3 null 8 33 -null 5 5 null 9 55 +3 3 2 32 NULL NULL +4 4 4 44 NULL NULL +NULL 3 3 NULL 8 33 +NULL 5 5 NULL 9 55 query IIIIII SELECT * FROM xyu INNER JOIN xyv ON xyu.x = xyv.x AND xyu.y = xyv.y WHERE xyu.x = 1 AND xyu.y < 10 @@ -798,27 +798,27 @@ SELECT * FROM xyu INNER JOIN xyv ON xyu.x = xyv.x AND xyu.y = xyv.y AND xyu.x = query IIITTT SELECT * FROM xyu LEFT OUTER JOIN xyv ON xyu.x = xyv.x AND xyu.y = xyv.y AND xyu.x = 1 AND xyu.y < 10 order by xyu.uid ---- -0 0 0 0 null null null null +0 0 0 0 NULL NULL NULL NULL 1 1 1 1 5 1 1 1 -2 3 1 31 null null null null -3 3 2 32 null null null null -4 4 4 44 null null null null +2 3 1 31 NULL NULL NULL NULL +3 3 2 32 NULL NULL NULL NULL +4 4 4 44 NULL NULL NULL NULL query IIIIII SELECT * FROM xyu RIGHT OUTER JOIN xyv ON xyu.x = xyv.x AND xyu.y = xyv.y AND xyu.x = 1 AND xyu.y < 10 order by v ---- 1 1 1 1 5 1 1 1 -null null null null 6 2 2 2 -null null null null 7 3 1 31 -null null null null 8 3 3 33 -null null null null 9 5 5 55 +NULL NULL NULL NULL 6 2 2 2 +NULL NULL NULL NULL 7 3 1 31 +NULL NULL NULL NULL 8 3 3 33 +NULL NULL NULL NULL 9 5 5 55 query IIII rowsort SELECT * FROM (SELECT * FROM xyu ORDER BY x, y) AS xyu LEFT OUTER JOIN (SELECT * FROM xyv ORDER BY x, y) AS xyv USING(x, y) WHERE x > 2 ---- 2 3 1 31 7 31 -3 3 2 32 null null -4 4 4 44 null null +3 3 2 32 NULL NULL +4 4 4 44 NULL NULL # TODO: x > 2 means x on both sides # query IIII @@ -835,20 +835,20 @@ SELECT * FROM (SELECT * FROM xyu ORDER BY x, y) AS xyu LEFT OUTER JOIN (SELECT * query IIITTT rowsort SELECT * FROM (SELECT * FROM xyu ORDER BY x, y) AS xyu LEFT OUTER JOIN (SELECT * FROM xyv ORDER BY x, y) AS xyv ON xyu.x = xyv.x AND xyu.y = xyv.y AND xyu.x = 1 AND xyu.y < 10 ---- -0 0 0 0 null null null null +0 0 0 0 NULL NULL NULL NULL 1 1 1 1 5 1 1 1 -2 3 1 31 null null null null -3 3 2 32 null null null null -4 4 4 44 null null null null +2 3 1 31 NULL NULL NULL NULL +3 3 2 32 NULL NULL NULL NULL +4 4 4 44 NULL NULL NULL NULL query IIIIII SELECT * FROM xyu RIGHT OUTER JOIN (SELECT * FROM xyv ORDER BY x, y) AS xyv ON xyu.x = xyv.x AND xyu.y = xyv.y AND xyu.x = 1 AND xyu.y < 10 ORDER BY v ---- 1 1 1 1 5 1 1 1 -null null null null 6 2 2 2 -null null null null 7 3 1 31 -null null null null 8 3 3 33 -null null null null 9 5 5 55 +NULL NULL NULL NULL 6 2 2 2 +NULL NULL NULL NULL 7 3 1 31 +NULL NULL NULL NULL 8 3 3 33 +NULL NULL NULL NULL 9 5 5 55 statement ok drop table if exists l @@ -876,27 +876,27 @@ SELECT a, l.a, r.a FROM l INNER JOIN r USING(a) WHERE a = 2 query III SELECT a, l.a, r.a FROM l LEFT OUTER JOIN r USING(a) WHERE a = 1 ---- -1 1 null +1 1 NULL query III SELECT a, l.a, r.a FROM l RIGHT OUTER JOIN r USING(a) WHERE a = 4 ---- -4 null 4 +4 NULL 4 query III SELECT a, l.a, r.a FROM l FULL OUTER JOIN r USING(a) WHERE a = 1 ---- -1 1 null +1 1 NULL query III SELECT a, l.a, r.a FROM l FULL OUTER JOIN r USING(a) WHERE a = 4 ---- -4 null 4 +4 NULL 4 query III SELECT * FROM l LEFT OUTER JOIN r USING(a) WHERE a = 1 ---- -1 1 null +1 1 NULL query III SELECT * FROM l LEFT OUTER JOIN r USING(a) WHERE a = 2 @@ -911,7 +911,7 @@ SELECT * FROM l RIGHT OUTER JOIN r USING(a) WHERE a = 3 query III SELECT * FROM l RIGHT OUTER JOIN r USING(a) WHERE a = 4 ---- -null 4 1 +NULL 4 1 statement ok drop table if exists foo @@ -998,10 +998,10 @@ drop table if exists bar query TII SELECT * FROM onecolumn AS a(aid, x) RIGHT JOIN twocolumn ON false order by y ---- -null null 3 45 45 -null null 0 44 51 -null null 1 null 52 -null null 2 42 53 +NULL NULL 3 45 45 +NULL NULL 0 44 51 +NULL NULL 1 NULL 52 +NULL NULL 2 42 53 statement ok SELECT * FROM onecolumn AS a(aid, x) RIGHT JOIN twocolumn ON true where false order by y diff --git a/tests/slt/crdb/natural_join.slt b/tests/slt/crdb/natural_join.slt index 505e9be1..b3cd1f4c 100644 --- a/tests/slt/crdb/natural_join.slt +++ b/tests/slt/crdb/natural_join.slt @@ -81,9 +81,9 @@ query II SELECT k, x FROM t1 NATURAL JOIN t2 ---- 0 0 -0 null -null 0 -null null +0 NULL +NULL 0 +NULL NULL query II SELECT k, x FROM t1 JOIN t2 ON k = x diff --git a/tests/slt/crdb/order_by.slt b/tests/slt/crdb/order_by.slt index f93ed8d8..734b9f58 100644 --- a/tests/slt/crdb/order_by.slt +++ b/tests/slt/crdb/order_by.slt @@ -12,21 +12,21 @@ SELECT c FROM t ORDER BY c ---- false true -null +NULL query I SELECT c FROM t ORDER BY c ---- false true -null +NULL query I SELECT c FROM t ORDER BY c DESC ---- true false -null +NULL query II SELECT a, b FROM t ORDER BY b @@ -166,18 +166,18 @@ SELECT * FROM t ORDER BY true,a ---- 1 9 true 2 8 false -3 7 null -4 7 null -5 7 null +3 7 NULL +4 7 NULL +5 7 NULL query III SELECT * FROM t ORDER BY "a" ---- 1 9 true 2 8 false -3 7 null -4 7 null -5 7 null +3 7 NULL +4 7 NULL +5 7 NULL query error SELECT * FROM t ORDER BY 2.5,a @@ -232,8 +232,8 @@ INSERT INTO bar VALUES (0, NULL), (1, NULL) query IT SELECT * FROM bar ORDER BY baz, id ---- -0 null -1 null +0 NULL +1 NULL statement ok drop table if exists abcd @@ -272,8 +272,8 @@ INSERT INTO store VALUES (0, NULL, 10), (1, NULL, 5) query ITI SELECT * FROM store ORDER BY baz, extra ---- -1 null 5 -0 null 10 +1 NULL 5 +0 NULL 10 statement ok drop table if exists xy @@ -287,42 +287,42 @@ INSERT INTO xy VALUES (0, 2, NULL), (1, NULL, 6), (2, 2, 5), (3, 4, 8) query IT SELECT x, y FROM xy ORDER BY y NULLS FIRST ---- -2 null +2 NULL 2 5 -null 6 +NULL 6 4 8 query II SELECT x, y FROM xy ORDER BY y NULLS LAST ---- 2 5 -null 6 +NULL 6 4 8 -2 null +2 NULL query IT SELECT x, y FROM xy ORDER BY y DESC NULLS FIRST ---- -2 null +2 NULL 4 8 -null 6 +NULL 6 2 5 query II SELECT x, y FROM xy ORDER BY y DESC NULLS LAST ---- 4 8 -null 6 +NULL 6 2 5 -2 null +2 NULL query II SELECT x, y FROM xy ORDER BY y NULLS LAST ---- 2 5 -null 6 +NULL 6 4 8 -2 null +2 NULL statement ok INSERT INTO xy VALUES (4, NULL, NULL) @@ -330,72 +330,72 @@ INSERT INTO xy VALUES (4, NULL, NULL) query TI SELECT x, y FROM xy ORDER BY x NULLS FIRST, y NULLS LAST ---- -null 6 -null null +NULL 6 +NULL NULL 2 5 -2 null +2 NULL 4 8 query IT SELECT x, y FROM xy ORDER BY x NULLS LAST, y DESC NULLS FIRST ---- -2 null +2 NULL 2 5 4 8 -null null -null 6 +NULL NULL +NULL 6 query II SELECT x, y FROM xy ORDER BY x, y ---- 2 5 -2 null +2 NULL 4 8 -null 6 -null null +NULL 6 +NULL NULL query IT SELECT x, y FROM xy ORDER BY x, y DESC NULLS FIRST ---- -2 null +2 NULL 2 5 4 8 -null null -null 6 +NULL NULL +NULL 6 query IT SELECT x, y FROM xy ORDER BY x NULLS LAST, y DESC NULLS FIRST ---- -2 null +2 NULL 2 5 4 8 -null null -null 6 +NULL NULL +NULL 6 query TT SELECT x, y FROM xy ORDER BY x NULLS FIRST, y DESC NULLS LAST ---- -null 6 -null null +NULL 6 +NULL NULL 2 5 -2 null +2 NULL 4 8 query TT SELECT x, y FROM xy ORDER BY x NULLS FIRST, y DESC ---- -null 6 -null null +NULL 6 +NULL NULL 2 5 -2 null +2 NULL 4 8 query TI SELECT x, y FROM xy ORDER BY x NULLS FIRST, y DESC NULLS FIRST ---- -null null -null 6 -2 null +NULL NULL +NULL 6 +2 NULL 2 5 4 8 diff --git a/tests/slt/crdb/select.slt b/tests/slt/crdb/select.slt index b4eba0be..be782049 100644 --- a/tests/slt/crdb/select.slt +++ b/tests/slt/crdb/select.slt @@ -25,7 +25,7 @@ SELECT * FROM abc query TIII SELECT NULL AS z, * FROM abc ---- -null 1 2 3 +NULL 1 2 3 statement ok SELECT * FROM abc WHERE NULL @@ -79,17 +79,17 @@ INSERT INTO kv (k, v) VALUES ('a', null) query TT SELECT * FROM kv ---- -a null +a NULL query TT SELECT k,v FROM kv ---- -a null +a NULL query T SELECT v||'foo' FROM kv ---- -null +NULL # TODO: lower # query T @@ -105,12 +105,12 @@ a query TT SELECT kv.K,KV.v FROM kv ---- -a null +a NULL query TT SELECT kv.* FROM kv ---- -a null +a NULL query T SELECT FOO.k FROM kv AS foo WHERE foo.k = 'a' @@ -120,7 +120,7 @@ a query T SELECT foo.v FROM kv AS foo WHERE foo.k = 'a' ---- -null +NULL onlyif mysql statement ok @@ -232,7 +232,7 @@ INSERT INTO boolean_table (id, value) VALUES (1, NULL) query T SELECT value FROM boolean_table ---- -null +NULL query I SELECT CASE WHEN NULL THEN 1 ELSE 2 END @@ -267,7 +267,7 @@ true query T SELECT NULL IN (1, 2) ---- -null +NULL query I SELECT 1 IN (1, NULL) @@ -277,7 +277,7 @@ true query I SELECT 1 IN (NULL, 2) ---- -null +NULL # statement error 1105 # SELECT NULL IN ((1, 1)) @@ -530,7 +530,7 @@ INSERT INTO wide(id, n) VALUES(0, 10) query IITITTITTTTIRRI SELECT * FROM wide ---- -0 null null null null null null null null null null null null null 10 +0 NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL 10 statement ok drop table xyzw diff --git a/tests/slt/crdb/where.slt b/tests/slt/crdb/where.slt index 48cd417f..fc6b216f 100644 --- a/tests/slt/crdb/where.slt +++ b/tests/slt/crdb/where.slt @@ -93,14 +93,14 @@ SELECT a, b FROM ab WHERE a IN (1, 3, 4) ---- 1 10 3 30 -4 null +4 NULL query II SELECT a, b FROM ab WHERE a IN (1, 3, 4, NULL) ---- 1 10 3 30 -4 null +4 NULL query SELECT a, b FROM ab WHERE (a, b) IN ((1, 10), (3, 30), (4, 40)) diff --git a/tests/slt/describe.slt b/tests/slt/describe.slt index e327103e..f3377433 100644 --- a/tests/slt/describe.slt +++ b/tests/slt/describe.slt @@ -4,9 +4,9 @@ create table t9 (c1 int primary key, c2 int default 0, c3 varchar unique); query TTTTTT describe t9; ---- -c1 Integer 4 false PRIMARY null +c1 Integer 4 false PRIMARY NULL c2 Integer 4 true EMPTY 0 -c3 Varchar(None, CHARACTERS) variable true UNIQUE null +c3 Varchar(None, CHARACTERS) variable true UNIQUE NULL statement ok drop table t9; @@ -17,9 +17,9 @@ create table t9_m (c1 int primary key, c2 int primary key, c3 varchar unique); query TTTTTT describe t9_m; ---- -c1 Integer 4 false PRIMARY null -c2 Integer 4 false PRIMARY null -c3 Varchar(None, CHARACTERS) variable true UNIQUE null +c1 Integer 4 false PRIMARY NULL +c2 Integer 4 false PRIMARY NULL +c3 Varchar(None, CHARACTERS) variable true UNIQUE NULL statement ok drop table t9_m; diff --git a/tests/slt/distinct.slt b/tests/slt/distinct.slt index 0ce6bab9..4606171f 100644 --- a/tests/slt/distinct.slt +++ b/tests/slt/distinct.slt @@ -53,9 +53,9 @@ insert into distinct_alias values (0, 1, 2, 'a', 1.5), (1, 2, null, null, null), query TT rowsort select distinct lower(c) as x, c from distinct_alias; ---- +NULL NULL a A a a -null null query T rowsort select distinct (a = 1) as x from distinct_alias; @@ -73,7 +73,7 @@ query T rowsort select distinct upper(c) as x from distinct_alias order by x; ---- A -null +NULL statement ok drop table distinct_alias diff --git a/tests/slt/dummy.slt b/tests/slt/dummy.slt index 4dcb3eba..cbdbf778 100644 --- a/tests/slt/dummy.slt +++ b/tests/slt/dummy.slt @@ -45,7 +45,7 @@ false query B SELECT NULL=NULL ---- -null +NULL query B SELECT NOT(1=1) @@ -95,12 +95,12 @@ SELECT NULLIF(1, 2) query I SELECT NULLIF(NULL, 2) ---- -null +NULL query I SELECT NULLIF(NULL, NULL) ---- -null +NULL query I SELECT IFNULL(1, NULL) @@ -115,7 +115,7 @@ SELECT IFNULL(NULL, 1) query I SELECT IFNULL(NULL, NULL) ---- -null +NULL query I SELECT COALESCE(1) @@ -140,12 +140,12 @@ SELECT COALESCE(NULL, 2, NULL) query I SELECT COALESCE(NULL, NULL) ---- -null +NULL query I SELECT COALESCE() ---- -null +NULL query I SELECT CASE 1 WHEN 0 THEN 0 WHEN 1 THEN 1 WHEN NULL THEN 9 ELSE 2 END @@ -160,7 +160,7 @@ SELECT CASE 3 WHEN 0 THEN 0 WHEN 1 THEN 1 WHEN NULL THEN 9 ELSE 2 END query I SELECT CASE 3 WHEN 0 THEN 0 WHEN 1 THEN 1 WHEN NULL THEN 9 END ---- -null +NULL query I SELECT CASE 3 WHEN 0 THEN 0 WHEN 1 THEN 1 WHEN NULL THEN 9 ELSE 2 END @@ -180,7 +180,7 @@ SELECT CASE WHEN FALSE THEN 0 WHEN FALSE THEN 1 WHEN FALSE THEN 9 ELSE 2 END query I SELECT CASE WHEN FALSE THEN 0 WHEN FALSE THEN 1 WHEN FALSE THEN 9 END ---- -null +NULL query I SELECT CASE WHEN FALSE THEN 0 WHEN 1=1 THEN 1 WHEN FALSE THEN 9 ELSE 2 END @@ -227,12 +227,12 @@ true query I select - (null) from case_ty; ---- -null +NULL query T select 3 = - null from case_ty; ---- -null +NULL statement ok drop table case_ty diff --git a/tests/slt/filter_null.slt b/tests/slt/filter_null.slt index 01d3bde9..4eb4a1c6 100644 --- a/tests/slt/filter_null.slt +++ b/tests/slt/filter_null.slt @@ -35,7 +35,7 @@ select * from t where v1 > 1 query III select * from t where v1 is null ---- -1 null 3 +1 NULL 3 query III select * from t where v1 is not null diff --git a/tests/slt/insert.slt b/tests/slt/insert.slt index 1f7d2424..ca48be81 100644 --- a/tests/slt/insert.slt +++ b/tests/slt/insert.slt @@ -42,17 +42,17 @@ select * from t ---- 0 1 10 100 1 9 9 9 -10 null 1 null -11 null 9 null +10 NULL 1 NULL +11 NULL 9 NULL 2 2 20 200 20 1 10 100 21 9 9 9 3 3 30 300 4 4 40 400 5 1 10 100 -6 1 10 null -7 10 1 null -8 null null null +6 1 10 NULL +7 10 1 NULL +8 NULL NULL NULL statement ok drop table t; diff --git a/tests/slt/insert_multiple_primary_keys.slt b/tests/slt/insert_multiple_primary_keys.slt index d20662d3..0e1be266 100644 --- a/tests/slt/insert_multiple_primary_keys.slt +++ b/tests/slt/insert_multiple_primary_keys.slt @@ -40,9 +40,9 @@ select * from t 3 3 3 30 300 4 4 4 40 400 5 5 1 10 100 -6 6 1 10 null -7 7 10 1 null -8 8 null null null +6 6 1 10 NULL +7 7 10 1 NULL +8 8 NULL NULL NULL statement ok drop table t; diff --git a/tests/slt/join.slt b/tests/slt/join.slt index 40c5bcc2..c65945a9 100644 --- a/tests/slt/join.slt +++ b/tests/slt/join.slt @@ -59,9 +59,9 @@ insert into a values (0, 1, 1), (1, 2, 2), (2, 3, 3); query IIII rowsort select v1, v2, v3, v4 from a left join b on v1 = v3; ---- -1 1 null null -2 2 null null -3 3 null null +1 1 NULL NULL +2 2 NULL NULL +3 3 NULL NULL statement ok insert into b values (0, 1, 100), (1, 3, 300), (2, 4, 400); @@ -71,22 +71,22 @@ select v1, v2, v3, v4 from a left join b on v1 = v3; ---- 1 1 1 100 3 3 3 300 -2 2 null null +2 2 NULL NULL query IIII rowsort select v1, v2, v3, v4 from a right join b on v1 = v3; ---- 1 1 1 100 3 3 3 300 -null null 4 400 +NULL NULL 4 400 query IIII rowsort select v1, v2, v3, v4 from a full join b on v1 = v3; ---- 1 1 1 100 -2 2 null null +2 2 NULL NULL 3 3 3 300 -null null 4 400 +NULL NULL 4 400 statement ok drop table a; @@ -164,7 +164,7 @@ select a.*, b.* from a left join b on v1 < v3 or v2 < v4 0 1 1 1 2 2 2 0 1 1 2 3 3 4 1 2 2 2 3 3 4 -2 3 3 null null null null +2 3 3 NULL NULL NULL NULL query IIIIIII rowsort select a.*, b.* from a right join b on v1 <> v3 and v2 < v4 @@ -172,8 +172,8 @@ select a.*, b.* from a right join b on v1 <> v3 and v2 < v4 0 1 1 1 2 2 2 0 1 1 2 3 3 4 1 2 2 2 3 3 4 -null null null 0 1 1 1 -null null null 3 1 1 5 +NULL NULL NULL 0 1 1 1 +NULL NULL NULL 3 1 1 5 query IIIIIII rowsort select a.*, b.* from a cross join b @@ -312,8 +312,8 @@ left join lookup_inner i on o.k=i.k order by o.id,i.id; 1 1 1 2 2 3 -3 null -4 null +3 NULL +4 NULL 5 1 5 2 @@ -564,16 +564,16 @@ insert into full_using_r values (5, 2, 200), (6, 3, 300) query IIIII rowsort select * from full_using_l as l (lid, x, lv) full join full_using_r using(x) ---- -0 1 10 null null +0 1 10 NULL NULL 1 2 20 5 200 -null 3 null 6 300 +NULL 3 NULL 6 300 query IIIII rowsort select * from full_using_l full join full_using_r using(x) ---- -0 1 10 null null +0 1 10 NULL NULL 1 2 20 5 200 -null 3 null 6 300 +NULL 3 NULL 6 300 statement ok drop table full_using_l diff --git a/tests/slt/join_left_inner.slt b/tests/slt/join_left_inner.slt index 7df11dc0..db879aa6 100644 --- a/tests/slt/join_left_inner.slt +++ b/tests/slt/join_left_inner.slt @@ -40,9 +40,9 @@ insert into a values (0, 1, 1), (1, 2, 2), (2, 3, 3); query IIII rowsort select v1, v2, v3, v4 from a left join b on v1 = v3; ---- -1 1 null null -2 2 null null -3 3 null null +1 1 NULL NULL +2 2 NULL NULL +3 3 NULL NULL statement ok insert into b values (0, 1, 100), (1, 3, 300), (2, 4, 400); @@ -51,7 +51,7 @@ query IIII rowsort select v1, v2, v3, v4 from a left join b on v1 = v3; ---- 1 1 1 100 -2 2 null null +2 2 NULL NULL 3 3 3 300 statement ok diff --git a/tests/slt/order_by.slt b/tests/slt/order_by.slt index 2ac16bea..cdd049dd 100644 --- a/tests/slt/order_by.slt +++ b/tests/slt/order_by.slt @@ -72,18 +72,18 @@ select v1, v2 from t order by v1 asc, v2 asc ---- 1 0 2 2 -2 null -null 5 -null null +2 NULL +NULL 5 +NULL NULL query II select v1, v2 from t order by v1 asc nulls last, v2 asc ---- 1 0 2 2 -2 null -null 5 -null null +2 NULL +NULL 5 +NULL NULL statement ok drop table t diff --git a/tests/slt/set_operation.slt b/tests/slt/set_operation.slt index 38a1ffcb..4b2eccf9 100644 --- a/tests/slt/set_operation.slt +++ b/tests/slt/set_operation.slt @@ -18,7 +18,7 @@ select v from set_left union select v from set_right 3 4 5 -null +NULL query I rowsort select v from set_left union all select v from set_right @@ -33,9 +33,9 @@ select v from set_left union all select v from set_right 3 4 5 -null -null -null +NULL +NULL +NULL # Edge case: tuple values should be compared as whole set values across set # operators, rather than flattening their fields or comparing only one slot. @@ -75,14 +75,14 @@ select v from set_left except all select v from set_right ---- 2 5 -null +NULL query I rowsort select v from set_left intersect select v from set_right ---- 1 3 -null +NULL # Edge case: INTERSECT ALL emits min(left_count, right_count), so value 1 # appears twice even though the right side has three copies. @@ -92,14 +92,14 @@ select v from set_left intersect all select v from set_right 1 1 3 -null +NULL # Edge case: set membership treats NULL as a set value for duplicate # elimination/matching, unlike ordinary NULL = NULL predicate evaluation. query I rowsort select v from set_left where v is null intersect select v from set_right where v is null ---- -null +NULL query I rowsort select v from set_left where v is null except select v from set_right where v is null @@ -110,7 +110,7 @@ select v from set_left intersect select v from set_right order by v desc ---- 3 1 -null +NULL # Regression: branch table names from a set operation must not leak to the # top-level ORDER BY scope. diff --git a/tests/slt/sql_2016/E091_01.slt b/tests/slt/sql_2016/E091_01.slt index 81415d59..8b8d1333 100644 --- a/tests/slt/sql_2016/E091_01.slt +++ b/tests/slt/sql_2016/E091_01.slt @@ -6,7 +6,7 @@ CREATE TABLE TABLE_E091_01_01_01 ( ID INT PRIMARY KEY, A FLOAT ); query F SELECT AVG ( A ) FROM TABLE_E091_01_01_01 ---- -null +NULL statement ok drop table table_e091_01_01_01 diff --git a/tests/slt/sql_2016/E091_03.slt b/tests/slt/sql_2016/E091_03.slt index 77cc285b..aba93b58 100644 --- a/tests/slt/sql_2016/E091_03.slt +++ b/tests/slt/sql_2016/E091_03.slt @@ -6,7 +6,7 @@ CREATE TABLE TABLE_E091_03_01_01 ( ID INT PRIMARY KEY, A FLOAT ); query F SELECT MAX ( A ) FROM TABLE_E091_03_01_01 ---- -null +NULL statement ok drop table table_e091_03_01_01 diff --git a/tests/slt/sql_2016/E091_04.slt b/tests/slt/sql_2016/E091_04.slt index 8bce7520..b836ae0c 100644 --- a/tests/slt/sql_2016/E091_04.slt +++ b/tests/slt/sql_2016/E091_04.slt @@ -6,7 +6,7 @@ CREATE TABLE TABLE_E091_04_01_01 ( ID INT PRIMARY KEY, A FLOAT ); query F SELECT MIN ( A ) FROM TABLE_E091_04_01_01 ---- -null +NULL statement ok drop table table_e091_04_01_01 diff --git a/tests/slt/sql_2016/E091_05.slt b/tests/slt/sql_2016/E091_05.slt index 7408593a..30a475e6 100644 --- a/tests/slt/sql_2016/E091_05.slt +++ b/tests/slt/sql_2016/E091_05.slt @@ -6,7 +6,7 @@ CREATE TABLE TABLE_E091_05_01_01 ( ID INT PRIMARY KEY, A FLOAT ); query F SELECT SUM ( A ) FROM TABLE_E091_05_01_01 ---- -null +NULL statement ok drop table table_e091_05_01_01 diff --git a/tests/slt/sql_2016/E091_06.slt b/tests/slt/sql_2016/E091_06.slt index 4aadf607..6945e065 100644 --- a/tests/slt/sql_2016/E091_06.slt +++ b/tests/slt/sql_2016/E091_06.slt @@ -6,7 +6,7 @@ CREATE TABLE TABLE_E091_06_01_01 ( ID INT PRIMARY KEY, A FLOAT ); query F SELECT AVG ( ALL A ) FROM TABLE_E091_06_01_01 ---- -null +NULL statement ok CREATE TABLE TABLE_E091_06_02_01 ( ID INT PRIMARY KEY, A FLOAT ); @@ -22,7 +22,7 @@ CREATE TABLE TABLE_E091_06_03_01 ( ID INT PRIMARY KEY, A FLOAT ); query F SELECT MAX ( ALL A ) FROM TABLE_E091_06_03_01 ---- -null +NULL statement ok CREATE TABLE TABLE_E091_06_04_01 ( ID INT PRIMARY KEY, A FLOAT ); @@ -30,7 +30,7 @@ CREATE TABLE TABLE_E091_06_04_01 ( ID INT PRIMARY KEY, A FLOAT ); query F SELECT MIN ( ALL A ) FROM TABLE_E091_06_04_01 ---- -null +NULL statement ok CREATE TABLE TABLE_E091_06_05_01 ( ID INT PRIMARY KEY, A FLOAT ); @@ -38,7 +38,7 @@ CREATE TABLE TABLE_E091_06_05_01 ( ID INT PRIMARY KEY, A FLOAT ); query F SELECT SUM ( ALL A ) FROM TABLE_E091_06_05_01 ---- -null +NULL statement ok drop table table_e091_06_05_01 diff --git a/tests/slt/sql_2016/E091_07.slt b/tests/slt/sql_2016/E091_07.slt index 3bb91f12..241c5b34 100644 --- a/tests/slt/sql_2016/E091_07.slt +++ b/tests/slt/sql_2016/E091_07.slt @@ -6,7 +6,7 @@ CREATE TABLE TABLE_E091_07_01_01 ( ID INT PRIMARY KEY, A FLOAT ); query F SELECT AVG ( DISTINCT A ) FROM TABLE_E091_07_01_01 ---- -null +NULL statement ok CREATE TABLE TABLE_E091_07_01_02 ( ID INT PRIMARY KEY, A FLOAT ); @@ -22,7 +22,7 @@ CREATE TABLE TABLE_E091_07_01_03 ( ID INT PRIMARY KEY, A FLOAT ); query F SELECT MAX ( DISTINCT A ) FROM TABLE_E091_07_01_03 ---- -null +NULL statement ok CREATE TABLE TABLE_E091_07_01_04 ( ID INT PRIMARY KEY, A FLOAT ); @@ -30,7 +30,7 @@ CREATE TABLE TABLE_E091_07_01_04 ( ID INT PRIMARY KEY, A FLOAT ); query F SELECT MIN ( DISTINCT A ) FROM TABLE_E091_07_01_04 ---- -null +NULL statement ok CREATE TABLE TABLE_E091_07_01_05 ( ID INT PRIMARY KEY, A FLOAT ); @@ -38,7 +38,7 @@ CREATE TABLE TABLE_E091_07_01_05 ( ID INT PRIMARY KEY, A FLOAT ); query F SELECT SUM ( DISTINCT A ) FROM TABLE_E091_07_01_05 ---- -null +NULL statement ok drop table table_e091_07_01_01 diff --git a/tests/slt/sql_2016/E131.slt b/tests/slt/sql_2016/E131.slt index cadc5ac4..59884344 100644 --- a/tests/slt/sql_2016/E131.slt +++ b/tests/slt/sql_2016/E131.slt @@ -3,4 +3,4 @@ query T SELECT NULL ---- -null +NULL diff --git a/tests/slt/sql_2016/F131_03.slt b/tests/slt/sql_2016/F131_03.slt index 0c302c08..d733b64a 100644 --- a/tests/slt/sql_2016/F131_03.slt +++ b/tests/slt/sql_2016/F131_03.slt @@ -11,7 +11,7 @@ CREATE VIEW VIEW_F131_03_01_01 AS SELECT A, MIN ( B ) AS C FROM TABLE_F131_03_01 query I SELECT SUM ( C ) FROM VIEW_F131_03_01_01 ---- -null +NULL statement ok CREATE TABLE TABLE_F131_03_01_021 ( ID INT PRIMARY KEY, A INTEGER, B INTEGER ); diff --git a/tests/slt/sql_2016/F261_01.slt b/tests/slt/sql_2016/F261_01.slt index a0ba2577..3cb6bf54 100644 --- a/tests/slt/sql_2016/F261_01.slt +++ b/tests/slt/sql_2016/F261_01.slt @@ -8,12 +8,12 @@ SELECT CASE 0 WHEN 2 THEN 1 ELSE 1 END query I SELECT CASE 0 WHEN 2 THEN 1 ELSE NULL END ---- -null +NULL query I SELECT CASE 0 WHEN 2 THEN 1 END ---- -null +NULL query I SELECT CASE 0 WHEN 2 THEN NULL ELSE 1 END @@ -23,12 +23,12 @@ SELECT CASE 0 WHEN 2 THEN NULL ELSE 1 END query I SELECT CASE 0 WHEN 2 THEN NULL ELSE NULL END ---- -null +NULL query I SELECT CASE 0 WHEN 2 THEN NULL END ---- -null +NULL query I SELECT CASE 0 WHEN 2 THEN 1 ELSE 1 END @@ -38,12 +38,12 @@ SELECT CASE 0 WHEN 2 THEN 1 ELSE 1 END query I SELECT CASE 0 WHEN 2 THEN 1 ELSE NULL END ---- -null +NULL query I SELECT CASE 0 WHEN 2 THEN 1 END ---- -null +NULL query I SELECT CASE 0 WHEN 2 THEN NULL ELSE 1 END @@ -53,9 +53,9 @@ SELECT CASE 0 WHEN 2 THEN NULL ELSE 1 END query T SELECT CASE 0 WHEN 2 THEN NULL ELSE NULL END ---- -null +NULL query T SELECT CASE 0 WHEN 2 THEN NULL END ---- -null +NULL diff --git a/tests/slt/sql_2016/F261_02.slt b/tests/slt/sql_2016/F261_02.slt index 46a5057d..f36088d4 100644 --- a/tests/slt/sql_2016/F261_02.slt +++ b/tests/slt/sql_2016/F261_02.slt @@ -8,12 +8,12 @@ SELECT CASE WHEN 0 = 1 THEN 1 ELSE 1 END query I SELECT CASE WHEN 0 = 1 THEN 1 ELSE NULL END ---- -null +NULL query I SELECT CASE WHEN 0 = 1 THEN 1 END ---- -null +NULL query I SELECT CASE WHEN 0 = 1 THEN NULL ELSE 1 END @@ -23,9 +23,9 @@ SELECT CASE WHEN 0 = 1 THEN NULL ELSE 1 END query T SELECT CASE WHEN 0 = 1 THEN NULL ELSE NULL END ---- -null +NULL query T SELECT CASE WHEN 0 = 1 THEN NULL END ---- -null +NULL diff --git a/tests/slt/sql_2016/F261_03.slt b/tests/slt/sql_2016/F261_03.slt index d0a2a165..ea2cf040 100644 --- a/tests/slt/sql_2016/F261_03.slt +++ b/tests/slt/sql_2016/F261_03.slt @@ -3,4 +3,4 @@ query I SELECT NULLIF ( 1 , 1 ) ---- -null +NULL diff --git a/tests/slt/string_length.slt b/tests/slt/string_length.slt index 8d12547c..73f7bd93 100644 --- a/tests/slt/string_length.slt +++ b/tests/slt/string_length.slt @@ -29,7 +29,7 @@ select id from s where char_length(c) + 1 > 3; query I select char_length(c) from s where id = 3; ---- -null +NULL statement ok drop table s diff --git a/tests/slt/subquery.slt b/tests/slt/subquery.slt index 75b65510..b0a665d3 100644 --- a/tests/slt/subquery.slt +++ b/tests/slt/subquery.slt @@ -398,8 +398,8 @@ insert into scalar_outer values (1), (2); query IT select id, (select v from scalar_inner where v = -1) from scalar_outer order by id; ---- -1 null -2 null +1 NULL +2 NULL statement ok drop table scalar_outer; @@ -539,9 +539,9 @@ query II select outer_row.id, (select v from scalar_eof where id < 0) from scalar_eof outer_row order by outer_row.id; ---- -1 null -2 null -3 null +1 NULL +2 NULL +3 NULL # Validate before accepting the cached scalar, including a single outer-row limit. statement error diff --git a/tests/slt/substring.slt b/tests/slt/substring.slt index d62ad552..a9f14298 100644 --- a/tests/slt/substring.slt +++ b/tests/slt/substring.slt @@ -26,17 +26,17 @@ apple query T select substring('pineapple' from 1 for null ) ---- -null +NULL query T select substring('pineapple' from null for 4 ) ---- -null +NULL query T select substring(null from 1 for 4 ) ---- -null +NULL # issue: https://github.com/KipData/KiteSQL/issues/160 query T diff --git a/tests/slt/values.slt b/tests/slt/values.slt index 990eb21d..95b2fbab 100644 --- a/tests/slt/values.slt +++ b/tests/slt/values.slt @@ -27,7 +27,7 @@ query I SELECT * FROM (VALUES (1), (NULL), (3)); ---- 1 -null +NULL 3 statement ok diff --git a/tests/slt/where.slt b/tests/slt/where.slt index d79fefb2..2db53f55 100644 --- a/tests/slt/where.slt +++ b/tests/slt/where.slt @@ -241,7 +241,7 @@ select id from in_ty where (b % 2) in (0, null); query T rowsort select (b % 2) in (0, null) from in_ty; ---- -null +NULL true true @@ -269,8 +269,8 @@ insert into between_ty values (0, 1e300), (1, 1.5), (2, null); query T rowsort select d not between 0 and 3 from between_ty; ---- +NULL false -null true query I rowsort @@ -290,7 +290,7 @@ drop table between_ty query TTTTT select 1 between 3 and null, 1 not between 3 and null, 5 between 3 and null, -1 not between 2 and null, 5 between null and 3; ---- -false true null true false +false true NULL true false statement ok create table between_null(id int primary key, a int); @@ -327,9 +327,9 @@ select id from tuple_in where (a, b) in ((1, 10), (4, null), (null, 50)); query T rowsort select (a, b) in ((1, 10), (4, null)) from tuple_in; ---- -null -null -null +NULL +NULL +NULL true query I rowsort diff --git a/tests/slt/where_by_index.slt b/tests/slt/where_by_index.slt index e5a7bd94..02d78e9b 100644 --- a/tests/slt/where_by_index.slt +++ b/tests/slt/where_by_index.slt @@ -202,7 +202,7 @@ select * from t1 where (c1 = 7 or c1 = 10) and c2 < 9; query IIT select * from t1 where c1 is null and c2 is null; ---- -100000000 null null +100000000 NULL NULL query IIT select * from t1 where c1 > 0 and c1 < 8; diff --git a/tests/slt/where_by_index_explain.slt b/tests/slt/where_by_index_explain.slt index f37d43c0..b24ada38 100644 --- a/tests/slt/where_by_index_explain.slt +++ b/tests/slt/where_by_index_explain.slt @@ -138,7 +138,7 @@ Projection [t1.id, t1.c1, t1.c2] [Project => (Sort Option: Follow)] Filter (t1.c query T explain select * from t1 where c1 is null and c2 is null; ---- -Projection [t1.id, t1.c1, t1.c2] [Project => (Sort Option: Follow)] Filter t1.c2 is null, Is Having: false [Filter => (Sort Option: Follow)] TableScan t1 -> [t1.id, t1.c1, t1.c2] [IndexScan By u_c1_index => null => (Sort Option: OrderBy: (t1.c1 Asc Nulls Last) ignore_prefix_len: 0)] +Projection [t1.id, t1.c1, t1.c2] [Project => (Sort Option: Follow)] Filter t1.c2 is null, Is Having: false [Filter => (Sort Option: Follow)] TableScan t1 -> [t1.id, t1.c1, t1.c2] [IndexScan By u_c1_index => NULL => (Sort Option: OrderBy: (t1.c1 Asc Nulls Last) ignore_prefix_len: 0)] query T explain select * from t1 where c1 > 0 and c1 < 8; From d06f370b162bcc947f86661d3b86c7d6b7edc4b5 Mon Sep 17 00:00:00 2001 From: kould Date: Wed, 7 Oct 2026 04:14:03 +0800 Subject: [PATCH 04/16] test: align ORM default expectation with uppercase NULL --- src/orm/mod.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/orm/mod.rs b/src/orm/mod.rs index da34d1e0..9ce6178a 100644 --- a/src/orm/mod.rs +++ b/src/orm/mod.rs @@ -3709,7 +3709,7 @@ mod tests { len: "4".to_string(), nullable: true, key: "PRI".to_string(), - default: "null".to_string(), + default: "NULL".to_string(), } ); assert!(tuple From b3a50437fc5ce2721ac1fe53b1cd3b3d8baba8c0 Mon Sep 17 00:00:00 2001 From: kould Date: Wed, 7 Oct 2026 04:14:40 +0800 Subject: [PATCH 05/16] refactor: initialize uncorrelated scalar subqueries as runtime values Replace ScalarApply and WHERE scalar joins with ScalarQueryInit and leaf Init expressions. Keep statement-local results in an owning ExecArenaView cache, including NULL results, and make them available before sorting, grouping and DISTINCT. Use arena-scoped references for initialization definitions and relocate view references before serialization. Add subquery SLT regressions and a database-level catalog reload test. Retain explicit rejection of correlated scalar subqueries. --- src/binder/aggregate.rs | 15 + src/binder/create_view.rs | 1 + src/binder/expr.rs | 28 +- src/binder/mod.rs | 15 +- src/binder/parser.rs | 8 +- src/binder/select.rs | 465 ++++-------------- src/catalog/column.rs | 2 +- src/db.rs | 74 ++- src/execution/ddl/create_index.rs | 3 +- src/execution/ddl/drop_view.rs | 2 +- src/execution/dql/aggregate/mod.rs | 4 +- .../dql/aggregate/stream_distinct.rs | 3 +- src/execution/dql/dummy.rs | 2 +- src/execution/dql/join/hash/mod.rs | 4 +- src/execution/dql/mod.rs | 2 +- src/execution/dql/projection.rs | 3 +- src/execution/dql/scalar_apply.rs | 280 ----------- src/execution/dql/scalar_query_init.rs | 158 ++++++ src/execution/dql/show_table.rs | 2 +- src/execution/dql/values.rs | 3 +- src/execution/mod.rs | 48 +- src/expression/eq_col.rs | 18 +- src/expression/evaluator.rs | 72 ++- src/expression/mod.rs | 11 +- src/expression/range_detacher.rs | 7 +- src/expression/visitor.rs | 12 +- src/expression/visitor_mut.rs | 12 +- src/function/numbers.rs | 5 +- src/macros/mod.rs | 2 +- src/optimizer/heuristic/optimizer.rs | 11 +- src/optimizer/rule/implementation/dql/mod.rs | 2 +- .../{scalar_apply.rs => scalar_query_init.rs} | 8 +- src/optimizer/rule/implementation/mod.rs | 20 +- .../rule/normalization/column_pruning.rs | 50 +- .../normalization/compilation_in_advance.rs | 2 +- src/optimizer/rule/normalization/mod.rs | 2 +- src/planner/arena.rs | 291 ++++++++++- src/planner/mod.rs | 10 +- src/planner/operator/mod.rs | 34 +- .../{scalar_apply.rs => scalar_query_init.rs} | 16 +- src/planner/operator/visitor.rs | 11 +- src/planner/operator/visitor_mut.rs | 6 +- src/planner/scalar_query_ref.rs | 101 ++++ src/types/tuple.rs | 130 +++-- tests/slt/exists.slt | 39 ++ tests/slt/subquery.slt | 176 ++++++- 46 files changed, 1335 insertions(+), 835 deletions(-) delete mode 100644 src/execution/dql/scalar_apply.rs create mode 100644 src/execution/dql/scalar_query_init.rs rename src/optimizer/rule/implementation/dql/{scalar_apply.rs => scalar_query_init.rs} (84%) rename src/planner/operator/{scalar_apply.rs => scalar_query_init.rs} (70%) create mode 100644 src/planner/scalar_query_ref.rs diff --git a/src/binder/aggregate.rs b/src/binder/aggregate.rs index e17dc898..36be4085 100644 --- a/src/binder/aggregate.rs +++ b/src/binder/aggregate.rs @@ -190,6 +190,15 @@ impl> Binder<'_, '_, T, A> { if expr.has_agg_call(arena)? { continue; } + if !expr.any_referenced_column(arena, |_, _| true)? { + if let Some(position) = unmatched_group_exprs + .iter() + .position(|group_expr| expr.eq_ignore_colref_pos(*group_expr, arena)) + { + unmatched_group_exprs.remove(position); + } + continue; + } let Some(position) = unmatched_group_exprs .iter() .position(|group_expr| expr.eq_ignore_colref_pos(*group_expr, arena)) @@ -202,6 +211,12 @@ impl> Binder<'_, '_, T, A> { unmatched_group_exprs.remove(position); } + unmatched_group_exprs.retain(|expr| { + !matches!( + arena.expression(expr.unpack_alias(arena)), + ScalarExpression::Init { .. } + ) + }); if !unmatched_group_exprs.is_empty() { return Err(DatabaseError::AggMiss( "in the GROUP BY clause the field must be in the select clause".to_string(), diff --git a/src/binder/create_view.rs b/src/binder/create_view.rs index 42a397aa..7f49753c 100644 --- a/src/binder/create_view.rs +++ b/src/binder/create_view.rs @@ -91,6 +91,7 @@ impl> Binder<'_, '_, T, A> { }; plan = self.bind_project(plan, exprs, arena)?; let schema = plan.output_schema(arena).clone(); + arena.materialize_scalar_queries(&mut plan)?; Ok(LogicalPlan::new( Operator::CreateView(CreateViewOperator { diff --git a/src/binder/expr.rs b/src/binder/expr.rs index 97b75fd4..1542808a 100644 --- a/src/binder/expr.rs +++ b/src/binder/expr.rs @@ -25,7 +25,7 @@ use crate::expression::function::FunctionSummary; use crate::expression::{AliasType, ScalarExpression, TypeCast}; use crate::planner::operator::mark_apply::MarkApplyQuantifier; use crate::planner::operator::scalar_subquery::ScalarSubqueryOperator; -use crate::planner::{ExprRef, LogicalPlan, PlanArena}; +use crate::planner::{ExprRef, LogicalPlan, MetaArena, PlanArena}; use crate::storage::Transaction; use crate::types::value::{DataValue, Utf8Type}; use crate::types::{CharLengthUnits, LogicalType}; @@ -208,21 +208,17 @@ impl<'a, T: Transaction, A: AsRef<[(usize, LogicalType)]>> Binder<'a, '_, T, A> { let (sub_query, column, correlated) = self.bind_subquery_plan_with_output(None, arena, build)?; - let sub_query = ScalarSubqueryOperator::build(sub_query); - let (expr, sub_query) = match self.context.step_now() { - QueryBindStep::Where => (column, sub_query), - QueryBindStep::Project => self.bind_temp_table(column, sub_query, arena)?, - _ => { - return Err(DatabaseError::UnsupportedStmt( - "scalar subqueries can only appear in `WHERE` or SELECT list".to_string(), - )) - } - }; - self.context.sub_query(SubQueryType::SubQuery { - plan: sub_query, - correlated, - }); - Ok(expr) + if correlated { + return Err(DatabaseError::UnsupportedStmt( + "correlated scalar subqueries are not supported".to_string(), + )); + } + let id = arena.alloc_scalar_query_ref(); + let ty = column.return_type(arena).into_owned(); + self.context + .scalar_queries + .push((id, ScalarSubqueryOperator::build(sub_query))); + Ok(ScalarExpression::Init { id, ty }) } pub(crate) fn bind_exists_subquery_plan<'arena, F>( diff --git a/src/binder/mod.rs b/src/binder/mod.rs index 8e61b81b..30aed1a2 100644 --- a/src/binder/mod.rs +++ b/src/binder/mod.rs @@ -68,7 +68,7 @@ use crate::errors::DatabaseError; use crate::expression::{ScalarExpression, TypeCast}; use crate::planner::operator::join::JoinType; use crate::planner::operator::mark_apply::MarkApplyQuantifier; -use crate::planner::{ExprRef, LogicalPlan, PlanArena, PlanRef}; +use crate::planner::{ExprRef, LogicalPlan, PlanArena, PlanRef, ScalarQueryRef}; use crate::storage::{TableCache, Transaction, ViewCache}; use crate::types::tuple::Schema; use crate::types::LogicalType; @@ -102,10 +102,6 @@ pub enum QueryBindStep { #[derive(Debug, Clone, Hash, Eq, PartialEq)] pub enum SubQueryType { - SubQuery { - plan: LogicalPlan, - correlated: bool, - }, ExistsSubQuery { plan: LogicalPlan, correlated: bool, @@ -151,8 +147,10 @@ pub(crate) struct CteCheckpoint { impl BoundSource<'_> { pub(crate) fn matches_name(&self, table_name: &str) -> bool { - self.table_name.as_ref() == table_name - || matches!(self.alias.as_ref(), Some(alias) if alias.as_ref() == table_name) + match &self.alias { + Some(alias) => alias.as_ref() == table_name, + None => self.table_name.as_ref() == table_name, + } } pub(crate) fn same_binding( @@ -259,6 +257,7 @@ pub struct BinderContext<'a, T: Transaction> { bind_step: QueryBindStep, sub_queries: HashMap>, + scalar_queries: Vec<(ScalarQueryRef, LogicalPlan)>, has_outer_refs: bool, pub(crate) allow_default: bool, @@ -321,6 +320,7 @@ impl<'a, T: Transaction> BinderContext<'a, T> { using: Default::default(), bind_step: QueryBindStep::From, sub_queries: Default::default(), + scalar_queries: Vec::new(), has_outer_refs: false, allow_default: false, } @@ -347,6 +347,7 @@ impl<'a, T: Transaction> BinderContext<'a, T> { using: self.using.clone(), bind_step: self.bind_step, sub_queries: Default::default(), + scalar_queries: Vec::new(), has_outer_refs: false, allow_default: self.allow_default, } diff --git a/src/binder/parser.rs b/src/binder/parser.rs index 5145ee9f..e79782f0 100644 --- a/src/binder/parser.rs +++ b/src/binder/parser.rs @@ -2870,7 +2870,8 @@ impl<'a, 'parent, T: Transaction, A: AsRef<[(usize, LogicalType)]>> Binder<'a, ' arena.alloc_expression(predicate) })?; - self.bind_where_expr(children, predicate, arena) + let plan = self.bind_where_expr(children, predicate, arena)?; + Ok(self.init_scalar_queries(plan)) } pub(crate) fn normalize_select_item( @@ -3174,7 +3175,7 @@ impl<'a, 'parent, T: Transaction, A: AsRef<[(usize, LogicalType)]>> Binder<'a, ' self.context.restore_ctes(cte_checkpoint); self.context.step(origin_step); - Ok(plan) + Ok(self.init_scalar_queries(plan)) } fn bind_non_negative_limit_value( @@ -3253,7 +3254,8 @@ impl<'a, 'parent, T: Transaction, A: AsRef<[(usize, LogicalType)]>> Binder<'a, ' stmt: &Statement, arena: &mut PlanArena, ) -> Result { - Ok(self.build_statement(arena).statement(stmt)?.finish()) + let plan = self.build_statement(arena).statement(stmt)?.finish(); + Ok(self.init_scalar_queries(plan)) } } diff --git a/src/binder/select.rs b/src/binder/select.rs index f7d4ebd4..b394cce0 100644 --- a/src/binder/select.rs +++ b/src/binder/select.rs @@ -19,15 +19,12 @@ use crate::{ operator::{ filter::FilterOperator, join::JoinOperator as LJoinOperator, limit::LimitOperator, mark_apply::MarkApplyOperator, project::ProjectOperator, - scalar_apply::ScalarApplyOperator, Operator, + scalar_query_init::ScalarQueryInitOperator, Operator, }, operator::{join::JoinType, table_scan::TableScanOperator}, }, }; -use std::{ - borrow::Cow, - collections::{HashMap, HashSet}, -}; +use std::{borrow::Cow, collections::HashSet}; use super::{Binder, BinderContext, QueryBindStep, SetOperatorKind, Source, SubQueryType}; @@ -71,76 +68,12 @@ impl ExprVisitorMut for RightSidePositionGlobalizer<'_> { } } -/// Whether any of `exprs` reads the output of a SELECT-list scalar subquery. -/// Those subqueries are only joined in at the Project step, above DISTINCT / -/// GROUP BY / ORDER BY, so such a reference would read another column. -/// -/// Takes the subquery map rather than the binder so callers can pass -/// expressions borrowed from the same binder context. -// TODO(#386): evaluate uncorrelated scalar subqueries once, as constants -// (like PostgreSQL's InitPlan), so these clauses can use them. -fn references_select_list_sub_query( - sub_queries: &mut HashMap>, - exprs: impl IntoIterator, - arena: &mut PlanArena, -) -> Result { - let Some(sub_queries) = sub_queries.get_mut(&QueryBindStep::Project) else { - return Ok(false); - }; - // `exprs` can only be traversed once, so it is the outer loop. - for expr in exprs { - for sub_query in sub_queries.iter_mut() { - let SubQueryType::SubQuery { plan, .. } = sub_query else { - continue; - }; - let schema = plan.output_schema(arena); - if expr.any_referenced_column(arena, |arena, candidate| { - schema - .iter() - .any(|column| arena.same_column(*column, *candidate)) - })? { - return Ok(true); - } - } - } - Ok(false) -} - struct AppendedRightOutput { column: ColumnRef, child_position: usize, output_position: usize, } -struct SplitScopePositionRebinder<'a> { - left_schema: &'a Schema, - right_schema: &'a Schema, -} - -impl ExprVisitorMut for SplitScopePositionRebinder<'_> { - fn visit_column_ref( - &mut self, - column: &mut ColumnRef, - position: &mut usize, - arena: &mut (dyn MetaArena + '_), - ) -> Result<(), DatabaseError> { - if let Some(left_position) = self - .left_schema - .iter() - .position(|candidate| arena.same_column(*candidate, *column)) - { - *position = left_position; - } else if let Some(right_position) = self - .right_schema - .iter() - .position(|candidate| arena.same_column(*candidate, *column)) - { - *position = right_position; - } - Ok(()) - } -} - struct MarkerPositionGlobalizer<'a> { output_column: &'a ColumnRef, left_len: usize, @@ -205,7 +138,7 @@ where A: AsRef<[(usize, LogicalType)]>, { pub(crate) binder: &'s mut Binder<'a, 'b, T, A>, - pub(crate) arena: &'s mut crate::planner::PlanArena<'arena>, + pub(crate) arena: &'s mut PlanArena<'arena>, } pub struct BindPlanFrom<'s, 'a, 'b, 'arena, T, A, M = ()> @@ -214,7 +147,7 @@ where A: AsRef<[(usize, LogicalType)]>, { pub(crate) binder: &'s mut Binder<'a, 'b, T, A>, - pub(crate) arena: &'s mut crate::planner::PlanArena<'arena>, + pub(crate) arena: &'s mut PlanArena<'arena>, pub(crate) plan: LogicalPlan, pub(crate) _marker: std::marker::PhantomData, } @@ -225,7 +158,7 @@ where A: AsRef<[(usize, LogicalType)]>, { pub(crate) binder: &'s mut Binder<'a, 'b, T, A>, - pub(crate) arena: &'s mut crate::planner::PlanArena<'arena>, + pub(crate) arena: &'s mut PlanArena<'arena>, pub(super) plan: LogicalPlan, pub(super) select_list: Vec, pub(crate) _marker: std::marker::PhantomData, @@ -237,7 +170,7 @@ where A: AsRef<[(usize, LogicalType)]>, { pub(super) binder: &'s mut Binder<'a, 'b, T, A>, - pub(super) arena: &'s mut crate::planner::PlanArena<'arena>, + pub(super) arena: &'s mut PlanArena<'arena>, pub(super) plan: LogicalPlan, pub(super) select_list: Vec, } @@ -248,7 +181,7 @@ where A: AsRef<[(usize, LogicalType)]>, { binder: &'s mut Binder<'a, 'b, T, A>, - arena: &'s mut crate::planner::PlanArena<'arena>, + arena: &'s mut PlanArena<'arena>, plan: LogicalPlan, select_list: Vec, having: Option, @@ -261,7 +194,7 @@ where A: AsRef<[(usize, LogicalType)]>, { binder: &'s mut Binder<'a, 'b, T, A>, - arena: &'s mut crate::planner::PlanArena<'arena>, + arena: &'s mut PlanArena<'arena>, plan: LogicalPlan, select_list: Vec, orderby: Option>, @@ -273,7 +206,7 @@ where A: AsRef<[(usize, LogicalType)]>, { binder: &'s mut Binder<'a, 'b, T, A>, - arena: &'s mut crate::planner::PlanArena<'arena>, + arena: &'s mut PlanArena<'arena>, plan: LogicalPlan, select_list: Vec, orderby: Option>, @@ -285,7 +218,7 @@ where A: AsRef<[(usize, LogicalType)]>, { binder: &'s mut Binder<'a, 'b, T, A>, - arena: &'s mut crate::planner::PlanArena<'arena>, + arena: &'s mut PlanArena<'arena>, plan: LogicalPlan, select_list: Vec, orderby: Option>, @@ -297,7 +230,7 @@ where A: AsRef<[(usize, LogicalType)]>, { binder: &'s mut Binder<'a, 'b, T, A>, - arena: &'s mut crate::planner::PlanArena<'arena>, + arena: &'s mut PlanArena<'arena>, plan: LogicalPlan, select_list: Vec, } @@ -483,8 +416,10 @@ where return self.aggregate_without_group()?.finish(); } } - self.binder - .bind_project(self.plan, self.select_list, self.arena) + let plan = self + .binder + .bind_project(self.plan, self.select_list, self.arena)?; + Ok(self.binder.init_scalar_queries(plan)) } } @@ -543,7 +478,7 @@ where orderby: Option>, mut bind_sort_field: impl FnMut( &mut Binder<'a, 'b, T, A>, - &mut crate::planner::PlanArena<'arena>, + &mut PlanArena<'arena>, &[ExprRef], O, ) -> Result, @@ -571,16 +506,7 @@ where self.arena, )?; } - let context = &mut self.binder.context; - if references_select_list_sub_query( - &mut context.sub_queries, - context.group_by_exprs.iter().copied(), - self.arena, - )? { - return Err(DatabaseError::UnsupportedStmt( - "GROUP BY over a scalar subquery in the SELECT list is not supported".to_string(), - )); - } + if !self.binder.context.agg_calls.is_empty() || !self.binder.context.group_by_exprs.is_empty() { @@ -675,16 +601,6 @@ where distinct: bool, ) -> Result, DatabaseError> { if distinct { - if references_select_list_sub_query( - &mut self.binder.context.sub_queries, - self.select_list.iter().copied(), - self.arena, - )? { - return Err(DatabaseError::UnsupportedStmt( - "DISTINCT over a scalar subquery in the SELECT list is not supported" - .to_string(), - )); - } let distinct_outputs = self.select_list.clone(); self.binder.bind_distinct_output_exprs( &distinct_outputs, @@ -717,16 +633,6 @@ where mut self, ) -> Result, DatabaseError> { if let Some(orderby) = self.orderby { - if references_select_list_sub_query( - &mut self.binder.context.sub_queries, - orderby.iter().map(|field| field.expr), - self.arena, - )? { - return Err(DatabaseError::UnsupportedStmt( - "ORDER BY over a scalar subquery in the SELECT list is not supported" - .to_string(), - )); - } self.plan = self.binder.bind_sort(self.plan, orderby, self.arena)?; } @@ -754,7 +660,7 @@ where } Ok(BindPlanProjected { - plan: self.plan, + plan: self.binder.init_scalar_queries(self.plan), _marker: std::marker::PhantomData, }) } @@ -791,9 +697,19 @@ impl BindPlanComplete { } impl<'a: 'b, 'b, T: Transaction, A: AsRef<[(usize, LogicalType)]>> Binder<'a, 'b, T, A> { + pub(crate) fn init_scalar_queries(&mut self, mut plan: LogicalPlan) -> LogicalPlan { + for (id, init) in std::mem::take(&mut self.context.scalar_queries) + .into_iter() + .rev() + { + plan = ScalarQueryInitOperator::build(plan, init, id); + } + plan + } + pub(crate) fn build_plan<'s, 'arena>( &'s mut self, - arena: &'s mut crate::planner::PlanArena<'arena>, + arena: &'s mut PlanArena<'arena>, ) -> BindPlanStart<'s, 'a, 'b, 'arena, T, A> { BindPlanStart { binder: self, @@ -801,7 +717,9 @@ impl<'a: 'b, 'b, T: Transaction, A: AsRef<[(usize, LogicalType)]>> Binder<'a, 'b } } - fn is_temp_alias_projection(exprs: &[ExprRef], arena: &crate::planner::PlanArena) -> bool { + /// Whether `exprs` only renames its input to an alias, as `bind_alias` builds for `FROM t AS x` + /// (`temp == false`) or for a temp table (`temp == true`). + fn is_alias_projection(exprs: &[ExprRef], temp: bool, arena: &PlanArena) -> bool { !exprs.is_empty() && exprs.iter().all(|expr| { matches!( @@ -814,7 +732,8 @@ impl<'a: 'b, 'b, T: Transaction, A: AsRef<[(usize, LogicalType)]>> Binder<'a, 'b ScalarExpression::ColumnRef { column, .. } if matches!( &arena.column(*column).summary().relation, - crate::catalog::ColumnRelation::Table { is_temp: true, .. } + crate::catalog::ColumnRelation::Table { is_temp, .. } + if *is_temp == temp ) ) ) @@ -824,7 +743,7 @@ impl<'a: 'b, 'b, T: Transaction, A: AsRef<[(usize, LogicalType)]>> Binder<'a, 'b pub(crate) fn is_joined_values_source( join_type: Option, source: &Source<'a>, - arena: &crate::planner::PlanArena, + arena: &PlanArena, ) -> bool { join_type.is_some() && matches!( @@ -876,31 +795,10 @@ impl<'a: 'b, 'b, T: Transaction, A: AsRef<[(usize, LogicalType)]>> Binder<'a, 'b Ok(()) } - fn globalize_join_filter_from_split_scope( - join_condition: &mut JoinCondition, - left_len: usize, - right_schema: &Schema, - arena: &mut crate::planner::PlanArena, - ) -> Result<(), DatabaseError> { - let JoinCondition::On { filter, .. } = join_condition else { - return Ok(()); - }; - - if let Some(expr) = filter { - RightSidePositionGlobalizer { - right_schema, - left_len, - } - .visit(expr, arena)?; - } - - Ok(()) - } - fn localize_appended_right_outputs<'expr>( exprs: impl Iterator, appended_outputs: &[AppendedRightOutput], - arena: &mut crate::planner::PlanArena, + arena: &mut PlanArena, ) -> Result<(), DatabaseError> { struct AppendedRightOutputBinder<'a> { appended_outputs: &'a [AppendedRightOutput], @@ -930,72 +828,11 @@ impl<'a: 'b, 'b, T: Transaction, A: AsRef<[(usize, LogicalType)]>> Binder<'a, 'b Ok(()) } - fn rebind_split_scope_positions( - mut expr: ExprRef, - left_schema: &Schema, - right_schema: &Schema, - arena: &mut crate::planner::PlanArena, - ) -> Result<(), DatabaseError> { - SplitScopePositionRebinder { - left_schema, - right_schema, - } - .visit(&mut expr, arena) - } - - fn build_join_from_split_scope_predicates( - &self, - mut children: LogicalPlan, - mut plan: LogicalPlan, - join_ty: JoinType, - predicates: impl IntoIterator, - rebind_positions: bool, - arena: &mut crate::planner::PlanArena, - ) -> Result { - let left_schema = children.output_schema(arena); - let right_schema = plan.output_schema(arena); - let mut on_keys = Vec::new(); - let mut filter = Vec::new(); - - for predicate in predicates { - if rebind_positions { - Self::rebind_split_scope_positions(predicate, left_schema, right_schema, arena)?; - } - Self::extract_join_keys( - predicate, - &mut on_keys, - &mut filter, - left_schema, - right_schema, - arena, - )?; - } - - let mut join_condition = JoinCondition::On { - on: on_keys, - filter: Self::combine_conjuncts(filter, arena), - }; - Self::globalize_join_filter_from_split_scope( - &mut join_condition, - left_schema.len(), - right_schema, - arena, - )?; - - Ok(LJoinOperator::build( - children, - plan, - join_condition, - join_ty, - self.force_nested_loop, - )) - } - fn bind_set_cast( &mut self, mut left_plan: LogicalPlan, mut right_plan: LogicalPlan, - arena: &mut crate::planner::PlanArena, + arena: &mut PlanArena, ) -> Result<(LogicalPlan, LogicalPlan), DatabaseError> { let mut left_cast = vec![]; let mut right_cast = vec![]; @@ -1044,7 +881,7 @@ impl<'a: 'b, 'b, T: Transaction, A: AsRef<[(usize, LogicalType)]>> Binder<'a, 'b is_all: bool, mut left_plan: LogicalPlan, mut right_plan: LogicalPlan, - arena: &mut crate::planner::PlanArena, + arena: &mut PlanArena, ) -> Result { let mut left_schema = left_plan.output_schema(arena); let mut right_schema = right_plan.output_schema(arena); @@ -1152,7 +989,7 @@ impl<'a: 'b, 'b, T: Transaction, A: AsRef<[(usize, LogicalType)]>> Binder<'a, 'b alias_column: &[String], table_alias: TableName, table_name: TableName, - arena: &mut crate::planner::PlanArena, + arena: &mut PlanArena, ) -> Result { let input_schema = plan.output_schema(arena); let input_schema_len = input_schema.len(); @@ -1201,7 +1038,7 @@ impl<'a: 'b, 'b, T: Transaction, A: AsRef<[(usize, LogicalType)]>> Binder<'a, 'b &mut self, mut plan: LogicalPlan, source_name: TableName, - arena: &mut crate::planner::PlanArena, + arena: &mut PlanArena, ) -> LogicalPlan { let input_schema = plan.output_schema(arena); let input_schema_len = input_schema.len(); @@ -1237,12 +1074,12 @@ impl<'a: 'b, 'b, T: Transaction, A: AsRef<[(usize, LogicalType)]>> Binder<'a, 'b join_type: Option, table_name: TableName, alias: Option, - arena: &mut crate::planner::PlanArena, + arena: &mut PlanArena, ) -> Result { let table_alias = alias.as_ref().map(|alias| alias.name.clone()); if let Some(plan_ref) = self.context.cte(&table_name).map(|cte| cte.plan_ref) { - let mut plan = arena.plan(plan_ref).clone(); + let mut plan = arena.plan(plan_ref).clone().clone_plan(arena)?; if let Some(alias) = alias { plan = self.bind_alias( plan, @@ -1307,7 +1144,7 @@ impl<'a: 'b, 'b, T: Transaction, A: AsRef<[(usize, LogicalType)]>> Binder<'a, 'b mut plan: LogicalPlan, alias: Option, joint_type: Option, - arena: &mut crate::planner::PlanArena, + arena: &mut PlanArena, ) -> Result { if let Some(alias) = alias { let source_name = arena.temp_table(); @@ -1362,7 +1199,7 @@ impl<'a: 'b, 'b, T: Transaction, A: AsRef<[(usize, LogicalType)]>> Binder<'a, 'b expr: ScalarExpression, alias: Option, joint_type: Option, - arena: &mut crate::planner::PlanArena, + arena: &mut PlanArena, ) -> Result { let ScalarExpression::TableFunction(function) = expr else { return Err(DatabaseError::UnsupportedStmt( @@ -1395,7 +1232,7 @@ impl<'a: 'b, 'b, T: Transaction, A: AsRef<[(usize, LogicalType)]>> Binder<'a, 'b #[allow(unused_assignments)] pub(crate) fn bind_table_column_refs( context: &BinderContext<'a, T>, - arena: &mut crate::planner::PlanArena, + arena: &mut PlanArena, exprs: &mut Vec, table_name: TableName, is_qualified_wildcard: bool, @@ -1403,7 +1240,7 @@ impl<'a: 'b, 'b, T: Transaction, A: AsRef<[(usize, LogicalType)]>> Binder<'a, 'b let (source, position_offset) = Self::resolve_source_columns_in_scope(context, table_name.as_ref())?; - let fn_not_on_using = |column: &ColumnRef, arena: &crate::planner::PlanArena<'_>| { + let fn_not_on_using = |column: &ColumnRef, arena: &PlanArena<'_>| { let column_catalog = arena.column(*column); if context.using.is_empty() { return Some(&table_name) == column_catalog.table_name(); @@ -1433,7 +1270,7 @@ impl<'a: 'b, 'b, T: Transaction, A: AsRef<[(usize, LogicalType)]>> Binder<'a, 'b fn wildcard_column_expr( context: &BinderContext<'a, T>, - arena: &mut crate::planner::PlanArena, + arena: &mut PlanArena, column: &ColumnRef, position: usize, is_qualified_wildcard: bool, @@ -1464,7 +1301,7 @@ impl<'a: 'b, 'b, T: Transaction, A: AsRef<[(usize, LogicalType)]>> Binder<'a, 'b mut right: LogicalPlan, join_type: JoinType, constraint: JoinConstraintInput, - arena: &mut crate::planner::PlanArena, + arena: &mut PlanArena, ) -> Result { let left_len = left.output_schema(arena).len(); right.output_schema(arena); @@ -1487,12 +1324,11 @@ impl<'a: 'b, 'b, T: Transaction, A: AsRef<[(usize, LogicalType)]>> Binder<'a, 'b &mut self, mut children: LogicalPlan, predicate: ExprRef, - arena: &mut crate::planner::PlanArena, + arena: &mut PlanArena, ) -> Result { self.context.step(QueryBindStep::Where); if let Some(sub_queries) = self.context.sub_queries_at_now() { - let mut uses_mark_apply = None; for sub_query in sub_queries { match sub_query { SubQueryType::ExistsSubQuery { @@ -1500,13 +1336,6 @@ impl<'a: 'b, 'b, T: Transaction, A: AsRef<[(usize, LogicalType)]>> Binder<'a, 'b correlated, output_column, } => { - if matches!(uses_mark_apply, Some(false)) { - return Err(DatabaseError::UnsupportedStmt( - "mixed EXISTS/IN with other WHERE subqueries is not supported yet" - .to_string(), - )); - } - uses_mark_apply = Some(true); let left_schema = children.output_schema(arena).clone(); let (plan, predicates) = Self::prepare_mark_apply( predicate, @@ -1533,13 +1362,6 @@ impl<'a: 'b, 'b, T: Transaction, A: AsRef<[(usize, LogicalType)]>> Binder<'a, 'b predicate: mut quantified_predicate, .. } => { - if matches!(uses_mark_apply, Some(false)) { - return Err(DatabaseError::UnsupportedStmt( - "mixed EXISTS/IN with other WHERE subqueries is not supported yet" - .to_string(), - )); - } - uses_mark_apply = Some(true); if correlated { quantified_predicate = Self::rewrite_correlated_quantified_predicate( quantified_predicate, @@ -1565,32 +1387,9 @@ impl<'a: 'b, 'b, T: Transaction, A: AsRef<[(usize, LogicalType)]>> Binder<'a, 'b predicates, ); } - SubQueryType::SubQuery { plan, correlated } => { - if matches!(uses_mark_apply, Some(true)) { - return Err(DatabaseError::UnsupportedStmt( - "mixed EXISTS/IN with other WHERE subqueries is not supported yet" - .to_string(), - )); - } - uses_mark_apply = Some(false); - if correlated { - return Err(DatabaseError::UnsupportedStmt( - "correlated scalar subqueries in WHERE are not supported" - .to_string(), - )); - } - children = self.build_join_from_split_scope_predicates( - children, - plan, - JoinType::Inner, - std::iter::once(predicate), - true, - arena, - )?; - } } } - if matches!(uses_mark_apply, Some(true)) { + { let passthrough_exprs = children .output_schema(arena) .iter() @@ -1608,7 +1407,6 @@ impl<'a: 'b, 'b, T: Transaction, A: AsRef<[(usize, LogicalType)]>> Binder<'a, 'b Childrens::Only(Box::new(filter)), )); } - return Ok(children); } Ok(FilterOperator::build(predicate, children, false)) } @@ -1616,7 +1414,7 @@ impl<'a: 'b, 'b, T: Transaction, A: AsRef<[(usize, LogicalType)]>> Binder<'a, 'b fn ensure_mark_apply_right_outputs( plan: &mut LogicalPlan, predicates: &[ExprRef], - arena: &mut crate::planner::PlanArena, + arena: &mut PlanArena, ) -> Result, DatabaseError> { let output_schema = plan.output_schema(arena).clone(); let output_len = output_schema.len(); @@ -1626,6 +1424,11 @@ impl<'a: 'b, 'b, T: Transaction, A: AsRef<[(usize, LogicalType)]>> Binder<'a, 'b .. } = plan { + // An alias projection already outputs every input column under the alias; its input + // columns may look like the outer query's (`t1` in `FROM t1 AS x`) but are not. + if Self::is_alias_projection(&op.exprs, false, arena) { + return Ok(Vec::new()); + } let Childrens::Only(child) = childrens.as_mut() else { return Ok(Vec::new()); }; @@ -1671,7 +1474,7 @@ impl<'a: 'b, 'b, T: Transaction, A: AsRef<[(usize, LogicalType)]>> Binder<'a, 'b correlated: bool, preserve_projection: bool, mut apply_predicates: Vec, - arena: &mut crate::planner::PlanArena, + arena: &mut PlanArena, ) -> Result<(LogicalPlan, Vec), DatabaseError> { let left_len = left_schema.len(); MarkerPositionGlobalizer { @@ -1743,7 +1546,7 @@ impl<'a: 'b, 'b, T: Transaction, A: AsRef<[(usize, LogicalType)]>> Binder<'a, 'b fn plan_has_correlated_refs( plan: &LogicalPlan, left_schema: &Schema, - arena: &mut crate::planner::PlanArena, + arena: &mut PlanArena, ) -> Result { if !plan .operator @@ -1772,7 +1575,7 @@ impl<'a: 'b, 'b, T: Transaction, A: AsRef<[(usize, LogicalType)]>> Binder<'a, 'b fn expr_has_correlated_refs( expr: ExprRef, left_schema: &Schema, - arena: &mut crate::planner::PlanArena, + arena: &mut PlanArena, ) -> Result { expr.any_referenced_column(arena, |arena, column| { left_schema @@ -1813,7 +1616,7 @@ impl<'a: 'b, 'b, T: Transaction, A: AsRef<[(usize, LogicalType)]>> Binder<'a, 'b plan: LogicalPlan, left_schema: &Schema, preserve_projection: bool, - arena: &mut crate::planner::PlanArena, + arena: &mut PlanArena, ) -> Result<(LogicalPlan, Vec), DatabaseError> { match plan.childrens.as_ref() { Childrens::Only(_) => {} @@ -1829,6 +1632,15 @@ impl<'a: 'b, 'b, T: Transaction, A: AsRef<[(usize, LogicalType)]>> Binder<'a, 'b } match plan { + // `FROM t AS x`: the alias gives the subquery's columns their own identity, which its + // predicates are bound to, so it is kept. It only reads its input, never the outer query. + plan if matches!( + &plan.operator, + Operator::Project(op) if Self::is_alias_projection(&op.exprs, false, arena) + ) => + { + Ok((plan, vec![])) + } LogicalPlan { operator: Operator::Filter(op), childrens, @@ -1871,7 +1683,7 @@ impl<'a: 'b, 'b, T: Transaction, A: AsRef<[(usize, LogicalType)]>> Binder<'a, 'b arena, )?; - if !preserve_projection || Self::is_temp_alias_projection(&op.exprs, arena) { + if !preserve_projection || Self::is_alias_projection(&op.exprs, true, arena) { Ok((child, correlated_filters)) } else { for expr in &op.exprs { @@ -1929,7 +1741,7 @@ impl<'a: 'b, 'b, T: Transaction, A: AsRef<[(usize, LogicalType)]>> Binder<'a, 'b &mut self, children: LogicalPlan, mut having: ExprRef, - arena: &mut crate::planner::PlanArena, + arena: &mut PlanArena, ) -> Result { self.context.step(QueryBindStep::Having); @@ -1950,44 +1762,12 @@ impl<'a: 'b, 'b, T: Transaction, A: AsRef<[(usize, LogicalType)]>> Binder<'a, 'b pub(crate) fn bind_project( &mut self, - mut children: LogicalPlan, - mut select_list: Vec, - arena: &mut crate::planner::PlanArena, + children: LogicalPlan, + select_list: Vec, + _arena: &mut PlanArena, ) -> Result { self.context.step(QueryBindStep::Project); - if let Some(sub_queries) = self.context.sub_queries_at_now() { - for sub_query in sub_queries { - let SubQueryType::SubQuery { - mut plan, - correlated, - } = sub_query - else { - return Err(DatabaseError::UnsupportedStmt( - "only scalar subqueries are supported in SELECT list".to_string(), - )); - }; - - if correlated { - return Err(DatabaseError::UnsupportedStmt( - "correlated scalar subqueries in SELECT list are not supported".to_string(), - )); - } - - let left_len = children.output_schema(arena).len(); - let right_schema = plan.output_schema(arena); - for expr in &mut select_list { - RightSidePositionGlobalizer { - right_schema, - left_len, - } - .visit(expr, arena)?; - } - - children = ScalarApplyOperator::build(children, plan); - } - } - Ok(Self::build_project_plan(children, select_list)) } @@ -1995,7 +1775,7 @@ impl<'a: 'b, 'b, T: Transaction, A: AsRef<[(usize, LogicalType)]>> Binder<'a, 'b &mut self, children: LogicalPlan, sort_fields: Vec, - _arena: &mut crate::planner::PlanArena, + _arena: &mut PlanArena, ) -> Result { self.context.step(QueryBindStep::Sort); @@ -2016,11 +1796,7 @@ impl<'a: 'b, 'b, T: Transaction, A: AsRef<[(usize, LogicalType)]>> Binder<'a, 'b Ok(LimitOperator::build(offset_value, limit_value, children)) } - pub fn extract_select_join( - &mut self, - select_items: &mut [ExprRef], - arena: &mut crate::planner::PlanArena, - ) { + pub fn extract_select_join(&mut self, select_items: &mut [ExprRef], arena: &mut PlanArena) { if self.context.bind_table.len() < 2 { return; } @@ -2075,7 +1851,7 @@ impl<'a: 'b, 'b, T: Transaction, A: AsRef<[(usize, LogicalType)]>> Binder<'a, 'b constraint: JoinConstraintInput, left_schema: &Schema, right_schema: &Schema, - arena: &mut crate::planner::PlanArena, + arena: &mut PlanArena, ) -> Result { match constraint { JoinConstraintInput::On(expr) => { @@ -2104,7 +1880,7 @@ impl<'a: 'b, 'b, T: Transaction, A: AsRef<[(usize, LogicalType)]>> Binder<'a, 'b fn find_column<'a>( schema: &'a Schema, name: &'a str, - arena: &crate::planner::PlanArena, + arena: &PlanArena, ) -> Option<(usize, &'a ColumnRef)> { schema .iter() @@ -2231,7 +2007,7 @@ impl<'a: 'b, 'b, T: Transaction, A: AsRef<[(usize, LogicalType)]>> Binder<'a, 'b accum_filter: &mut Vec, left_schema: &Schema, right_schema: &Schema, - arena: &mut crate::planner::PlanArena, + arena: &mut PlanArena, ) -> Result<(), DatabaseError> { let fn_contains = |schema: &Schema, column: ColumnRef| { let summary = arena.column(column).summary(); @@ -2372,7 +2148,7 @@ mod tests { MarkApplyKind, MarkApplyOperator, MarkApplyQuantifier, }; use crate::planner::operator::Operator; - use crate::planner::{Childrens, ExprRef, LogicalPlan, PlanArena}; + use crate::planner::{Childrens, ExprRef, LogicalPlan, PlanArena, TableArenaCell}; use crate::types::LogicalType; fn test_column(arena: &mut PlanArena, name: &str, position: usize) -> ExprRef { @@ -2417,7 +2193,7 @@ mod tests { #[test] fn test_right_side_position_globalizer_only_shifts_right_columns() -> Result<(), DatabaseError> { - let table_arena = crate::planner::TableArenaCell::default(); + let table_arena = TableArenaCell::default(); let mut arena = PlanArena::new(&table_arena); let left_column = arena.alloc_column(ColumnCatalog::new( "left".to_string(), @@ -2475,7 +2251,7 @@ mod tests { #[test] fn test_projection_output_binder_rewrites_to_project_slot() -> Result<(), DatabaseError> { - let table_arena = crate::planner::TableArenaCell::default(); + let table_arena = TableArenaCell::default(); let mut arena = PlanArena::new(&table_arena); let project_inner = test_column(&mut arena, "c1", 0); let project_output = arena.alloc_expression(ScalarExpression::Alias { @@ -2537,16 +2313,21 @@ mod tests { } #[test] - fn test_scalar_subquery_in_where_binds_as_inner_join() -> Result<(), DatabaseError> { + fn test_scalar_subquery_in_where_binds_as_init() -> Result<(), DatabaseError> { let table_states = build_t1_table()?; - let plan = table_states.plan("select * from t1 where c1 = (select max(c3) from t2)")?; - let Some((join_type, join_condition)) = find_join(&plan) else { - panic!("expected scalar subquery to introduce a join") + let mut arena = PlanArena::new(&table_states.table_arena); + let mut plan = table_states.plan_with_arena( + "select * from t1 where c1 = (select max(c3) from t2)", + &mut arena, + )?; + assert!(matches!(plan.operator, Operator::ScalarQueryInit(_))); + assert!(find_join(&plan).is_none()); + let Childrens::Twins { left, right } = plan.childrens.as_mut() else { + panic!("expected init children") }; - - assert_eq!(*join_type, JoinType::Inner); - assert!(matches!(join_condition, JoinCondition::On { .. })); - + assert_eq!(left.output_schema(&mut arena).len(), 2); + assert_eq!(right.output_schema(&mut arena).len(), 1); + assert_eq!(plan.output_schema(&mut arena).len(), 2); Ok(()) } @@ -2616,63 +2397,25 @@ mod tests { Ok(()) } - fn find_top_join(plan: &LogicalPlan) -> Option<&LogicalPlan> { - if matches!(plan.operator, Operator::Join(_)) { - return Some(plan); - } - - match plan.childrens.as_ref() { - Childrens::Only(child) => find_top_join(child), - Childrens::Twins { .. } | Childrens::None => None, - } - } - - fn collect_column_positions(expr: ExprRef, arena: &PlanArena, positions: &mut Vec) { - match arena.expression(expr.unpack_alias(arena)) { - ScalarExpression::ColumnRef { position, .. } => positions.push(*position), - ScalarExpression::Binary { - left_expr, - right_expr, - .. - } => { - collect_column_positions(*left_expr, arena, positions); - collect_column_positions(*right_expr, arena, positions); - } - _ => {} - } - } - #[test] - fn test_multiple_scalar_subqueries_in_where_rebind_positions() -> Result<(), DatabaseError> { + fn test_multiple_scalar_subqueries_have_independent_slots() -> Result<(), DatabaseError> { let table_states = build_t1_table()?; let mut arena = PlanArena::new(&table_states.table_arena); let plan = table_states.plan_with_arena( "select * from t1 where c1 <= (select 4) and c1 > (select 1)", &mut arena, )?; - let outer_join = - find_top_join(&plan).expect("expected scalar subqueries to introduce a join"); - let Operator::Join(op) = &outer_join.operator else { - panic!("expected join plan") + let Operator::ScalarQueryInit(first) = &plan.operator else { + panic!("expected scalar init") }; - let Childrens::Twins { left, .. } = outer_join.childrens.as_ref() else { - panic!("expected binary join") + let Childrens::Twins { left, .. } = plan.childrens.as_ref() else { + panic!("expected init children") }; - let JoinCondition::On { - filter: Some(filter), - .. - } = &op.on - else { - panic!("expected join filter") + let Operator::ScalarQueryInit(second) = &left.operator else { + panic!("expected second scalar init") }; - let mut left_plan = left.as_ref().clone(); - let left_len = left_plan.output_schema(&mut arena).len(); - - let mut positions = Vec::new(); - collect_column_positions(*filter, &arena, &mut positions); - - assert_eq!(positions, vec![0, left_len - 1, 0, left_len]); - + assert_ne!(first.id, second.id); + assert!(find_join(&plan).is_none()); Ok(()) } } diff --git a/src/catalog/column.rs b/src/catalog/column.rs index 04b20526..a3448b7c 100644 --- a/src/catalog/column.rs +++ b/src/catalog/column.rs @@ -183,7 +183,7 @@ impl ColumnCatalog { .map(|expr| { arena .expression(*expr) - .eval(arena, None) + .eval::<[DataValue]>(arena, None) .map(Cow::into_owned) }) .transpose() diff --git a/src/db.rs b/src/db.rs index cce05bb8..12fbb68a 100644 --- a/src/db.rs +++ b/src/db.rs @@ -42,7 +42,9 @@ use crate::optimizer::rule::normalization::NormalizationRuleImpl; #[cfg(feature = "orm")] use crate::orm::FromQueryRow; use crate::planner::operator::Operator; -use crate::planner::{LogicalPlan, MetaArena, PlanArena, PlanInput, PlanKeeper, TableArenaCell}; +use crate::planner::{ + ExecArenaView, LogicalPlan, MetaArena, PlanArena, PlanInput, PlanKeeper, TableArenaCell, +}; #[cfg(all(not(target_arch = "wasm32"), feature = "lmdb"))] use crate::storage::lmdb::{LmdbConfig, LmdbStorage}; use crate::storage::memory::MemoryStorage; @@ -582,7 +584,14 @@ impl State { transaction: &'a mut S::TransactionType<'txn>, plan: PlanInput<'a>, mut plan_arena: A, - ) -> Result<(Schema, A, Executor<'a, S::TransactionType<'txn>>), DatabaseError> + ) -> Result< + ( + Schema, + ExecArenaView, + Executor<'a, S::TransactionType<'txn>>, + ), + DatabaseError, + > where S: 'txn, { @@ -599,6 +608,7 @@ impl State { &self.table_functions, ); let root = build_write(&mut arena, &mut plan_arena, plan, read_context, transaction); + let plan_arena = ExecArenaView::new(plan_arena); let executor = Executor::new(arena, root, keeper); Ok((schema, plan_arena, executor)) @@ -612,7 +622,7 @@ impl State { ) -> Result< ( Schema, - PlanArena<'a>, + ExecArenaView>, Executor<'a, S::TransactionType<'txn>>, ), DatabaseError, @@ -675,6 +685,7 @@ impl State { table_functions, ); let root = build_write(&mut arena, &mut plan_arena, plan, cache, transaction); + let plan_arena = ExecArenaView::new(plan_arena); let executor = Executor::new(arena, root, keeper); Ok((schema, plan_arena, executor)) @@ -692,7 +703,7 @@ impl DDLApply { fn apply_to( self, state: &mut State, - plan_arena: &PlanArena, + plan_arena: &mut PlanArena, ) -> Result { let mut catalog_changed = false; match self { @@ -772,7 +783,7 @@ impl Database { return Err(err.with_sql_context(context)); } }; - let (plan_arena, apply) = + let (mut plan_arena, apply) = match TransactionIter::new(schema, plan_arena, executor, transaction) .done_with_ddl_apply() { @@ -789,7 +800,7 @@ impl Database { let mut catalog_changed = false; for apply in apply { - catalog_changed |= unsafe { apply.apply_to(&mut *state, &plan_arena) } + catalog_changed |= unsafe { apply.apply_to(&mut *state, &mut plan_arena) } .map_err(|err| err.with_sql_context(context))?; } if catalog_changed { @@ -1243,13 +1254,14 @@ impl<'a, T: Transaction + 'a, A: MetaArena + 'a> TransactionIter<'a, T, A> { } } -impl<'a, T: Transaction + 'a> TransactionIter<'a, T, PlanArena<'a>> { +impl<'a, T: Transaction + 'a> TransactionIter<'a, T, ExecArenaView>> { fn done_with_ddl_apply(mut self) -> Result<(PlanArena<'a>, Vec), DatabaseError> { while self.next_tuple(|_, _| ())?.is_some() {} Ok(( self.plan_arena .take() - .expect("DDL apply plan arena is unavailable after statement completion"), + .expect("DDL apply plan arena is unavailable after statement completion") + .into_parent(), std::mem::take(&mut self.ddl_apply), )) } @@ -1282,13 +1294,12 @@ impl ResultIter for TransactionIter<'_, T, A> { pub(crate) mod test { use crate::binder::{Binder, BinderContext}; use crate::catalog::{ColumnCatalog, ColumnDesc}; - #[cfg(feature = "unsafe_txdb_checkpoint")] - use crate::db::CatalogKind; - use crate::db::{DataBaseBuilder, DatabaseError, ResultIter}; + use crate::db::{CatalogKind, DataBaseBuilder, DatabaseError, ResultIter}; use crate::expression::ScalarExpression; use crate::planner::operator::join::JoinCondition; use crate::planner::operator::Operator; use crate::planner::PlanArena; + use crate::storage::memory::MemoryStorage; use crate::storage::{ table_codec::TableCodec, Storage, TableCache, Transaction, TransactionIsolationLevel, }; @@ -1319,6 +1330,47 @@ pub(crate) mod test { assert_send_sync::>(); } + #[test] + fn scalar_query_view_references_survive_catalog_reload() -> Result<(), DatabaseError> { + let storage = MemoryStorage::new(); + { + let mut db = DataBaseBuilder::path(".").build_with_storage(storage.clone())?; + db.ddl("create view scalar_a as select (select 11) as v")?; + db.ddl("create view scalar_b as select (select 22) as v")?; + db.ddl("create view scalar_nested as select (select (select 33)) as v")?; + } + + let mut db = DataBaseBuilder::path(".").build_with_storage(storage)?; + // Loading in a different order must preserve the saved initialization references. + for name in ["scalar_b", "scalar_a", "scalar_nested"] { + db.load(CatalogKind::View(name.into()))?; + } + let sql = "select a.v, b.v, c.v, (select 44) from scalar_a a \ + cross join scalar_b b cross join scalar_nested c"; + let mut iter = db.run(sql)?; + assert_eq!( + iter.next_tuple(|_, row| row.values.clone())?, + Some(vec![ + DataValue::Int32(11), + DataValue::Int32(22), + DataValue::Int32(33), + DataValue::Int32(44), + ]) + ); + assert!(iter.next_tuple(|_, _| ())?.is_none()); + iter.done()?; + + // New definitions must not reuse references registered by the loaded views. + db.ddl("create view scalar_new as select (select 55) as v")?; + let mut iter = db.run("select a.v, b.v from scalar_a a cross join scalar_new b")?; + assert_eq!( + iter.next_tuple(|_, row| row.values.clone())?, + Some(vec![DataValue::Int32(11), DataValue::Int32(55)]) + ); + iter.done()?; + Ok(()) + } + pub(crate) fn build_table( table_cache: &mut TableCache, transaction: &mut T, diff --git a/src/execution/ddl/create_index.rs b/src/execution/ddl/create_index.rs index 7705f1ec..89dd3446 100644 --- a/src/execution/ddl/create_index.rs +++ b/src/execution/ddl/create_index.rs @@ -24,6 +24,7 @@ use crate::planner::MetaArena; use crate::storage::Transaction; use crate::types::index::Index; use crate::types::tuple::Schema; +use crate::types::value::DataValue; use crate::types::ColumnId; pub struct CreateIndex<'a> { @@ -119,7 +120,7 @@ impl<'a, T: Transaction + 'a> ExecutorNode<'a, T> for CreateIndex<'a> { if arena.result_tuple().pk.is_none() { continue; } - arena.rewrite(&column_exprs, plan_arena, None)?; + arena.rewrite(&column_exprs, plan_arena, None::<&[DataValue]>)?; { let mut state = arena.local_state(plan_arena); let (tuple, transaction, table_codec) = state.tuple_transaction_codec_mut(); diff --git a/src/execution/ddl/drop_view.rs b/src/execution/ddl/drop_view.rs index 30785ad6..a2e75fc0 100644 --- a/src/execution/ddl/drop_view.rs +++ b/src/execution/ddl/drop_view.rs @@ -42,7 +42,7 @@ impl<'a, T: Transaction + 'a> ExecutorNode<'a, T> for DropView<'a> { fn next_tuple( &mut self, arena: &mut ExecArena<'a, T>, - _: &mut (dyn MetaArena + 'a), + _: &mut (dyn MetaArena + '_), ) -> Result<(), DatabaseError> { let DropViewOperator { view_name, diff --git a/src/execution/dql/aggregate/mod.rs b/src/execution/dql/aggregate/mod.rs index 96ebdf52..2735f699 100644 --- a/src/execution/dql/aggregate/mod.rs +++ b/src/execution/dql/aggregate/mod.rs @@ -87,10 +87,10 @@ pub(crate) fn create_accumulators( .try_collect() } -pub(crate) fn update_accumulators( +pub(crate) fn update_accumulators( accs: &mut [Box], agg_calls: &[ExprRef], - tuple: &dyn TupleLike, + tuple: &T, arena: &(dyn MetaArena + '_), ) -> Result<(), DatabaseError> { for (acc, expr) in accs.iter_mut().zip(agg_calls.iter()) { diff --git a/src/execution/dql/aggregate/stream_distinct.rs b/src/execution/dql/aggregate/stream_distinct.rs index 87aab16f..46c82bf4 100644 --- a/src/execution/dql/aggregate/stream_distinct.rs +++ b/src/execution/dql/aggregate/stream_distinct.rs @@ -22,6 +22,7 @@ use crate::planner::LogicalPlan; use crate::planner::MetaArena; use crate::storage::Transaction; use crate::types::tuple::Tuple; +use crate::types::value::DataValue; pub struct StreamDistinctExecutor<'a> { groupby_exprs: &'a [ExprRef], @@ -64,7 +65,7 @@ impl<'a, T: Transaction + 'a> ExecutorNode<'a, T> for StreamDistinctExecutor<'a> } return Ok(()); } - arena.rewrite(self.groupby_exprs, plan_arena, None)?; + arena.rewrite(self.groupby_exprs, plan_arena, None::<&[DataValue]>)?; if let Some(last_keys) = &mut self.last_keys { if last_keys.values == arena.result_tuple().values { diff --git a/src/execution/dql/dummy.rs b/src/execution/dql/dummy.rs index ae0860ca..66c7647c 100644 --- a/src/execution/dql/dummy.rs +++ b/src/execution/dql/dummy.rs @@ -49,7 +49,7 @@ impl<'a, T: Transaction + 'a> ExecutorNode<'a, T> for Dummy { fn next_tuple( &mut self, arena: &mut ExecArena<'a, T>, - _: &mut (dyn MetaArena + 'a), + _: &mut (dyn MetaArena + '_), ) -> Result<(), DatabaseError> { let Some(row) = self.row.take() else { arena.finish(); diff --git a/src/execution/dql/join/hash/mod.rs b/src/execution/dql/join/hash/mod.rs index 228cd36f..6f22e876 100644 --- a/src/execution/dql/join/hash/mod.rs +++ b/src/execution/dql/join/hash/mod.rs @@ -121,14 +121,14 @@ impl JoinProbeState for JoinProbeStateImpl { } } -pub(crate) fn filter( +pub(crate) fn filter( values: &T, filter_expr: &ExprRef, plan_arena: &(dyn MetaArena + '_), ) -> Result { match &*plan_arena .expression(*filter_expr) - .eval(plan_arena, Some(values as &dyn TupleLike))? + .eval(plan_arena, Some(values))? { DataValue::Boolean(false) | DataValue::Null => Ok(false), DataValue::Boolean(true) => Ok(true), diff --git a/src/execution/dql/mod.rs b/src/execution/dql/mod.rs index 03e8ac43..0bab43cd 100644 --- a/src/execution/dql/mod.rs +++ b/src/execution/dql/mod.rs @@ -26,7 +26,7 @@ pub(crate) mod limit; pub(crate) mod mark_apply; pub(crate) mod projection; pub(crate) mod recursive_cte; -pub(crate) mod scalar_apply; +pub(crate) mod scalar_query_init; pub(crate) mod scalar_subquery; pub(crate) mod seq_scan; pub(crate) mod set_membership; diff --git a/src/execution/dql/projection.rs b/src/execution/dql/projection.rs index 8a98d768..f663c86e 100644 --- a/src/execution/dql/projection.rs +++ b/src/execution/dql/projection.rs @@ -21,6 +21,7 @@ use crate::planner::ExprRef; use crate::planner::LogicalPlan; use crate::planner::MetaArena; use crate::storage::Transaction; +use crate::types::value::DataValue; pub struct Projection<'a> { exprs: &'a [ExprRef], @@ -53,7 +54,7 @@ impl<'a, T: Transaction + 'a> ExecutorNode<'a, T> for Projection<'a> { return Ok(()); } - arena.rewrite(self.exprs, plan_arena, None)?; + arena.rewrite(self.exprs, plan_arena, None::<&[DataValue]>)?; arena.resume(); Ok(()) } diff --git a/src/execution/dql/scalar_apply.rs b/src/execution/dql/scalar_apply.rs deleted file mode 100644 index b506d68f..00000000 --- a/src/execution/dql/scalar_apply.rs +++ /dev/null @@ -1,280 +0,0 @@ -// Copyright 2024 KipData/KiteSQL -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use crate::errors::DatabaseError; -use crate::execution::{ - build_read, ExecArena, ExecId, ExecNode, ExecutionContext, ExecutorNode, ReadExecutor, -}; -use crate::planner::operator::scalar_apply::ScalarApplyOperator; -use crate::planner::LogicalPlan; -use crate::planner::MetaArena; -use crate::storage::Transaction; -use crate::types::tuple::Tuple; - -pub struct ScalarApply { - left_input: ExecId, - right_input: ExecId, - cached_right: Option, -} - -impl<'a, T: Transaction + 'a> ReadExecutor<'a, T> for ScalarApply { - type Input = (&'a ScalarApplyOperator, &'a LogicalPlan, &'a LogicalPlan); - - fn into_executor( - (_, left_input, right_input): Self::Input, - arena: &mut ExecArena<'a, T>, - plan_arena: &mut (dyn MetaArena + 'a), - cache: ExecutionContext<'_>, - transaction: &T, - ) -> ExecId { - let left_input = build_read(arena, plan_arena, left_input, cache, transaction); - let right_input = build_read(arena, plan_arena, right_input, cache, transaction); - arena.push(ExecNode::ScalarApply(Self { - left_input, - right_input, - cached_right: None, - })) - } -} - -impl<'a, T: Transaction + 'a> ExecutorNode<'a, T> for ScalarApply { - fn next_tuple( - &mut self, - arena: &mut ExecArena<'a, T>, - plan_arena: &mut (dyn MetaArena + 'a), - ) -> Result<(), DatabaseError> { - let right_tuple = - Self::load_right_once(&mut self.cached_right, self.right_input, arena, plan_arena)?; - if !arena.next_tuple(self.left_input, plan_arena)? { - arena.finish(); - return Ok(()); - } - arena - .result_tuple_mut() - .values - .extend(right_tuple.values.iter().cloned()); - arena.resume(); - Ok(()) - } -} - -impl ScalarApply { - fn load_right_once<'a, 'c, T: Transaction + 'a>( - cached_right: &'c mut Option, - right_input: ExecId, - arena: &mut ExecArena<'a, T>, - plan_arena: &mut (dyn MetaArena + 'a), - ) -> Result<&'c Tuple, DatabaseError> { - match cached_right { - Some(tuple) => Ok(tuple), - None => { - if !arena.next_tuple(right_input, plan_arena)? { - return Err(DatabaseError::InvalidValue( - "scalar apply right input returned no rows".to_string(), - )); - } - let first = arena.materialize_tuple(); - if arena.next_tuple(right_input, plan_arena)? { - return Err(DatabaseError::InvalidValue( - "scalar apply right input returned more than one row".to_string(), - )); - } - Ok(cached_right.insert(first)) - } - } - } -} - -#[cfg(all(test, not(target_arch = "wasm32")))] -mod tests { - use super::*; - use crate::catalog::{ColumnCatalog, ColumnDesc}; - use crate::execution::{execute_input, try_collect}; - use crate::planner::operator::scalar_subquery::ScalarSubqueryOperator; - use crate::planner::operator::values::ValuesOperator; - use crate::planner::operator::Operator; - use crate::planner::test::PlanArenaTestExt; - use crate::planner::{Childrens, LogicalPlan}; - use crate::storage::rocksdb::RocksStorage; - use crate::storage::{StatisticsMetaCache, Storage, TableCache, ViewCache}; - use crate::types::value::DataValue; - use crate::types::LogicalType; - use tempfile::TempDir; - - fn build_values( - arena: &mut crate::planner::PlanArena, - name: &str, - rows: Vec>, - ) -> LogicalPlan { - let desc = ColumnDesc::new(LogicalType::Integer, None, false, None).unwrap(); - let schema_ref = vec![arena.alloc_column(ColumnCatalog::new(name.to_string(), true, desc))]; - - LogicalPlan::new( - Operator::Values(ValuesOperator::new( - arena.alloc_expression_rows(&rows), - rows.len(), - schema_ref, - )), - Childrens::None, - ) - } - - fn build_test_storage() -> Result< - ( - TableCache, - ViewCache, - StatisticsMetaCache, - TempDir, - RocksStorage, - ), - DatabaseError, - > { - let meta_cache = crate::storage::StatisticsMetaCache::default(); - let view_cache = crate::storage::ViewCache::default(); - let table_cache = crate::storage::TableCache::default(); - - let temp_dir = TempDir::new().expect("unable to create temporary working directory"); - let storage = RocksStorage::new(temp_dir.path())?; - - Ok((table_cache, view_cache, meta_cache, temp_dir, storage)) - } - - #[test] - fn scalar_apply_repeats_scalar_result_for_each_left_row() -> Result<(), DatabaseError> { - let table_arena = crate::planner::TableArenaCell::default(); - let mut plan_arena = crate::planner::PlanArena::new(&table_arena); - let mut left = build_values( - &mut plan_arena, - "left_c1", - vec![ - vec![crate::types::value::DataValue::Int32(1)], - vec![crate::types::value::DataValue::Int32(2)], - ], - ); - let mut right = ScalarSubqueryOperator::build(build_values( - &mut plan_arena, - "right_c1", - vec![vec![crate::types::value::DataValue::Int32(7)]], - )); - - let (table_cache, view_cache, meta_cache, _temp_dir, storage) = build_test_storage()?; - let transaction = storage.transaction()?; - left.populate_output_schema_recursive(&mut plan_arena); - right.populate_output_schema_recursive(&mut plan_arena); - let tuples = try_collect(execute_input::<_, ScalarApply>( - (&ScalarApplyOperator, &left, &right), - crate::execution::empty_context(&table_cache, &view_cache, &meta_cache), - plan_arena, - &transaction, - ))?; - - let actual = tuples - .into_iter() - .flat_map(|tuple| tuple.values) - .collect::>(); - assert_eq!( - actual, - vec![ - DataValue::Int32(1), - DataValue::Int32(7), - DataValue::Int32(2), - DataValue::Int32(7), - ] - ); - - Ok(()) - } - - #[test] - fn scalar_subquery_checks_cardinality_on_second_call() -> Result<(), DatabaseError> { - for rows in [ - vec![], - vec![vec![DataValue::Int32(7)]], - vec![vec![DataValue::Int32(7)], vec![DataValue::Int32(8)]], - ] { - let row_count = rows.len(); - let table_arena = crate::planner::TableArenaCell::default(); - let mut plan_arena = crate::planner::PlanArena::new(&table_arena); - let mut input = build_values(&mut plan_arena, "right_c1", rows); - input.populate_output_schema_recursive(&mut plan_arena); - let (table_cache, view_cache, meta_cache, _temp_dir, storage) = build_test_storage()?; - let transaction = storage.transaction()?; - let mut executor = - execute_input::<_, crate::execution::dql::scalar_subquery::ScalarSubquery>( - (&ScalarSubqueryOperator, &input), - crate::execution::empty_context(&table_cache, &view_cache, &meta_cache), - plan_arena, - &transaction, - ); - let expected = if row_count == 0 { - DataValue::Null - } else { - DataValue::Int32(7) - }; - assert_eq!(executor.next_tuple()?.unwrap().values, vec![expected]); - if row_count > 1 { - assert!( - matches!(executor.next_tuple(), Err(DatabaseError::InvalidValue(message)) - if message == "scalar subquery returned more than one row") - ); - } else { - assert!(executor.next_tuple()?.is_none()); - assert!(executor.next_tuple()?.is_none()); - } - } - Ok(()) - } - - #[test] - fn scalar_apply_repeats_null_scalar_result_for_each_left_row() -> Result<(), DatabaseError> { - let table_arena = crate::planner::TableArenaCell::default(); - let mut plan_arena = crate::planner::PlanArena::new(&table_arena); - let mut left = build_values( - &mut plan_arena, - "left_c1", - vec![vec![DataValue::Int32(1)], vec![DataValue::Int32(2)]], - ); - let mut right = ScalarSubqueryOperator::build(build_values( - &mut plan_arena, - "right_c1", - vec![vec![DataValue::Null]], - )); - - let (table_cache, view_cache, meta_cache, _temp_dir, storage) = build_test_storage()?; - let transaction = storage.transaction()?; - left.populate_output_schema_recursive(&mut plan_arena); - right.populate_output_schema_recursive(&mut plan_arena); - let tuples = try_collect(execute_input::<_, ScalarApply>( - (&ScalarApplyOperator, &left, &right), - crate::execution::empty_context(&table_cache, &view_cache, &meta_cache), - plan_arena, - &transaction, - ))?; - - assert_eq!( - tuples - .into_iter() - .flat_map(|tuple| tuple.values) - .collect::>(), - vec![ - DataValue::Int32(1), - DataValue::Null, - DataValue::Int32(2), - DataValue::Null, - ] - ); - - Ok(()) - } -} diff --git a/src/execution/dql/scalar_query_init.rs b/src/execution/dql/scalar_query_init.rs new file mode 100644 index 00000000..17e13e8f --- /dev/null +++ b/src/execution/dql/scalar_query_init.rs @@ -0,0 +1,158 @@ +// Copyright 2024 KipData/KiteSQL +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use crate::errors::DatabaseError; +use crate::execution::{ + build_read, ExecArena, ExecId, ExecNode, ExecutionContext, ExecutorNode, ReadExecutor, +}; +use crate::planner::operator::scalar_query_init::ScalarQueryInitOperator; +use crate::planner::{LogicalPlan, MetaArena, ScalarQueryRef}; +use crate::storage::Transaction; +use crate::types::value::DataValue; + +pub struct ScalarQueryInit { + input: ExecId, + init: ExecId, + reference: ScalarQueryRef, +} + +impl ScalarQueryInit { + pub(crate) fn new(input: ExecId, init: ExecId, reference: ScalarQueryRef) -> Self { + Self { + input, + init, + reference, + } + } +} + +impl<'a, T: Transaction + 'a> ReadExecutor<'a, T> for ScalarQueryInit { + type Input = ( + &'a ScalarQueryInitOperator, + &'a LogicalPlan, + &'a LogicalPlan, + ); + + fn into_executor( + (op, input, init): Self::Input, + arena: &mut ExecArena<'a, T>, + plan_arena: &mut (dyn MetaArena + 'a), + cache: ExecutionContext<'_>, + transaction: &T, + ) -> ExecId { + let input = build_read(arena, plan_arena, input, cache, transaction); + let init = build_read(arena, plan_arena, init, cache, transaction); + arena.push(ExecNode::ScalarQueryInit(Self::new(input, init, op.id))) + } +} + +impl<'a, T: Transaction + 'a> ExecutorNode<'a, T> for ScalarQueryInit { + fn next_tuple( + &mut self, + arena: &mut ExecArena<'a, T>, + plan_arena: &mut (dyn MetaArena + 'a), + ) -> Result<(), DatabaseError> { + if plan_arena.init_value(self.reference).is_none() { + let value = if arena.next_tuple(self.init, plan_arena)? { + arena.result_tuple().values[0].clone() + } else { + DataValue::Null + }; + // Drain the scalar wrapper to validate cardinality before publishing its value. + if arena.next_tuple(self.init, plan_arena)? { + return Err(DatabaseError::InvalidValue( + "scalar subquery returned more than one row".into(), + )); + } + plan_arena.set_init_value(self.reference, value); + } + arena.next_tuple(self.input, plan_arena)?; + Ok(()) + } +} + +#[cfg(all(test, not(target_arch = "wasm32")))] +mod tests { + use super::*; + use crate::catalog::{ColumnCatalog, ColumnDesc}; + use crate::execution::empty_context; + use crate::expression::ScalarExpression; + use crate::planner::operator::scalar_subquery::ScalarSubqueryOperator; + use crate::planner::operator::values::ValuesOperator; + use crate::planner::operator::Operator; + use crate::planner::{Childrens, ExecArenaView, PlanArena, TableArenaCell}; + use crate::storage::memory::MemoryStorage; + use crate::storage::{StatisticsMetaCache, Storage, TableCache, ViewCache}; + use crate::types::LogicalType; + + #[test] + fn scalar_init_shared_slot_caches_null_and_checks_cardinality() -> Result<(), DatabaseError> { + let storage = MemoryStorage::new(); + let transaction = storage.transaction()?; + let tables = TableCache::default(); + let views = ViewCache::default(); + let stats = StatisticsMetaCache::default(); + let cache = empty_context(&tables, &views, &stats); + for (value, row_count) in [ + (DataValue::Null, 1), + (DataValue::Int32(7), 1), + (DataValue::Int32(7), 2), + ] { + let table_arena = TableArenaCell::default(); + let mut metadata = PlanArena::new(&table_arena); + let reference = metadata.alloc_scalar_query_ref(); + let column = metadata.alloc_column(ColumnCatalog::new( + "v".into(), + true, + ColumnDesc::new(LogicalType::Integer, None, false, None)?, + )); + let expr = metadata.alloc_expression(ScalarExpression::Constant(value.clone())); + let make_init = |rows| { + ScalarSubqueryOperator::build(LogicalPlan::new( + Operator::Values(ValuesOperator::new(vec![expr; rows], rows, vec![column])), + Childrens::None, + )) + }; + let mut first = ScalarQueryInitOperator::build( + LogicalPlan::new(Operator::Dummy, Childrens::None), + make_init(row_count), + reference, + ); + // This duplicate would error if executed, making cache reuse observable even for NULL. + let mut duplicate = ScalarQueryInitOperator::build( + LogicalPlan::new(Operator::Dummy, Childrens::None), + make_init(2), + reference, + ); + first.populate_output_schema_recursive(&mut metadata); + duplicate.populate_output_schema_recursive(&mut metadata); + let mut arena = ExecArena::with_capacity(0); + arena.init_context(cache, &transaction); + let first_root = build_read(&mut arena, &mut metadata, &first, cache, &transaction); + let duplicate_root = + build_read(&mut arena, &mut metadata, &duplicate, cache, &transaction); + let mut view = ExecArenaView::new(metadata); + if row_count == 2 { + assert!(arena.next_tuple(first_root, &mut view).is_err()); + assert_eq!(view.init_value(reference), None); + } else { + assert!(arena.next_tuple(first_root, &mut view)?); + assert_eq!(view.init_value(reference), Some(&value)); + assert!(arena.next_tuple(duplicate_root, &mut view)?); + assert!(!arena.next_tuple(duplicate_root, &mut view)?); + } + } + Ok(()) + } +} diff --git a/src/execution/dql/show_table.rs b/src/execution/dql/show_table.rs index 9d0a2b60..3c7fbab2 100644 --- a/src/execution/dql/show_table.rs +++ b/src/execution/dql/show_table.rs @@ -29,7 +29,7 @@ impl<'a, T: Transaction + 'a> ReadExecutor<'a, T> for ShowTables<'a, T> { fn into_executor( _: Self::Input, arena: &mut ExecArena<'a, T>, - _: &mut (dyn MetaArena + 'a), + _: &mut (dyn MetaArena + '_), _: ExecutionContext<'_>, _: &T, ) -> ExecId { diff --git a/src/execution/dql/values.rs b/src/execution/dql/values.rs index d76df905..b872d7a1 100644 --- a/src/execution/dql/values.rs +++ b/src/execution/dql/values.rs @@ -19,6 +19,7 @@ use crate::planner::ExprRef; use crate::planner::MetaArena; use crate::storage::Transaction; use crate::types::tuple::Schema; +use crate::types::value::DataValue; pub struct Values<'a> { rows: std::slice::Iter<'a, ExprRef>, @@ -78,7 +79,7 @@ impl<'a, T: Transaction + 'a> ExecutorNode<'a, T> for Values<'a> { output.values.push( plan_arena .expression(*expr) - .eval(plan_arena, None)? + .eval::<[DataValue]>(plan_arena, None)? .into_owned() .cast(ty)?, ); diff --git a/src/execution/mod.rs b/src/execution/mod.rs index f08780cd..18210827 100644 --- a/src/execution/mod.rs +++ b/src/execution/mod.rs @@ -27,7 +27,7 @@ use self::ddl::add_column::AddColumn; use self::ddl::change_column::ChangeColumn; use self::dql::join::nested_loop_join::NestedLoopJoin; use self::dql::mark_apply::MarkApply; -use self::dql::scalar_apply::ScalarApply; +use self::dql::scalar_query_init::ScalarQueryInit; use crate::db::{ScalaFunctions, TableFunctions}; use crate::errors::DatabaseError; use crate::execution::ddl::create_index::CreateIndex; @@ -228,7 +228,7 @@ pub(crate) enum ExecNode<'a, T: Transaction + 'a> { Projection(Projection<'a>), RecursiveCte(RecursiveCte<'a, T>), RecursiveScan(RecursiveScan), - ScalarApply(ScalarApply), + ScalarQueryInit(ScalarQueryInit), ScalarSubquery(ScalarSubquery), SetMembership(SetMembership), SeqScan(SeqScan<'a, T>), @@ -351,8 +351,8 @@ impl<'a, T: Transaction + 'a> ExecNode<'a, T> { ExecNode::RecursiveScan(exec) => { >::next_tuple(exec, arena, plan_arena) } - ExecNode::ScalarApply(exec) => { - >::next_tuple(exec, arena, plan_arena) + ExecNode::ScalarQueryInit(exec) => { + >::next_tuple(exec, arena, plan_arena) } ExecNode::ScalarSubquery(exec) => { >::next_tuple(exec, arena, plan_arena) @@ -638,11 +638,11 @@ impl<'a, T: Transaction + 'a> ExecArena<'a, T> { std::mem::take(&mut self.result.tuple) } - pub(crate) fn rewrite( + pub(crate) fn rewrite( &mut self, exprs: &[E], arena: &dyn MetaArena, - input: Option<&dyn TupleLike>, + input: Option<&TInput>, ) -> Result<(), DatabaseError> { let values = &mut self.result.tuple.values; let base = values.len(); @@ -651,10 +651,12 @@ impl<'a, T: Transaction + 'a> ExecArena<'a, T> { for expr in exprs { let value = { let input_values = &values[..base]; - let current: &dyn TupleLike = input.unwrap_or(&input_values); - expr.expression(arena) - .eval(arena, Some(current)) - .map(|value| value.into_owned()) + let expression = expr.expression(arena); + match input { + Some(input) => expression.eval(arena, Some(input)), + None => expression.eval(arena, Some(&input_values)), + } + .map(|value| value.into_owned()) }; match value { Ok(value) => values.push(value), @@ -789,9 +791,9 @@ where } } Operator::Filter(op) => read!(Filter, (op, plan.childrens.only())), - Operator::ScalarApply(op) => { + Operator::ScalarQueryInit(op) => { let (left, right) = plan.childrens.twins(); - read!(ScalarApply, (op, left, right)) + read!(ScalarQueryInit, (op, left, right)) } Operator::MarkApply(op) => { let (left, right) = plan.childrens.twins(); @@ -869,7 +871,7 @@ pub(crate) fn build_write<'a, T>( plan_arena: &mut (dyn MetaArena + 'a), plan: &'a LogicalPlan, cache: ExecutionContext<'a>, - transaction: &'a mut T, + transaction: &'a T, ) -> ExecId where T: Transaction + 'a, @@ -889,6 +891,14 @@ where } match &plan.operator { + Operator::ScalarQueryInit(op) => { + let (input, init) = plan.childrens.twins(); + let init = build_read(arena, plan_arena, init, cache, transaction); + let input = build_write(arena, plan_arena, input, cache, transaction); + arena.push(ExecNode::ScalarQueryInit(ScalarQueryInit::new( + input, init, op.id, + ))) + } Operator::Insert(op) => write!(Insert<'a>, (op, plan.childrens.only())), Operator::Update(op) => write!(Update<'a>, (op, plan.childrens.only())), Operator::Delete(op) => write!(Delete<'a>, (op, plan.childrens.only())), @@ -920,6 +930,7 @@ where #[cfg(all(test, not(target_arch = "wasm32")))] mod test_utils { use super::*; + use crate::planner::ExecArenaView; static EMPTY_SCALA_FUNCTIONS: std::sync::LazyLock = std::sync::LazyLock::new(ScalaFunctions::default); @@ -942,7 +953,7 @@ mod test_utils { pub(crate) struct TestExecutor<'a, T: Transaction + 'a> { executor: Executor<'a, T>, - plan_arena: PlanArena<'a>, + plan_arena: ExecArenaView>, } impl TestExecutor<'_, T> { @@ -971,8 +982,8 @@ mod test_utils { transaction, ); TestExecutor { + plan_arena: ExecArenaView::new(plan_arena), executor: Executor::new(arena, root, PlanKeeper::empty()), - plan_arena, } } @@ -997,8 +1008,8 @@ mod test_utils { transaction, ); TestExecutor { + plan_arena: ExecArenaView::new(plan_arena), executor: Executor::new(arena, root, PlanKeeper::empty()), - plan_arena, } } @@ -1022,13 +1033,14 @@ pub(crate) use test_utils::{empty_context, execute_input, execute_input_mut, try #[cfg(test)] mod test { use super::*; + use crate::planner::TableArenaCell; use crate::storage::memory::MemoryTransaction; use std::panic::{catch_unwind, AssertUnwindSafe}; #[test] fn active_nodes_cannot_be_overwritten_or_relocated() { - let table_arena = crate::planner::TableArenaCell::default(); - let mut plan_arena = crate::planner::PlanArena::new(&table_arena); + let table_arena = TableArenaCell::default(); + let mut plan_arena = PlanArena::new(&table_arena); let mut arena = ExecArena::<'_, MemoryTransaction>::with_capacity(0); arena.push(ExecNode::Dummy(Dummy::default())); let slot = diff --git a/src/expression/eq_col.rs b/src/expression/eq_col.rs index 73817db9..e1707421 100644 --- a/src/expression/eq_col.rs +++ b/src/expression/eq_col.rs @@ -22,8 +22,8 @@ use crate::expression::function::table::TableFunction; use crate::expression::visitor::{walk_expr, ExprVisitor}; use crate::expression::window::WindowCall; use crate::expression::{BinaryOperator, ScalarExpression, TrimWhereField, UnaryOperator}; -use crate::planner::ExprRef; use crate::planner::MetaArena; +use crate::planner::{ExprRef, ScalarQueryRef}; use crate::types::evaluator::{BinaryEvaluatorRef, CastEvaluatorRef, UnaryEvaluatorRef}; use crate::types::value::DataValue; use crate::types::LogicalType; @@ -83,6 +83,22 @@ impl ExprVisitor for EqIgnoreColRefPosVisitor<'_, '_> { walk_expr(self, lhs, arena) } + fn visit_init( + &mut self, + id: ScalarQueryRef, + ty: &LogicalType, + _arena: &(dyn MetaArena + '_), + ) -> Result<(), DatabaseError> { + self.equal = match self.rhs() { + ScalarExpression::Init { + id: rhs_id, + ty: rhs_ty, + } => id == *rhs_id && ty == rhs_ty, + _ => false, + }; + Ok(()) + } + fn visit_constant(&mut self, lhs: &DataValue) -> Result<(), DatabaseError> { self.equal = matches!(self.rhs(), ScalarExpression::Constant(rhs) if lhs == rhs); Ok(()) diff --git a/src/expression/evaluator.rs b/src/expression/evaluator.rs index b0be5fbd..e34ca5c8 100644 --- a/src/expression/evaluator.rs +++ b/src/expression/evaluator.rs @@ -41,10 +41,10 @@ macro_rules! eval_to_num { } impl ScalarExpression { - pub fn eval<'a>( + pub fn eval<'a, T: TupleLike + ?Sized>( &'a self, arena: &'a (dyn MetaArena + '_), - tuple: Option<&'a dyn TupleLike>, + tuple: Option<&'a T>, ) -> Result, DatabaseError> { match self { ScalarExpression::Constant(val) => match val { @@ -53,6 +53,12 @@ impl ScalarExpression { } val => Ok(Cow::Borrowed(val)), }, + ScalarExpression::Init { id, .. } => { + let value = arena.init_value(*id).ok_or_else(|| { + DatabaseError::InvalidValue(format!("scalar query {id} is not initialized")) + })?; + Ok(Cow::Borrowed(value)) + } ScalarExpression::ColumnRef { position, .. } => { let Some(tuple) = tuple else { return Ok(Cow::Owned(DataValue::Null)); @@ -262,7 +268,7 @@ impl ScalarExpression { } ScalarExpression::ScalaFunction(ScalarFunction { inner, args, .. }) => { let value = match tuple { - Some(tuple) => inner.eval(args, arena, Some(tuple))?, + Some(tuple) => inner.eval(args, arena, Some(&tuple))?, None => inner.eval(args, arena, None)?, }; value.cast(inner.return_type()).map(Cow::Owned) @@ -401,7 +407,7 @@ fn trim_string(value: &str, trim_what: &str, trim_where: Option) mod tests { use super::*; use crate::planner::test::PlanArenaTestExt; - use crate::planner::ExprRef; + use crate::planner::{ExecArenaView, ExprRef, TableArenaCell}; fn const_in( arena: &mut PlanArena<'_>, @@ -422,12 +428,45 @@ mod tests { }) } + #[test] + fn init_reads_execution_view_and_preserves_slot_position() -> Result<(), DatabaseError> { + use crate::expression::visitor_mut::{ExprVisitorMut, PositionShift}; + use crate::types::tuple::Tuple; + + let table_arena = TableArenaCell::default(); + let mut arena = PlanArena::new(&table_arena); + let reference = arena.alloc_scalar_query_ref(); + let mut init = arena.alloc_expression(ScalarExpression::Init { + id: reference, + ty: LogicalType::Integer, + }); + PositionShift { delta: 5 }.visit(&mut init, &mut arena)?; + assert!(matches!( + arena.expression(init), + ScalarExpression::Init { id, ty: LogicalType::Integer } if *id == reference + )); + assert!(!init.any_referenced_column(&arena, |_, _| true)?); + let row = Tuple::new(None, vec![DataValue::Int32(99)]); + assert!(matches!( + arena.expression(init).eval(&arena, Some(&row)), + Err(DatabaseError::InvalidValue(message)) if message.contains("is not initialized") + )); + let mut view = ExecArenaView::new(arena); + assert!(view.expression(init).eval(&view, Some(&row)).is_err()); + view.set_init_value(reference, DataValue::Null); + let value = view.expression(init).eval(&view, Some(&row))?; + assert!(matches!(value, Cow::Borrowed(DataValue::Null))); + let value = view.expression(init).eval::(&view, None)?; + assert!(matches!(value, Cow::Borrowed(DataValue::Null))); + Ok(()) + } + #[test] fn eval_borrows_leaf_values_and_owns_binary_results() -> Result<(), DatabaseError> { use crate::types::evaluator::binary_create; use crate::types::tuple::Tuple; - let table_arena = crate::planner::TableArenaCell::default(); + let table_arena = TableArenaCell::default(); let mut arena = PlanArena::new(&table_arena); let value = DataValue::Int64(42); let constant = arena.alloc_expression(ScalarExpression::Constant(value.clone())); @@ -472,7 +511,7 @@ mod tests { use crate::types::evaluator::cast_create; use crate::types::tuple::Tuple; - let table_arena = crate::planner::TableArenaCell::default(); + let table_arena = TableArenaCell::default(); let mut arena = PlanArena::new(&table_arena); let value = DataValue::Int32(7); let column_ref = arena.alloc_column(crate::catalog::ColumnCatalog::new( @@ -532,7 +571,7 @@ mod tests { #[test] fn in_eval_matches_even_if_null_appears_first() -> Result<(), DatabaseError> { - let table_arena = crate::planner::TableArenaCell::default(); + let table_arena = TableArenaCell::default(); let mut arena = PlanArena::new(&table_arena); let expr = const_in( &mut arena, @@ -542,7 +581,10 @@ mod tests { ); assert_eq!( - arena.expression(expr).eval(&arena, None)?.into_owned(), + arena + .expression(expr) + .eval::(&arena, None)? + .into_owned(), DataValue::Boolean(true) ); Ok(()) @@ -550,7 +592,7 @@ mod tests { #[test] fn in_eval_returns_null_when_only_null_blocks_non_match() -> Result<(), DatabaseError> { - let table_arena = crate::planner::TableArenaCell::default(); + let table_arena = TableArenaCell::default(); let mut arena = PlanArena::new(&table_arena); let expr = const_in( &mut arena, @@ -560,7 +602,10 @@ mod tests { ); assert_eq!( - arena.expression(expr).eval(&arena, None)?.into_owned(), + arena + .expression(expr) + .eval::(&arena, None)? + .into_owned(), DataValue::Null ); Ok(()) @@ -568,7 +613,7 @@ mod tests { #[test] fn not_in_eval_matches_even_if_null_appears_first() -> Result<(), DatabaseError> { - let table_arena = crate::planner::TableArenaCell::default(); + let table_arena = TableArenaCell::default(); let mut arena = PlanArena::new(&table_arena); let expr = const_in( &mut arena, @@ -578,7 +623,10 @@ mod tests { ); assert_eq!( - arena.expression(expr).eval(&arena, None)?.into_owned(), + arena + .expression(expr) + .eval::(&arena, None)? + .into_owned(), DataValue::Boolean(false) ); Ok(()) diff --git a/src/expression/mod.rs b/src/expression/mod.rs index f65f8ad5..08b1129f 100644 --- a/src/expression/mod.rs +++ b/src/expression/mod.rs @@ -20,7 +20,7 @@ use crate::expression::function::table::TableFunction; use crate::expression::visitor::{walk_expr, ExprVisitor}; use crate::expression::visitor_mut::ExprVisitorMut; use crate::planner::operator::sort::SortField; -use crate::planner::{Explain, ExprRef, MetaArena}; +use crate::planner::{Explain, ExprRef, MetaArena, ScalarQueryRef}; use crate::types::evaluator::{ binary_create, cast_create, unary_create, BinaryEvaluatorRef, CastEvaluatorRef, UnaryEvaluatorRef, @@ -66,6 +66,10 @@ pub enum AliasType { #[derive(Debug, PartialEq, Eq, Clone, Hash, ReferenceSerialization)] pub enum ScalarExpression { Constant(DataValue), + Init { + id: ScalarQueryRef, + ty: LogicalType, + }, ColumnRef { column: ColumnRef, position: usize, @@ -481,6 +485,7 @@ impl TypeCast for ScalarExpression { } ScalarExpression::Position { .. } => Cow::Owned(LogicalType::Integer), ScalarExpression::Alias { expr, .. } => expr.return_type(arena), + ScalarExpression::Init { ty, .. } => Cow::Borrowed(ty), ScalarExpression::Empty | ScalarExpression::TableFunction(_) => unreachable!(), ScalarExpression::Tuple(exprs) => Cow::Owned(LogicalType::Tuple( exprs @@ -551,6 +556,7 @@ impl Explain for ExprRef { match arena.expression(*self) { ScalarExpression::Constant(value) => write!(f, "{value}"), + ScalarExpression::Init { id, .. } => write!(f, "Init({id})"), ScalarExpression::ColumnRef { column, .. } => Explain::fmt(column, arena, f), ScalarExpression::Alias { alias, expr } => match alias { AliasType::Name(alias) => f.write_str(alias), @@ -1084,6 +1090,7 @@ mod test { use crate::expression::{AliasType, BinaryOperator, ScalarExpression, UnaryOperator}; use crate::function::current_date::CurrentDate; use crate::function::numbers::Numbers; + use crate::planner::operator::sort::SortField; use crate::planner::{ExprRef, MetaArena, PlanArena, TableArenaCell}; use crate::serdes::{ReferenceDecodeContext, ReferenceSerialization, ReferenceTables}; use crate::storage::rocksdb::RocksStorage; @@ -1544,7 +1551,7 @@ mod test { }, spec: WindowSpec { partition_by: vec![two], - order_by: vec![crate::planner::operator::sort::SortField::from(three).desc()], + order_by: vec![SortField::from(three).desc()], }, }), Some(&context), diff --git a/src/expression/range_detacher.rs b/src/expression/range_detacher.rs index 2829d0fa..d132b71c 100644 --- a/src/expression/range_detacher.rs +++ b/src/expression/range_detacher.rs @@ -530,7 +530,8 @@ impl<'a, M: RangeColumnMatcher, A: MetaArena + ?Sized> RangeDetacher<'a, M, A> { None } - ScalarExpression::Constant(_) + ScalarExpression::Init { .. } + | ScalarExpression::Constant(_) | ScalarExpression::Alias { .. } | ScalarExpression::TypeCast { .. } | ScalarExpression::IsNull { .. } @@ -553,7 +554,9 @@ impl<'a, M: RangeColumnMatcher, A: MetaArena + ?Sized> RangeDetacher<'a, M, A> { | ScalarExpression::TableFunction(_) | ScalarExpression::Empty => unreachable!(), }, - ScalarExpression::Constant(_) | ScalarExpression::ColumnRef { .. } => None, + ScalarExpression::Init { .. } + | ScalarExpression::Constant(_) + | ScalarExpression::ColumnRef { .. } => None, // FIXME: support [RangeDetacher::_detach] ScalarExpression::Unary { .. } | ScalarExpression::In { .. } diff --git a/src/expression/visitor.rs b/src/expression/visitor.rs index 3f4f586f..9e269d3e 100644 --- a/src/expression/visitor.rs +++ b/src/expression/visitor.rs @@ -21,7 +21,7 @@ use crate::expression::window::WindowCall; use crate::expression::{ AliasType, BinaryOperator, ScalarExpression, TrimWhereField, UnaryOperator, }; -use crate::planner::{ExprRef, MetaArena}; +use crate::planner::{ExprRef, MetaArena, ScalarQueryRef}; use crate::types::evaluator::{BinaryEvaluatorRef, CastEvaluatorRef, UnaryEvaluatorRef}; use crate::types::value::DataValue; use crate::types::LogicalType; @@ -49,6 +49,15 @@ pub trait ExprVisitor: Sized { Ok(true) } + fn visit_init( + &mut self, + _id: ScalarQueryRef, + _ty: &LogicalType, + _arena: &A, + ) -> Result<(), DatabaseError> { + Ok(()) + } + fn visit_constant(&mut self, _value: &DataValue) -> Result<(), DatabaseError> { Ok(()) } @@ -294,6 +303,7 @@ pub fn walk_expr>( ) -> Result<(), DatabaseError> { match arena.expression(expr) { ScalarExpression::Constant(value) => visitor.visit_constant(value), + ScalarExpression::Init { id, ty } => visitor.visit_init(*id, ty, arena), ScalarExpression::ColumnRef { column, .. } => visitor.visit_column_ref(column), ScalarExpression::Alias { expr, alias } => visitor.visit_alias(*expr, alias, arena), ScalarExpression::TypeCast { diff --git a/src/expression/visitor_mut.rs b/src/expression/visitor_mut.rs index 439d9e4a..2903058f 100644 --- a/src/expression/visitor_mut.rs +++ b/src/expression/visitor_mut.rs @@ -21,8 +21,8 @@ use crate::expression::window::WindowCall; use crate::expression::{ AliasType, BinaryOperator, ScalarExpression, TrimWhereField, UnaryOperator, }; -use crate::planner::ExprRef; use crate::planner::MetaArena; +use crate::planner::{ExprRef, ScalarQueryRef}; use crate::types::evaluator::{BinaryEvaluatorRef, CastEvaluatorRef, UnaryEvaluatorRef}; use crate::types::value::DataValue; use crate::types::LogicalType; @@ -61,6 +61,15 @@ impl ExprVisitorMut for PositionShift { } pub trait ExprVisitorMut: Sized { + fn visit_init( + &mut self, + _id: &mut ScalarQueryRef, + _ty: &mut LogicalType, + _arena: &mut dyn MetaArena, + ) -> Result<(), DatabaseError> { + Ok(()) + } + fn visit( &mut self, expr: &mut ExprRef, @@ -395,6 +404,7 @@ pub fn walk_mut_expr( std::mem::replace(&mut *arena.expression_mut(*expr), ScalarExpression::Empty); let result = match &mut expression { ScalarExpression::Constant(value) => visitor.visit_constant(value, arena), + ScalarExpression::Init { id, ty } => visitor.visit_init(id, ty, arena), ScalarExpression::ColumnRef { column, position } => { visitor.visit_column_ref(column, position, arena) } diff --git a/src/function/numbers.rs b/src/function/numbers.rs index f5c844e8..6613ad45 100644 --- a/src/function/numbers.rs +++ b/src/function/numbers.rs @@ -50,7 +50,10 @@ impl TableFunctionImpl for Numbers { args: &[ExprRef], arena: &(dyn MetaArena + '_), ) -> Result>>, DatabaseError> { - let mut value = arena.expression(args[0]).eval(arena, None)?.into_owned(); + let mut value = arena + .expression(args[0]) + .eval::<[DataValue]>(arena, None)? + .into_owned(); value = value.cast(&LogicalType::Integer)?; let num = value diff --git a/src/macros/mod.rs b/src/macros/mod.rs index 9db99307..18a87a1d 100644 --- a/src/macros/mod.rs +++ b/src/macros/mod.rs @@ -179,7 +179,7 @@ macro_rules! table_function { let mut _index = 0; $closure($({ - let mut value = arena.expression(args[_index]).eval(arena, None)?.into_owned(); + let mut value = arena.expression(args[_index]).eval::<[::kite_sql::types::value::DataValue]>(arena, None)?.into_owned(); _index += 1; value = value.cast(&$arg_ty)?; diff --git a/src/optimizer/heuristic/optimizer.rs b/src/optimizer/heuristic/optimizer.rs index a95778e9..b0b455c9 100644 --- a/src/optimizer/heuristic/optimizer.rs +++ b/src/optimizer/heuristic/optimizer.rs @@ -512,9 +512,14 @@ impl ImplementationRuleIndex { Operator::Project(_) if self.contains(ImplementationRuleImpl::Projection) => { Some(PhysicalOption::new(PlanImpl::Project, SortOption::Follow)) } - Operator::ScalarApply(_) if self.contains(ImplementationRuleImpl::ScalarApply) => Some( - PhysicalOption::new(PlanImpl::ScalarApply, SortOption::Follow), - ), + Operator::ScalarQueryInit(_) + if self.contains(ImplementationRuleImpl::ScalarQueryInit) => + { + Some(PhysicalOption::new( + PlanImpl::ScalarQueryInit, + SortOption::Follow, + )) + } Operator::ScalarSubquery(_) if self.contains(ImplementationRuleImpl::ScalarSubquery) => { diff --git a/src/optimizer/rule/implementation/dql/mod.rs b/src/optimizer/rule/implementation/dql/mod.rs index 9258b248..1d96ea47 100644 --- a/src/optimizer/rule/implementation/dql/mod.rs +++ b/src/optimizer/rule/implementation/dql/mod.rs @@ -20,7 +20,7 @@ pub(crate) mod join; pub(crate) mod limit; pub(crate) mod mark_apply; pub(crate) mod projection; -pub(crate) mod scalar_apply; +pub(crate) mod scalar_query_init; pub(crate) mod scalar_subquery; pub(crate) mod sort; pub(crate) mod table_scan; diff --git a/src/optimizer/rule/implementation/dql/scalar_apply.rs b/src/optimizer/rule/implementation/dql/scalar_query_init.rs similarity index 84% rename from src/optimizer/rule/implementation/dql/scalar_apply.rs rename to src/optimizer/rule/implementation/dql/scalar_query_init.rs index 55151508..9be962ba 100644 --- a/src/optimizer/rule/implementation/dql/scalar_apply.rs +++ b/src/optimizer/rule/implementation/dql/scalar_query_init.rs @@ -22,14 +22,14 @@ use crate::single_mapping; use std::sync::LazyLock; static SCALAR_APPLY_PATTERN: LazyLock = LazyLock::new(|| Pattern { - predicate: |op| matches!(op, Operator::ScalarApply(_)), + predicate: |op| matches!(op, Operator::ScalarQueryInit(_)), }); #[derive(Clone)] -pub struct ScalarApplyImplementation; +pub struct ScalarQueryInitImplementation; single_mapping!( - ScalarApplyImplementation, + ScalarQueryInitImplementation, SCALAR_APPLY_PATTERN, - PhysicalOption::new(PlanImpl::ScalarApply, SortOption::Follow) + PhysicalOption::new(PlanImpl::ScalarQueryInit, SortOption::Follow) ); diff --git a/src/optimizer/rule/implementation/mod.rs b/src/optimizer/rule/implementation/mod.rs index 6f9fc067..f31ae4cc 100644 --- a/src/optimizer/rule/implementation/mod.rs +++ b/src/optimizer/rule/implementation/mod.rs @@ -45,7 +45,7 @@ use crate::optimizer::rule::implementation::dql::join::JoinImplementation; use crate::optimizer::rule::implementation::dql::limit::LimitImplementation; use crate::optimizer::rule::implementation::dql::mark_apply::MarkApplyImplementation; use crate::optimizer::rule::implementation::dql::projection::ProjectionImplementation; -use crate::optimizer::rule::implementation::dql::scalar_apply::ScalarApplyImplementation; +use crate::optimizer::rule::implementation::dql::scalar_query_init::ScalarQueryInitImplementation; use crate::optimizer::rule::implementation::dql::scalar_subquery::ScalarSubqueryImplementation; use crate::optimizer::rule::implementation::dql::sort::SortImplementation; use crate::optimizer::rule::implementation::dql::table_scan::{ @@ -66,7 +66,7 @@ pub enum ImplementationRuleRootTag { Limit, MarkApply, Project, - ScalarApply, + ScalarQueryInit, ScalarSubquery, TableScan, FunctionScan, @@ -102,7 +102,7 @@ impl ImplementationRuleRootTag { Operator::Limit(_) => Some(Self::Limit), Operator::MarkApply(_) => Some(Self::MarkApply), Operator::Project(_) => Some(Self::Project), - Operator::ScalarApply(_) => Some(Self::ScalarApply), + Operator::ScalarQueryInit(_) => Some(Self::ScalarQueryInit), Operator::ScalarSubquery(_) => Some(Self::ScalarSubquery), Operator::TableScan(_) => Some(Self::TableScan), Operator::FunctionScan(_) => Some(Self::FunctionScan), @@ -150,7 +150,7 @@ pub enum ImplementationRuleImpl { Limit, MarkApply, Projection, - ScalarApply, + ScalarQueryInit, ScalarSubquery, SeqScan, FunctionScan, @@ -188,7 +188,7 @@ impl MatchPattern for ImplementationRuleImpl { ImplementationRuleImpl::Limit => LimitImplementation.pattern(), ImplementationRuleImpl::MarkApply => MarkApplyImplementation.pattern(), ImplementationRuleImpl::Projection => ProjectionImplementation.pattern(), - ImplementationRuleImpl::ScalarApply => ScalarApplyImplementation.pattern(), + ImplementationRuleImpl::ScalarQueryInit => ScalarQueryInitImplementation.pattern(), ImplementationRuleImpl::ScalarSubquery => ScalarSubqueryImplementation.pattern(), ImplementationRuleImpl::SeqScan => SeqScanImplementation.pattern(), ImplementationRuleImpl::IndexScan => IndexScanImplementation.pattern(), @@ -227,7 +227,7 @@ impl ImplementationRuleImpl { ImplementationRuleImpl::Limit => ImplementationRuleRootTag::Limit, ImplementationRuleImpl::MarkApply => ImplementationRuleRootTag::MarkApply, ImplementationRuleImpl::Projection => ImplementationRuleRootTag::Project, - ImplementationRuleImpl::ScalarApply => ImplementationRuleRootTag::ScalarApply, + ImplementationRuleImpl::ScalarQueryInit => ImplementationRuleRootTag::ScalarQueryInit, ImplementationRuleImpl::ScalarSubquery => ImplementationRuleRootTag::ScalarSubquery, ImplementationRuleImpl::SeqScan | ImplementationRuleImpl::IndexScan => { ImplementationRuleRootTag::TableScan @@ -278,7 +278,7 @@ impl ImplementationRule for ImplementationRuleImpl { ImplementationRuleImpl::Limit => update!(LimitImplementation), ImplementationRuleImpl::MarkApply => update!(MarkApplyImplementation), ImplementationRuleImpl::Projection => update!(ProjectionImplementation), - ImplementationRuleImpl::ScalarApply => update!(ScalarApplyImplementation), + ImplementationRuleImpl::ScalarQueryInit => update!(ScalarQueryInitImplementation), ImplementationRuleImpl::ScalarSubquery => update!(ScalarSubqueryImplementation), ImplementationRuleImpl::SeqScan => update!(SeqScanImplementation), ImplementationRuleImpl::IndexScan => update!(IndexScanImplementation), @@ -443,9 +443,9 @@ mod tests { )?; assert_sql_rule( "select (select c3 from t2 limit 1) from t1", - ImplementationRuleImpl::ScalarApply, - |op| matches!(op, Operator::ScalarApply(_)), - PlanImpl::ScalarApply, + ImplementationRuleImpl::ScalarQueryInit, + |op| matches!(op, Operator::ScalarQueryInit(_)), + PlanImpl::ScalarQueryInit, )?; assert_sql_rule( "select (select c3 from t2 limit 1) from t1", diff --git a/src/optimizer/rule/normalization/column_pruning.rs b/src/optimizer/rule/normalization/column_pruning.rs index a17a1007..82bf63be 100644 --- a/src/optimizer/rule/normalization/column_pruning.rs +++ b/src/optimizer/rule/normalization/column_pruning.rs @@ -518,7 +518,7 @@ impl ColumnPruning { } Operator::Sort(_) | Operator::Limit(_) - | Operator::ScalarApply(_) + | Operator::ScalarQueryInit(_) | Operator::MarkApply(_) | Operator::ScalarSubquery(_) | Operator::Join(_) @@ -527,11 +527,30 @@ impl ColumnPruning { | Operator::SetMembership(_) | Operator::TopK(_) | Operator::Window(_) => { - if matches!(operator, Operator::ScalarApply(_) | Operator::MarkApply(_)) { - let mut child_required = required_columns; - Self::extend_operator_referenced_columns(operator, &mut child_required, arena)?; + if matches!(operator, Operator::ScalarQueryInit(_)) { + let Childrens::Twins { left, right } = childrens else { + unreachable!("scalar initialization requires two children"); + }; + Self::_apply_appending(required_columns, all_referenced, left, outcome, arena)?; + changed |= outcome.changed; + let main_removed_end = outcome.removed_positions.len(); + Self::_apply_appending( + ReferencedColumns::default(), + true, + right, + outcome, + arena, + )?; + changed |= outcome.changed; + outcome.removed_positions.truncate(main_removed_end); + } else if matches!(operator, Operator::MarkApply(_)) { + Self::extend_operator_referenced_columns( + operator, + &mut required_columns, + arena, + )?; changed |= Self::apply_twins( - child_required, + required_columns, true, childrens, outcome, @@ -541,10 +560,9 @@ impl ColumnPruning { outcome.removed_positions.truncate(output_start); } else if matches!(operator, Operator::Join(_)) { let (old_left_outputs_len, left_removed_start, right_removed_start) = { - let mut child_required = required_columns; Self::extend_operator_referenced_columns( operator, - &mut child_required, + &mut required_columns, arena, )?; let old_left_outputs_len = match childrens { @@ -559,7 +577,7 @@ impl ColumnPruning { let left_removed_start = outcome.removed_positions.len(); Self::_apply_appending( - child_required.clone(), + required_columns.clone(), all_referenced, left.as_mut(), outcome, @@ -568,7 +586,7 @@ impl ColumnPruning { let left_changed = outcome.changed; let right_removed_start = outcome.removed_positions.len(); Self::_apply_appending( - child_required, + required_columns, all_referenced, right.as_mut(), outcome, @@ -657,10 +675,13 @@ impl ColumnPruning { outcome.removed_positions.truncate(output_start); } } else if matches!(operator, Operator::Union(_) | Operator::SetMembership(_)) { - let mut child_required = required_columns; - Self::extend_operator_referenced_columns(operator, &mut child_required, arena)?; + Self::extend_operator_referenced_columns( + operator, + &mut required_columns, + arena, + )?; changed |= Self::apply_twins( - child_required, + required_columns, all_referenced, childrens, outcome, @@ -671,14 +692,13 @@ impl ColumnPruning { } else { let child_start = outcome.removed_positions.len(); let child_changed = { - let mut child_required = required_columns; Self::extend_operator_referenced_columns( operator, - &mut child_required, + &mut required_columns, arena, )?; Self::apply_only_child( - child_required, + required_columns, all_referenced, childrens, outcome, diff --git a/src/optimizer/rule/normalization/compilation_in_advance.rs b/src/optimizer/rule/normalization/compilation_in_advance.rs index 7423f969..1fe28b46 100644 --- a/src/optimizer/rule/normalization/compilation_in_advance.rs +++ b/src/optimizer/rule/normalization/compilation_in_advance.rs @@ -38,7 +38,7 @@ impl EvaluatorBind { Self::_apply(left, arena)?; let bind_right = matches!( plan.operator, - Operator::ScalarApply(_) + Operator::ScalarQueryInit(_) | Operator::MarkApply(_) | Operator::Join(_) | Operator::Union(_) diff --git a/src/optimizer/rule/normalization/mod.rs b/src/optimizer/rule/normalization/mod.rs index f6ff5ad8..87eea173 100644 --- a/src/optimizer/rule/normalization/mod.rs +++ b/src/optimizer/rule/normalization/mod.rs @@ -108,7 +108,7 @@ impl NormalizationRuleRootTag { match operator { Operator::Aggregate(_) => Some(Self::Aggregate), Operator::MarkApply(_) => Some(Self::MarkApply), - Operator::ScalarApply(_) => Some(Self::Any), + Operator::ScalarQueryInit(_) => Some(Self::Any), Operator::Filter(_) => Some(Self::Filter), Operator::Join(_) => Some(Self::Join), Operator::Limit(_) => Some(Self::Limit), diff --git a/src/planner/arena.rs b/src/planner/arena.rs index 4c801b85..59d8f899 100644 --- a/src/planner/arena.rs +++ b/src/planner/arena.rs @@ -14,20 +14,80 @@ use crate::catalog::{ColumnCatalog, ColumnRef, TableName}; use crate::errors::DatabaseError; +use crate::expression::visitor_mut::ExprVisitorMut; use crate::expression::ScalarExpression; -use crate::planner::LogicalPlan; +use crate::planner::operator::scalar_query_init::ScalarQueryInitOperator; +use crate::planner::operator::visitor_mut::{ + walk_mut_operator, OperatorExprVisitorMut, OperatorVisitorMut, +}; +use crate::planner::operator::{Operator, PhysicalOption}; +use crate::planner::{LogicalPlan, ScalarQueryRef}; use crate::types::index::{IndexMeta, IndexMetaRef}; use crate::types::tuple::Schema; use crate::types::value::DataValue; +use crate::types::LogicalType; use std::cell::UnsafeCell; -use std::collections::HashSet; +use std::collections::{HashMap, HashSet}; use std::fmt; +struct ScalarQueryRelocator { + source_arena: usize, + target_arena: usize, + base: usize, +} + +impl ScalarQueryRelocator { + fn relocate(&self, reference: &mut ScalarQueryRef) { + if reference.arena_id == self.source_arena { + reference.arena_id = self.target_arena; + reference.pos += self.base; + } + } +} + +impl ExprVisitorMut for ScalarQueryRelocator { + fn visit_init( + &mut self, + reference: &mut ScalarQueryRef, + _ty: &mut LogicalType, + _arena: &mut dyn MetaArena, + ) -> Result<(), DatabaseError> { + self.relocate(reference); + Ok(()) + } +} + +struct ScalarQueryPlanRelocator<'a> { + references: ScalarQueryRelocator, + arena: &'a mut dyn MetaArena, +} + +impl<'a> OperatorVisitorMut<'a> for ScalarQueryPlanRelocator<'_> { + fn visit_operator( + &mut self, + operator: &'a mut Operator, + physical_option: Option<&'a mut PhysicalOption>, + ) -> Result<(), DatabaseError> { + OperatorExprVisitorMut::new(&mut self.references, self.arena) + .visit_operator(operator, physical_option)?; + walk_mut_operator(self, operator) + } + + fn visit_scalar_query_init( + &mut self, + op: &'a mut ScalarQueryInitOperator, + ) -> Result<(), DatabaseError> { + self.references.relocate(&mut op.id); + Ok(()) + } +} + pub struct TableArena { dummy_columns: [ColumnCatalog; DUMMY_COLUMN_COUNT], columns: Vec, indexes: Vec, expressions: Vec, + scalar_query_count: usize, version: usize, } @@ -62,6 +122,8 @@ pub struct PlanArena<'a> { table_arena_version: usize, allocated_columns_len: usize, temp_table_id: usize, + arena_id: usize, + scalar_query_count: usize, columns: Vec, indexes: Vec, expressions: Vec, @@ -153,6 +215,20 @@ pub trait MetaArena { None } + fn arena_id(&self) -> usize; + + fn alloc_scalar_query_ref(&mut self) -> ScalarQueryRef; + + fn reserve_scalar_query_ref(&mut self, reference: ScalarQueryRef); + + fn init_value(&self, _reference: ScalarQueryRef) -> Option<&DataValue> { + None + } + + fn set_init_value(&mut self, _reference: ScalarQueryRef, _value: DataValue) { + panic!("scalar query initialization requires an execution arena view") + } + fn alloc_dummy(&mut self, name: &str) -> ColumnRef { self.table_arena_cell().borrow().alloc_dummy(name) } @@ -206,6 +282,21 @@ impl MetaArena for Box { fn bound_param(&self, id: usize) -> Option<&DataValue> { (**self).bound_param(id) } + fn arena_id(&self) -> usize { + (**self).arena_id() + } + fn alloc_scalar_query_ref(&mut self) -> ScalarQueryRef { + (**self).alloc_scalar_query_ref() + } + fn reserve_scalar_query_ref(&mut self, reference: ScalarQueryRef) { + (**self).reserve_scalar_query_ref(reference) + } + fn init_value(&self, reference: ScalarQueryRef) -> Option<&DataValue> { + (**self).init_value(reference) + } + fn set_init_value(&mut self, reference: ScalarQueryRef, value: DataValue) { + (**self).set_init_value(reference, value) + } fn alloc_column(&mut self, column: ColumnCatalog) -> ColumnRef { (**self).alloc_column(column) } @@ -235,6 +326,82 @@ impl MetaArena for Box { } } +/// Owns the metadata and scalar-query cache for one statement execution. +pub(crate) struct ExecArenaView { + parent: A, + init_values: HashMap>, +} + +impl ExecArenaView { + pub(crate) fn new(parent: A) -> Self { + Self { + parent, + init_values: HashMap::new(), + } + } + + pub(crate) fn into_parent(self) -> A { + self.parent + } +} + +impl MetaArena for ExecArenaView { + fn arena_id(&self) -> usize { + self.parent.arena_id() + } + fn alloc_scalar_query_ref(&mut self) -> ScalarQueryRef { + self.parent.alloc_scalar_query_ref() + } + fn reserve_scalar_query_ref(&mut self, reference: ScalarQueryRef) { + self.parent.reserve_scalar_query_ref(reference) + } + fn init_value(&self, reference: ScalarQueryRef) -> Option<&DataValue> { + self.init_values.get(&reference).and_then(Option::as_ref) + } + fn table_arena_cell<'a>(&self) -> &'a TableArenaCell + where + Self: 'a, + { + self.parent.table_arena_cell() + } + fn has_bound_params(&self) -> bool { + self.parent.has_bound_params() + } + fn bound_param(&self, id: usize) -> Option<&DataValue> { + self.parent.bound_param(id) + } + fn set_init_value(&mut self, reference: ScalarQueryRef, value: DataValue) { + self.init_values.insert(reference, Some(value)); + } + fn expression_mut(&mut self, expr: ExprRef) -> ArenaExprMut<'_> { + self.parent.expression_mut(expr) + } + fn alloc_column(&mut self, column: ColumnCatalog) -> ColumnRef { + self.parent.alloc_column(column) + } + fn alloc_index(&mut self, index: IndexMeta) -> IndexMetaRef { + self.parent.alloc_index(index) + } + fn alloc_expression(&mut self, expression: ScalarExpression) -> ExprRef { + self.parent.alloc_expression(expression) + } + fn column(&self, column: ColumnRef) -> &ColumnCatalog { + self.parent.column(column) + } + fn index(&self, index: IndexMetaRef) -> &IndexMeta { + self.parent.index(index) + } + fn expression(&self, expr: ExprRef) -> &ScalarExpression { + self.parent.expression(expr) + } + fn find_column(&self, column: &ColumnCatalog) -> Option { + self.parent.find_column(column) + } + fn find_index(&self, index: &IndexMeta) -> Option { + self.parent.find_index(index) + } +} + const DUMMY_COLUMN_NAMES: [&str; DUMMY_COLUMN_COUNT] = [ "TABLE", "VIEW", @@ -297,6 +464,7 @@ impl Default for TableArena { columns: Vec::new(), indexes: Vec::new(), expressions: Vec::new(), + scalar_query_count: 0, version: 0, } } @@ -396,6 +564,21 @@ impl fmt::Debug for TableArena { } impl MetaArena for TableArena { + fn arena_id(&self) -> usize { + 0 + } + fn alloc_scalar_query_ref(&mut self) -> ScalarQueryRef { + let reference = ScalarQueryRef { + arena_id: 0, + pos: self.scalar_query_count, + }; + self.scalar_query_count += 1; + reference + } + fn reserve_scalar_query_ref(&mut self, reference: ScalarQueryRef) { + assert_eq!(reference.arena_id, 0); + self.scalar_query_count = self.scalar_query_count.max(reference.pos + 1); + } fn alloc_column(&mut self, column: ColumnCatalog) -> ColumnRef { if let Some(column_ref) = self.find_column(&column) { return column_ref; @@ -526,6 +709,8 @@ impl<'a> PlanArena<'a> { table_arena_version, allocated_columns_len: 0, temp_table_id: 0, + arena_id: table_arena.borrow().arena_id() + 1, + scalar_query_count: 0, columns: Vec::new(), indexes: Vec::new(), expressions: Vec::new(), @@ -589,6 +774,26 @@ impl<'a> PlanArena<'a> { table_arena.increment_version(); } + /// Moves scalar definitions with a cached plan into the persistent namespace. + pub(crate) fn materialize_scalar_queries( + &mut self, + plan: &mut LogicalPlan, + ) -> Result<(), DatabaseError> { + self.assert_table_arena_unchanged(); + let target = self.table_arena.borrow_mut(); + let references = ScalarQueryRelocator { + source_arena: self.arena_id, + target_arena: target.arena_id(), + base: target.scalar_query_count, + }; + target.scalar_query_count += self.scalar_query_count; + ScalarQueryPlanRelocator { + references, + arena: self, + } + .visit_plan(plan) + } + pub(crate) fn materialize_into_table_arena(&self) { self.assert_table_arena_unchanged(); @@ -715,6 +920,25 @@ impl<'a> PlanArena<'a> { } impl MetaArena for PlanArena<'_> { + fn arena_id(&self) -> usize { + self.arena_id + } + fn alloc_scalar_query_ref(&mut self) -> ScalarQueryRef { + let reference = ScalarQueryRef { + arena_id: self.arena_id, + pos: self.scalar_query_count, + }; + self.scalar_query_count += 1; + reference + } + fn reserve_scalar_query_ref(&mut self, reference: ScalarQueryRef) { + if reference.arena_id == self.arena_id { + self.scalar_query_count = self.scalar_query_count.max(reference.pos + 1); + } else { + assert_eq!(reference.arena_id, 0); + assert!(reference.pos < self.table_arena.borrow().scalar_query_count); + } + } fn table_arena_cell<'a>(&self) -> &'a TableArenaCell where Self: 'a, @@ -820,6 +1044,7 @@ pub(crate) struct ParamArena<'a> { expressions: Vec<(Option, ArenaExpr)>, parameter_count: usize, parent_end: usize, + scalar_query_count: usize, } impl<'a> ParamArena<'a> { @@ -850,11 +1075,28 @@ impl<'a> ParamArena<'a> { expressions, parameter_count: parameter_expressions.len(), parent_end: parent.expression_end(), + scalar_query_count: 0, }) } } impl MetaArena for ParamArena<'_> { + fn arena_id(&self) -> usize { + self.parent.arena_id() + 1 + } + fn alloc_scalar_query_ref(&mut self) -> ScalarQueryRef { + let reference = ScalarQueryRef { + arena_id: self.arena_id(), + pos: self.scalar_query_count, + }; + self.scalar_query_count += 1; + reference + } + fn reserve_scalar_query_ref(&mut self, reference: ScalarQueryRef) { + if reference.arena_id == self.arena_id() { + self.scalar_query_count = self.scalar_query_count.max(reference.pos + 1); + } + } fn table_arena_cell<'a>(&self) -> &'a TableArenaCell where Self: 'a, @@ -931,6 +1173,51 @@ mod tests { use crate::types::LogicalType; use std::panic::{catch_unwind, AssertUnwindSafe}; + #[test] + fn scalar_query_namespaces_follow_arena_parent() -> Result<(), DatabaseError> { + let root = TableArenaCell::default(); + let catalog = root.borrow_mut().alloc_scalar_query_ref(); + assert_eq!( + catalog, + ScalarQueryRef { + arena_id: 0, + pos: 0 + } + ); + let mut plan = PlanArena::new(&root); + let local = plan.alloc_scalar_query_ref(); + assert_eq!( + local, + ScalarQueryRef { + arena_id: 1, + pos: 0 + } + ); + assert_ne!(catalog, local); + let mut params = ParamArena::new(&plan, &[], &[])?; + assert_eq!(params.arena_id(), 2); + let param_local = params.alloc_scalar_query_ref(); + assert_eq!( + param_local, + ScalarQueryRef { + arena_id: 2, + pos: 0 + } + ); + let mut view = ExecArenaView::new(params); + view.set_init_value(catalog, DataValue::Int32(11)); + assert_eq!(view.arena_id(), 2); + assert_eq!(view.init_value(catalog), Some(&DataValue::Int32(11))); + assert_eq!(view.init_value(local), None); + assert_eq!(view.init_value(param_local), None); + view.set_init_value(local, DataValue::Null); + assert_eq!(view.init_value(local), Some(&DataValue::Null)); + view.set_init_value(local, DataValue::Int32(33)); + assert_eq!(view.init_value(catalog), Some(&DataValue::Int32(11))); + assert_eq!(view.init_value(local), Some(&DataValue::Int32(33))); + Ok(()) + } + fn column(name: &str) -> ColumnCatalog { ColumnCatalog::new( name.to_string(), diff --git a/src/planner/mod.rs b/src/planner/mod.rs index d1ed25d2..090b6328 100644 --- a/src/planner/mod.rs +++ b/src/planner/mod.rs @@ -15,6 +15,7 @@ mod arena; pub mod operator; mod plan_keeper; +mod scalar_query_ref; use crate::catalog::TableName; use crate::errors::DatabaseError; @@ -29,9 +30,10 @@ use kite_sql_serde_macros::ReferenceSerialization; use std::fmt; use std::hash::{Hash, Hasher}; +pub(crate) use arena::{ExecArenaView, ParamArena, PlanRef}; pub use arena::{ExprRef, MetaArena, PlanArena, TableArena, TableArenaCell}; -pub(crate) use arena::{ParamArena, PlanRef}; pub(crate) use plan_keeper::{PlanInput, PlanKeeper}; +pub use scalar_query_ref::ScalarQueryRef; pub(crate) trait Explain { fn fmt(&self, arena: &(dyn MetaArena + '_), f: &mut fmt::Formatter<'_>) -> fmt::Result; @@ -253,7 +255,11 @@ impl LogicalPlan { } _ => unreachable!(), }, - Operator::ScalarApply(_) | Operator::Join(_) => match childrens { + Operator::ScalarQueryInit(_) => match childrens { + Childrens::Twins { left, .. } => left.output_schema(arena).clone(), + _ => unreachable!(), + }, + Operator::Join(_) => match childrens { Childrens::Twins { left, right } => { let mut schema = left.output_schema(arena).clone(); schema.extend_from_slice(right.output_schema(arena)); diff --git a/src/planner/operator/mod.rs b/src/planner/operator/mod.rs index beefe902..fa129e89 100644 --- a/src/planner/operator/mod.rs +++ b/src/planner/operator/mod.rs @@ -37,7 +37,7 @@ pub mod limit; pub mod mark_apply; pub mod project; pub mod recursive_cte; -pub mod scalar_apply; +pub mod scalar_query_init; pub mod scalar_subquery; pub mod set_membership; pub mod sort; @@ -56,8 +56,8 @@ use self::{ aggregate::AggregateOperator, alter_table::add_column::AddColumnOperator, alter_table::change_column::ChangeColumnOperator, filter::FilterOperator, join::JoinOperator, limit::LimitOperator, mark_apply::MarkApplyOperator, project::ProjectOperator, - scalar_apply::ScalarApplyOperator, scalar_subquery::ScalarSubqueryOperator, sort::SortOperator, - table_scan::TableScanOperator, + scalar_query_init::ScalarQueryInitOperator, scalar_subquery::ScalarSubqueryOperator, + sort::SortOperator, table_scan::TableScanOperator, }; use crate::catalog::ColumnRef; use crate::errors::DatabaseError; @@ -97,7 +97,7 @@ pub enum Operator { // DQL Dummy, Aggregate(AggregateOperator), - ScalarApply(ScalarApplyOperator), + ScalarQueryInit(ScalarQueryInitOperator), MarkApply(MarkApplyOperator), Filter(FilterOperator), Join(JoinOperator), @@ -177,7 +177,7 @@ pub enum PlanImpl { HashAggregate, StreamAggregate, StreamDistinct, - ScalarApply, + ScalarQueryInit, MarkApply, Filter, HashJoin, @@ -248,7 +248,7 @@ macro_rules! impl_display_explain { } impl_display_explain!( - ScalarApplyOperator, + ScalarQueryInitOperator, MarkApplyOperator, ScalarSubqueryOperator, FunctionScanOperator, @@ -277,7 +277,7 @@ impl Explain for Operator { match self { Operator::Dummy => f.write_str("Dummy"), Operator::Aggregate(op) => Explain::fmt(op, arena, f), - Operator::ScalarApply(op) => Explain::fmt(op, arena, f), + Operator::ScalarQueryInit(op) => Explain::fmt(op, arena, f), Operator::MarkApply(op) => Explain::fmt(op, arena, f), Operator::Filter(op) => Explain::fmt(op, arena, f), Operator::Join(op) => Explain::fmt(op, arena, f), @@ -595,7 +595,7 @@ impl Explain for PlanImpl { PlanImpl::HashAggregate => f.write_str("HashAggregate"), PlanImpl::StreamAggregate => f.write_str("StreamAggregate"), PlanImpl::StreamDistinct => f.write_str("StreamDistinct"), - PlanImpl::ScalarApply => f.write_str("ScalarApply"), + PlanImpl::ScalarQueryInit => f.write_str("ScalarQueryInit"), PlanImpl::MarkApply => f.write_str("MarkApply"), PlanImpl::Filter => f.write_str("Filter"), PlanImpl::HashJoin => f.write_str("HashJoin"), @@ -767,7 +767,7 @@ mod tests { (PlanImpl::HashAggregate, "HashAggregate"), (PlanImpl::StreamAggregate, "StreamAggregate"), (PlanImpl::StreamDistinct, "StreamDistinct"), - (PlanImpl::ScalarApply, "ScalarApply"), + (PlanImpl::ScalarQueryInit, "ScalarQueryInit"), (PlanImpl::MarkApply, "MarkApply"), (PlanImpl::Filter, "Filter"), (PlanImpl::HashJoin, "HashJoin"), @@ -1003,7 +1003,9 @@ mod tests { assert_eq!(referenced_columns(&delete, &mut arena)?, vec![a]); let no_reference_operators = [ - Operator::ScalarApply(ScalarApplyOperator), + Operator::ScalarQueryInit(ScalarQueryInitOperator { + id: arena.alloc_scalar_query_ref(), + }), Operator::ScalarSubquery(ScalarSubqueryOperator), Operator::Analyze(AnalyzeOperator { table_name: "users".into(), @@ -1382,14 +1384,18 @@ mod tests { } #[test] - fn scalar_apply_and_subquery_build_expected_child_shapes() { + fn scalar_query_init_and_subquery_build_expected_child_shapes() { let table_arena = TableArenaCell::default(); - let arena = PlanArena::new(&table_arena); + let mut arena = PlanArena::new(&table_arena); let left = LogicalPlan::new(Operator::ShowTable, Childrens::None); let right = LogicalPlan::new(Operator::ShowView, Childrens::None); - let apply = ScalarApplyOperator::build(left.clone(), right); - assert_eq!(apply.operator.explain(&arena).to_string(), "ScalarApply"); + let apply = + ScalarQueryInitOperator::build(left.clone(), right, arena.alloc_scalar_query_ref()); + assert_eq!( + apply.operator.explain(&arena).to_string(), + "ScalarQueryInit #1:0" + ); assert!(matches!(*apply.childrens, Childrens::Twins { .. })); let subquery = ScalarSubqueryOperator::build(left); diff --git a/src/planner/operator/scalar_apply.rs b/src/planner/operator/scalar_query_init.rs similarity index 70% rename from src/planner/operator/scalar_apply.rs rename to src/planner/operator/scalar_query_init.rs index 99df8d84..0f4bf462 100644 --- a/src/planner/operator/scalar_apply.rs +++ b/src/planner/operator/scalar_query_init.rs @@ -13,18 +13,20 @@ // limitations under the License. use super::Operator; -use crate::planner::{Childrens, LogicalPlan}; +use crate::planner::{Childrens, LogicalPlan, ScalarQueryRef}; use kite_sql_serde_macros::ReferenceSerialization; use std::fmt; use std::fmt::Formatter; #[derive(Debug, PartialEq, Eq, Clone, Hash, ReferenceSerialization)] -pub struct ScalarApplyOperator; +pub struct ScalarQueryInitOperator { + pub id: ScalarQueryRef, +} -impl ScalarApplyOperator { - pub fn build(left: LogicalPlan, right: LogicalPlan) -> LogicalPlan { +impl ScalarQueryInitOperator { + pub fn build(left: LogicalPlan, right: LogicalPlan, id: ScalarQueryRef) -> LogicalPlan { LogicalPlan::new( - Operator::ScalarApply(ScalarApplyOperator), + Operator::ScalarQueryInit(ScalarQueryInitOperator { id }), Childrens::Twins { left: Box::new(left), right: Box::new(right), @@ -33,8 +35,8 @@ impl ScalarApplyOperator { } } -impl fmt::Display for ScalarApplyOperator { +impl fmt::Display for ScalarQueryInitOperator { fn fmt(&self, f: &mut Formatter) -> fmt::Result { - write!(f, "ScalarApply") + write!(f, "ScalarQueryInit #{}", self.id) } } diff --git a/src/planner/operator/visitor.rs b/src/planner/operator/visitor.rs index b1512982..0cf604bc 100644 --- a/src/planner/operator/visitor.rs +++ b/src/planner/operator/visitor.rs @@ -31,7 +31,10 @@ pub trait OperatorVisitor<'a>: Sized { Ok(()) } - fn visit_scalar_apply(&mut self, _op: &'a ScalarApplyOperator) -> Result<(), DatabaseError> { + fn visit_scalar_query_init( + &mut self, + _op: &'a ScalarQueryInitOperator, + ) -> Result<(), DatabaseError> { Ok(()) } @@ -338,7 +341,7 @@ pub fn walk_operator<'a, V: OperatorVisitor<'a>>( match operator { Operator::Dummy => visitor.visit_dummy(), Operator::Aggregate(op) => visitor.visit_aggregate(op), - Operator::ScalarApply(op) => visitor.visit_scalar_apply(op), + Operator::ScalarQueryInit(op) => visitor.visit_scalar_query_init(op), Operator::MarkApply(op) => visitor.visit_mark_apply(op), Operator::Filter(op) => visitor.visit_filter(op), Operator::Join(op) => visitor.visit_join(op), @@ -448,7 +451,9 @@ pub(crate) mod tests { is_distinct: false, force_spill: false, }), - Operator::ScalarApply(ScalarApplyOperator), + Operator::ScalarQueryInit(ScalarQueryInitOperator { + id: arena.alloc_scalar_query_ref(), + }), Operator::MarkApply(mark_apply), Operator::Filter(FilterOperator { predicate: expr(5), diff --git a/src/planner/operator/visitor_mut.rs b/src/planner/operator/visitor_mut.rs index d1d101ee..b7fd5825 100644 --- a/src/planner/operator/visitor_mut.rs +++ b/src/planner/operator/visitor_mut.rs @@ -67,9 +67,9 @@ pub trait OperatorVisitorMut<'a>: Sized { Ok(()) } - fn visit_scalar_apply( + fn visit_scalar_query_init( &mut self, - _op: &'a mut ScalarApplyOperator, + _op: &'a mut ScalarQueryInitOperator, ) -> Result<(), DatabaseError> { Ok(()) } @@ -403,7 +403,7 @@ pub fn walk_mut_operator<'a, V: OperatorVisitorMut<'a>>( match operator { Operator::Dummy => visitor.visit_dummy(), Operator::Aggregate(op) => visitor.visit_aggregate(op), - Operator::ScalarApply(op) => visitor.visit_scalar_apply(op), + Operator::ScalarQueryInit(op) => visitor.visit_scalar_query_init(op), Operator::MarkApply(op) => visitor.visit_mark_apply(op), Operator::Filter(op) => visitor.visit_filter(op), Operator::Join(op) => visitor.visit_join(op), diff --git a/src/planner/scalar_query_ref.rs b/src/planner/scalar_query_ref.rs new file mode 100644 index 00000000..debf2bcc --- /dev/null +++ b/src/planner/scalar_query_ref.rs @@ -0,0 +1,101 @@ +// Copyright 2024 KipData/KiteSQL +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use crate::errors::DatabaseError; +use crate::planner::MetaArena; +use crate::serdes::{ReferenceDecodeContext, ReferenceSerialization, ReferenceTables}; +use crate::storage::Transaction; +use std::io::{Read, Write}; + +/// A definition's position in an Arena namespace, independent of execution-local slots. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub struct ScalarQueryRef { + pub(crate) arena_id: usize, + pub(crate) pos: usize, +} + +impl std::fmt::Display for ScalarQueryRef { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{}:{}", self.arena_id, self.pos) + } +} + +impl ReferenceSerialization for ScalarQueryRef { + fn encode( + &self, + writer: &mut W, + direct: bool, + tables: &mut ReferenceTables, + arena: &A, + ) -> Result<(), DatabaseError> { + self.arena_id.encode(writer, direct, tables, arena)?; + self.pos.encode(writer, direct, tables, arena) + } + + fn decode( + reader: &mut R, + context: Option<&ReferenceDecodeContext<'_, T>>, + tables: &ReferenceTables, + arena: &mut A, + ) -> Result { + let reference = Self { + arena_id: usize::decode(reader, context, tables, arena)?, + pos: usize::decode(reader, context, tables, arena)?, + }; + arena.reserve_scalar_query_ref(reference); + Ok(reference) + } +} + +#[cfg(all(test, not(target_arch = "wasm32")))] +mod tests { + use super::*; + use crate::planner::TableArena; + use crate::storage::rocksdb::RocksTransaction; + + #[test] + fn decoding_preserves_source_reference() -> Result<(), DatabaseError> { + let mut source = TableArena::default(); + let reference = source.alloc_scalar_query_ref(); + let mut bytes = Vec::new(); + let mut tables = ReferenceTables::new(); + reference.encode(&mut bytes, false, &mut tables, &source)?; + let mut target = TableArena::default(); + let context = ReferenceDecodeContext::::new(None); + let first = ScalarQueryRef::decode::( + &mut bytes.as_slice(), + Some(&context), + &tables, + &mut target, + )?; + let repeated = ScalarQueryRef::decode::( + &mut bytes.as_slice(), + Some(&context), + &tables, + &mut target, + )?; + assert_eq!(first, repeated); + assert_eq!(first, reference); + let next_view = ReferenceDecodeContext::::new(None); + let second = ScalarQueryRef::decode::( + &mut bytes.as_slice(), + Some(&next_view), + &tables, + &mut target, + )?; + assert_eq!(second, reference); + assert_ne!(target.alloc_scalar_query_ref(), reference); + Ok(()) + } +} diff --git a/src/types/tuple.rs b/src/types/tuple.rs index abbc9611..9745cb31 100644 --- a/src/types/tuple.rs +++ b/src/types/tuple.rs @@ -76,8 +76,20 @@ impl<'a, 'p> SchemaView<'a, 'p> { } pub trait TupleLike { + fn len(&self) -> usize; + + fn is_empty(&self) -> bool { + self.len() == 0 + } + fn value_at(&self, index: usize) -> &DataValue; + /// The boundary between the outermost split's left and right inputs. + #[inline] + fn split_point(&self) -> Option { + None + } + #[inline] fn as_slice(&self) -> Option<&[DataValue]> { None @@ -85,19 +97,15 @@ pub trait TupleLike { } #[derive(Clone, Copy)] -pub struct SplitTupleRef<'a> { - left: &'a [DataValue], - right: &'a [DataValue], +pub struct SplitTupleRef<'a, L: ?Sized, R: ?Sized> { + left: &'a L, + right: &'a R, left_len: usize, } -impl<'a> SplitTupleRef<'a> { - pub fn new(left: &'a Tuple, right: &'a Tuple) -> Self { - Self::from_slices(left.values.as_slice(), right.values.as_slice()) - } - - pub fn from_slices(left: &'a [DataValue], right: &'a [DataValue]) -> Self { - SplitTupleRef { +impl<'a, L: TupleLike + ?Sized, R: TupleLike + ?Sized> SplitTupleRef<'a, L, R> { + pub fn new(left: &'a L, right: &'a R) -> Self { + Self { left, right, left_len: left.len(), @@ -105,6 +113,12 @@ impl<'a> SplitTupleRef<'a> { } } +impl<'a> SplitTupleRef<'a, [DataValue], [DataValue]> { + pub fn from_slices(left: &'a [DataValue], right: &'a [DataValue]) -> Self { + Self::new(left, right) + } +} + #[derive(Clone, Debug, PartialEq, Eq, Hash, Default)] pub struct Tuple { pub pk: Option, @@ -112,6 +126,11 @@ pub struct Tuple { } impl TupleLike for Tuple { + #[inline] + fn len(&self) -> usize { + self.values.len() + } + #[inline] fn value_at(&self, index: usize) -> &DataValue { &self.values[index] @@ -124,6 +143,11 @@ impl TupleLike for Tuple { } impl TupleLike for [DataValue] { + #[inline] + fn len(&self) -> usize { + <[DataValue]>::len(self) + } + #[inline] fn value_at(&self, index: usize) -> &DataValue { &self[index] @@ -135,49 +159,45 @@ impl TupleLike for [DataValue] { } } -impl TupleLike for &Tuple { +impl TupleLike for &T { #[inline] - fn value_at(&self, index: usize) -> &DataValue { - &self.values[index] + fn len(&self) -> usize { + (**self).len() } #[inline] - fn as_slice(&self) -> Option<&[DataValue]> { - Some(self.values.as_slice()) + fn value_at(&self, index: usize) -> &DataValue { + (**self).value_at(index) } -} -impl TupleLike for &[DataValue] { #[inline] - fn value_at(&self, index: usize) -> &DataValue { - &self[index] + fn split_point(&self) -> Option { + (**self).split_point() } #[inline] fn as_slice(&self) -> Option<&[DataValue]> { - Some(self) + (**self).as_slice() } } -impl TupleLike for &dyn TupleLike { +impl TupleLike for SplitTupleRef<'_, L, R> { #[inline] - fn value_at(&self, index: usize) -> &DataValue { - (*self).value_at(index) + fn len(&self) -> usize { + self.left_len + self.right.len() } #[inline] - fn as_slice(&self) -> Option<&[DataValue]> { - (*self).as_slice() + fn split_point(&self) -> Option { + Some(self.left_len) } -} -impl TupleLike for SplitTupleRef<'_> { #[inline] fn value_at(&self, index: usize) -> &DataValue { if index < self.left_len { - &self.left[index] + self.left.value_at(index) } else { - &self.right[index - self.left_len] + self.right.value_at(index - self.left_len) } } } @@ -299,6 +319,10 @@ mod tests { struct OneValueTupleLike(DataValue); impl super::TupleLike for OneValueTupleLike { + fn len(&self) -> usize { + 1 + } + fn value_at(&self, index: usize) -> &DataValue { assert_eq!(index, 0); &self.0 @@ -676,6 +700,27 @@ mod tests { assert_eq!(split.value_at(1), &DataValue::Int32(2)); let split = super::SplitTupleRef::from_slices(&left.values, &right.values); assert_eq!(split.value_at(1), &DataValue::Int32(2)); + assert_eq!(split.len(), 2); + assert!(!split.is_empty()); + assert_eq!(tuple_like.len(), 2); + assert_eq!(one.len(), 1); + let empty: &[DataValue] = &[]; + assert!(empty.is_empty()); + let tail = OneValueTupleLike(DataValue::Int32(9)); + let nested = super::SplitTupleRef::new(&split, &tail); + assert_eq!(nested.len(), 3); + assert_eq!(nested.value_at(0), &DataValue::Int32(1)); + assert_eq!(nested.value_at(1), &DataValue::Int32(2)); + assert_eq!(nested.value_at(2), &DataValue::Int32(9)); + assert_eq!(nested.as_slice(), None); + let empty_left = super::SplitTupleRef::new(empty, &tail); + assert_eq!(empty_left.len(), 1); + assert_eq!(empty_left.value_at(0), &DataValue::Int32(9)); + let empty_right = super::SplitTupleRef::new(&tail, empty); + assert_eq!(empty_right.len(), 1); + assert_eq!(empty_right.value_at(0), &DataValue::Int32(9)); + let empty_split = super::SplitTupleRef::new(empty, empty); + assert!(super::TupleLike::is_empty(&empty_split)); let tuple_slice: &[DataValue] = (&tuple).into(); assert_eq!(tuple_slice, tuple.values.as_slice()); assert_eq!( @@ -688,6 +733,33 @@ mod tests { ); } + #[test] + fn test_tuple_split_point() { + use super::{SplitTupleRef, TupleLike}; + + let left = Tuple::new(None, vec![DataValue::Int32(1)]); + let right = Tuple::new(None, vec![DataValue::Int32(2)]); + let tail = [DataValue::Int32(3)]; + assert_eq!(left.split_point(), None); + assert_eq!(tail.as_slice().split_point(), None); + assert_eq!(OneValueTupleLike(DataValue::Null).split_point(), None); + + let split = SplitTupleRef::new(&left, &right); + assert_eq!(split.split_point(), Some(1)); + let nested = SplitTupleRef::new(&split, tail.as_slice()); + assert_eq!(nested.split_point(), Some(2)); + assert_eq!(nested.value_at(nested.split_point().unwrap()), &tail[0]); + let dynamic: &dyn TupleLike = &nested; + assert_eq!(dynamic.split_point(), Some(2)); + assert_eq!(TupleLike::split_point(&dynamic), Some(2)); + assert_eq!(TupleLike::split_point(&&&nested), Some(2)); + + let empty: &[DataValue] = &[]; + assert_eq!(SplitTupleRef::new(empty, &left).split_point(), Some(0)); + assert_eq!(SplitTupleRef::new(&left, empty).split_point(), Some(1)); + assert_eq!(SplitTupleRef::new(empty, empty).split_point(), Some(0)); + } + #[test] fn test_tuple_deserialize_null_bits_and_primary_projection() { let serializers = [ diff --git a/tests/slt/exists.slt b/tests/slt/exists.slt index 80cadb32..496370bd 100644 --- a/tests/slt/exists.slt +++ b/tests/slt/exists.slt @@ -38,3 +38,42 @@ drop table t1 statement ok drop table t2 + +# A table alias hides the table name, and a correlated EXISTS keeps the alias identity of its +# columns, so non-equality correlations compare inner and outer rows. +statement ok +create table t_corr(pk int primary key, b int) + +statement ok +insert into t_corr values (0, 10), (1, 20), (2, 30), (3, null) + +statement error +select t_corr.b from t_corr as x + +query I rowsort +select b from t_corr where exists(select 1 from t_corr as x where x.b < t_corr.b) +---- +20 +30 + +query I rowsort +select b from t_corr where exists(select 1 from t_corr as x where x.b = t_corr.b) +---- +10 +20 +30 + +query I rowsort +select y.b from t_corr as y where exists(select 1 from t_corr as x where x.b > y.b) +---- +10 +20 + +query I rowsort +select b from t_corr where not exists(select 1 from t_corr as x where x.b < t_corr.b) +---- +10 +NULL + +statement ok +drop table t_corr diff --git a/tests/slt/subquery.slt b/tests/slt/subquery.slt index b0a665d3..872df2d1 100644 --- a/tests/slt/subquery.slt +++ b/tests/slt/subquery.slt @@ -488,11 +488,13 @@ statement error select * from users where id = (select user_id from orders where amount = age); -# Mark-apply and scalar-join subqueries cannot share one WHERE scope yet. -statement error +# Scalar initialization and EXISTS can share a WHERE scope. +query IT select * from users where exists(select 1 from orders) and id = (select max(user_id) from orders); +---- +2 30 statement ok drop table users; @@ -500,7 +502,7 @@ drop table users; statement ok drop table orders; -# ScalarApply must preserve its cached first result while validating scalar cardinality. +# ScalarQueryInit validates cardinality before publishing the cached result. statement ok create table scalar_eof (id int primary key, v int, keep_row boolean); @@ -551,26 +553,36 @@ from scalar_eof outer_row order by outer_row.id limit 1; statement ok drop table scalar_eof; -# SELECT-list scalar subqueries are joined in above DISTINCT / GROUP BY / -# ORDER BY, so those clauses cannot read them yet: reject instead of reading -# another column (which crashed, or silently returned wrong results). +# Scalar results are initialized before sorting, grouping and DISTINCT. statement ok create table sq_ref(id int primary key, c varchar); statement ok insert into sq_ref values (0, 'a'), (1, 'b'), (2, 'a'); -statement error +query II select id, (select count(*) from sq_ref) - id as x from sq_ref order by x; +---- +2 1 +1 2 +0 3 -statement error +query I select (select max(id) from sq_ref) + 1 as x from sq_ref order by x; +---- +3 +3 +3 -statement error +query I select distinct (select max(id) from sq_ref) from sq_ref; +---- +2 -statement error +query I select (select max(id) from sq_ref) as x from sq_ref group by x; +---- +2 # Not referenced by those clauses: still supported. query II @@ -636,3 +648,147 @@ drop table sq_outer; statement ok drop table sq_inner; + +# Uncorrelated scalar initialization across query clauses and DML. +statement ok +create table init_rows(id int primary key, v int) + +statement ok +insert into init_rows values (1, 10), (2, 20), (3, 30) + +query II +select id, v + (select max(v) from init_rows) as x from init_rows order by x desc +---- +3 60 +2 50 +1 40 + +query II +select id, v + (select max(v) from init_rows) as x from init_rows order by x desc limit 2 +---- +3 60 +2 50 + +query I +select (select (select max(v) from init_rows)) + (select min(v) from init_rows) +---- +40 + +query I +select distinct (select max(v) from init_rows) from init_rows +---- +30 + +query II +select (select max(v) from init_rows), count(*) from init_rows group by (select max(v) from init_rows) +---- +30 3 + +query I +select sum((select max(v) from init_rows)) from init_rows +---- +90 + +query I +select count(*) from init_rows having count(*) > (select 2) +---- +3 + +query I +select id from init_rows where v < (select max(v) from init_rows) order by id +---- +1 +2 + +query I +select id from init_rows where id in (select id from init_rows) and v = (select max(v) from init_rows) +---- +3 + +query I +select id from init_rows where exists(select 1 from init_rows) and v = (select max(v) from init_rows) +---- +3 + +query I +select coalesce((select 9), 0) +---- +9 + +query I +select coalesce((select v from init_rows where id < 0), -1) +---- +-1 + +statement error +select (select v from init_rows) + +statement error +select id from init_rows where v = (select v from init_rows) + +statement error +select (select x.v from init_rows as x where x.id = init_rows.id) from init_rows + +statement ok +create view init_max as select (select max(v) from init_rows) as m + +statement ok +create view init_min as select (select min(v) from init_rows) as m + +query II +select a.m, b.m from init_max as a cross join init_min as b +---- +30 10 + +query II +with q as (select (select max(v) from init_rows) as m) select a.m, b.m from q as a cross join q as b +---- +30 30 + +query II +select id, sum((select 2)) over (order by id) from init_rows order by id +---- +1 2 +2 4 +3 6 + +statement ok +update init_rows set v = 40 where id = (select max(id) from init_rows) + +query I +select (select max(v) from init_rows) +---- +40 + +statement ok +update init_rows set v = (select max(v) from init_rows) + 1 where id = 1 + +query I +select v from init_rows where id = 1 +---- +41 + +statement ok +insert into init_rows values (4, (select min(v) from init_rows)) + +query I +select v from init_rows where id = 4 +---- +20 + +statement ok +delete from init_rows where id = (select max(id) from init_rows) + +query I +select count(*) from init_rows +---- +3 + +statement ok +drop view init_max + +statement ok +drop view init_min + +statement ok +drop table init_rows From e35b8314265f289996236f198c5fab7b4f7e7bcd Mon Sep 17 00:00:00 2001 From: kould Date: Wed, 7 Oct 2026 04:31:28 +0800 Subject: [PATCH 06/16] feat: add abs for integer and floating-point values Register signed and unsigned integer and floating-point overloads. Preserve NULL values and reject signed integer absolute-value overflow. Cover numeric columns, arithmetic arguments, predicates, NULL and overflow in SLT. --- src/db.rs | 4 ++ src/function/abs.rs | 91 +++++++++++++++++++++++++++++++++++++++++++++ src/function/mod.rs | 1 + tests/slt/abs.slt | 29 +++++++++++++++ 4 files changed, 125 insertions(+) create mode 100644 src/function/abs.rs create mode 100644 tests/slt/abs.slt diff --git a/src/db.rs b/src/db.rs index 12fbb68a..f71a1c1e 100644 --- a/src/db.rs +++ b/src/db.rs @@ -25,6 +25,7 @@ use crate::expression::function::table::{ ArcTableFunctionImpl, TableFunctionCatalog, TableFunctionImpl, }; use crate::expression::function::FunctionSummary; +use crate::function::abs::Abs; use crate::function::char_length::CharLength; #[cfg(feature = "time")] use crate::function::current_date::CurrentDate; @@ -297,6 +298,9 @@ impl DataBaseBuilder { _p: Default::default(), }; + for ty in Abs::types() { + state.load_scalar_function(Abs::new(ty)); + } state.load_scalar_function(CharLength::new("char_length".to_lowercase())); state.load_scalar_function(CharLength::new("character_length".to_lowercase())); #[cfg(feature = "time")] diff --git a/src/function/abs.rs b/src/function/abs.rs new file mode 100644 index 00000000..9f5cd34c --- /dev/null +++ b/src/function/abs.rs @@ -0,0 +1,91 @@ +// Copyright 2024 KipData/KiteSQL +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use crate::errors::DatabaseError; +use crate::expression::function::scala::FuncMonotonicity; +use crate::expression::function::scala::ScalarFunctionImpl; +use crate::expression::function::FunctionSummary; +use crate::planner::ExprRef; +use crate::planner::MetaArena; +use crate::types::tuple::TupleLike; +use crate::types::value::DataValue; +use crate::types::LogicalType; +use ordered_float::OrderedFloat; +use std::sync::Arc; + +#[derive(Debug)] +pub(crate) struct Abs { + summary: FunctionSummary, + return_type: LogicalType, +} + +impl Abs { + pub(crate) fn new(ty: LogicalType) -> Arc { + Arc::new(Self { + summary: FunctionSummary { + name: "abs".into(), + arg_types: vec![ty.clone()], + }, + return_type: ty, + }) + } + + pub(crate) fn types() -> [LogicalType; 10] { + [ + LogicalType::Tinyint, + LogicalType::Smallint, + LogicalType::Integer, + LogicalType::Bigint, + LogicalType::UTinyint, + LogicalType::USmallint, + LogicalType::UInteger, + LogicalType::UBigint, + LogicalType::Float, + LogicalType::Double, + ] + } +} + +impl ScalarFunctionImpl for Abs { + fn eval( + &self, + exprs: &[ExprRef], + arena: &(dyn MetaArena + '_), + tuples: Option<&dyn TupleLike>, + ) -> Result { + let value = arena.expression(exprs[0]).eval(arena, tuples)?; + let overflow = || DatabaseError::OverFlow; + Ok(match value.as_ref() { + DataValue::Int8(v) => DataValue::Int8(v.checked_abs().ok_or_else(overflow)?), + DataValue::Int16(v) => DataValue::Int16(v.checked_abs().ok_or_else(overflow)?), + DataValue::Int32(v) => DataValue::Int32(v.checked_abs().ok_or_else(overflow)?), + DataValue::Int64(v) => DataValue::Int64(v.checked_abs().ok_or_else(overflow)?), + DataValue::Float32(v) => DataValue::Float32(OrderedFloat(v.0.abs())), + DataValue::Float64(v) => DataValue::Float64(OrderedFloat(v.0.abs())), + _ => value.into_owned(), + }) + } + + fn monotonicity(&self) -> Option { + None + } + + fn return_type(&self) -> &LogicalType { + &self.return_type + } + + fn summary(&self) -> &FunctionSummary { + &self.summary + } +} diff --git a/src/function/mod.rs b/src/function/mod.rs index 77982a03..9fe10775 100644 --- a/src/function/mod.rs +++ b/src/function/mod.rs @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +pub(crate) mod abs; pub(crate) mod char_length; #[cfg(feature = "time")] pub(crate) mod current_date; diff --git a/tests/slt/abs.slt b/tests/slt/abs.slt new file mode 100644 index 00000000..30d5ef97 --- /dev/null +++ b/tests/slt/abs.slt @@ -0,0 +1,29 @@ +statement ok +create table t_abs(id int primary key, i int, b bigint, s smallint, d double, f float); + +statement ok +insert into t_abs values (0, -3, -9000000000, -7, -1.5, -2.5), (1, 4, 5, 6, 0.25, 1.5), (2, null, null, null, null, null); + +query IIIRR rowsort +select abs(i), abs(b), abs(s), abs(d), abs(f) from t_abs; +---- +3 9000000000 7 1.5 2.5 +4 5 6 0.25 1.5 +NULL NULL NULL NULL NULL + +query I rowsort +select abs(i - b) from t_abs where id < 2; +---- +1 +8999999997 + +query I +select id from t_abs where abs(i) > 3; +---- +1 + +statement error +select abs(-2147483648); + +statement ok +drop table t_abs; From d00de4330146062ed8c7827ff18ddb7ff4287804 Mon Sep 17 00:00:00 2001 From: kould Date: Wed, 7 Oct 2026 07:06:42 +0800 Subject: [PATCH 07/16] feat: add callback-based mutable SQL execution Expose Database::run_mut to consume the final statement's results inside a callback, reusing DDL execution, commit and catalog publication. Move statement scope cleanup into TransactionGuard so arena and DDL changes can be returned by value. Use run_mut in the SLT harness to avoid SQL parse-display-parse roundtrips. Reject aggregates in WHERE, prefer input columns over conflicting SELECT aliases in SELECT/WHERE, and correct floating-point integer-cast range checks with regression coverage. --- src/binder/expr.rs | 15 ++++ src/binder/parser.rs | 42 +++++++++-- src/binder/select.rs | 5 ++ src/db.rs | 135 +++++++++++++++++++++------------- src/planner/arena.rs | 2 +- src/serdes/mod.rs | 3 +- src/types/evaluator/cast.rs | 90 ++++++++++++++++++++++- tests/slt/aggregation.slt | 23 ++++++ tests/slt/where.slt | 75 +++++++++++++++++++ tests/sqllogictest/src/lib.rs | 94 ++++++++--------------- 10 files changed, 358 insertions(+), 126 deletions(-) diff --git a/src/binder/expr.rs b/src/binder/expr.rs index 1542808a..e874b274 100644 --- a/src/binder/expr.rs +++ b/src/binder/expr.rs @@ -326,6 +326,21 @@ impl<'a, T: Transaction, A: AsRef<[(usize, LogicalType)]>> Binder<'a, '_, T, A> arena: &mut PlanArena, ) -> Result { if table_name.is_none() { + // SELECT expressions and WHERE read input columns, not a same-named output alias. + if bind_table_name.is_none() + && matches!( + self.context.step_now(), + QueryBindStep::Project | QueryBindStep::Where + ) + { + let input = match self.context.using.get(column_name) { + Some(column) => Some(column.visible_expr(arena)?), + None => Self::find_column_in_scope(&self.context, arena, column_name)?, + }; + if let Some(input) = input { + return Ok(input); + } + } if let Some((_, expr)) = self .context .expr_aliases diff --git a/src/binder/parser.rs b/src/binder/parser.rs index e79782f0..702248b4 100644 --- a/src/binder/parser.rs +++ b/src/binder/parser.rs @@ -37,7 +37,7 @@ use crate::planner::operator::recursive_cte::{RecursiveCteOperator, RecursiveSca use crate::planner::operator::sort::SortField; use crate::planner::operator::Operator; use crate::planner::MetaArena; -use crate::planner::{Childrens, ExprRef, LogicalPlan, PlanArena}; +use crate::planner::{Childrens, ExecArenaView, ExprRef, LogicalPlan, PlanArena}; use crate::storage::{Storage, Transaction}; use crate::types::value::{DataValue, Utf8Type}; use crate::types::{CharLengthUnits, ColumnId, LogicalType}; @@ -164,18 +164,50 @@ impl Database { let sql = sql.as_ref(); let statements = prepare_all(sql).map_err(|err| err.with_sql_context(sql))?; - for statement in statements { - if !matches!(command_type(&statement)?, CommandType::DDL) { + for statement in &statements { + if !matches!(command_type(statement)?, CommandType::DDL) { return Err(DatabaseError::UnsupportedStmt( "`Database::ddl` only accepts DDL statements".to_string(), ) .with_sql_context(sql)); } + } + self.run_mut_statements(sql, statements, |_| Ok(())) + } + + /// Executes SQL with catalog mutations allowed, lending the last statement's results + /// to `consume`. On success, unread rows are drained before committing and publishing + /// catalog updates. A callback or execution error rolls the current statement back. + /// Earlier statements are completed and committed in order before the callback runs. + pub fn run_mut(&mut self, sql: impl AsRef, consume: C) -> Result + where + C: for<'a> FnOnce( + &mut TransactionIter<'a, S::TransactionType<'a>, ExecArenaView>>, + ) -> Result, + { + let sql = sql.as_ref(); + let statements = prepare_all(sql).map_err(|err| err.with_sql_context(sql))?; + self.run_mut_statements(sql, statements, consume) + } + fn run_mut_statements( + &mut self, + sql: &str, + mut statements: Vec, + consume: C, + ) -> Result + where + C: for<'a> FnOnce( + &mut TransactionIter<'a, S::TransactionType<'a>, ExecArenaView>>, + ) -> Result, + { + let last = statements + .pop() + .ok_or_else(|| DatabaseError::EmptyStatement.with_sql_context(sql))?; + for statement in statements { self.execute_mut(sql, &[], |binder, arena| binder.bind(&statement, arena))?; } - - Ok(()) + self.execute_mut_with(sql, &[], |binder, arena| binder.bind(&last, arena), consume) } /// Runs one or more SQL statements and returns an iterator for the final result set. diff --git a/src/binder/select.rs b/src/binder/select.rs index b394cce0..d185d3b8 100644 --- a/src/binder/select.rs +++ b/src/binder/select.rs @@ -1327,6 +1327,11 @@ impl<'a: 'b, 'b, T: Transaction, A: AsRef<[(usize, LogicalType)]>> Binder<'a, 'b arena: &mut PlanArena, ) -> Result { self.context.step(QueryBindStep::Where); + if predicate.has_agg_call(arena)? { + return Err(DatabaseError::AggMiss( + "aggregate functions are not allowed in WHERE".into(), + )); + } if let Some(sub_queries) = self.context.sub_queries_at_now() { for sub_query in sub_queries { diff --git a/src/db.rs b/src/db.rs index f71a1c1e..c2c4b135 100644 --- a/src/db.rs +++ b/src/db.rs @@ -773,6 +773,26 @@ impl Database { &mut Binder<'bind, '_, S::TransactionType<'txn>, A>, &mut PlanArena<'a>, ) -> Result, + { + self.execute_mut_with(context, params, build, |_| Ok(())) + } + + pub(crate) fn execute_mut_with( + &mut self, + context: &str, + params: A, + build: F, + consume: C, + ) -> Result + where + A: AsRef<[(usize, LogicalType)]>, + F: for<'a, 'txn, 'bind> FnOnce( + &mut Binder<'bind, '_, S::TransactionType<'txn>, A>, + &mut PlanArena<'a>, + ) -> Result, + C: for<'a> FnOnce( + &mut TransactionIter<'a, S::TransactionType<'a>, ExecArenaView>>, + ) -> Result, { let transaction = Box::into_raw(Box::new( self.storage @@ -787,16 +807,18 @@ impl Database { return Err(err.with_sql_context(context)); } }; - let (mut plan_arena, apply) = - match TransactionIter::new(schema, plan_arena, executor, transaction) - .done_with_ddl_apply() - { - Ok(apply) => apply, - Err(err) => { - unsafe { drop(Box::from_raw(transaction)) }; - return Err(err.with_sql_context(context)); - } - }; + let (result, mut plan_arena, apply) = match (|| { + let mut iter = TransactionIter::new(schema, plan_arena, executor, transaction); + let result = consume(&mut iter)?; + let (arena, apply) = iter.done_with_ddl_apply()?; + Ok::<_, DatabaseError>((result, arena, apply)) + })() { + Ok(result) => result, + Err(err) => { + unsafe { drop(Box::from_raw(transaction)) }; + return Err(err.with_sql_context(context)); + } + }; if let Err(err) = unsafe { Box::from_raw(transaction).commit() } { return Err(err.with_sql_context(context)); @@ -810,7 +832,7 @@ impl Database { if catalog_changed { unsafe { (&mut *state).recycle_table_arena() }?; } - Ok(()) + Ok(result) } pub fn analyze(&mut self, table_name: impl AsRef) -> Result<(), DatabaseError> { @@ -1172,13 +1194,42 @@ impl<'a, 'txn, S: Storage> BindSource<'a> for &'a mut DBTransaction<'txn, S> { } } -/// Raw result iterator returned by transaction execution APIs. -pub struct TransactionIter<'a, T: Transaction + 'a, A: MetaArena + 'a = Box> { +struct TransactionGuard<'a, T: Transaction + 'a> { executor: Option>, - plan_arena: Option, - schema: Schema, transaction: *mut T, statement_scope_active: bool, +} + +impl TransactionGuard<'_, T> { + #[inline] + fn finish_statement_scope( + &mut self, + ddl_apply: &mut Vec, + ) -> Result<(), DatabaseError> { + if !self.statement_scope_active { + return Ok(()); + } + + if let Some(mut executor) = self.executor.take() { + ddl_apply.extend(executor.take_ddl_apply()); + } + self.statement_scope_active = false; + unsafe { (*self.transaction).end_statement_scope() } + } +} + +impl Drop for TransactionGuard<'_, T> { + fn drop(&mut self) { + let _ = self.finish_statement_scope(&mut Vec::new()); + } +} + +/// Raw result iterator returned by transaction execution APIs. +pub struct TransactionIter<'a, T: Transaction + 'a, A: MetaArena + 'a = Box> { + // Drop execution before the metadata arena it borrows from. + guard: TransactionGuard<'a, T>, + plan_arena: A, + schema: Schema, ddl_apply: Vec, } @@ -1190,35 +1241,25 @@ impl<'a, T: Transaction + 'a, A: MetaArena + 'a> TransactionIter<'a, T, A> { transaction: *mut T, ) -> Self { Self { - executor: Some(executor), - plan_arena: Some(plan_arena), + guard: TransactionGuard { + executor: Some(executor), + transaction, + statement_scope_active: true, + }, + plan_arena, schema, - transaction, - statement_scope_active: true, ddl_apply: Vec::new(), } } #[inline] fn finish_statement_scope(&mut self) -> Result<(), DatabaseError> { - if !self.statement_scope_active { - return Ok(()); - } - - if let Some(mut executor) = self.executor.take() { - self.ddl_apply.extend(executor.take_ddl_apply()); - } - self.statement_scope_active = false; - unsafe { (*self.transaction).end_statement_scope() } + self.guard.finish_statement_scope(&mut self.ddl_apply) } #[inline] pub fn schema(&self, f: impl FnOnce(&SchemaView<'_, '_>) -> R) -> R { - let plan_arena = self - .plan_arena - .as_ref() - .expect("result iterator schema is unavailable after statement completion"); - let schema = SchemaView::new(&self.schema, plan_arena); + let schema = SchemaView::new(&self.schema, &self.plan_arena); f(&schema) } @@ -1227,14 +1268,11 @@ impl<'a, T: Transaction + 'a, A: MetaArena + 'a> TransactionIter<'a, T, A> { &mut self, f: impl FnOnce(&SchemaView<'_, '_>, &mut Tuple) -> R, ) -> Result, DatabaseError> { - let Some(executor) = self.executor.as_mut() else { + let Some(executor) = self.guard.executor.as_mut() else { return Ok(None); }; let executor_ptr = std::ptr::from_mut(executor); - let plan_arena = self - .plan_arena - .as_mut() - .expect("result iterator plan arena is unavailable after statement completion"); + let plan_arena = &mut self.plan_arena; match unsafe { (*executor_ptr).next_tuple(plan_arena) } { Ok(Some(tuple)) => { let schema = SchemaView::new(&self.schema, plan_arena); @@ -1261,19 +1299,14 @@ impl<'a, T: Transaction + 'a, A: MetaArena + 'a> TransactionIter<'a, T, A> { impl<'a, T: Transaction + 'a> TransactionIter<'a, T, ExecArenaView>> { fn done_with_ddl_apply(mut self) -> Result<(PlanArena<'a>, Vec), DatabaseError> { while self.next_tuple(|_, _| ())?.is_some() {} - Ok(( - self.plan_arena - .take() - .expect("DDL apply plan arena is unavailable after statement completion") - .into_parent(), - std::mem::take(&mut self.ddl_apply), - )) - } -} - -impl Drop for TransactionIter<'_, T, A> { - fn drop(&mut self) { - let _ = self.finish_statement_scope(); + let Self { + guard, + plan_arena, + ddl_apply, + .. + } = self; + drop(guard); + Ok((plan_arena.into_parent(), ddl_apply)) } } diff --git a/src/planner/arena.rs b/src/planner/arena.rs index 59d8f899..8ca8fefa 100644 --- a/src/planner/arena.rs +++ b/src/planner/arena.rs @@ -327,7 +327,7 @@ impl MetaArena for Box { } /// Owns the metadata and scalar-query cache for one statement execution. -pub(crate) struct ExecArenaView { +pub struct ExecArenaView { parent: A, init_values: HashMap>, } diff --git a/src/serdes/mod.rs b/src/serdes/mod.rs index a07c8341..93f068ee 100644 --- a/src/serdes/mod.rs +++ b/src/serdes/mod.rs @@ -180,6 +180,7 @@ impl ReferenceTables { mod tests { use crate::errors::DatabaseError; use crate::expression::range_detacher::Range; + use crate::planner::TableArena; use crate::serdes::{ReferenceSerialization, ReferenceTables}; use crate::storage::rocksdb::RocksTransaction; use crate::types::value::DataValue; @@ -195,7 +196,7 @@ mod tests { S: ReferenceSerialization + PartialEq + Debug, { let mut reference_tables = ReferenceTables::new(); - let mut arena = crate::planner::TableArena::default(); + let mut arena = TableArena::default(); let mut cursor = Cursor::new(Vec::new()); source.encode(&mut cursor, false, &mut reference_tables, &arena)?; diff --git a/src/types/evaluator/cast.rs b/src/types/evaluator/cast.rs index 60ec2a43..c7bd0262 100644 --- a/src/types/evaluator/cast.rs +++ b/src/types/evaluator/cast.rs @@ -178,10 +178,15 @@ macro_rules! float_to_int_cast { let float_value: $float_type = $float_value; if float_value.is_nan() { Ok(0) - } else if float_value <= 0.0 || float_value > <$int_type>::MAX as $float_type { - Err($crate::errors::DatabaseError::OverFlow) } else { - Ok(float_value as $int_type) + let truncated = float_value.trunc(); + // Use an exclusive upper bound: wide integer MAX rounds up to 2^N in floats. + let upper = <$int_type>::MAX as $float_type + 1.0; + if truncated < <$int_type>::MIN as $float_type || truncated >= upper { + Err($crate::errors::DatabaseError::OverFlow) + } else { + Ok(truncated as $int_type) + } } }}; } @@ -1163,6 +1168,85 @@ mod test { cast_create(&from, &to) } + #[test] + fn float_integer_cast_accepts_negative_and_zero_values() -> Result<(), DatabaseError> { + for (from, value) in [ + (LogicalType::Float, DataValue::Float32(OrderedFloat(-12.75))), + ( + LogicalType::Double, + DataValue::Float64(OrderedFloat(-12.75)), + ), + ] { + for (to, expected) in [ + (LogicalType::Tinyint, DataValue::Int8(-12)), + (LogicalType::Smallint, DataValue::Int16(-12)), + (LogicalType::Integer, DataValue::Int32(-12)), + (LogicalType::Bigint, DataValue::Int64(-12)), + ] { + assert_eq!(cast_eval(from.clone(), to, &value)?, expected); + } + } + for (from, zero) in [ + (LogicalType::Float, DataValue::Float32(OrderedFloat(0.0))), + (LogicalType::Double, DataValue::Float64(OrderedFloat(-0.0))), + ] { + assert_eq!( + cast_eval(from.clone(), LogicalType::Integer, &zero)?, + DataValue::Int32(0) + ); + assert_eq!( + cast_eval(from, LogicalType::UInteger, &zero)?, + DataValue::UInt32(0) + ); + } + Ok(()) + } + + #[test] + fn float_integer_cast_checks_target_bounds() { + macro_rules! check_bounds { + ($float:ty, $int:ty) => {{ + let min = <$int>::MIN as $float; + let upper = <$int>::MAX as $float + 1.0; + assert_eq!(float_to_int_cast!(min, $int, $float).unwrap(), <$int>::MIN); + assert!(matches!( + float_to_int_cast!(upper, $int, $float), + Err(DatabaseError::OverFlow) + )); + assert!(matches!( + float_to_int_cast!(min * 2.0 - 1.0, $int, $float), + Err(DatabaseError::OverFlow) + )); + assert!(matches!( + float_to_int_cast!(<$float>::INFINITY, $int, $float), + Err(DatabaseError::OverFlow) + )); + assert!(matches!( + float_to_int_cast!(<$float>::NEG_INFINITY, $int, $float), + Err(DatabaseError::OverFlow) + )); + }}; + } + check_bounds!(f32, i8); + check_bounds!(f32, i16); + check_bounds!(f32, i32); + check_bounds!(f32, i64); + check_bounds!(f64, i8); + check_bounds!(f64, i16); + check_bounds!(f64, i32); + check_bounds!(f64, i64); + check_bounds!(f32, u8); + check_bounds!(f32, u16); + check_bounds!(f32, u32); + check_bounds!(f32, u64); + check_bounds!(f64, u8); + check_bounds!(f64, u16); + check_bounds!(f64, u32); + check_bounds!(f64, u64); + assert_eq!(float_to_int_cast!(127.9_f64, i8, f64).unwrap(), 127); + assert_eq!(float_to_int_cast!(-128.9_f64, i8, f64).unwrap(), -128); + } + fn utf8(value: &str) -> DataValue { DataValue::Utf8 { value: value.to_string(), diff --git a/tests/slt/aggregation.slt b/tests/slt/aggregation.slt index 0d6b0b1c..70bd2efa 100644 --- a/tests/slt/aggregation.slt +++ b/tests/slt/aggregation.slt @@ -98,5 +98,28 @@ select (select sum(null) from sn) from sn where id = 1; ---- NULL +# Aggregates, including SELECT aliases containing aggregates, cannot run in WHERE. +statement error agg miss: aggregate functions are not allowed in WHERE +select count(*) as n from sn where n is not null; + +statement error agg miss: aggregate functions are not allowed in WHERE +select + + 23 * + count(*) * - + 69 as n from sn where not n is null; + +statement error agg miss: aggregate functions are not allowed in WHERE +select count(*) as n from sn where coalesce(n, 0) > 0; + +statement error agg miss: aggregate functions are not allowed in WHERE +select id from sn where count(*) > 0; + +query I +select count(*) as n from sn having n > 0; +---- +2 + +query I +select count(*) from sn where a > 0; +---- +2 + statement ok drop table sn diff --git a/tests/slt/where.slt b/tests/slt/where.slt index 2db53f55..71340b73 100644 --- a/tests/slt/where.slt +++ b/tests/slt/where.slt @@ -345,3 +345,78 @@ select id from tuple_in where (a, b) <> (1, 10); statement ok drop table tuple_in + +# A SELECT alias must not hide a same-named input column in SELECT or WHERE. +statement ok +create table alias_input(id int primary key, v int); + +statement ok +insert into alias_input values (1, 10), (2, 20), (3, null); + +query II +select id, -v as v from alias_input where v > 15 order by id; +---- +2 -20 + +query II +select id, cast(null as int) as v from alias_input where v is null; +---- +3 NULL + +query II +select id, 57 as v from alias_input where v < 15 order by id; +---- +1 57 + +query II +select 57 as v, v from alias_input order by id; +---- +57 10 +57 20 +57 NULL + +# ORDER BY still resolves the output alias before the input column. +query II +select id, -v as v from alias_input where id < 3 order by v; +---- +2 -20 +1 -10 + +# Retain existing alias fallback when there is no input column of that name. +query II +select id, v + 1 as x from alias_input where x = 11; +---- +1 11 + +query II +select id, -v as v from alias_input where exists(select 1 where alias_input.v > 15); +---- +2 -20 + +statement ok +create table alias_other(id int primary key, v int); + +statement error is ambiguous +select a.id as v from alias_input a cross join alias_other b where v > 0; + +statement ok +drop table alias_other; + +query IIII +select cast(-12.75 as int), cast(cast(-12.75 as float) as int), cast(0.0 as int), cast(-0.0 as int); +---- +-12 -12 0 0 + +query I +select id from alias_input where cast(-v as int) = -20; +---- +2 + +statement error over flow +select cast(2147483648.0 as int); + +statement error over flow +select cast(-2147483649.0 as int); + +statement ok +drop table alias_input; diff --git a/tests/sqllogictest/src/lib.rs b/tests/sqllogictest/src/lib.rs index 72f681f7..dc125001 100644 --- a/tests/sqllogictest/src/lib.rs +++ b/tests/sqllogictest/src/lib.rs @@ -12,10 +12,10 @@ // See the License for the specific language governing permissions and // limitations under the License. -use kite_sql::binder::{command_type, CommandType}; -use kite_sql::db::{prepare_all, Database, DatabaseIter, Statement}; +use kite_sql::db::Database; use kite_sql::errors::DatabaseError; use kite_sql::storage::lmdb::LmdbStorage; +use kite_sql::types::value::DataValue; use sqllogictest::{DBOutput, DefaultColumnType, DB}; use std::time::Instant; @@ -30,73 +30,37 @@ impl DB for SQLBase { fn run(&mut self, sql: &str) -> Result, Self::Error> { let start = Instant::now(); println!("|— Input SQL: {}", sql); - let mut statements = prepare_all(sql)?.into_iter().peekable(); - - let output = loop { - let Some(statement) = statements.next() else { - break DBOutput::StatementComplete(0); - }; - let is_last = statements.peek().is_none(); - match command_type(&statement)? { - CommandType::DDL => { - self.db.ddl(statement.to_string())?; - if is_last { - break DBOutput::StatementComplete(0); - } - } - CommandType::Analyze => { - execute_analyze_statement(&mut self.db, &statement)?; - if is_last { - break DBOutput::StatementComplete(0); - } - } - _ => { - let iter = self.db.run(statement.to_string())?; - if is_last { - break collect_output(iter)?; - } - iter.done()?; - } - } - }; - + let output = run_sql(&mut self.db, sql, |_, value| value.to_string())?; println!(" |— time spent: {:?}", start.elapsed()); Ok(output) } } -fn collect_output( - mut iter: DatabaseIter<'_, LmdbStorage>, -) -> Result, DatabaseError> { - let types = vec![DefaultColumnType::Any; iter.schema(|schema| schema.len())]; - let mut rows = Vec::new(); - - while let Some(row) = iter.next_tuple(|_, tuple| { - tuple - .values - .iter() - .map(|value| format!("{}", value)) - .collect() - })? { - rows.push(row); - } - iter.done()?; - if rows.is_empty() { - return Ok(DBOutput::StatementComplete(0)); - } - Ok(DBOutput::Rows { types, rows }) -} - -fn execute_analyze_statement( +/// Runs every statement in `sql` and returns the output of the last one, rendering each value of +/// column `i` with `format(i, value)`. +pub fn run_sql( db: &mut Database, - statement: &Statement, -) -> Result<(), DatabaseError> { - let Statement::Analyze(analyze) = statement else { - unreachable!("execute_analyze_statement only accepts ANALYZE") - }; - let table_name = analyze - .table_name - .as_ref() - .ok_or_else(|| DatabaseError::UnsupportedStmt("ANALYZE requires table name".to_string()))?; - db.analyze(table_name.to_string()) + sql: &str, + mut format: impl FnMut(usize, &DataValue) -> String, +) -> Result, DatabaseError> { + db.run_mut(sql, |iter| { + let width = iter.schema(|schema| schema.len()); + let types = vec![DefaultColumnType::Any; width]; + let mut rows = Vec::new(); + while let Some(row) = iter.next_tuple(|_, tuple| { + tuple + .values + .iter() + .enumerate() + .map(|(i, value)| format(i, value)) + .collect() + })? { + rows.push(row); + } + if rows.is_empty() { + Ok(DBOutput::StatementComplete(0)) + } else { + Ok(DBOutput::Rows { types, rows }) + } + }) } From 83db95f2c0c1b597431da0aa003397d75f44ec32 Mon Sep 17 00:00:00 2001 From: kould Date: Wed, 7 Oct 2026 07:52:10 +0800 Subject: [PATCH 08/16] feat: use truncating integer division and eliminate unary plus Keep integer division results integral with checked zero and overflow handling, while preserving fractional AVG and floating-point results. Eliminate unary plus during binding and remove its evaluator dispatch. Short-circuit decisive boolean operands so guarded division is not evaluated. Update division expectations and cover text identity, signed division, overflow, NULL propagation and mixed floating-point expressions. --- src/binder/expr.rs | 20 ++++- src/db/prepared.rs | 8 +- src/execution/dql/aggregate/avg.rs | 21 ++---- src/expression/evaluator.rs | 8 ++ src/types/evaluator/binary.rs | 73 +++++++++++++++++- src/types/evaluator/float32.rs | 4 - src/types/evaluator/float64.rs | 4 - src/types/evaluator/unary.rs | 114 ++++++----------------------- tests/slt/basic_test.slt | 26 +++++++ tests/slt/char.slt | 25 +++++++ tests/slt/crdb/and_or.slt | 10 +++ tests/slt/where.slt | 1 + tests/sqllogictest/Cargo.toml | 1 + 13 files changed, 193 insertions(+), 122 deletions(-) diff --git a/src/binder/expr.rs b/src/binder/expr.rs index e874b274..5b6d9c3c 100644 --- a/src/binder/expr.rs +++ b/src/binder/expr.rs @@ -420,10 +420,12 @@ impl<'a, T: Transaction, A: AsRef<[(usize, LogicalType)]>> Binder<'a, '_, T, A> LogicalType::max_logical_type(&left_ty, &right_ty)?.into_owned() } expression::BinaryOperator::Divide => { - if let LogicalType::Decimal(precision, scale) = - LogicalType::max_logical_type(&left_ty, &right_ty)?.into_owned() + let ty = LogicalType::max_logical_type(&left_ty, &right_ty)?.into_owned(); + if ty.is_signed_numeric() + || ty.is_unsigned_numeric() + || matches!(ty, LogicalType::Decimal(..)) { - LogicalType::Decimal(precision, scale) + ty } else { LogicalType::Double } @@ -459,6 +461,18 @@ impl<'a, T: Transaction, A: AsRef<[(usize, LogicalType)]>> Binder<'a, '_, T, A> op: expression::UnaryOperator, arena: &mut PlanArena, ) -> Result { + if op == expression::UnaryOperator::Plus { + let ty = expr.return_type(arena); + if ty.is_numeric() + || matches!( + ty.as_ref(), + LogicalType::SqlNull | LogicalType::Char(..) | LogicalType::Varchar(..) + ) + { + return Ok(arena.expression(expr).clone()); + } + return Err(DatabaseError::UnsupportedUnaryOperator(ty.into_owned(), op)); + } let ty = if let expression::UnaryOperator::Not = op { LogicalType::Boolean } else { diff --git a/src/db/prepared.rs b/src/db/prepared.rs index 380f61c1..2f61233c 100644 --- a/src/db/prepared.rs +++ b/src/db/prepared.rs @@ -155,18 +155,18 @@ mod tests { "select (($1 * 3 + 7) % 97) + ($1 / 2)", &[(1, LogicalType::Bigint)], )?; - for (value, expected) in [(7, 31.5), (23, 87.5)] { + for (value, expected) in [(7, 31_i64), (23, 87_i64)] { let mut iter = db.execute(&plan, [(1, DataValue::Int64(value))])?; iter.schema(|schema| { assert_eq!(schema.len(), 1); assert_eq!( schema.iter().next().unwrap().datatype(), - &LogicalType::Double + &LogicalType::Bigint ); }); assert_eq!( iter.next_tuple(|_, row| row.values.clone())?, - Some(vec![DataValue::Float64(expected.into())]) + Some(vec![DataValue::Int64(expected)]) ); assert!(iter.next_tuple(|_, _| ())?.is_none()); iter.done()?; @@ -175,7 +175,7 @@ mod tests { let mut iter = tx.execute(&plan, [(1, DataValue::Int64(7))])?; assert_eq!( iter.next_tuple(|_, row| row.values.clone())?, - Some(vec![DataValue::Float64(31.5.into())]) + Some(vec![DataValue::Int64(31)]) ); iter.done()?; tx.commit()?; diff --git a/src/execution/dql/aggregate/avg.rs b/src/execution/dql/aggregate/avg.rs index c47461de..7e4b2702 100644 --- a/src/execution/dql/aggregate/avg.rs +++ b/src/execution/dql/aggregate/avg.rs @@ -18,6 +18,7 @@ use crate::execution::dql::aggregate::Accumulator; use crate::expression::BinaryOperator; use crate::types::evaluator::binary_create; use crate::types::value::DataValue; +use crate::types::LogicalType; use std::borrow::Cow; pub struct AvgAccumulator { @@ -60,24 +61,14 @@ impl Accumulator for AvgAccumulator { let Some(acc) = &self.inner else { return Ok(()); }; - let mut value = Cow::Borrowed(acc.result()); - let value_ty = value.logical_type(); - if self.count == 0 { return Ok(()); } - let quantity = if value_ty.is_signed_numeric() { - DataValue::Int64(self.count as i64) - } else { - DataValue::UInt32(self.count as u32) - }; - let quantity_ty = quantity.logical_type(); - - if value_ty != quantity_ty { - value = Cow::Owned(value.into_owned().cast(&quantity_ty)?) - } - let evaluator = binary_create(Cow::Owned(quantity_ty), BinaryOperator::Divide)?; - self.result = evaluator.binary_eval(value.as_ref(), &quantity)?; + // AVG returns Double even when ordinary integer division truncates. + let value = acc.result().clone().cast(&LogicalType::Double)?; + let quantity = DataValue::Float64((self.count as f64).into()); + let evaluator = binary_create(Cow::Owned(LogicalType::Double), BinaryOperator::Divide)?; + self.result = evaluator.binary_eval(&value, &quantity)?; Ok(()) } diff --git a/src/expression/evaluator.rs b/src/expression/evaluator.rs index e34ca5c8..54d77aef 100644 --- a/src/expression/evaluator.rs +++ b/src/expression/evaluator.rs @@ -86,12 +86,20 @@ impl ScalarExpression { } } ScalarExpression::Binary { + op, left_expr, right_expr, evaluator, .. } => { let left = arena.expression(*left_expr).eval(arena, tuple)?; + if matches!( + (op, left.as_ref()), + (BinaryOperator::And, DataValue::Boolean(false)) + | (BinaryOperator::Or, DataValue::Boolean(true)) + ) { + return Ok(left); + } let right = arena.expression(*right_expr).eval(arena, tuple)?; evaluator diff --git a/src/types/evaluator/binary.rs b/src/types/evaluator/binary.rs index 6a7a20d3..26cd8252 100644 --- a/src/types/evaluator/binary.rs +++ b/src/types/evaluator/binary.rs @@ -561,7 +561,12 @@ macro_rules! numeric_binary_evaluator_definition { right: &$crate::types::value::DataValue, ) -> Result<$crate::types::value::DataValue, $crate::errors::DatabaseError> { Ok(match (left, right) { - ($compute_type(v1), $compute_type(v2)) => $crate::types::value::DataValue::Float64(ordered_float::OrderedFloat(*v1 as f64 / *v2 as f64)), + ($compute_type(v1), $compute_type(v2)) => { + if *v2 == 0 { + return Err($crate::errors::DatabaseError::InvalidValue("division by zero".into())); + } + $compute_type(v1.checked_div(*v2).ok_or($crate::errors::DatabaseError::OverFlow)?) + }, ($compute_type(_), $crate::types::value::DataValue::Null) | ($crate::types::value::DataValue::Null, $compute_type(_)) | ($crate::types::value::DataValue::Null, $crate::types::value::DataValue::Null) => $crate::types::value::DataValue::Null, @@ -721,6 +726,72 @@ mod test { Ok(()) } + #[test] + fn integer_division_truncates_and_checks_errors() -> Result<(), DatabaseError> { + macro_rules! check_signed { + ($ty:expr, $variant:ident, $int:ty) => {{ + let divide = binary_create(Cow::Owned($ty), BinaryOperator::Divide)?; + for (left, right, expected) in [(5, 2, 2), (-5, 2, -2), (5, -2, -2), (-5, -2, 2)] { + assert_eq!( + divide + .binary_eval(&DataValue::$variant(left), &DataValue::$variant(right))?, + DataValue::$variant(expected) + ); + } + assert!(matches!( + divide.binary_eval(&DataValue::$variant(<$int>::MIN), &DataValue::$variant(-1)), + Err(DatabaseError::OverFlow) + )); + assert!(matches!( + divide.binary_eval(&DataValue::$variant(1), &DataValue::$variant(0)), + Err(DatabaseError::InvalidValue(_)) + )); + assert_eq!( + divide.binary_eval(&DataValue::Null, &DataValue::$variant(0))?, + DataValue::Null + ); + }}; + } + check_signed!(LogicalType::Tinyint, Int8, i8); + check_signed!(LogicalType::Smallint, Int16, i16); + check_signed!(LogicalType::Integer, Int32, i32); + check_signed!(LogicalType::Bigint, Int64, i64); + for (ty, left, right, expected) in [ + ( + LogicalType::UTinyint, + DataValue::UInt8(5), + DataValue::UInt8(2), + DataValue::UInt8(2), + ), + ( + LogicalType::USmallint, + DataValue::UInt16(5), + DataValue::UInt16(2), + DataValue::UInt16(2), + ), + ( + LogicalType::UInteger, + DataValue::UInt32(5), + DataValue::UInt32(2), + DataValue::UInt32(2), + ), + ( + LogicalType::UBigint, + DataValue::UInt64(9007199254740993), + DataValue::UInt64(1), + DataValue::UInt64(9007199254740993), + ), + ] { + let divide = binary_create(Cow::Owned(ty), BinaryOperator::Divide)?; + assert_eq!(divide.binary_eval(&left, &right)?, expected); + assert_eq!( + divide.binary_eval(&left, &DataValue::Null)?, + DataValue::Null + ); + } + Ok(()) + } + #[test] fn test_binary_evaluator_serialization() -> Result<(), DatabaseError> { let evaluator = create(LogicalType::Boolean, BinaryOperator::NotEq)?; diff --git a/src/types/evaluator/float32.rs b/src/types/evaluator/float32.rs index 61ea730e..b7620cc5 100644 --- a/src/types/evaluator/float32.rs +++ b/src/types/evaluator/float32.rs @@ -19,9 +19,6 @@ use crate::types::LogicalType; #[cfg(feature = "decimal")] use rust_decimal::prelude::FromPrimitive; use std::hint; -pub fn float32_plus_unary_eval(value: &DataValue) -> DataValue { - value.clone() -} pub fn float32_minus_unary_eval(value: &DataValue) -> DataValue { match value { DataValue::Float32(value) => DataValue::Float32(-value), @@ -195,7 +192,6 @@ mod test { let left = DataValue::Float32(OrderedFloat(5.5)); let right = DataValue::Float32(OrderedFloat(2.0)); - assert_eq!(float32_plus_unary_eval(&left), left); assert_eq!( float32_minus_unary_eval(&left), DataValue::Float32(OrderedFloat(-5.5)) diff --git a/src/types/evaluator/float64.rs b/src/types/evaluator/float64.rs index 71e8679b..a1bc68d5 100644 --- a/src/types/evaluator/float64.rs +++ b/src/types/evaluator/float64.rs @@ -26,9 +26,6 @@ use rust_decimal::prelude::FromPrimitive; #[cfg(feature = "decimal")] use rust_decimal::Decimal; use std::hint; -pub fn float64_plus_unary_eval(value: &DataValue) -> DataValue { - value.clone() -} pub fn float64_minus_unary_eval(value: &DataValue) -> DataValue { match value { DataValue::Float64(value) => DataValue::Float64(-value), @@ -194,7 +191,6 @@ mod test { fn test_float64_binary_and_cast_evaluators() { let value = DataValue::Float64(ordered_float::OrderedFloat(1.5)); - assert_eq!(float64_plus_unary_eval(&value), value); assert_eq!( float64_minus_unary_eval(&value), DataValue::Float64(ordered_float::OrderedFloat(-1.5)) diff --git a/src/types/evaluator/unary.rs b/src/types/evaluator/unary.rs index 79e553ad..6c081616 100644 --- a/src/types/evaluator/unary.rs +++ b/src/types/evaluator/unary.rs @@ -25,58 +25,31 @@ use crate::types::evaluator::UnaryEvaluatorRef; use crate::types::LogicalType; use std::borrow::Cow; -const UNARY_INT8_PLUS: u16 = 0; -const UNARY_INT8_MINUS: u16 = 1; -const UNARY_INT16_PLUS: u16 = 2; -const UNARY_INT16_MINUS: u16 = 3; -const UNARY_INT32_PLUS: u16 = 4; -const UNARY_INT32_MINUS: u16 = 5; -const UNARY_INT64_PLUS: u16 = 6; -const UNARY_INT64_MINUS: u16 = 7; -const UNARY_BOOLEAN_NOT: u16 = 8; -const UNARY_FLOAT32_PLUS: u16 = 9; -const UNARY_FLOAT32_MINUS: u16 = 10; -const UNARY_FLOAT64_PLUS: u16 = 11; -const UNARY_FLOAT64_MINUS: u16 = 12; -// Any unary operator on the NULL literal yields NULL, as in PostgreSQL. -const UNARY_SQL_NULL: u16 = 13; - -// Evaluator positions are serialized ABI. Do not reorder or reuse existing -// positions; only append new positions at the end of the current layout. - -fn numeric_unary_ref( - plus: u16, - minus: u16, - ty: &LogicalType, - op: UnaryOperator, -) -> Result { - let pos = match op { - UnaryOperator::Plus => plus, - UnaryOperator::Minus => minus, - _ => return Err(DatabaseError::UnsupportedUnaryOperator(ty.clone(), op)), - }; - Ok(UnaryEvaluatorRef::new(pos)) -} +const UNARY_INT8_MINUS: u16 = 0; +const UNARY_INT16_MINUS: u16 = 1; +const UNARY_INT32_MINUS: u16 = 2; +const UNARY_INT64_MINUS: u16 = 3; +const UNARY_BOOLEAN_NOT: u16 = 4; +const UNARY_FLOAT32_MINUS: u16 = 5; +const UNARY_FLOAT64_MINUS: u16 = 6; +const UNARY_SQL_NULL: u16 = 7; pub fn unary_create( ty: Cow<'_, LogicalType>, op: UnaryOperator, ) -> Result { - let ty = ty.as_ref(); - match ty { - LogicalType::Tinyint => numeric_unary_ref(UNARY_INT8_PLUS, UNARY_INT8_MINUS, ty, op), - LogicalType::Smallint => numeric_unary_ref(UNARY_INT16_PLUS, UNARY_INT16_MINUS, ty, op), - LogicalType::Integer => numeric_unary_ref(UNARY_INT32_PLUS, UNARY_INT32_MINUS, ty, op), - LogicalType::Bigint => numeric_unary_ref(UNARY_INT64_PLUS, UNARY_INT64_MINUS, ty, op), - LogicalType::Boolean => match op { - UnaryOperator::Not => Ok(UnaryEvaluatorRef::new(UNARY_BOOLEAN_NOT)), - _ => Err(DatabaseError::UnsupportedUnaryOperator(ty.clone(), op)), - }, - LogicalType::Float => numeric_unary_ref(UNARY_FLOAT32_PLUS, UNARY_FLOAT32_MINUS, ty, op), - LogicalType::Double => numeric_unary_ref(UNARY_FLOAT64_PLUS, UNARY_FLOAT64_MINUS, ty, op), - LogicalType::SqlNull => Ok(UnaryEvaluatorRef::new(UNARY_SQL_NULL)), - _ => Err(DatabaseError::UnsupportedUnaryOperator(ty.clone(), op)), - } + let pos = match (ty.as_ref(), op) { + (LogicalType::Tinyint, UnaryOperator::Minus) => UNARY_INT8_MINUS, + (LogicalType::Smallint, UnaryOperator::Minus) => UNARY_INT16_MINUS, + (LogicalType::Integer, UnaryOperator::Minus) => UNARY_INT32_MINUS, + (LogicalType::Bigint, UnaryOperator::Minus) => UNARY_INT64_MINUS, + (LogicalType::Boolean, UnaryOperator::Not) => UNARY_BOOLEAN_NOT, + (LogicalType::Float, UnaryOperator::Minus) => UNARY_FLOAT32_MINUS, + (LogicalType::Double, UnaryOperator::Minus) => UNARY_FLOAT64_MINUS, + (LogicalType::SqlNull, UnaryOperator::Minus | UnaryOperator::Not) => UNARY_SQL_NULL, + _ => return Err(DatabaseError::UnsupportedUnaryOperator(ty.into_owned(), op)), + }; + Ok(UnaryEvaluatorRef::new(pos)) } pub(crate) fn eval_unary( @@ -84,18 +57,12 @@ pub(crate) fn eval_unary( value: &crate::types::value::DataValue, ) -> Result { Ok(match pos { - UNARY_INT8_PLUS => int8_plus_unary_eval(value), UNARY_INT8_MINUS => int8_minus_unary_eval(value)?, - UNARY_INT16_PLUS => int16_plus_unary_eval(value), UNARY_INT16_MINUS => int16_minus_unary_eval(value)?, - UNARY_INT32_PLUS => int32_plus_unary_eval(value), UNARY_INT32_MINUS => int32_minus_unary_eval(value)?, - UNARY_INT64_PLUS => int64_plus_unary_eval(value), UNARY_INT64_MINUS => int64_minus_unary_eval(value)?, UNARY_BOOLEAN_NOT => boolean_not_unary_eval(value), - UNARY_FLOAT32_PLUS => float32_plus_unary_eval(value), UNARY_FLOAT32_MINUS => float32_minus_unary_eval(value), - UNARY_FLOAT64_PLUS => float64_plus_unary_eval(value), UNARY_FLOAT64_MINUS => float64_minus_unary_eval(value), UNARY_SQL_NULL => crate::types::value::DataValue::Null, _ => unreachable!("unknown unary evaluator position {pos}"), @@ -106,11 +73,6 @@ pub(crate) fn eval_unary( macro_rules! numeric_unary_evaluator_definition { ($value_type:ident, $compute_type:path) => { paste::paste! { - pub fn [<$value_type:snake _plus_unary_eval>]( - value: &$crate::types::value::DataValue, - ) -> $crate::types::value::DataValue { - value.clone() - } // `-MIN` does not fit the type: report it like binary `+`/`-` do // instead of panicking (debug) or wrapping (release). pub fn [<$value_type:snake _minus_unary_eval>]( @@ -154,7 +116,7 @@ mod test { } #[test] - fn test_unary_evaluator_positions_are_stable() -> Result<(), DatabaseError> { + fn test_unary_evaluator_positions() -> Result<(), DatabaseError> { assert_eq!( create(LogicalType::Integer, UnaryOperator::Minus)?.pos, UNARY_INT32_MINUS @@ -164,8 +126,8 @@ mod test { UNARY_BOOLEAN_NOT ); assert_eq!( - create(LogicalType::Double, UnaryOperator::Plus)?.pos, - UNARY_FLOAT64_PLUS + create(LogicalType::Double, UnaryOperator::Minus)?.pos, + UNARY_FLOAT64_MINUS ); Ok(()) @@ -182,60 +144,30 @@ mod test { #[test] fn test_numeric_unary_evaluators() -> Result<(), DatabaseError> { let cases = vec![ - ( - LogicalType::Tinyint, - UnaryOperator::Plus, - DataValue::Int8(7), - DataValue::Int8(7), - ), ( LogicalType::Tinyint, UnaryOperator::Minus, DataValue::Int8(7), DataValue::Int8(-7), ), - ( - LogicalType::Smallint, - UnaryOperator::Plus, - DataValue::Int16(7), - DataValue::Int16(7), - ), ( LogicalType::Smallint, UnaryOperator::Minus, DataValue::Int16(7), DataValue::Int16(-7), ), - ( - LogicalType::Integer, - UnaryOperator::Plus, - DataValue::Int32(7), - DataValue::Int32(7), - ), ( LogicalType::Integer, UnaryOperator::Minus, DataValue::Int32(7), DataValue::Int32(-7), ), - ( - LogicalType::Bigint, - UnaryOperator::Plus, - DataValue::Int64(7), - DataValue::Int64(7), - ), ( LogicalType::Bigint, UnaryOperator::Minus, DataValue::Int64(7), DataValue::Int64(-7), ), - ( - LogicalType::Float, - UnaryOperator::Plus, - DataValue::Float32(OrderedFloat(1.5)), - DataValue::Float32(OrderedFloat(1.5)), - ), ( LogicalType::Double, UnaryOperator::Minus, diff --git a/tests/slt/basic_test.slt b/tests/slt/basic_test.slt index 6a8e6c72..48ef1f7e 100644 --- a/tests/slt/basic_test.slt +++ b/tests/slt/basic_test.slt @@ -3,6 +3,32 @@ select 1 ---- 1 +query IIIII +select 5/2, -5/2, 5/-2, -5/-2, 66 + 15/-71; +---- +2 -2 -2 2 66 + +query I +select cast(9007199254740993 as bigint)/1; +---- +9007199254740993 + +query RRR +select 5.0/2, 5/cast(2 as double), cast(5 as float)/cast(2 as float) + cast(1 as float); +---- +2.5 2.5 3.5 + +query II +select null/2, 2/null; +---- +NULL NULL + +statement error division by zero +select 1/0; + +statement error over flow +select cast(-2147483648 as int)/-1; + query R select 10000.00::FLOAT + 234.567::FLOAT ---- diff --git a/tests/slt/char.slt b/tests/slt/char.slt index 75938baf..12bbe883 100644 --- a/tests/slt/char.slt +++ b/tests/slt/char.slt @@ -40,6 +40,31 @@ select * from t2; 0 🪁 🪁🪁🪁🪁🪁 🪁 1 NULL NULL NULL +query TTT +select +v1, ++v2, +v3 from t1 order by id; +---- +🪁 🪁🪁🪁🪁🪁 🪁 +NULL NULL NULL + +query TTT +select +v1, ++v2, +v3 from t2 order by id; +---- +🪁 🪁🪁🪁🪁🪁 🪁 +NULL NULL NULL + +query TTT +select +'abc', +'12', +cast('abc' as char(3)); +---- +abc 12 abc + +query I +select id from t2 where +v1 is null; +---- +1 + +statement error +select -v1 from t2; + statement ok drop table t1; diff --git a/tests/slt/crdb/and_or.slt b/tests/slt/crdb/and_or.slt index a54e57da..32ed8e0d 100644 --- a/tests/slt/crdb/and_or.slt +++ b/tests/slt/crdb/and_or.slt @@ -7,6 +7,16 @@ CREATE TABLE t (k INT PRIMARY KEY null, a INT null, b INT null) statement ok INSERT INTO t VALUES (1, NULL, NULL), (2, NULL, 1), (3, 1, NULL), (4, 2, 0), (5, 3, 3) +statement error division by zero +SELECT a = 2 AND 3 / b = 1 FROM t WHERE k = 4; + +statement error division by zero +SELECT a <> 2 OR 3 / b = 1 FROM t WHERE k = 4; + +statement error division by zero +SELECT NULL AND 3 / b = 1 FROM t WHERE k = 4; + +# The zero divisor on k=4 is skipped for a decisive AND/OR left operand. query T SELECT a <> 2 AND 3 / b = 1 FROM t ORDER BY k ---- diff --git a/tests/slt/where.slt b/tests/slt/where.slt index 71340b73..82c6ee46 100644 --- a/tests/slt/where.slt +++ b/tests/slt/where.slt @@ -146,6 +146,7 @@ query I rowsort select id from t where b / 2 = 1; ---- 1 +2 # more than one column query I rowsort diff --git a/tests/sqllogictest/Cargo.toml b/tests/sqllogictest/Cargo.toml index 9ee0f0c6..1c22a89d 100644 --- a/tests/sqllogictest/Cargo.toml +++ b/tests/sqllogictest/Cargo.toml @@ -2,6 +2,7 @@ name = "sqllogictest-test" version = "0.4.0" edition = "2021" +default-run = "sqllogictest-test" [dependencies] clap = { version = "4" } From 64822cfb205e5a674df9dd23870637f4cec05ad3 Mon Sep 17 00:00:00 2001 From: kould Date: Wed, 7 Oct 2026 08:51:15 +0800 Subject: [PATCH 09/16] perf: recognize equijoins in filtered cross joins Push single-input predicates through cross joins and extract conjunctive equality predicates as inner-join keys, retaining residual filters and localizing right-side positions. Share join-key extraction with explicit JOIN binding. Raise the predicate-pushdown iteration cap to 30 for deeper join trees. Cover plans and query results with join SLT regressions. This does not implement join-order optimization. --- src/binder/select.rs | 147 +----------------- src/db.rs | 2 +- .../rule/normalization/pushdown_predicates.rs | 80 +++++++++- src/planner/operator/join.rs | 147 +++++++++++++++++- tests/slt/crdb/join.slt | 81 ++++++++++ 5 files changed, 307 insertions(+), 150 deletions(-) diff --git a/src/binder/select.rs b/src/binder/select.rs index d185d3b8..c93f54e4 100644 --- a/src/binder/select.rs +++ b/src/binder/select.rs @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +use crate::planner::operator::join::extract_join_keys; use crate::planner::MetaArena; use crate::{ expression::ScalarExpression, @@ -1865,7 +1866,7 @@ impl<'a: 'b, 'b, T: Transaction, A: AsRef<[(usize, LogicalType)]>> Binder<'a, 'b // expression that didn't match equi-join pattern let mut filter = vec![]; - Self::extract_join_keys( + extract_join_keys( expr, &mut on_keys, &mut filter, @@ -1994,150 +1995,6 @@ impl<'a: 'b, 'b, T: Transaction, A: AsRef<[(usize, LogicalType)]>> Binder<'a, 'b } } } - - /// for sqlrs - /// original idea from datafusion planner.rs - /// Extracts equijoin ON condition be a single Eq or multiple conjunctive Eqs - /// Filters matching this pattern are added to `accum` - /// Filters that don't match this pattern are added to `accum_filter` - /// Examples: - /// ```text - /// foo = bar => accum=[(foo, bar)] accum_filter=[] - /// foo = bar AND bar = baz => accum=[(foo, bar), (bar, baz)] accum_filter=[] - /// foo = bar AND baz > 1 => accum=[(foo, bar)] accum_filter=[baz > 1] - /// ``` - fn extract_join_keys( - expr: ExprRef, - accum: &mut Vec<(ExprRef, ExprRef)>, - accum_filter: &mut Vec, - left_schema: &Schema, - right_schema: &Schema, - arena: &mut PlanArena, - ) -> Result<(), DatabaseError> { - let fn_contains = |schema: &Schema, column: ColumnRef| { - let summary = arena.column(column).summary(); - schema - .iter() - .any(|candidate| arena.column(*candidate).summary() == summary) - }; - let fn_or_contains = |column: ColumnRef| { - fn_contains(left_schema, column) || fn_contains(right_schema, column) - }; - - let expr = expr.unpack_alias(arena); - match arena.expression(expr) { - ScalarExpression::Binary { - left_expr, - right_expr, - op, - .. - } => { - match op { - BinaryOperator::Eq => { - match ( - left_expr.unpack_alias_ref(arena), - right_expr.unpack_alias_ref(arena), - ) { - // example: foo = bar - ( - ScalarExpression::ColumnRef { column: l, .. }, - ScalarExpression::ColumnRef { column: r, .. }, - ) => { - // reorder left and right joins keys to pattern: (left, right) - let key = if fn_contains(left_schema, *l) - && fn_contains(right_schema, *r) - { - Some((*left_expr, *right_expr)) - } else if fn_contains(left_schema, *r) - && fn_contains(right_schema, *l) - { - Some((*right_expr, *left_expr)) - } else { - if fn_or_contains(*l) || fn_or_contains(*r) { - accum_filter.push(expr); - } - None - }; - // Join keys are compared (and hashed) directly, so cast - // both to one type like `l = r` in a filter; otherwise - // e.g. `bigint = int` never matches. - if let Some((left, right)) = key { - let ty = LogicalType::max_logical_type( - &left.return_type(arena), - &right.return_type(arena), - )? - .into_owned(); - accum.push(( - left.type_cast(Cow::Borrowed(&ty), arena)?, - right.type_cast(Cow::Borrowed(&ty), arena)?, - )); - } - } - (ScalarExpression::ColumnRef { column, .. }, _) - | (_, ScalarExpression::ColumnRef { column, .. }) => { - if fn_or_contains(*column) { - accum_filter.push(expr); - } - } - _other => { - // example: baz > 1 - if left_expr.all_referenced_columns(arena, |_, column| { - fn_or_contains(*column) - })? && right_expr - .all_referenced_columns(arena, |_, column| { - fn_or_contains(*column) - })? - { - accum_filter.push(expr); - } - } - } - } - BinaryOperator::And => { - // example: foo = bar AND baz > 1 - let (left_expr, right_expr) = (*left_expr, *right_expr); - Self::extract_join_keys( - left_expr, - accum, - accum_filter, - left_schema, - right_schema, - arena, - )?; - Self::extract_join_keys( - right_expr, - accum, - accum_filter, - left_schema, - right_schema, - arena, - )?; - } - BinaryOperator::Or => { - accum_filter.push(expr); - } - _ => { - if left_expr - .all_referenced_columns(arena, |_, column| fn_or_contains(*column))? - && right_expr.all_referenced_columns(arena, |_, column| { - fn_or_contains(*column) - })? - { - accum_filter.push(expr); - } - } - } - } - _ => { - if expr.all_referenced_columns(arena, |_, column| fn_or_contains(*column))? { - // example: baz > 1 - accum_filter.push(expr); - } - } - } - - Ok(()) - } } #[cfg(all(test, not(target_arch = "wasm32")))] diff --git a/src/db.rs b/src/db.rs index c2c4b135..e2a5cc7a 100644 --- a/src/db.rs +++ b/src/db.rs @@ -339,7 +339,7 @@ fn optimizer_pipeline() -> HepOptimizerPipeline { ) .before_batch( "Predicate Pushdown".to_string(), - HepBatchStrategy::fix_point_topdown(10), + HepBatchStrategy::fix_point_topdown(30), vec![ NormalizationRuleImpl::PushPredicateThroughJoin, NormalizationRuleImpl::PushJoinPredicateIntoScan, diff --git a/src/optimizer/rule/normalization/pushdown_predicates.rs b/src/optimizer/rule/normalization/pushdown_predicates.rs index 8eb9ade8..16fd7c0a 100644 --- a/src/optimizer/rule/normalization/pushdown_predicates.rs +++ b/src/optimizer/rule/normalization/pushdown_predicates.rs @@ -19,6 +19,7 @@ use crate::expression::{BinaryOperator, ScalarExpression}; use crate::optimizer::core::rule::NormalizationRule; use crate::optimizer::plan_utils::{replace_with_only_child, wrap_child_with}; use crate::planner::operator::filter::FilterOperator; +use crate::planner::operator::join::extract_join_keys; use crate::planner::operator::join::{JoinCondition, JoinType}; use crate::planner::operator::{Operator, SortOption}; use crate::planner::{Childrens, ExprRef, LogicalPlan, PlanArena}; @@ -159,17 +160,57 @@ impl NormalizationRule for PushPredicateThroughJoin { if !matches!( join_type, - JoinType::Inner | JoinType::LeftOuter | JoinType::RightOuter + JoinType::Inner | JoinType::Cross | JoinType::LeftOuter | JoinType::RightOuter ) { return Ok(false); } - let (left_filter, mut right_filter, common_filter, left_len) = + let (left_filter, mut right_filter, mut common_filter, left_len) = classify_join_filters(filter_op.predicate, join_plan.childrens.as_mut(), arena)?; + if join_type == JoinType::Cross { + if let Some(predicate) = common_filter { + let Childrens::Twins { left, right } = join_plan.childrens.as_mut() else { + return Ok(false); + }; + let left_schema = left.output_schema(arena).clone(); + let right_schema = right.output_schema(arena).clone(); + let mut keys = Vec::new(); + let mut residual = Vec::new(); + extract_join_keys( + predicate, + &mut keys, + &mut residual, + &left_schema, + &right_schema, + arena, + )?; + if !keys.is_empty() { + for (_, right) in &mut keys { + // WHERE uses joined positions; join keys use each input's local positions. + *right = right.clone_expression(arena)?; + PositionShift { + delta: -(left_len as isize), + } + .visit(right, arena)?; + } + if let Operator::Join(op) = &mut join_plan.operator { + op.join_type = JoinType::Inner; + op.on = JoinCondition::On { + on: keys, + filter: None, + }; + } + common_filter = reduce_filters(residual, filter_op.having, arena) + .map(|op| op.predicate); + applied = true; + } + } + } + let mut new_ops = (None, None, None); match join_type { - JoinType::Inner => { + JoinType::Inner | JoinType::Cross => { if let Some(left_filter_op) = reduce_filters(left_filter, filter_op.having, arena) { @@ -1036,6 +1077,39 @@ mod tests { Ok(()) } + #[test] + fn cross_join_without_conjunctive_keys_stays_cross() -> Result<(), DatabaseError> { + let tables = build_t1_table()?; + for predicate in ["c1 = c3 or c2 = c4", "c1 < c3", "c1 > 0 and c3 < 9"] { + let mut arena = PlanArena::new(&tables.table_arena); + let plan = tables.plan_with_arena( + &format!("select * from t1, t2 where {predicate}"), + &mut arena, + )?; + let plan = apply_pipeline( + plan, + HepOptimizerPipeline::builder().before_batch( + "cross join pushdown".into(), + HepBatchStrategy::once_topdown(), + vec![NormalizationRuleImpl::PushPredicateThroughJoin], + ), + &mut arena, + )?; + let child = plan.childrens.only(); + let join = if matches!(child.operator, Operator::Filter(_)) { + child.childrens.only() + } else { + child + }; + let Operator::Join(op) = &join.operator else { + panic!("expected join") + }; + assert_eq!(op.join_type, JoinType::Cross, "{predicate}"); + assert!(matches!(op.on, JoinCondition::None)); + } + Ok(()) + } + #[test] fn test_push_predicate_through_join_in_inner_join() -> Result<(), DatabaseError> { let table_state = build_t1_table()?; diff --git a/src/planner/operator/join.rs b/src/planner/operator/join.rs index ea46b9d2..b2e52293 100644 --- a/src/planner/operator/join.rs +++ b/src/planner/operator/join.rs @@ -13,9 +13,15 @@ // limitations under the License. use super::{Operator, PlanImpl}; +use crate::catalog::ColumnRef; +use crate::errors::DatabaseError; +use crate::expression::{BinaryOperator, ScalarExpression, TypeCast}; use crate::planner::MetaArena; -use crate::planner::{Childrens, Explain, ExprRef, LogicalPlan}; +use crate::planner::{Childrens, Explain, ExprRef, LogicalPlan, PlanArena}; +use crate::types::tuple::Schema; +use crate::types::LogicalType; use kite_sql_serde_macros::ReferenceSerialization; +use std::borrow::Cow; use std::fmt; use std::fmt::Formatter; @@ -127,6 +133,145 @@ impl fmt::Display for JoinType { } } +/// for sqlrs +/// original idea from datafusion planner.rs +/// Extracts equijoin ON condition be a single Eq or multiple conjunctive Eqs +/// Filters matching this pattern are added to `accum` +/// Filters that don't match this pattern are added to `accum_filter` +/// Examples: +/// ```text +/// foo = bar => accum=[(foo, bar)] accum_filter=[] +/// foo = bar AND bar = baz => accum=[(foo, bar), (bar, baz)] accum_filter=[] +/// foo = bar AND baz > 1 => accum=[(foo, bar)] accum_filter=[baz > 1] +/// ``` +pub(crate) fn extract_join_keys( + expr: ExprRef, + accum: &mut Vec<(ExprRef, ExprRef)>, + accum_filter: &mut Vec, + left_schema: &Schema, + right_schema: &Schema, + arena: &mut PlanArena, +) -> Result<(), DatabaseError> { + let fn_contains = |schema: &Schema, column: ColumnRef| { + let summary = arena.column(column).summary(); + schema + .iter() + .any(|candidate| arena.column(*candidate).summary() == summary) + }; + let fn_or_contains = + |column: ColumnRef| fn_contains(left_schema, column) || fn_contains(right_schema, column); + + let expr = expr.unpack_alias(arena); + match arena.expression(expr) { + ScalarExpression::Binary { + left_expr, + right_expr, + op, + .. + } => { + match op { + BinaryOperator::Eq => { + match ( + left_expr.unpack_alias_ref(arena), + right_expr.unpack_alias_ref(arena), + ) { + // example: foo = bar + ( + ScalarExpression::ColumnRef { column: l, .. }, + ScalarExpression::ColumnRef { column: r, .. }, + ) => { + // reorder left and right joins keys to pattern: (left, right) + let key = if fn_contains(left_schema, *l) + && fn_contains(right_schema, *r) + { + Some((*left_expr, *right_expr)) + } else if fn_contains(left_schema, *r) && fn_contains(right_schema, *l) + { + Some((*right_expr, *left_expr)) + } else { + if fn_or_contains(*l) || fn_or_contains(*r) { + accum_filter.push(expr); + } + None + }; + // Join keys are compared (and hashed) directly, so cast + // both to one type like `l = r` in a filter; otherwise + // e.g. `bigint = int` never matches. + if let Some((left, right)) = key { + let ty = LogicalType::max_logical_type( + &left.return_type(arena), + &right.return_type(arena), + )? + .into_owned(); + accum.push(( + left.type_cast(Cow::Borrowed(&ty), arena)?, + right.type_cast(Cow::Borrowed(&ty), arena)?, + )); + } + } + (ScalarExpression::ColumnRef { column, .. }, _) + | (_, ScalarExpression::ColumnRef { column, .. }) => { + if fn_or_contains(*column) { + accum_filter.push(expr); + } + } + _other => { + // example: baz > 1 + if left_expr.all_referenced_columns(arena, |_, column| { + fn_or_contains(*column) + })? && right_expr.all_referenced_columns(arena, |_, column| { + fn_or_contains(*column) + })? { + accum_filter.push(expr); + } + } + } + } + BinaryOperator::And => { + // example: foo = bar AND baz > 1 + let (left_expr, right_expr) = (*left_expr, *right_expr); + extract_join_keys( + left_expr, + accum, + accum_filter, + left_schema, + right_schema, + arena, + )?; + extract_join_keys( + right_expr, + accum, + accum_filter, + left_schema, + right_schema, + arena, + )?; + } + BinaryOperator::Or => { + accum_filter.push(expr); + } + _ => { + if left_expr + .all_referenced_columns(arena, |_, column| fn_or_contains(*column))? + && right_expr + .all_referenced_columns(arena, |_, column| fn_or_contains(*column))? + { + accum_filter.push(expr); + } + } + } + } + _ => { + if expr.all_referenced_columns(arena, |_, column| fn_or_contains(*column))? { + // example: baz > 1 + accum_filter.push(expr); + } + } + } + + Ok(()) +} + #[cfg(test)] mod tests { use super::*; diff --git a/tests/slt/crdb/join.slt b/tests/slt/crdb/join.slt index 3b89057a..b867d43e 100644 --- a/tests/slt/crdb/join.slt +++ b/tests/slt/crdb/join.slt @@ -1187,3 +1187,84 @@ drop table onecolumn_w statement ok drop table str1 + +# Comma/CROSS joins must recognize conjunctive equality keys without losing residual filters. +statement ok +create table ij_a(id int primary key, k int, v int); + +statement ok +create table ij_b(id int primary key, k bigint, v int); + +statement ok +create table ij_c(id int primary key, k int); + +statement ok +insert into ij_a values (1,1,10),(2,2,20),(3,null,30),(4,2,25); + +statement ok +insert into ij_b values (1,2,100),(2,1,200),(3,2,300),(4,null,400); + +statement ok +insert into ij_c values (1,1),(2,2); + +# Both side filters are pushed down, while the cross-side inequality stays above the equijoin. +query T +explain select a.id,b.id from ij_a a,ij_b b where b.k=a.k and a.v>15 and b.v<250 and a.v (Sort Option: Follow)] Filter (a.v < b.v), Is Having: false [Filter => (Sort Option: Follow)] Inner Join On cast (a.k as Bigint) = b.k [HashJoin => (Sort Option: None)] Filter ((ij_a.v) as (a.v) > 15), Is Having: false [Filter => (Sort Option: Follow)] TableScan ij_a -> [ij_a.id, ij_a.k, ij_a.v] [SeqScan => (Sort Option: None)] Filter ((ij_b.v) as (b.v) < 250), Is Having: false [Filter => (Sort Option: Follow)] TableScan ij_b -> [ij_b.id, ij_b.k, ij_b.v] [SeqScan => (Sort Option: None)] + +query II +select a.id,b.id from ij_a a,ij_b b where b.k=a.k and a.v>15 and b.v<250 and a.v320 order by a.id,b.id; +---- +4 3 + +query II +select a.id,b.id from ij_a a,ij_b b where a.k=b.k or a.id=b.id order by a.id,b.id; +---- +1 1 +1 2 +2 1 +2 2 +2 3 +3 3 +4 1 +4 3 +4 4 + +query II +select a.id,b.id from ij_a a,ij_b b where a.id=2 and b.id=1; +---- +2 1 + +query III +select a.id,b.id,c.id from ij_a a,ij_b b,ij_c c where c.k=b.k and b.k=a.k and a.v>15 and c.id=2 order by a.id,b.id,c.id; +---- +2 1 2 +2 3 2 +4 1 2 +4 3 2 + +query II +select a.id,b.id from ij_a a,ij_b b where a.v>b.v order by a.id,b.id; +---- + +query II +select a.id,b.id from ij_a a,ij_a b where b.k=a.k and a.id=2 order by b.id; +---- +2 2 +2 4 + +statement ok +drop table ij_c; + +statement ok +drop table ij_b; + +statement ok +drop table ij_a; From e61fc353714ef6004d32b601d724de78f90a3e46 Mon Sep 17 00:00:00 2001 From: kould Date: Thu, 8 Oct 2026 02:25:52 +0800 Subject: [PATCH 10/16] feat: evaluate correlated scalar subqueries per input row Capture outer parameters separately from scalar results and use OuterParam, OuterValue and InitValue expressions to read execution-local values. Attach scalar initializers before their consumers without tuple result-column placeholders or post-hoc projections. Use an explicit ExecMetaArena on execution paths and confine value updates to it. Model scalar initialization with states and reuse a scratch tuple buffer while running child plans. Retain explicit rejection of correlated ORDER BY, DISTINCT and unsupported outer scopes. Add subquery SLT coverage for stage attachment, missing/NULL/text results, multiple correlations and unsupported boundaries. Gate the LMDB bounds helper to its supported build configurations. --- src/binder/aggregate.rs | 2 +- src/binder/expr.rs | 77 ++++-- src/binder/mod.rs | 33 ++- src/binder/parser.rs | 27 ++- src/binder/select.rs | 161 +++++++++++-- src/db.rs | 22 +- src/execution/ddl/add_column.rs | 6 +- src/execution/ddl/change_column.rs | 6 +- src/execution/ddl/create_index.rs | 6 +- src/execution/ddl/create_table.rs | 6 +- src/execution/ddl/create_view.rs | 6 +- src/execution/ddl/drop_column.rs | 6 +- src/execution/ddl/drop_index.rs | 6 +- src/execution/ddl/drop_table.rs | 6 +- src/execution/ddl/drop_view.rs | 6 +- src/execution/ddl/truncate.rs | 6 +- src/execution/dml/analyze.rs | 6 +- src/execution/dml/copy_from_file.rs | 6 +- src/execution/dml/copy_to_file.rs | 6 +- src/execution/dml/delete.rs | 6 +- src/execution/dml/insert.rs | 6 +- src/execution/dml/update.rs | 6 +- src/execution/dql/aggregate/hash_agg.rs | 6 +- src/execution/dql/aggregate/simple_agg.rs | 6 +- src/execution/dql/aggregate/stream_agg.rs | 6 +- .../dql/aggregate/stream_distinct.rs | 6 +- src/execution/dql/describe.rs | 6 +- src/execution/dql/dummy.rs | 6 +- src/execution/dql/explain.rs | 6 +- src/execution/dql/external_sort.rs | 6 +- src/execution/dql/filter.rs | 6 +- src/execution/dql/function_scan.rs | 6 +- src/execution/dql/index_scan.rs | 6 +- src/execution/dql/join/hash_join.rs | 10 +- src/execution/dql/join/nested_loop_join.rs | 7 +- src/execution/dql/limit.rs | 6 +- src/execution/dql/mark_apply.rs | 23 +- src/execution/dql/projection.rs | 6 +- src/execution/dql/recursive_cte.rs | 14 +- src/execution/dql/scalar_query_init.rs | 163 ++++++++++--- src/execution/dql/scalar_subquery.rs | 6 +- src/execution/dql/seq_scan.rs | 6 +- src/execution/dql/set_membership.rs | 6 +- src/execution/dql/show_table.rs | 6 +- src/execution/dql/show_view.rs | 6 +- src/execution/dql/sort.rs | 6 +- src/execution/dql/top_k.rs | 6 +- src/execution/dql/union.rs | 6 +- src/execution/dql/values.rs | 6 +- src/execution/dql/window.rs | 6 +- src/execution/mod.rs | 38 ++- src/expression/eq_col.rs | 24 +- src/expression/evaluator.rs | 12 +- src/expression/mod.rs | 18 +- src/expression/range_detacher.rs | 8 +- src/expression/visitor.rs | 24 +- src/expression/visitor_mut.rs | 24 +- .../rule/normalization/column_pruning.rs | 14 ++ src/orm/mod.rs | 4 +- src/planner/arena.rs | 58 +++-- src/planner/mod.rs | 2 +- src/planner/operator/mod.rs | 29 ++- src/planner/operator/scalar_query_init.rs | 32 ++- src/planner/operator/visitor.rs | 16 +- src/planner/operator/visitor_mut.rs | 13 +- src/storage/mod.rs | 1 + tests/slt/subquery.slt | 219 +++++++++++++++++- 67 files changed, 996 insertions(+), 313 deletions(-) diff --git a/src/binder/aggregate.rs b/src/binder/aggregate.rs index 36be4085..5fb781ab 100644 --- a/src/binder/aggregate.rs +++ b/src/binder/aggregate.rs @@ -214,7 +214,7 @@ impl> Binder<'_, '_, T, A> { unmatched_group_exprs.retain(|expr| { !matches!( arena.expression(expr.unpack_alias(arena)), - ScalarExpression::Init { .. } + ScalarExpression::InitValue { .. } ) }); if !unmatched_group_exprs.is_empty() { diff --git a/src/binder/expr.rs b/src/binder/expr.rs index 5b6d9c3c..f994a8d6 100644 --- a/src/binder/expr.rs +++ b/src/binder/expr.rs @@ -18,7 +18,7 @@ use crate::expression; use crate::expression::agg::AggKind; use crate::iter_ext::Itertools; -use super::{Binder, BinderContext, QueryBindStep, SubQueryType}; +use super::{Binder, BinderContext, BoundScalarQuery, QueryBindStep, SubQueryType}; use crate::expression::function::scala::{ArcScalarFunctionImpl, ScalarFunction}; use crate::expression::function::table::TableFunction; use crate::expression::function::FunctionSummary; @@ -147,7 +147,7 @@ impl<'a, T: Transaction, A: AsRef<[(usize, LogicalType)]>> Binder<'a, '_, T, A> { let mut binder = Binder::new(self.context.fork_empty(), self.args, Some(&self.context)); let sub_query = build(&mut binder, arena)?; - let correlated = binder.context.has_outer_refs(); + let correlated = binder.context.has_join_outer_ref(); Ok((sub_query, correlated)) } @@ -206,19 +206,42 @@ impl<'a, T: Transaction, A: AsRef<[(usize, LogicalType)]>> Binder<'a, '_, T, A> &mut PlanArena<'arena>, ) -> Result, { - let (sub_query, column, correlated) = - self.bind_subquery_plan_with_output(None, arena, build)?; - if correlated { + let mut child_context = self.context.fork_empty(); + child_context.capture_scalar_outer = true; + let mut binder = Binder::new(child_context, self.args, Some(&self.context)); + let mut sub_query = build(&mut binder, arena)?; + let param_bindings = binder.context.scalar_outer_bindings; + if !param_bindings.is_empty() && self.context.capture_scalar_outer { return Err(DatabaseError::UnsupportedStmt( - "correlated scalar subqueries are not supported".to_string(), + "nested correlated scalar queries are not supported yet".into(), + )); + } + if !param_bindings.is_empty() && self.context.step_now() != QueryBindStep::Project { + return Err(DatabaseError::UnsupportedStmt( + "correlated scalar subqueries currently require the SELECT list".into(), + )); + } + let schema = sub_query.output_schema(arena); + if schema.len() != 1 { + return Err(DatabaseError::MisMatch( + "expects only one expression to be returned", + "the expression returned by the subquery", )); } + let ty = arena.column(schema[0]).datatype().clone(); let id = arena.alloc_scalar_query_ref(); - let ty = column.return_type(arena).into_owned(); - self.context - .scalar_queries - .push((id, ScalarSubqueryOperator::build(sub_query))); - Ok(ScalarExpression::Init { id, ty }) + let value = arena.alloc_expression(if param_bindings.is_empty() { + ScalarExpression::InitValue { id, ty: ty.clone() } + } else { + ScalarExpression::OuterValue { id, ty: ty.clone() } + }); + self.context.scalar_queries.push(BoundScalarQuery { + value, + step: self.context.step_now(), + plan: ScalarSubqueryOperator::build(sub_query), + param_bindings, + }); + Ok(arena.expression(value).clone()) } pub(crate) fn bind_exists_subquery_plan<'arena, F>( @@ -360,12 +383,14 @@ impl<'a, T: Transaction, A: AsRef<[(usize, LogicalType)]>> Binder<'a, '_, T, A> try_default!(&table_name, column_name); } if let Some(table) = table_name.or(bind_table_name) { + let mut is_outer = false; let (source, position_offset) = match Self::resolve_source_columns_in_scope(&self.context, table) { Ok(source) => source, Err(err) => { if let Some(parent) = self.parent { - self.context.mark_outer_ref(); + self.context.mark_join_outer_ref(); + is_outer = true; Self::resolve_source_columns_in_scope(parent, table).map_err(|_| err)? } else { return Err(err); @@ -376,10 +401,11 @@ impl<'a, T: Transaction, A: AsRef<[(usize, LogicalType)]>> Binder<'a, '_, T, A> Self::find_column_in_schema(source.schema().iter(), arena, column_name) .ok_or_else(|| DatabaseError::column_not_found(column_name.to_string()))?; - Ok(ScalarExpression::column_expr( - column, - position_offset + position, - )) + let expr = ScalarExpression::column_expr(column, position_offset + position); + if is_outer && self.context.capture_scalar_outer { + return self.capture_scalar_outer(expr, arena); + } + Ok(expr) } else { // handle col syntax let mut find_visible_column = @@ -392,8 +418,13 @@ impl<'a, T: Transaction, A: AsRef<[(usize, LogicalType)]>> Binder<'a, '_, T, A> let mut got_column = find_visible_column(&self.context)?; if got_column.is_none() { if let Some(parent) = self.parent { - self.context.mark_outer_ref(); + self.context.mark_join_outer_ref(); got_column = find_visible_column(parent)?; + if self.context.capture_scalar_outer { + if let Some(expr) = got_column { + return self.capture_scalar_outer(expr, arena); + } + } } } match got_column { @@ -403,6 +434,18 @@ impl<'a, T: Transaction, A: AsRef<[(usize, LogicalType)]>> Binder<'a, '_, T, A> } } + fn capture_scalar_outer( + &mut self, + expr: ScalarExpression, + arena: &mut PlanArena, + ) -> Result { + let ty = expr.return_type(arena).into_owned(); + let id = arena.alloc_scalar_query_ref(); + let source = arena.alloc_expression(expr); + self.context.scalar_outer_bindings.push((id, source)); + Ok(ScalarExpression::OuterParam { id, ty }) + } + pub(crate) fn bind_binary_op_expr( &mut self, left_expr: ExprRef, diff --git a/src/binder/mod.rs b/src/binder/mod.rs index 30aed1a2..aae02188 100644 --- a/src/binder/mod.rs +++ b/src/binder/mod.rs @@ -235,6 +235,13 @@ impl UsingColumn { } } +pub(crate) struct BoundScalarQuery { + value: ExprRef, + step: QueryBindStep, + plan: LogicalPlan, + param_bindings: Vec<(ScalarQueryRef, ExprRef)>, +} + pub struct BinderContext<'a, T: Transaction> { pub(crate) scala_functions: &'a ScalaFunctions, pub(crate) table_functions: &'a TableFunctions, @@ -254,12 +261,14 @@ pub struct BinderContext<'a, T: Transaction> { pub(crate) agg_calls: Vec, // join using: HashMap, - - bind_step: QueryBindStep, + has_join_outer_ref: bool, + // subquery sub_queries: HashMap>, - scalar_queries: Vec<(ScalarQueryRef, LogicalPlan)>, - has_outer_refs: bool, + scalar_queries: Vec, + capture_scalar_outer: bool, + scalar_outer_bindings: Vec<(ScalarQueryRef, ExprRef)>, + bind_step: QueryBindStep, pub(crate) allow_default: bool, } @@ -321,7 +330,9 @@ impl<'a, T: Transaction> BinderContext<'a, T> { bind_step: QueryBindStep::From, sub_queries: Default::default(), scalar_queries: Vec::new(), - has_outer_refs: false, + capture_scalar_outer: false, + scalar_outer_bindings: Vec::new(), + has_join_outer_ref: false, allow_default: false, } } @@ -348,7 +359,9 @@ impl<'a, T: Transaction> BinderContext<'a, T> { bind_step: self.bind_step, sub_queries: Default::default(), scalar_queries: Vec::new(), - has_outer_refs: false, + capture_scalar_outer: false, + scalar_outer_bindings: Vec::new(), + has_join_outer_ref: false, allow_default: self.allow_default, } } @@ -429,12 +442,12 @@ impl<'a, T: Transaction> BinderContext<'a, T> { self.sub_queries.remove(&self.bind_step) } - pub fn mark_outer_ref(&mut self) { - self.has_outer_refs = true; + pub fn mark_join_outer_ref(&mut self) { + self.has_join_outer_ref = true; } - pub fn has_outer_refs(&self) -> bool { - self.has_outer_refs + pub fn has_join_outer_ref(&self) -> bool { + self.has_join_outer_ref } pub fn table(&self, table_name: TableName) -> Result, DatabaseError> { diff --git a/src/binder/parser.rs b/src/binder/parser.rs index 702248b4..2dbafd27 100644 --- a/src/binder/parser.rs +++ b/src/binder/parser.rs @@ -37,7 +37,7 @@ use crate::planner::operator::recursive_cte::{RecursiveCteOperator, RecursiveSca use crate::planner::operator::sort::SortField; use crate::planner::operator::Operator; use crate::planner::MetaArena; -use crate::planner::{Childrens, ExecArenaView, ExprRef, LogicalPlan, PlanArena}; +use crate::planner::{Childrens, ExprRef, LogicalPlan, PlanArena}; use crate::storage::{Storage, Transaction}; use crate::types::value::{DataValue, Utf8Type}; use crate::types::{CharLengthUnits, ColumnId, LogicalType}; @@ -182,7 +182,7 @@ impl Database { pub fn run_mut(&mut self, sql: impl AsRef, consume: C) -> Result where C: for<'a> FnOnce( - &mut TransactionIter<'a, S::TransactionType<'a>, ExecArenaView>>, + &mut TransactionIter<'a, S::TransactionType<'a>, PlanArena<'a>>, ) -> Result, { let sql = sql.as_ref(); @@ -198,7 +198,7 @@ impl Database { ) -> Result where C: for<'a> FnOnce( - &mut TransactionIter<'a, S::TransactionType<'a>, ExecArenaView>>, + &mut TransactionIter<'a, S::TransactionType<'a>, PlanArena<'a>>, ) -> Result, { let last = statements @@ -277,7 +277,7 @@ impl Database { } else { let inner = Box::into_raw(Box::new(TransactionIter::new( schema, - Box::new(plan_arena) as Box, + plan_arena.boxed(), executor, transaction, ))); @@ -1256,6 +1256,12 @@ where Childrens::Only(Box::new(plan)), ); } + let steps = [ + QueryBindStep::From, + QueryBindStep::Where, + QueryBindStep::Project, + ]; + plan = self.binder.bind_scalar_queries(plan, &steps, self.arena)?; self.binder.bind_update(table_name, value_exprs, plan) } else { Err(DatabaseError::UnsupportedStmt(format!( @@ -2720,8 +2726,8 @@ impl<'a, 'parent, T: Transaction, A: AsRef<[(usize, LogicalType)]>> Binder<'a, ' let left_plan = { let mut left_binder = Binder::new(self.context.fork(), self.args, self.parent); let plan = left_binder.bind_set_expr(left, arena)?; - if left_binder.context.has_outer_refs() { - self.context.mark_outer_ref(); + if left_binder.context.has_join_outer_ref() { + self.context.mark_join_outer_ref(); } plan }; @@ -2729,8 +2735,8 @@ impl<'a, 'parent, T: Transaction, A: AsRef<[(usize, LogicalType)]>> Binder<'a, ' let right_plan = { let mut right_binder = Binder::new(self.context.fork(), self.args, self.parent); let plan = right_binder.bind_set_expr(right, arena)?; - if right_binder.context.has_outer_refs() { - self.context.mark_outer_ref(); + if right_binder.context.has_join_outer_ref() { + self.context.mark_join_outer_ref(); } plan }; @@ -2902,8 +2908,7 @@ impl<'a, 'parent, T: Transaction, A: AsRef<[(usize, LogicalType)]>> Binder<'a, ' arena.alloc_expression(predicate) })?; - let plan = self.bind_where_expr(children, predicate, arena)?; - Ok(self.init_scalar_queries(plan)) + self.bind_where_expr(children, predicate, arena) } pub(crate) fn normalize_select_item( @@ -3207,7 +3212,7 @@ impl<'a, 'parent, T: Transaction, A: AsRef<[(usize, LogicalType)]>> Binder<'a, ' self.context.restore_ctes(cte_checkpoint); self.context.step(origin_step); - Ok(self.init_scalar_queries(plan)) + Ok(plan) } fn bind_non_negative_limit_value( diff --git a/src/binder/select.rs b/src/binder/select.rs index c93f54e4..fa20484b 100644 --- a/src/binder/select.rs +++ b/src/binder/select.rs @@ -32,6 +32,7 @@ use super::{Binder, BinderContext, QueryBindStep, SetOperatorKind, Source, SubQu use crate::catalog::{ColumnRef, ColumnRelation, TableName}; use crate::errors::DatabaseError; use crate::execution::dql::join::joins_nullable; +use crate::expression::visitor::ExprVisitor; use crate::expression::visitor_mut::{walk_mut_expr, ExprVisitorMut, PositionShift}; use crate::expression::{AliasType, BinaryOperator, TypeCast}; use crate::iter_ext::Itertools; @@ -41,7 +42,7 @@ use crate::planner::operator::join::JoinCondition; use crate::planner::operator::set_membership::{SetMembershipKind, SetMembershipOperator}; use crate::planner::operator::sort::{SortField, SortOperator}; use crate::planner::operator::union::UnionOperator; -use crate::planner::{Childrens, ExprRef, LogicalPlan, PlanArena}; +use crate::planner::{Childrens, ExprRef, LogicalPlan, PlanArena, ScalarQueryRef}; use crate::storage::Transaction; use crate::types::tuple::Schema; use crate::types::{ColumnId, LogicalType}; @@ -412,6 +413,17 @@ where #[cfg(feature = "orm")] pub fn finish(self) -> Result { + if self + .binder + .context + .scalar_queries + .iter() + .any(|query| !query.param_bindings.is_empty()) + { + return Err(DatabaseError::UnsupportedStmt( + "correlated scalar queries in ORM projections are not supported yet".into(), + )); + } for expr in &self.select_list { if expr.has_agg_call(self.arena)? || expr.has_window_call(self.arena)? { return self.aggregate_without_group()?.finish(); @@ -420,7 +432,7 @@ where let plan = self .binder .bind_project(self.plan, self.select_list, self.arena)?; - Ok(self.binder.init_scalar_queries(plan)) + Ok(plan) } } @@ -489,6 +501,30 @@ where self.binder .extract_select_aggregate(&mut self.select_list, self.arena)?; + // Statement-constant scalar values needed by grouping/aggregate arguments are initialized + // before the aggregate, without turning them into input columns. + self.plan = + self.binder + .bind_scalar_queries(self.plan, &[QueryBindStep::Agg], self.arena)?; + if self + .binder + .context + .scalar_queries + .iter() + .any(|query| !query.param_bindings.is_empty()) + { + if !group_by.is_empty() || !self.binder.context.agg_calls.is_empty() { + return Err(DatabaseError::UnsupportedStmt( + "correlated scalar queries with outer aggregation are not supported yet".into(), + )); + } + } else if !group_by.is_empty() || !self.binder.context.agg_calls.is_empty() { + self.plan = self.binder.bind_scalar_queries( + self.plan, + &[QueryBindStep::Project], + self.arena, + )?; + } if !group_by.is_empty() { self.binder.extract_group_by_aggregate_exprs( &mut self.select_list, @@ -511,6 +547,17 @@ where if !self.binder.context.agg_calls.is_empty() || !self.binder.context.group_by_exprs.is_empty() { + if self + .binder + .context + .scalar_queries + .iter() + .any(|q| !q.param_bindings.is_empty()) + { + return Err(DatabaseError::UnsupportedStmt( + "correlated scalar queries with outer aggregation are not supported yet".into(), + )); + } let agg_calls = std::mem::take(&mut self.binder.context.agg_calls); let group_by_exprs = std::mem::take(&mut self.binder.context.group_by_exprs); let output_exprs = self @@ -575,6 +622,43 @@ where pub(crate) fn window( mut self, ) -> Result, DatabaseError> { + if self + .binder + .context + .scalar_queries + .iter() + .any(|query| !query.param_bindings.is_empty()) + { + for expr in self.select_list.iter().chain( + self.orderby + .iter() + .flat_map(|fields| fields.iter().map(|field| &field.expr)), + ) { + if expr.has_window_call(self.arena)? { + return Err(DatabaseError::UnsupportedStmt( + "correlated scalar queries with outer windows are not supported yet".into(), + )); + } + } + } + if self.orderby.is_some() + && self + .binder + .context + .scalar_queries + .iter() + .any(|query| !query.param_bindings.is_empty()) + { + return Err(DatabaseError::UnsupportedStmt( + "correlated scalar values across ORDER BY require a row-scoped execution context" + .into(), + )); + } + self.plan = self.binder.bind_scalar_queries( + self.plan, + &[QueryBindStep::Project, QueryBindStep::Sort], + self.arena, + )?; self.plan = self.binder.bind_window( self.plan, &mut self.select_list, @@ -602,6 +686,25 @@ where distinct: bool, ) -> Result, DatabaseError> { if distinct { + struct OuterResult(bool); + impl ExprVisitor for OuterResult { + fn visit_outer_value( + &mut self, + _id: ScalarQueryRef, + _ty: &LogicalType, + _arena: &(dyn MetaArena + '_), + ) -> Result<(), DatabaseError> { + self.0 = true; + Ok(()) + } + } + let mut outer = OuterResult(false); + for expr in &self.select_list { + ExprVisitor::visit(&mut outer, *expr, self.arena)?; + } + if outer.0 { + return Err(DatabaseError::UnsupportedStmt("correlated scalar values across DISTINCT require a row-scoped execution context".into())); + } let distinct_outputs = self.select_list.clone(); self.binder.bind_distinct_output_exprs( &distinct_outputs, @@ -661,7 +764,7 @@ where } Ok(BindPlanProjected { - plan: self.binder.init_scalar_queries(self.plan), + plan: self.plan, _marker: std::marker::PhantomData, }) } @@ -698,12 +801,30 @@ impl BindPlanComplete { } impl<'a: 'b, 'b, T: Transaction, A: AsRef<[(usize, LogicalType)]>> Binder<'a, 'b, T, A> { + pub(crate) fn bind_scalar_queries( + &mut self, + mut plan: LogicalPlan, + on_steps: &[QueryBindStep], + _arena: &mut PlanArena, + ) -> Result { + for query in self + .context + .scalar_queries + .extract_if(.., |query| on_steps.contains(&query.step)) + { + plan = + ScalarQueryInitOperator::build(plan, query.plan, query.value, query.param_bindings); + } + Ok(plan) + } + pub(crate) fn init_scalar_queries(&mut self, mut plan: LogicalPlan) -> LogicalPlan { - for (id, init) in std::mem::take(&mut self.context.scalar_queries) + for query in std::mem::take(&mut self.context.scalar_queries) .into_iter() .rev() { - plan = ScalarQueryInitOperator::build(plan, init, id); + plan = + ScalarQueryInitOperator::build(plan, query.plan, query.value, query.param_bindings); } plan } @@ -1327,6 +1448,7 @@ impl<'a: 'b, 'b, T: Transaction, A: AsRef<[(usize, LogicalType)]>> Binder<'a, 'b predicate: ExprRef, arena: &mut PlanArena, ) -> Result { + children = self.bind_scalar_queries(children, &[QueryBindStep::Where], arena)?; self.context.step(QueryBindStep::Where); if predicate.has_agg_call(arena)? { return Err(DatabaseError::AggMiss( @@ -1745,10 +1867,11 @@ impl<'a: 'b, 'b, T: Transaction, A: AsRef<[(usize, LogicalType)]>> Binder<'a, 'b fn bind_having( &mut self, - children: LogicalPlan, + mut children: LogicalPlan, mut having: ExprRef, arena: &mut PlanArena, ) -> Result { + children = self.bind_scalar_queries(children, &[QueryBindStep::Having], arena)?; self.context.step(QueryBindStep::Having); self.validate_having_orderby(having, arena)?; @@ -1770,10 +1893,10 @@ impl<'a: 'b, 'b, T: Transaction, A: AsRef<[(usize, LogicalType)]>> Binder<'a, 'b &mut self, children: LogicalPlan, select_list: Vec, - _arena: &mut PlanArena, + arena: &mut PlanArena, ) -> Result { + let children = self.bind_scalar_queries(children, &[QueryBindStep::Project], arena)?; self.context.step(QueryBindStep::Project); - Ok(Self::build_project_plan(children, select_list)) } @@ -1781,10 +1904,10 @@ impl<'a: 'b, 'b, T: Transaction, A: AsRef<[(usize, LogicalType)]>> Binder<'a, 'b &mut self, children: LogicalPlan, sort_fields: Vec, - _arena: &mut PlanArena, + arena: &mut PlanArena, ) -> Result { + let children = self.bind_scalar_queries(children, &[QueryBindStep::Sort], arena)?; self.context.step(QueryBindStep::Sort); - Ok(LogicalPlan::new( Operator::Sort(SortOperator { sort_fields }), Childrens::Only(Box::new(children)), @@ -2182,9 +2305,16 @@ mod tests { "select * from t1 where c1 = (select max(c3) from t2)", &mut arena, )?; - assert!(matches!(plan.operator, Operator::ScalarQueryInit(_))); + assert!(matches!(plan.operator, Operator::Project(_))); assert!(find_join(&plan).is_none()); - let Childrens::Twins { left, right } = plan.childrens.as_mut() else { + let Childrens::Only(filter) = plan.childrens.as_mut() else { + panic!("expected project input") + }; + let Childrens::Only(init) = filter.childrens.as_mut() else { + panic!("expected filter input") + }; + assert!(matches!(init.operator, Operator::ScalarQueryInit(_))); + let Childrens::Twins { left, right } = init.childrens.as_mut() else { panic!("expected init children") }; assert_eq!(left.output_schema(&mut arena).len(), 2); @@ -2267,16 +2397,17 @@ mod tests { "select * from t1 where c1 <= (select 4) and c1 > (select 1)", &mut arena, )?; - let Operator::ScalarQueryInit(first) = &plan.operator else { + let init = plan.childrens.only().childrens.only(); + let Operator::ScalarQueryInit(first) = &init.operator else { panic!("expected scalar init") }; - let Childrens::Twins { left, .. } = plan.childrens.as_ref() else { + let Childrens::Twins { left, .. } = init.childrens.as_ref() else { panic!("expected init children") }; let Operator::ScalarQueryInit(second) = &left.operator else { panic!("expected second scalar init") }; - assert_ne!(first.id, second.id); + assert_ne!(first.reference(&arena), second.reference(&arena)); assert!(find_join(&plan).is_none()); Ok(()) } diff --git a/src/db.rs b/src/db.rs index e2a5cc7a..74cf4775 100644 --- a/src/db.rs +++ b/src/db.rs @@ -44,7 +44,7 @@ use crate::optimizer::rule::normalization::NormalizationRuleImpl; use crate::orm::FromQueryRow; use crate::planner::operator::Operator; use crate::planner::{ - ExecArenaView, LogicalPlan, MetaArena, PlanArena, PlanInput, PlanKeeper, TableArenaCell, + ExecMetaArena, LogicalPlan, MetaArena, PlanArena, PlanInput, PlanKeeper, TableArenaCell, }; #[cfg(all(not(target_arch = "wasm32"), feature = "lmdb"))] use crate::storage::lmdb::{LmdbConfig, LmdbStorage}; @@ -591,7 +591,7 @@ impl State { ) -> Result< ( Schema, - ExecArenaView, + ExecMetaArena, Executor<'a, S::TransactionType<'txn>>, ), DatabaseError, @@ -612,7 +612,7 @@ impl State { &self.table_functions, ); let root = build_write(&mut arena, &mut plan_arena, plan, read_context, transaction); - let plan_arena = ExecArenaView::new(plan_arena); + let plan_arena = ExecMetaArena::new(plan_arena); let executor = Executor::new(arena, root, keeper); Ok((schema, plan_arena, executor)) @@ -626,7 +626,7 @@ impl State { ) -> Result< ( Schema, - ExecArenaView>, + ExecMetaArena>, Executor<'a, S::TransactionType<'txn>>, ), DatabaseError, @@ -689,7 +689,7 @@ impl State { table_functions, ); let root = build_write(&mut arena, &mut plan_arena, plan, cache, transaction); - let plan_arena = ExecArenaView::new(plan_arena); + let plan_arena = ExecMetaArena::new(plan_arena); let executor = Executor::new(arena, root, keeper); Ok((schema, plan_arena, executor)) @@ -791,7 +791,7 @@ impl Database { &mut PlanArena<'a>, ) -> Result, C: for<'a> FnOnce( - &mut TransactionIter<'a, S::TransactionType<'a>, ExecArenaView>>, + &mut TransactionIter<'a, S::TransactionType<'a>, PlanArena<'a>>, ) -> Result, { let transaction = Box::into_raw(Box::new( @@ -963,7 +963,7 @@ impl<'a, S: Storage> BindSource<'a> for &'a Database { }; let inner = Box::into_raw(Box::new(TransactionIter::new( schema, - Box::new(arena) as Box, + arena.boxed(), executor, transaction, ))); @@ -1173,7 +1173,7 @@ impl<'a, 'txn, S: Storage> BindSource<'a> for &'a mut DBTransaction<'txn, S> { .execute(unsafe { &mut *transaction }, plan.into(), arena)?; Ok(TransactionIter::new( schema, - Box::new(arena) as Box, + arena.boxed(), executor, transaction, )) @@ -1228,7 +1228,7 @@ impl Drop for TransactionGuard<'_, T> { pub struct TransactionIter<'a, T: Transaction + 'a, A: MetaArena + 'a = Box> { // Drop execution before the metadata arena it borrows from. guard: TransactionGuard<'a, T>, - plan_arena: A, + plan_arena: ExecMetaArena, schema: Schema, ddl_apply: Vec, } @@ -1236,7 +1236,7 @@ pub struct TransactionIter<'a, T: Transaction + 'a, A: MetaArena + 'a = Box TransactionIter<'a, T, A> { pub(crate) fn new( schema: Schema, - plan_arena: A, + plan_arena: ExecMetaArena, executor: Executor<'a, T>, transaction: *mut T, ) -> Self { @@ -1296,7 +1296,7 @@ impl<'a, T: Transaction + 'a, A: MetaArena + 'a> TransactionIter<'a, T, A> { } } -impl<'a, T: Transaction + 'a> TransactionIter<'a, T, ExecArenaView>> { +impl<'a, T: Transaction + 'a> TransactionIter<'a, T, PlanArena<'a>> { fn done_with_ddl_apply(mut self) -> Result<(PlanArena<'a>, Vec), DatabaseError> { while self.next_tuple(|_, _| ())?.is_some() {} let Self { diff --git a/src/execution/ddl/add_column.rs b/src/execution/ddl/add_column.rs index 878eff21..c0b78dcb 100644 --- a/src/execution/ddl/add_column.rs +++ b/src/execution/ddl/add_column.rs @@ -19,7 +19,7 @@ use crate::execution::{ }; use crate::iter_ext::Itertools; use crate::planner::operator::alter_table::add_column::AddColumnOperator; -use crate::planner::MetaArena; +use crate::planner::{ExecMetaArena, MetaArena}; use crate::storage::Transaction; use crate::types::index::{Index, IndexType}; use crate::types::value::DataValue; @@ -43,10 +43,10 @@ impl<'a, T: Transaction + 'a> WriteExecutor<'a, T> for AddColumn<'a> { } impl<'a, T: Transaction + 'a> ExecutorNode<'a, T> for AddColumn<'a> { - fn next_tuple( + fn next_tuple( &mut self, arena: &mut ExecArena<'a, T>, - plan_arena: &mut (dyn MetaArena + 'a), + plan_arena: &mut ExecMetaArena, ) -> Result<(), DatabaseError> { let table_cache = arena.table_cache(); let AddColumnOperator { diff --git a/src/execution/ddl/change_column.rs b/src/execution/ddl/change_column.rs index 7405111b..ccddade6 100644 --- a/src/execution/ddl/change_column.rs +++ b/src/execution/ddl/change_column.rs @@ -19,7 +19,7 @@ use crate::execution::{ }; use crate::iter_ext::Itertools; use crate::planner::operator::alter_table::change_column::{ChangeColumnOperator, NotNullChange}; -use crate::planner::MetaArena; +use crate::planner::{ExecMetaArena, MetaArena}; use crate::storage::Transaction; use crate::types::value::DataValue; @@ -42,10 +42,10 @@ impl<'a, T: Transaction + 'a> WriteExecutor<'a, T> for ChangeColumn<'a> { } impl<'a, T: Transaction + 'a> ExecutorNode<'a, T> for ChangeColumn<'a> { - fn next_tuple( + fn next_tuple( &mut self, arena: &mut ExecArena<'a, T>, - plan_arena: &mut (dyn MetaArena + 'a), + plan_arena: &mut ExecMetaArena, ) -> Result<(), DatabaseError> { let table_cache = arena.table_cache(); let ChangeColumnOperator { diff --git a/src/execution/ddl/create_index.rs b/src/execution/ddl/create_index.rs index 89dd3446..2dd43683 100644 --- a/src/execution/ddl/create_index.rs +++ b/src/execution/ddl/create_index.rs @@ -20,7 +20,7 @@ use crate::execution::{ use crate::expression::ScalarExpression; use crate::planner::operator::create_index::CreateIndexOperator; use crate::planner::LogicalPlan; -use crate::planner::MetaArena; +use crate::planner::{ExecMetaArena, MetaArena}; use crate::storage::Transaction; use crate::types::index::Index; use crate::types::tuple::Schema; @@ -54,10 +54,10 @@ impl<'a, T: Transaction + 'a> WriteExecutor<'a, T> for CreateIndex<'a> { } impl<'a, T: Transaction + 'a> ExecutorNode<'a, T> for CreateIndex<'a> { - fn next_tuple( + fn next_tuple( &mut self, arena: &mut ExecArena<'a, T>, - plan_arena: &mut (dyn MetaArena + 'a), + plan_arena: &mut ExecMetaArena, ) -> Result<(), DatabaseError> { let CreateIndexOperator { table_name, diff --git a/src/execution/ddl/create_table.rs b/src/execution/ddl/create_table.rs index 7633693c..9f398f8f 100644 --- a/src/execution/ddl/create_table.rs +++ b/src/execution/ddl/create_table.rs @@ -17,7 +17,7 @@ use crate::execution::{ DDLApply, ExecArena, ExecId, ExecNode, ExecutionContext, ExecutorNode, WriteExecutor, }; use crate::planner::operator::create_table::CreateTableOperator; -use crate::planner::MetaArena; +use crate::planner::{ExecMetaArena, MetaArena}; use crate::storage::Transaction; pub struct CreateTable<'a> { @@ -39,10 +39,10 @@ impl<'a, T: Transaction + 'a> WriteExecutor<'a, T> for CreateTable<'a> { } impl<'a, T: Transaction + 'a> ExecutorNode<'a, T> for CreateTable<'a> { - fn next_tuple( + fn next_tuple( &mut self, arena: &mut ExecArena<'a, T>, - plan_arena: &mut (dyn MetaArena + 'a), + plan_arena: &mut ExecMetaArena, ) -> Result<(), DatabaseError> { let CreateTableOperator { table_name, diff --git a/src/execution/ddl/create_view.rs b/src/execution/ddl/create_view.rs index 8933d153..f7184891 100644 --- a/src/execution/ddl/create_view.rs +++ b/src/execution/ddl/create_view.rs @@ -17,7 +17,7 @@ use crate::execution::{ DDLApply, ExecArena, ExecId, ExecNode, ExecutionContext, ExecutorNode, WriteExecutor, }; use crate::planner::operator::create_view::CreateViewOperator; -use crate::planner::MetaArena; +use crate::planner::{ExecMetaArena, MetaArena}; use crate::storage::Transaction; pub struct CreateView<'a> { @@ -39,10 +39,10 @@ impl<'a, T: Transaction + 'a> WriteExecutor<'a, T> for CreateView<'a> { } impl<'a, T: Transaction + 'a> ExecutorNode<'a, T> for CreateView<'a> { - fn next_tuple( + fn next_tuple( &mut self, arena: &mut ExecArena<'a, T>, - plan_arena: &mut (dyn MetaArena + 'a), + plan_arena: &mut ExecMetaArena, ) -> Result<(), DatabaseError> { let CreateViewOperator { view, or_replace } = self.op; let (transaction, table_codec) = arena.transaction_codec_mut(); diff --git a/src/execution/ddl/drop_column.rs b/src/execution/ddl/drop_column.rs index 31a981d3..db7c2c7e 100644 --- a/src/execution/ddl/drop_column.rs +++ b/src/execution/ddl/drop_column.rs @@ -19,7 +19,7 @@ use crate::execution::{ }; use crate::iter_ext::Itertools; use crate::planner::operator::alter_table::drop_column::DropColumnOperator; -use crate::planner::MetaArena; +use crate::planner::{ExecMetaArena, MetaArena}; use crate::storage::Transaction; pub struct DropColumn<'a> { @@ -41,10 +41,10 @@ impl<'a, T: Transaction + 'a> WriteExecutor<'a, T> for DropColumn<'a> { } impl<'a, T: Transaction + 'a> ExecutorNode<'a, T> for DropColumn<'a> { - fn next_tuple( + fn next_tuple( &mut self, arena: &mut ExecArena<'a, T>, - plan_arena: &mut (dyn MetaArena + 'a), + plan_arena: &mut ExecMetaArena, ) -> Result<(), DatabaseError> { let table_cache = arena.table_cache(); let DropColumnOperator { diff --git a/src/execution/ddl/drop_index.rs b/src/execution/ddl/drop_index.rs index 683fb42e..a4c53446 100644 --- a/src/execution/ddl/drop_index.rs +++ b/src/execution/ddl/drop_index.rs @@ -17,7 +17,7 @@ use crate::execution::{ DDLApply, ExecArena, ExecId, ExecNode, ExecutionContext, ExecutorNode, WriteExecutor, }; use crate::planner::operator::drop_index::DropIndexOperator; -use crate::planner::MetaArena; +use crate::planner::{ExecMetaArena, MetaArena}; use crate::storage::Transaction; pub struct DropIndex<'a> { @@ -39,10 +39,10 @@ impl<'a, T: Transaction + 'a> WriteExecutor<'a, T> for DropIndex<'a> { } impl<'a, T: Transaction + 'a> ExecutorNode<'a, T> for DropIndex<'a> { - fn next_tuple( + fn next_tuple( &mut self, arena: &mut ExecArena<'a, T>, - plan_arena: &mut (dyn MetaArena + 'a), + plan_arena: &mut ExecMetaArena, ) -> Result<(), DatabaseError> { let DropIndexOperator { table_name, diff --git a/src/execution/ddl/drop_table.rs b/src/execution/ddl/drop_table.rs index 9467418b..3df6c183 100644 --- a/src/execution/ddl/drop_table.rs +++ b/src/execution/ddl/drop_table.rs @@ -17,7 +17,7 @@ use crate::execution::{ DDLApply, ExecArena, ExecId, ExecNode, ExecutionContext, ExecutorNode, WriteExecutor, }; use crate::planner::operator::drop_table::DropTableOperator; -use crate::planner::MetaArena; +use crate::planner::{ExecMetaArena, MetaArena}; use crate::storage::Transaction; pub struct DropTable<'a> { @@ -39,10 +39,10 @@ impl<'a, T: Transaction + 'a> WriteExecutor<'a, T> for DropTable<'a> { } impl<'a, T: Transaction + 'a> ExecutorNode<'a, T> for DropTable<'a> { - fn next_tuple( + fn next_tuple( &mut self, arena: &mut ExecArena<'a, T>, - plan_arena: &mut (dyn MetaArena + 'a), + plan_arena: &mut ExecMetaArena, ) -> Result<(), DatabaseError> { let DropTableOperator { table_name, diff --git a/src/execution/ddl/drop_view.rs b/src/execution/ddl/drop_view.rs index a2e75fc0..ce2b1277 100644 --- a/src/execution/ddl/drop_view.rs +++ b/src/execution/ddl/drop_view.rs @@ -17,7 +17,7 @@ use crate::execution::{ DDLApply, ExecArena, ExecId, ExecNode, ExecutionContext, ExecutorNode, WriteExecutor, }; use crate::planner::operator::drop_view::DropViewOperator; -use crate::planner::MetaArena; +use crate::planner::{ExecMetaArena, MetaArena}; use crate::storage::Transaction; pub struct DropView<'a> { @@ -39,10 +39,10 @@ impl<'a, T: Transaction + 'a> WriteExecutor<'a, T> for DropView<'a> { } impl<'a, T: Transaction + 'a> ExecutorNode<'a, T> for DropView<'a> { - fn next_tuple( + fn next_tuple( &mut self, arena: &mut ExecArena<'a, T>, - _: &mut (dyn MetaArena + '_), + _: &mut ExecMetaArena, ) -> Result<(), DatabaseError> { let DropViewOperator { view_name, diff --git a/src/execution/ddl/truncate.rs b/src/execution/ddl/truncate.rs index 32a5dc4c..ffc54ee5 100644 --- a/src/execution/ddl/truncate.rs +++ b/src/execution/ddl/truncate.rs @@ -17,7 +17,7 @@ use crate::execution::{ ExecArena, ExecId, ExecNode, ExecutionContext, ExecutorNode, WriteExecutor, }; use crate::planner::operator::truncate::TruncateOperator; -use crate::planner::MetaArena; +use crate::planner::{ExecMetaArena, MetaArena}; use crate::storage::Transaction; pub struct Truncate<'a> { @@ -39,10 +39,10 @@ impl<'a, T: Transaction + 'a> WriteExecutor<'a, T> for Truncate<'a> { } impl<'a, T: Transaction + 'a> ExecutorNode<'a, T> for Truncate<'a> { - fn next_tuple( + fn next_tuple( &mut self, arena: &mut ExecArena<'a, T>, - plan_arena: &mut (dyn MetaArena + 'a), + plan_arena: &mut ExecMetaArena, ) -> Result<(), DatabaseError> { let TruncateOperator { table_name } = self.op; let mut state = arena.local_state(plan_arena); diff --git a/src/execution/dml/analyze.rs b/src/execution/dml/analyze.rs index 213d87a1..5b2bb92d 100644 --- a/src/execution/dml/analyze.rs +++ b/src/execution/dml/analyze.rs @@ -24,7 +24,7 @@ use crate::optimizer::core::histogram::{HistogramBuilder, ANALYZE_STATISTICS_REL use crate::optimizer::core::statistics_meta::StatisticsMeta; use crate::planner::operator::analyze::AnalyzeOperator; use crate::planner::LogicalPlan; -use crate::planner::MetaArena; +use crate::planner::{ExecMetaArena, MetaArena}; use crate::storage::{table_codec::TableCodec, Transaction}; use crate::types::index::IndexId; use crate::types::value::{DataValue, Utf8Type}; @@ -72,10 +72,10 @@ impl<'a, T: Transaction + 'a> WriteExecutor<'a, T> for Analyze<'a> { } impl<'a, T: Transaction + 'a> ExecutorNode<'a, T> for Analyze<'a> { - fn next_tuple( + fn next_tuple( &mut self, arena: &mut ExecArena<'a, T>, - plan_arena: &mut (dyn MetaArena + 'a), + plan_arena: &mut ExecMetaArena, ) -> Result<(), DatabaseError> { let Some(input) = self.input.take() else { arena.finish(); diff --git a/src/execution/dml/copy_from_file.rs b/src/execution/dml/copy_from_file.rs index c1d25aa8..697f00b5 100644 --- a/src/execution/dml/copy_from_file.rs +++ b/src/execution/dml/copy_from_file.rs @@ -19,7 +19,7 @@ use crate::execution::{ }; use crate::iter_ext::Itertools; use crate::planner::operator::copy_from_file::CopyFromFileOperator; -use crate::planner::MetaArena; +use crate::planner::{ExecMetaArena, MetaArena}; use crate::storage::Transaction; use crate::types::tuple::Tuple; use crate::types::tuple_builder::TupleBuilder; @@ -45,10 +45,10 @@ impl<'a, T: Transaction + 'a> WriteExecutor<'a, T> for CopyFromFile<'a> { } impl<'a, T: Transaction + 'a> ExecutorNode<'a, T> for CopyFromFile<'a> { - fn next_tuple( + fn next_tuple( &mut self, arena: &mut ExecArena<'a, T>, - plan_arena: &mut (dyn MetaArena + 'a), + plan_arena: &mut ExecMetaArena, ) -> Result<(), DatabaseError> { let op = self.op; let column_types = op diff --git a/src/execution/dml/copy_to_file.rs b/src/execution/dml/copy_to_file.rs index 07e02865..31e2ce0f 100644 --- a/src/execution/dml/copy_to_file.rs +++ b/src/execution/dml/copy_to_file.rs @@ -20,7 +20,7 @@ use crate::execution::{ use crate::iter_ext::Itertools; use crate::planner::operator::copy_to_file::CopyToFileOperator; use crate::planner::LogicalPlan; -use crate::planner::MetaArena; +use crate::planner::{ExecMetaArena, MetaArena}; use crate::storage::Transaction; use crate::types::tuple_builder::TupleBuilder; @@ -84,10 +84,10 @@ impl CopyToFile<'_> { } impl<'a, T: Transaction + 'a> ExecutorNode<'a, T> for CopyToFile<'a> { - fn next_tuple( + fn next_tuple( &mut self, arena: &mut ExecArena<'a, T>, - plan_arena: &mut (dyn MetaArena + 'a), + plan_arena: &mut ExecMetaArena, ) -> Result<(), DatabaseError> { let Some(input) = self.input.take() else { arena.finish(); diff --git a/src/execution/dml/delete.rs b/src/execution/dml/delete.rs index be80def3..301e6c98 100644 --- a/src/execution/dml/delete.rs +++ b/src/execution/dml/delete.rs @@ -19,7 +19,7 @@ use crate::execution::{ }; use crate::planner::operator::delete::DeleteOperator; use crate::planner::LogicalPlan; -use crate::planner::MetaArena; +use crate::planner::{ExecMetaArena, MetaArena}; use crate::storage::Transaction; use crate::types::index::{Index, IndexType}; use crate::types::tuple_builder::TupleBuilder; @@ -51,10 +51,10 @@ impl<'a, T: Transaction + 'a> WriteExecutor<'a, T> for Delete<'a> { } impl<'a, T: Transaction + 'a> ExecutorNode<'a, T> for Delete<'a> { - fn next_tuple( + fn next_tuple( &mut self, arena: &mut ExecArena<'a, T>, - plan_arena: &mut (dyn MetaArena + 'a), + plan_arena: &mut ExecMetaArena, ) -> Result<(), DatabaseError> { let Some(input) = self.input.take() else { arena.finish(); diff --git a/src/execution/dml/insert.rs b/src/execution/dml/insert.rs index edc9e84d..a2838170 100644 --- a/src/execution/dml/insert.rs +++ b/src/execution/dml/insert.rs @@ -20,7 +20,7 @@ use crate::execution::{ use crate::iter_ext::Itertools; use crate::planner::operator::insert::InsertOperator; use crate::planner::LogicalPlan; -use crate::planner::MetaArena; +use crate::planner::{ExecMetaArena, MetaArena}; use crate::storage::Transaction; use crate::types::index::Index; use crate::types::tuple::{Schema, Tuple}; @@ -92,10 +92,10 @@ impl Insert<'_> { } impl<'a, T: Transaction + 'a> ExecutorNode<'a, T> for Insert<'a> { - fn next_tuple( + fn next_tuple( &mut self, arena: &mut ExecArena<'a, T>, - plan_arena: &mut (dyn MetaArena + 'a), + plan_arena: &mut ExecMetaArena, ) -> Result<(), DatabaseError> { let Some(input) = self.input.take() else { arena.finish(); diff --git a/src/execution/dml/update.rs b/src/execution/dml/update.rs index 3b052a9e..2a722afa 100644 --- a/src/execution/dml/update.rs +++ b/src/execution/dml/update.rs @@ -21,7 +21,7 @@ use crate::iter_ext::Itertools; use crate::planner::operator::update::UpdateOperator; use crate::planner::ExprRef; use crate::planner::LogicalPlan; -use crate::planner::MetaArena; +use crate::planner::{ExecMetaArena, MetaArena}; use crate::storage::Transaction; use crate::types::index::{Index, IndexMeta, IndexType}; use crate::types::tuple::{Schema, Tuple}; @@ -88,10 +88,10 @@ impl Update<'_> { } impl<'a, T: Transaction + 'a> ExecutorNode<'a, T> for Update<'a> { - fn next_tuple( + fn next_tuple( &mut self, arena: &mut ExecArena<'a, T>, - plan_arena: &mut (dyn MetaArena + 'a), + plan_arena: &mut ExecMetaArena, ) -> Result<(), DatabaseError> { let Some(input) = self.input.take() else { arena.finish(); diff --git a/src/execution/dql/aggregate/hash_agg.rs b/src/execution/dql/aggregate/hash_agg.rs index 5028e343..7880d79a 100644 --- a/src/execution/dql/aggregate/hash_agg.rs +++ b/src/execution/dql/aggregate/hash_agg.rs @@ -22,7 +22,7 @@ use crate::execution::{ use crate::planner::operator::aggregate::AggregateOperator; use crate::planner::ExprRef; use crate::planner::LogicalPlan; -use crate::planner::MetaArena; +use crate::planner::{ExecMetaArena, MetaArena}; use crate::storage::Transaction; use crate::types::value::DataValue; use std::collections::hash_map::IntoIter as HashMapIntoIter; @@ -65,10 +65,10 @@ impl<'a, T: Transaction + 'a> ReadExecutor<'a, T> for HashAggExecutor<'a> { } impl<'a, T: Transaction + 'a> ExecutorNode<'a, T> for HashAggExecutor<'a> { - fn next_tuple( + fn next_tuple( &mut self, arena: &mut ExecArena<'a, T>, - plan_arena: &mut (dyn MetaArena + 'a), + plan_arena: &mut ExecMetaArena, ) -> Result<(), DatabaseError> { if self.output.is_none() { let mut group_hash_accs: HashMap, Vec>> = diff --git a/src/execution/dql/aggregate/simple_agg.rs b/src/execution/dql/aggregate/simple_agg.rs index 7d0e889b..64e1fe2b 100644 --- a/src/execution/dql/aggregate/simple_agg.rs +++ b/src/execution/dql/aggregate/simple_agg.rs @@ -21,7 +21,7 @@ use crate::expression::ScalarExpression; use crate::planner::operator::aggregate::AggregateOperator; use crate::planner::ExprRef; use crate::planner::LogicalPlan; -use crate::planner::MetaArena; +use crate::planner::{ExecMetaArena, MetaArena}; use crate::storage::Transaction; pub struct SimpleAggExecutor<'a> { agg_calls: &'a [ExprRef], @@ -49,10 +49,10 @@ impl<'a, T: Transaction + 'a> ReadExecutor<'a, T> for SimpleAggExecutor<'a> { } impl<'a, T: Transaction + 'a> ExecutorNode<'a, T> for SimpleAggExecutor<'a> { - fn next_tuple( + fn next_tuple( &mut self, arena: &mut ExecArena<'a, T>, - plan_arena: &mut (dyn MetaArena + 'a), + plan_arena: &mut ExecMetaArena, ) -> Result<(), DatabaseError> { if self.returned { arena.finish(); diff --git a/src/execution/dql/aggregate/stream_agg.rs b/src/execution/dql/aggregate/stream_agg.rs index 2f238578..a9cb91af 100644 --- a/src/execution/dql/aggregate/stream_agg.rs +++ b/src/execution/dql/aggregate/stream_agg.rs @@ -22,7 +22,7 @@ use crate::execution::{ use crate::planner::operator::aggregate::AggregateOperator; use crate::planner::ExprRef; use crate::planner::LogicalPlan; -use crate::planner::MetaArena; +use crate::planner::{ExecMetaArena, MetaArena}; use crate::storage::Transaction; use crate::types::value::DataValue; use std::mem; @@ -65,10 +65,10 @@ impl<'a, T: Transaction + 'a> ReadExecutor<'a, T> for StreamAggExecutor<'a> { } impl<'a, T: Transaction + 'a> ExecutorNode<'a, T> for StreamAggExecutor<'a> { - fn next_tuple( + fn next_tuple( &mut self, arena: &mut ExecArena<'a, T>, - plan_arena: &mut (dyn MetaArena + 'a), + plan_arena: &mut ExecMetaArena, ) -> Result<(), DatabaseError> { loop { if !arena.next_tuple(self.input, plan_arena)? { diff --git a/src/execution/dql/aggregate/stream_distinct.rs b/src/execution/dql/aggregate/stream_distinct.rs index 46c82bf4..1702fe10 100644 --- a/src/execution/dql/aggregate/stream_distinct.rs +++ b/src/execution/dql/aggregate/stream_distinct.rs @@ -19,7 +19,7 @@ use crate::execution::{ use crate::planner::operator::aggregate::AggregateOperator; use crate::planner::ExprRef; use crate::planner::LogicalPlan; -use crate::planner::MetaArena; +use crate::planner::{ExecMetaArena, MetaArena}; use crate::storage::Transaction; use crate::types::tuple::Tuple; use crate::types::value::DataValue; @@ -50,10 +50,10 @@ impl<'a, T: Transaction + 'a> ReadExecutor<'a, T> for StreamDistinctExecutor<'a> } impl<'a, T: Transaction + 'a> ExecutorNode<'a, T> for StreamDistinctExecutor<'a> { - fn next_tuple( + fn next_tuple( &mut self, arena: &mut ExecArena<'a, T>, - plan_arena: &mut (dyn MetaArena + 'a), + plan_arena: &mut ExecMetaArena, ) -> Result<(), DatabaseError> { loop { if !arena.next_tuple(self.input, plan_arena)? { diff --git a/src/execution/dql/describe.rs b/src/execution/dql/describe.rs index 2c64137c..cbd18eff 100644 --- a/src/execution/dql/describe.rs +++ b/src/execution/dql/describe.rs @@ -16,7 +16,7 @@ use crate::catalog::{ColumnCatalog, ColumnRef, TableName}; use crate::errors::DatabaseError; use crate::execution::{ExecArena, ExecId, ExecNode, ExecutionContext, ExecutorNode, ReadExecutor}; use crate::planner::operator::describe::DescribeOperator; -use crate::planner::MetaArena; +use crate::planner::{ExecMetaArena, MetaArena}; use crate::storage::Transaction; use crate::types::value::{DataValue, Utf8Type}; use crate::types::CharLengthUnits; @@ -72,10 +72,10 @@ impl<'a, T: Transaction + 'a> ReadExecutor<'a, T> for Describe { } impl<'a, T: Transaction + 'a> ExecutorNode<'a, T> for Describe { - fn next_tuple( + fn next_tuple( &mut self, arena: &mut ExecArena<'a, T>, - plan_arena: &mut (dyn MetaArena + 'a), + plan_arena: &mut ExecMetaArena, ) -> Result<(), DatabaseError> { if self.columns.is_none() { let table = arena diff --git a/src/execution/dql/dummy.rs b/src/execution/dql/dummy.rs index 66c7647c..019053dc 100644 --- a/src/execution/dql/dummy.rs +++ b/src/execution/dql/dummy.rs @@ -14,7 +14,7 @@ use crate::errors::DatabaseError; use crate::execution::{ExecArena, ExecId, ExecNode, ExecutionContext, ExecutorNode, ReadExecutor}; -use crate::planner::MetaArena; +use crate::planner::{ExecMetaArena, MetaArena}; use crate::storage::Transaction; use crate::types::tuple::Tuple; @@ -46,10 +46,10 @@ impl<'a, T: Transaction + 'a> ReadExecutor<'a, T> for Dummy { } impl<'a, T: Transaction + 'a> ExecutorNode<'a, T> for Dummy { - fn next_tuple( + fn next_tuple( &mut self, arena: &mut ExecArena<'a, T>, - _: &mut (dyn MetaArena + '_), + _: &mut ExecMetaArena, ) -> Result<(), DatabaseError> { let Some(row) = self.row.take() else { arena.finish(); diff --git a/src/execution/dql/explain.rs b/src/execution/dql/explain.rs index 05b113eb..dd999aba 100644 --- a/src/execution/dql/explain.rs +++ b/src/execution/dql/explain.rs @@ -15,7 +15,7 @@ use crate::errors::DatabaseError; use crate::execution::{ExecArena, ExecId, ExecNode, ExecutionContext, ExecutorNode, ReadExecutor}; use crate::planner::LogicalPlan; -use crate::planner::MetaArena; +use crate::planner::{ExecMetaArena, MetaArena}; use crate::storage::Transaction; use crate::types::value::{DataValue, Utf8Type}; use crate::types::CharLengthUnits; @@ -43,10 +43,10 @@ impl<'a, T: Transaction + 'a> ReadExecutor<'a, T> for Explain<'a> { } impl<'a, T: Transaction + 'a> ExecutorNode<'a, T> for Explain<'a> { - fn next_tuple( + fn next_tuple( &mut self, arena: &mut ExecArena<'a, T>, - plan_arena: &mut (dyn MetaArena + 'a), + plan_arena: &mut ExecMetaArena, ) -> Result<(), DatabaseError> { if self.emitted { arena.finish(); diff --git a/src/execution/dql/external_sort.rs b/src/execution/dql/external_sort.rs index 8a276fba..6fcbdf68 100644 --- a/src/execution/dql/external_sort.rs +++ b/src/execution/dql/external_sort.rs @@ -20,7 +20,7 @@ use crate::execution::{ }; use crate::planner::operator::sort::{SortField, SortOperator}; use crate::planner::LogicalPlan; -use crate::planner::MetaArena; +use crate::planner::{ExecMetaArena, MetaArena}; use crate::storage::Transaction; use std::fs::File; use std::io::BufReader; @@ -69,10 +69,10 @@ impl<'a, T: Transaction + 'a> ReadExecutor<'a, T> for ExternalSort<'a> { } impl<'a, T: Transaction + 'a> ExecutorNode<'a, T> for ExternalSort<'a> { - fn next_tuple( + fn next_tuple( &mut self, arena: &mut ExecArena<'a, T>, - plan_arena: &mut (dyn MetaArena + 'a), + plan_arena: &mut ExecMetaArena, ) -> Result<(), DatabaseError> { loop { if let Some(rows) = &mut self.rows { diff --git a/src/execution/dql/filter.rs b/src/execution/dql/filter.rs index 3d6c0e51..757ed6ec 100644 --- a/src/execution/dql/filter.rs +++ b/src/execution/dql/filter.rs @@ -19,7 +19,7 @@ use crate::execution::{ use crate::planner::operator::filter::FilterOperator; use crate::planner::ExprRef; use crate::planner::LogicalPlan; -use crate::planner::MetaArena; +use crate::planner::{ExecMetaArena, MetaArena}; use crate::storage::Transaction; pub struct Filter { predicate: ExprRef, @@ -45,10 +45,10 @@ impl<'a, T: Transaction + 'a> ReadExecutor<'a, T> for Filter { } impl<'a, T: Transaction + 'a> ExecutorNode<'a, T> for Filter { - fn next_tuple( + fn next_tuple( &mut self, arena: &mut ExecArena<'a, T>, - plan_arena: &mut (dyn MetaArena + 'a), + plan_arena: &mut ExecMetaArena, ) -> Result<(), DatabaseError> { loop { if !arena.next_tuple(self.input, plan_arena)? { diff --git a/src/execution/dql/function_scan.rs b/src/execution/dql/function_scan.rs index e09455c3..e7b206c1 100644 --- a/src/execution/dql/function_scan.rs +++ b/src/execution/dql/function_scan.rs @@ -16,7 +16,7 @@ use crate::errors::DatabaseError; use crate::execution::{ExecArena, ExecId, ExecNode, ExecutionContext, ExecutorNode, ReadExecutor}; use crate::expression::function::table::TableFunction; use crate::planner::operator::function_scan::FunctionScanOperator; -use crate::planner::MetaArena; +use crate::planner::{ExecMetaArena, MetaArena}; use crate::storage::Transaction; use crate::types::tuple::Tuple; @@ -50,10 +50,10 @@ impl<'a, T: Transaction + 'a> ReadExecutor<'a, T> for FunctionScan<'a> { } impl<'a, T: Transaction + 'a> ExecutorNode<'a, T> for FunctionScan<'a> { - fn next_tuple( + fn next_tuple( &mut self, arena: &mut ExecArena<'a, T>, - plan_arena: &mut (dyn MetaArena + 'a), + plan_arena: &mut ExecMetaArena, ) -> Result<(), DatabaseError> { if self.iter.is_none() { let TableFunction { args, catalog } = self.table_function; diff --git a/src/execution/dql/index_scan.rs b/src/execution/dql/index_scan.rs index fcd38bd6..164fafa4 100644 --- a/src/execution/dql/index_scan.rs +++ b/src/execution/dql/index_scan.rs @@ -17,7 +17,7 @@ use crate::execution::{ExecArena, ExecId, ExecNode, ExecutionContext, ExecutorNo use crate::expression::range_detacher::{IndexRangeColumn, Range, RangeDetacher}; use crate::planner::operator::table_scan::TableScanOperator; use crate::planner::operator::SortOption; -use crate::planner::MetaArena; +use crate::planner::{ExecMetaArena, MetaArena}; use crate::storage::{IndexIter, IndexRanges, Iter, Transaction}; use crate::types::index::{IndexInfo, IndexLookup, RuntimeIndexProbe}; use std::borrow::Cow; @@ -99,10 +99,10 @@ impl<'a, T: Transaction + 'a> ReadExecutor<'a, T> for IndexScan<'a, T> { } impl<'a, T: Transaction + 'a> ExecutorNode<'a, T> for IndexScan<'a, T> { - fn next_tuple( + fn next_tuple( &mut self, arena: &mut ExecArena<'a, T>, - plan_arena: &mut (dyn MetaArena + 'a), + plan_arena: &mut ExecMetaArena, ) -> Result<(), DatabaseError> { let iter = match &mut self.iter { Some(iter) => iter, diff --git a/src/execution/dql/join/hash_join.rs b/src/execution/dql/join/hash_join.rs index 52f05b9e..d2f42819 100644 --- a/src/execution/dql/join/hash_join.rs +++ b/src/execution/dql/join/hash_join.rs @@ -28,7 +28,7 @@ use crate::execution::{ use crate::planner::operator::join::{JoinCondition, JoinOperator, JoinType}; use crate::planner::ExprRef; use crate::planner::LogicalPlan; -use crate::planner::MetaArena; +use crate::planner::{ExecMetaArena, MetaArena}; use crate::storage::Transaction; use crate::types::tuple::Tuple; use crate::types::value::DataValue; @@ -143,10 +143,10 @@ impl HashJoin { Ok(()) } - fn initialize_build<'a, T: Transaction + 'a>( + fn initialize_build<'a, T: Transaction + 'a, A: MetaArena + 'a>( &mut self, arena: &mut ExecArena<'a, T>, - plan_arena: &mut (dyn MetaArena + 'a), + plan_arena: &mut ExecMetaArena, ) -> Result<(), DatabaseError> { if !matches!(self.state, HashJoinState::Build) { return Ok(()); @@ -251,10 +251,10 @@ impl<'a, T: Transaction + 'a> ReadExecutor<'a, T> for HashJoin { } impl<'a, T: Transaction + 'a> ExecutorNode<'a, T> for HashJoin { - fn next_tuple( + fn next_tuple( &mut self, arena: &mut ExecArena<'a, T>, - plan_arena: &mut (dyn MetaArena + 'a), + plan_arena: &mut ExecMetaArena, ) -> Result<(), DatabaseError> { if let Some(err) = self.init_error.take() { return Err(err); diff --git a/src/execution/dql/join/nested_loop_join.rs b/src/execution/dql/join/nested_loop_join.rs index ef5b94e5..72e4a9b0 100644 --- a/src/execution/dql/join/nested_loop_join.rs +++ b/src/execution/dql/join/nested_loop_join.rs @@ -16,7 +16,7 @@ //! [`JoinType::RightOuter`], [`JoinType::Cross`], [`JoinType::Full`]. use crate::planner::LogicalPlan; -use crate::planner::MetaArena; +use crate::planner::{ExecMetaArena, MetaArena}; use crate::errors::DatabaseError; use crate::execution::dql::join::RowBitmap; @@ -154,10 +154,10 @@ impl<'a, T: Transaction + 'a> ReadExecutor<'a, T> for NestedLoopJoin<'a> { } impl<'a, T: Transaction + 'a> ExecutorNode<'a, T> for NestedLoopJoin<'a> { - fn next_tuple( + fn next_tuple( &mut self, arena: &mut ExecArena<'a, T>, - plan_arena: &mut (dyn MetaArena + 'a), + plan_arena: &mut ExecMetaArena, ) -> Result<(), DatabaseError> { let mut state = std::mem::replace(&mut self.state, NestedLoopJoinState::End); @@ -579,6 +579,7 @@ mod test { let address = arena.nodes.items.as_ptr(); assert_eq!(count, 5); let mut rows = 0; + let mut plan_arena = ExecMetaArena::new(plan_arena); while arena.next_tuple(root, &mut plan_arena)? { rows += 1; assert_eq!(arena.result_tuple().values.len(), 9); diff --git a/src/execution/dql/limit.rs b/src/execution/dql/limit.rs index 9c9f6874..a49a0665 100644 --- a/src/execution/dql/limit.rs +++ b/src/execution/dql/limit.rs @@ -18,7 +18,7 @@ use crate::execution::{ }; use crate::planner::operator::limit::LimitOperator; use crate::planner::LogicalPlan; -use crate::planner::MetaArena; +use crate::planner::{ExecMetaArena, MetaArena}; use crate::storage::Transaction; pub struct Limit { offset: Option, @@ -50,10 +50,10 @@ impl<'a, T: Transaction + 'a> ReadExecutor<'a, T> for Limit { } impl<'a, T: Transaction + 'a> ExecutorNode<'a, T> for Limit { - fn next_tuple( + fn next_tuple( &mut self, arena: &mut ExecArena<'a, T>, - plan_arena: &mut (dyn MetaArena + 'a), + plan_arena: &mut ExecMetaArena, ) -> Result<(), DatabaseError> { let offset = self.offset.unwrap_or(0); let limit = self.limit.unwrap_or(usize::MAX); diff --git a/src/execution/dql/mark_apply.rs b/src/execution/dql/mark_apply.rs index cd17b9d4..4d9f6c6d 100644 --- a/src/execution/dql/mark_apply.rs +++ b/src/execution/dql/mark_apply.rs @@ -18,7 +18,7 @@ use crate::execution::{ }; use crate::planner::operator::mark_apply::{MarkApplyKind, MarkApplyOperator, MarkApplyQuantifier}; use crate::planner::LogicalPlan; -use crate::planner::MetaArena; +use crate::planner::{ExecMetaArena, MetaArena}; use crate::storage::Transaction; use crate::types::index::RuntimeIndexProbe; use crate::types::tuple::{SplitTupleRef, Tuple}; @@ -65,10 +65,10 @@ impl<'a, T: Transaction + 'a> ReadExecutor<'a, T> for MarkApply<'a> { } impl<'a, T: Transaction + 'a> ExecutorNode<'a, T> for MarkApply<'a> { - fn next_tuple( + fn next_tuple( &mut self, arena: &mut ExecArena<'a, T>, - plan_arena: &mut (dyn MetaArena + 'a), + plan_arena: &mut ExecMetaArena, ) -> Result<(), DatabaseError> { if matches!(self.op.kind, MarkApplyKind::InnerJoin) { return self.next_join_tuple(arena, plan_arena); @@ -88,10 +88,10 @@ impl<'a, T: Transaction + 'a> ExecutorNode<'a, T> for MarkApply<'a> { } impl<'a> MarkApply<'a> { - fn next_join_tuple( + fn next_join_tuple( &mut self, arena: &mut ExecArena<'a, T>, - plan_arena: &mut (dyn MetaArena + 'a), + plan_arena: &mut ExecMetaArena, ) -> Result<(), DatabaseError> { loop { if let Some((root, left)) = &mut self.join_input { @@ -174,10 +174,10 @@ impl<'a> MarkApply<'a> { .transpose() } - fn mark_value( + fn mark_value( &self, arena: &mut ExecArena<'a, T>, - plan_arena: &mut (dyn MetaArena + 'a), + plan_arena: &mut ExecMetaArena, left_tuple: &Tuple, ) -> Result { let probe = self.parameterized_probe_value(left_tuple, plan_arena)?; @@ -255,10 +255,10 @@ impl<'a> MarkApply<'a> { } } - fn scan_quantified_right_input( + fn scan_quantified_right_input( &self, arena: &mut ExecArena<'a, T>, - plan_arena: &mut (dyn MetaArena + 'a), + plan_arena: &mut ExecMetaArena, right_input: ExecId, quantifier: MarkApplyQuantifier, left_tuple: &Tuple, @@ -521,6 +521,7 @@ mod tests { let address = arena.nodes.items.as_ptr(); let count = arena.nodes.items.len(); assert_eq!(count, 3); + let mut plan_arena = ExecMetaArena::new(plan_arena); for _ in 0..4 { assert!(arena.next_tuple(root, &mut plan_arena)?); assert_eq!( @@ -611,6 +612,7 @@ mod tests { let address = arena.nodes.items.as_ptr(); assert_eq!(count, 3); let mut tuples = Vec::new(); + let mut plan_arena = ExecMetaArena::new(plan_arena); while arena.next_tuple(root, &mut plan_arena)? { tuples.push(arena.materialize_tuple()); assert_eq!(arena.nodes.items.len(), count); @@ -739,6 +741,7 @@ mod tests { }; let left_tuple = Tuple::new(None, vec![DataValue::Int32(2), DataValue::Int32(1)]); + let mut plan_arena = ExecMetaArena::new(plan_arena); assert_eq!( exec.mark_value(&mut arena, &mut plan_arena, &left_tuple)?, DataValue::Boolean(true) @@ -792,6 +795,7 @@ mod tests { }; let left_tuple = Tuple::new(None, vec![DataValue::Int32(2)]); + let mut plan_arena = ExecMetaArena::new(plan_arena); assert_eq!( exec.mark_value(&mut arena, &mut plan_arena, &left_tuple)?, DataValue::Boolean(true) @@ -845,6 +849,7 @@ mod tests { }; let left_tuple = Tuple::new(None, vec![DataValue::Null]); + let mut plan_arena = ExecMetaArena::new(plan_arena); assert_eq!( exec.mark_value(&mut arena, &mut plan_arena, &left_tuple)?, DataValue::Null diff --git a/src/execution/dql/projection.rs b/src/execution/dql/projection.rs index f663c86e..5c9f03a3 100644 --- a/src/execution/dql/projection.rs +++ b/src/execution/dql/projection.rs @@ -19,7 +19,7 @@ use crate::execution::{ use crate::planner::operator::project::ProjectOperator; use crate::planner::ExprRef; use crate::planner::LogicalPlan; -use crate::planner::MetaArena; +use crate::planner::{ExecMetaArena, MetaArena}; use crate::storage::Transaction; use crate::types::value::DataValue; @@ -44,10 +44,10 @@ impl<'a, T: Transaction + 'a> ReadExecutor<'a, T> for Projection<'a> { } impl<'a, T: Transaction + 'a> ExecutorNode<'a, T> for Projection<'a> { - fn next_tuple( + fn next_tuple( &mut self, arena: &mut ExecArena<'a, T>, - plan_arena: &mut (dyn MetaArena + 'a), + plan_arena: &mut ExecMetaArena, ) -> Result<(), DatabaseError> { if !arena.next_tuple(self.input, plan_arena)? { arena.finish(); diff --git a/src/execution/dql/recursive_cte.rs b/src/execution/dql/recursive_cte.rs index 9563baef..654ec268 100644 --- a/src/execution/dql/recursive_cte.rs +++ b/src/execution/dql/recursive_cte.rs @@ -22,7 +22,7 @@ use crate::execution::{ }; use crate::planner::operator::recursive_cte::RecursiveScanOperator; use crate::planner::LogicalPlan; -use crate::planner::MetaArena; +use crate::planner::{ExecMetaArena, MetaArena}; use crate::storage::Transaction; use crate::types::tuple::Tuple; use std::mem; @@ -201,9 +201,9 @@ impl<'a, T: Transaction + 'a> RecursiveCte<'a, T> { } } - fn start_recursive( + fn start_recursive( &mut self, - plan_arena: &mut (dyn MetaArena + 'a), + plan_arena: &mut ExecMetaArena, ) -> Result { let Some(input) = mem::take(&mut self.working).into_input()? else { return Ok(false); @@ -246,10 +246,10 @@ impl<'a, T: Transaction + 'a> ReadExecutor<'a, T> for RecursiveCte<'a, T> { } impl<'a, T: Transaction + 'a> ExecutorNode<'a, T> for RecursiveCte<'a, T> { - fn next_tuple( + fn next_tuple( &mut self, arena: &mut ExecArena<'a, T>, - plan_arena: &mut (dyn MetaArena + 'a), + plan_arena: &mut ExecMetaArena, ) -> Result<(), DatabaseError> { loop { match self.phase { @@ -307,10 +307,10 @@ impl<'a, T: Transaction + 'a> ReadExecutor<'a, T> for RecursiveScan { } impl<'a, T: Transaction + 'a> ExecutorNode<'a, T> for RecursiveScan { - fn next_tuple( + fn next_tuple( &mut self, arena: &mut ExecArena<'a, T>, - _plan_arena: &mut (dyn MetaArena + 'a), + _plan_arena: &mut ExecMetaArena, ) -> Result<(), DatabaseError> { match self.input.next().transpose()? { Some(tuple) => arena.produce_tuple(tuple), diff --git a/src/execution/dql/scalar_query_init.rs b/src/execution/dql/scalar_query_init.rs index 17e13e8f..78d3ac70 100644 --- a/src/execution/dql/scalar_query_init.rs +++ b/src/execution/dql/scalar_query_init.rs @@ -16,28 +16,85 @@ use crate::errors::DatabaseError; use crate::execution::{ build_read, ExecArena, ExecId, ExecNode, ExecutionContext, ExecutorNode, ReadExecutor, }; +use crate::expression::ScalarExpression; use crate::planner::operator::scalar_query_init::ScalarQueryInitOperator; -use crate::planner::{LogicalPlan, MetaArena, ScalarQueryRef}; +use crate::planner::ExecMetaArena; +use crate::planner::{ExprRef, LogicalPlan, MetaArena, ScalarQueryRef}; use crate::storage::Transaction; +use crate::types::tuple::Tuple; use crate::types::value::DataValue; -pub struct ScalarQueryInit { +enum ScalarQueryInitState { + Initialize, + ReadInput, + ReadOuter, + EvaluateOuter, + Finished, +} + +pub struct ScalarQueryInit<'a> { input: ExecId, - init: ExecId, reference: ScalarQueryRef, + state: ScalarQueryInitState, + param_bindings: &'a [(ScalarQueryRef, ExprRef)], + scratch_tuple: Tuple, + + init_plan: &'a LogicalPlan, + init_pos: ExecId, + init: ExecId, } -impl ScalarQueryInit { - pub(crate) fn new(input: ExecId, init: ExecId, reference: ScalarQueryRef) -> Self { - Self { +impl<'a> ScalarQueryInit<'a> { + fn evaluate_init( + &self, + arena: &mut ExecArena<'a, T>, + plan_arena: &mut ExecMetaArena, + ) -> Result<(), DatabaseError> { + let mut value = DataValue::Null; + if arena.next_tuple(self.init, plan_arena)? { + std::mem::swap(&mut value, &mut arena.result_tuple_mut().values[0]); + } + if arena.next_tuple(self.init, plan_arena)? { + return Err(DatabaseError::InvalidValue( + "scalar subquery returned more than one row".into(), + )); + } + plan_arena.set_init_value(self.reference, value); + Ok(()) + } + + pub(crate) fn build( + op: &'a ScalarQueryInitOperator, + input: ExecId, + init_plan: &'a LogicalPlan, + arena: &mut ExecArena<'a, T>, + plan_arena: &mut (dyn MetaArena + 'a), + cache: ExecutionContext<'_>, + transaction: &T, + ) -> ExecId { + let init_pos = arena.nodes.position(); + let init = build_read(arena, plan_arena, init_plan, cache, transaction); + arena.push(ExecNode::ScalarQueryInit(Self { input, init, - reference, - } + reference: op.reference(plan_arena), + state: if matches!( + plan_arena.expression(op.value), + ScalarExpression::OuterValue { .. } + ) { + ScalarQueryInitState::ReadOuter + } else { + ScalarQueryInitState::Initialize + }, + param_bindings: &op.param_bindings, + scratch_tuple: Tuple::default(), + init_plan, + init_pos, + })) } } -impl<'a, T: Transaction + 'a> ReadExecutor<'a, T> for ScalarQueryInit { +impl<'a, T: Transaction + 'a> ReadExecutor<'a, T> for ScalarQueryInit<'a> { type Input = ( &'a ScalarQueryInitOperator, &'a LogicalPlan, @@ -52,33 +109,69 @@ impl<'a, T: Transaction + 'a> ReadExecutor<'a, T> for ScalarQueryInit { transaction: &T, ) -> ExecId { let input = build_read(arena, plan_arena, input, cache, transaction); - let init = build_read(arena, plan_arena, init, cache, transaction); - arena.push(ExecNode::ScalarQueryInit(Self::new(input, init, op.id))) + Self::build(op, input, init, arena, plan_arena, cache, transaction) } } -impl<'a, T: Transaction + 'a> ExecutorNode<'a, T> for ScalarQueryInit { - fn next_tuple( +impl<'a, T: Transaction + 'a> ExecutorNode<'a, T> for ScalarQueryInit<'a> { + fn next_tuple( &mut self, arena: &mut ExecArena<'a, T>, - plan_arena: &mut (dyn MetaArena + 'a), + plan_arena: &mut ExecMetaArena, ) -> Result<(), DatabaseError> { - if plan_arena.init_value(self.reference).is_none() { - let value = if arena.next_tuple(self.init, plan_arena)? { - arena.result_tuple().values[0].clone() - } else { - DataValue::Null - }; - // Drain the scalar wrapper to validate cardinality before publishing its value. - if arena.next_tuple(self.init, plan_arena)? { - return Err(DatabaseError::InvalidValue( - "scalar subquery returned more than one row".into(), - )); + loop { + match self.state { + ScalarQueryInitState::Initialize => { + if plan_arena.init_value(self.reference).is_none() { + self.evaluate_init(arena, plan_arena)?; + } + self.state = ScalarQueryInitState::ReadInput; + } + ScalarQueryInitState::ReadInput => { + if !arena.next_tuple(self.input, plan_arena)? { + self.state = ScalarQueryInitState::Finished; + } + return Ok(()); + } + ScalarQueryInitState::ReadOuter => { + if !arena.next_tuple(self.input, plan_arena)? { + self.state = ScalarQueryInitState::Finished; + return Ok(()); + } + // Preserve the outer row and lend the previous subquery buffer to the arena. + std::mem::swap(arena.result_tuple_mut(), &mut self.scratch_tuple); + for (reference, expr) in self.param_bindings { + let value = plan_arena + .expression(*expr) + .eval(plan_arena, Some(&self.scratch_tuple))? + .into_owned(); + plan_arena.set_init_value(*reference, value); + } + self.state = ScalarQueryInitState::EvaluateOuter; + } + ScalarQueryInitState::EvaluateOuter => { + let previous = arena.nodes.position(); + arena.nodes.seek(self.init_pos); + self.init = build_read( + arena, + plan_arena, + self.init_plan, + arena.context(), + arena.transaction(), + ); + arena.nodes.seek(previous); + self.evaluate_init(arena, plan_arena)?; + std::mem::swap(arena.result_tuple_mut(), &mut self.scratch_tuple); + self.state = ScalarQueryInitState::ReadOuter; + arena.resume(); + return Ok(()); + } + ScalarQueryInitState::Finished => { + arena.finish(); + return Ok(()); + } } - plan_arena.set_init_value(self.reference, value); } - arena.next_tuple(self.input, plan_arena)?; - Ok(()) } } @@ -91,7 +184,7 @@ mod tests { use crate::planner::operator::scalar_subquery::ScalarSubqueryOperator; use crate::planner::operator::values::ValuesOperator; use crate::planner::operator::Operator; - use crate::planner::{Childrens, ExecArenaView, PlanArena, TableArenaCell}; + use crate::planner::{Childrens, ExecMetaArena, PlanArena, TableArenaCell}; use crate::storage::memory::MemoryStorage; use crate::storage::{StatisticsMetaCache, Storage, TableCache, ViewCache}; use crate::types::LogicalType; @@ -118,6 +211,10 @@ mod tests { ColumnDesc::new(LogicalType::Integer, None, false, None)?, )); let expr = metadata.alloc_expression(ScalarExpression::Constant(value.clone())); + let marker = metadata.alloc_expression(ScalarExpression::InitValue { + id: reference, + ty: LogicalType::Integer, + }); let make_init = |rows| { ScalarSubqueryOperator::build(LogicalPlan::new( Operator::Values(ValuesOperator::new(vec![expr; rows], rows, vec![column])), @@ -127,13 +224,15 @@ mod tests { let mut first = ScalarQueryInitOperator::build( LogicalPlan::new(Operator::Dummy, Childrens::None), make_init(row_count), - reference, + marker, + Vec::new(), ); // This duplicate would error if executed, making cache reuse observable even for NULL. let mut duplicate = ScalarQueryInitOperator::build( LogicalPlan::new(Operator::Dummy, Childrens::None), make_init(2), - reference, + marker, + Vec::new(), ); first.populate_output_schema_recursive(&mut metadata); duplicate.populate_output_schema_recursive(&mut metadata); @@ -142,7 +241,7 @@ mod tests { let first_root = build_read(&mut arena, &mut metadata, &first, cache, &transaction); let duplicate_root = build_read(&mut arena, &mut metadata, &duplicate, cache, &transaction); - let mut view = ExecArenaView::new(metadata); + let mut view = ExecMetaArena::new(metadata); if row_count == 2 { assert!(arena.next_tuple(first_root, &mut view).is_err()); assert_eq!(view.init_value(reference), None); diff --git a/src/execution/dql/scalar_subquery.rs b/src/execution/dql/scalar_subquery.rs index 822f81cb..98d060a4 100644 --- a/src/execution/dql/scalar_subquery.rs +++ b/src/execution/dql/scalar_subquery.rs @@ -18,7 +18,7 @@ use crate::execution::{ }; use crate::planner::operator::scalar_subquery::ScalarSubqueryOperator; use crate::planner::LogicalPlan; -use crate::planner::MetaArena; +use crate::planner::{ExecMetaArena, MetaArena}; use crate::storage::Transaction; use crate::types::value::DataValue; @@ -49,10 +49,10 @@ impl<'a, T: Transaction + 'a> ReadExecutor<'a, T> for ScalarSubquery { } impl<'a, T: Transaction + 'a> ExecutorNode<'a, T> for ScalarSubquery { - fn next_tuple( + fn next_tuple( &mut self, arena: &mut ExecArena<'a, T>, - plan_arena: &mut (dyn MetaArena + 'a), + plan_arena: &mut ExecMetaArena, ) -> Result<(), DatabaseError> { let has_next = arena.next_tuple(self.input, plan_arena)?; if self.returned { diff --git a/src/execution/dql/seq_scan.rs b/src/execution/dql/seq_scan.rs index 6effc8e3..2e237084 100644 --- a/src/execution/dql/seq_scan.rs +++ b/src/execution/dql/seq_scan.rs @@ -15,7 +15,7 @@ use crate::errors::DatabaseError; use crate::execution::{ExecArena, ExecId, ExecNode, ExecutionContext, ExecutorNode, ReadExecutor}; use crate::planner::operator::table_scan::TableScanOperator; -use crate::planner::MetaArena; +use crate::planner::{ExecMetaArena, MetaArena}; use crate::storage::{Iter, Transaction, TupleIter}; pub(crate) struct SeqScan<'a, T: Transaction + 'a> { @@ -38,10 +38,10 @@ impl<'a, T: Transaction + 'a> ReadExecutor<'a, T> for SeqScan<'a, T> { } impl<'a, T: Transaction + 'a> ExecutorNode<'a, T> for SeqScan<'a, T> { - fn next_tuple( + fn next_tuple( &mut self, arena: &mut ExecArena<'a, T>, - plan_arena: &mut (dyn MetaArena + 'a), + plan_arena: &mut ExecMetaArena, ) -> Result<(), DatabaseError> { let state = arena.local_state(plan_arena); let iter = match &mut self.iter { diff --git a/src/execution/dql/set_membership.rs b/src/execution/dql/set_membership.rs index 75c38726..a83a8679 100644 --- a/src/execution/dql/set_membership.rs +++ b/src/execution/dql/set_membership.rs @@ -18,7 +18,7 @@ use crate::execution::{ }; use crate::planner::operator::set_membership::SetMembershipKind; use crate::planner::LogicalPlan; -use crate::planner::MetaArena; +use crate::planner::{ExecMetaArena, MetaArena}; use crate::storage::Transaction; use crate::types::tuple::Tuple; use std::collections::HashMap; @@ -54,10 +54,10 @@ impl<'a, T: Transaction + 'a> ReadExecutor<'a, T> for SetMembership { } impl<'a, T: Transaction + 'a> ExecutorNode<'a, T> for SetMembership { - fn next_tuple( + fn next_tuple( &mut self, arena: &mut ExecArena<'a, T>, - plan_arena: &mut (dyn MetaArena + 'a), + plan_arena: &mut ExecMetaArena, ) -> Result<(), DatabaseError> { if !self.built { while arena.next_tuple(self.right_input, plan_arena)? { diff --git a/src/execution/dql/show_table.rs b/src/execution/dql/show_table.rs index 3c7fbab2..ebd617ad 100644 --- a/src/execution/dql/show_table.rs +++ b/src/execution/dql/show_table.rs @@ -14,7 +14,7 @@ use crate::errors::DatabaseError; use crate::execution::{ExecArena, ExecId, ExecNode, ExecutionContext, ExecutorNode, ReadExecutor}; -use crate::planner::MetaArena; +use crate::planner::{ExecMetaArena, MetaArena}; use crate::storage::{TableIter, Transaction}; use crate::types::value::{DataValue, Utf8Type}; use crate::types::CharLengthUnits; @@ -38,10 +38,10 @@ impl<'a, T: Transaction + 'a> ReadExecutor<'a, T> for ShowTables<'a, T> { } impl<'a, T: Transaction + 'a> ExecutorNode<'a, T> for ShowTables<'a, T> { - fn next_tuple( + fn next_tuple( &mut self, arena: &mut ExecArena<'a, T>, - plan_arena: &mut (dyn MetaArena + 'a), + plan_arena: &mut ExecMetaArena, ) -> Result<(), DatabaseError> { if self.metas.is_none() { let mut state = arena.local_state(plan_arena); diff --git a/src/execution/dql/show_view.rs b/src/execution/dql/show_view.rs index 12f337a7..dfeef23d 100644 --- a/src/execution/dql/show_view.rs +++ b/src/execution/dql/show_view.rs @@ -14,7 +14,7 @@ use crate::errors::DatabaseError; use crate::execution::{ExecArena, ExecId, ExecNode, ExecutionContext, ExecutorNode, ReadExecutor}; -use crate::planner::MetaArena; +use crate::planner::{ExecMetaArena, MetaArena}; use crate::storage::{Transaction, ViewIter}; use crate::types::value::{DataValue, Utf8Type}; use crate::types::CharLengthUnits; @@ -38,10 +38,10 @@ impl<'a, T: Transaction + 'a> ReadExecutor<'a, T> for ShowViews<'a, T> { } impl<'a, T: Transaction + 'a> ExecutorNode<'a, T> for ShowViews<'a, T> { - fn next_tuple( + fn next_tuple( &mut self, arena: &mut ExecArena<'a, T>, - plan_arena: &mut (dyn MetaArena + 'a), + plan_arena: &mut ExecMetaArena, ) -> Result<(), DatabaseError> { if self.metas.is_none() { let context = arena.context(); diff --git a/src/execution/dql/sort.rs b/src/execution/dql/sort.rs index 85257185..66db8873 100644 --- a/src/execution/dql/sort.rs +++ b/src/execution/dql/sort.rs @@ -18,7 +18,7 @@ use crate::execution::{ }; use crate::planner::operator::sort::{SortField, SortOperator}; use crate::planner::LogicalPlan; -use crate::planner::MetaArena; +use crate::planner::{ExecMetaArena, MetaArena}; use crate::storage::Transaction; use crate::types::tuple::Tuple; use crate::types::value::DataValue; @@ -182,10 +182,10 @@ impl<'a, T: Transaction + 'a> ReadExecutor<'a, T> for Sort<'a> { } impl<'a, T: Transaction + 'a> ExecutorNode<'a, T> for Sort<'a> { - fn next_tuple( + fn next_tuple( &mut self, arena: &mut ExecArena<'a, T>, - plan_arena: &mut (dyn MetaArena + 'a), + plan_arena: &mut ExecMetaArena, ) -> Result<(), DatabaseError> { loop { if let Some((_, tuple)) = self.rows.pop() { diff --git a/src/execution/dql/top_k.rs b/src/execution/dql/top_k.rs index 108a772b..cdeaaed0 100644 --- a/src/execution/dql/top_k.rs +++ b/src/execution/dql/top_k.rs @@ -20,7 +20,7 @@ use crate::execution::{ use crate::planner::operator::sort::SortField; use crate::planner::operator::top_k::TopKOperator; use crate::planner::LogicalPlan; -use crate::planner::MetaArena; +use crate::planner::{ExecMetaArena, MetaArena}; use crate::storage::table_codec::BumpBytes; use crate::storage::Transaction; use crate::types::tuple::Tuple; @@ -143,10 +143,10 @@ impl<'a, T: Transaction + 'a> ReadExecutor<'a, T> for TopK<'a> { } impl<'a, T: Transaction + 'a> ExecutorNode<'a, T> for TopK<'a> { - fn next_tuple( + fn next_tuple( &mut self, arena: &mut ExecArena<'a, T>, - plan_arena: &mut (dyn MetaArena + 'a), + plan_arena: &mut ExecMetaArena, ) -> Result<(), DatabaseError> { if self.output.is_none() { let keep_count = self.offset.unwrap_or(0) + self.limit; diff --git a/src/execution/dql/union.rs b/src/execution/dql/union.rs index 12374e68..1d409344 100644 --- a/src/execution/dql/union.rs +++ b/src/execution/dql/union.rs @@ -17,7 +17,7 @@ use crate::execution::{ build_read, ExecArena, ExecId, ExecNode, ExecutionContext, ExecutorNode, ReadExecutor, }; use crate::planner::LogicalPlan; -use crate::planner::MetaArena; +use crate::planner::{ExecMetaArena, MetaArena}; use crate::storage::Transaction; pub struct Union { left_input: ExecId, @@ -46,10 +46,10 @@ impl<'a, T: Transaction + 'a> ReadExecutor<'a, T> for Union { } impl<'a, T: Transaction + 'a> ExecutorNode<'a, T> for Union { - fn next_tuple( + fn next_tuple( &mut self, arena: &mut ExecArena<'a, T>, - plan_arena: &mut (dyn MetaArena + 'a), + plan_arena: &mut ExecMetaArena, ) -> Result<(), DatabaseError> { if self.reading_left { if arena.next_tuple(self.left_input, plan_arena)? { diff --git a/src/execution/dql/values.rs b/src/execution/dql/values.rs index b872d7a1..9d75aa43 100644 --- a/src/execution/dql/values.rs +++ b/src/execution/dql/values.rs @@ -16,7 +16,7 @@ use crate::errors::DatabaseError; use crate::execution::{ExecArena, ExecId, ExecNode, ExecutionContext, ExecutorNode, ReadExecutor}; use crate::planner::operator::values::ValuesOperator; use crate::planner::ExprRef; -use crate::planner::MetaArena; +use crate::planner::{ExecMetaArena, MetaArena}; use crate::storage::Transaction; use crate::types::tuple::Schema; use crate::types::value::DataValue; @@ -59,10 +59,10 @@ impl<'a, T: Transaction + 'a> ReadExecutor<'a, T> for Values<'a> { } impl<'a, T: Transaction + 'a> ExecutorNode<'a, T> for Values<'a> { - fn next_tuple( + fn next_tuple( &mut self, arena: &mut ExecArena<'a, T>, - plan_arena: &mut (dyn MetaArena + 'a), + plan_arena: &mut ExecMetaArena, ) -> Result<(), DatabaseError> { if self.remaining_rows == 0 { arena.finish(); diff --git a/src/execution/dql/window.rs b/src/execution/dql/window.rs index cf5453da..32cc9aa1 100644 --- a/src/execution/dql/window.rs +++ b/src/execution/dql/window.rs @@ -20,7 +20,7 @@ use crate::expression::window::WindowFunctionKind; use crate::planner::operator::sort::SortField; use crate::planner::operator::window::WindowOperator; use crate::planner::LogicalPlan; -use crate::planner::MetaArena; +use crate::planner::{ExecMetaArena, MetaArena}; use crate::storage::Transaction; use crate::types::tuple::Tuple; use crate::types::value::DataValue; @@ -219,10 +219,10 @@ impl<'a> Window<'a> { } impl<'a, T: Transaction + 'a> ExecutorNode<'a, T> for Window<'a> { - fn next_tuple( + fn next_tuple( &mut self, arena: &mut ExecArena<'a, T>, - plan_arena: &mut (dyn MetaArena + 'a), + plan_arena: &mut ExecMetaArena, ) -> Result<(), DatabaseError> { let mut output_ready = true; loop { diff --git a/src/execution/mod.rs b/src/execution/mod.rs index 18210827..a43b2421 100644 --- a/src/execution/mod.rs +++ b/src/execution/mod.rs @@ -76,7 +76,7 @@ use crate::expression::ScalarExpression; use crate::planner::operator::join::JoinCondition; use crate::planner::operator::{Operator, PhysicalOption, PlanImpl}; use crate::planner::MetaArena; -use crate::planner::{ExprRef, LogicalPlan, PlanKeeper}; +use crate::planner::{ExecMetaArena, ExprRef, LogicalPlan, PlanKeeper}; use crate::storage::table_codec::TableCodec; use crate::storage::{StatisticsMetaCache, TableCache, Transaction, ViewCache}; use crate::types::index::RuntimeIndexProbe; @@ -179,9 +179,9 @@ impl<'a, T: Transaction + 'a> Executor<'a, T> { } } - pub(crate) fn next_tuple( + pub(crate) fn next_tuple( &mut self, - plan_arena: &mut (dyn MetaArena + 'a), + plan_arena: &mut ExecMetaArena, ) -> Result, DatabaseError> { if !self.arena.next_tuple(self.root, plan_arena)? { return Ok(None); @@ -228,7 +228,7 @@ pub(crate) enum ExecNode<'a, T: Transaction + 'a> { Projection(Projection<'a>), RecursiveCte(RecursiveCte<'a, T>), RecursiveScan(RecursiveScan), - ScalarQueryInit(ScalarQueryInit), + ScalarQueryInit(ScalarQueryInit<'a>), ScalarSubquery(ScalarSubquery), SetMembership(SetMembership), SeqScan(SeqScan<'a, T>), @@ -247,18 +247,18 @@ pub(crate) enum ExecNode<'a, T: Transaction + 'a> { } pub(crate) trait ExecutorNode<'a, T: Transaction + 'a>: Sized { - fn next_tuple( + fn next_tuple( &mut self, arena: &mut ExecArena<'a, T>, - plan_arena: &mut (dyn MetaArena + 'a), + plan_arena: &mut ExecMetaArena, ) -> Result<(), DatabaseError>; } impl<'a, T: Transaction + 'a> ExecNode<'a, T> { - fn next_tuple( + fn next_tuple( &mut self, arena: &mut ExecArena<'a, T>, - plan_arena: &mut (dyn MetaArena + 'a), + plan_arena: &mut ExecMetaArena, ) -> Result<(), DatabaseError> { match self { ExecNode::AddColumn(exec) => { @@ -352,7 +352,7 @@ impl<'a, T: Transaction + 'a> ExecNode<'a, T> { >::next_tuple(exec, arena, plan_arena) } ExecNode::ScalarQueryInit(exec) => { - >::next_tuple(exec, arena, plan_arena) + as ExecutorNode<'a, T>>::next_tuple(exec, arena, plan_arena) } ExecNode::ScalarSubquery(exec) => { >::next_tuple(exec, arena, plan_arena) @@ -688,10 +688,10 @@ impl<'a, T: Transaction + 'a> ExecArena<'a, T> { self.resume(); } - pub(crate) fn next_tuple( + pub(crate) fn next_tuple( &mut self, id: ExecId, - plan_arena: &mut (dyn MetaArena + 'a), + plan_arena: &mut ExecMetaArena, ) -> Result { self.result.status = None; let slot = &self.nodes.items[id] as *const std::cell::RefCell>; @@ -793,7 +793,7 @@ where Operator::Filter(op) => read!(Filter, (op, plan.childrens.only())), Operator::ScalarQueryInit(op) => { let (left, right) = plan.childrens.twins(); - read!(ScalarQueryInit, (op, left, right)) + read!(ScalarQueryInit<'a>, (op, left, right)) } Operator::MarkApply(op) => { let (left, right) = plan.childrens.twins(); @@ -893,11 +893,8 @@ where match &plan.operator { Operator::ScalarQueryInit(op) => { let (input, init) = plan.childrens.twins(); - let init = build_read(arena, plan_arena, init, cache, transaction); let input = build_write(arena, plan_arena, input, cache, transaction); - arena.push(ExecNode::ScalarQueryInit(ScalarQueryInit::new( - input, init, op.id, - ))) + ScalarQueryInit::build(op, input, init, arena, plan_arena, cache, transaction) } Operator::Insert(op) => write!(Insert<'a>, (op, plan.childrens.only())), Operator::Update(op) => write!(Update<'a>, (op, plan.childrens.only())), @@ -930,7 +927,6 @@ where #[cfg(all(test, not(target_arch = "wasm32")))] mod test_utils { use super::*; - use crate::planner::ExecArenaView; static EMPTY_SCALA_FUNCTIONS: std::sync::LazyLock = std::sync::LazyLock::new(ScalaFunctions::default); @@ -953,7 +949,7 @@ mod test_utils { pub(crate) struct TestExecutor<'a, T: Transaction + 'a> { executor: Executor<'a, T>, - plan_arena: ExecArenaView>, + plan_arena: ExecMetaArena>, } impl TestExecutor<'_, T> { @@ -982,7 +978,7 @@ mod test_utils { transaction, ); TestExecutor { - plan_arena: ExecArenaView::new(plan_arena), + plan_arena: ExecMetaArena::new(plan_arena), executor: Executor::new(arena, root, PlanKeeper::empty()), } } @@ -1008,7 +1004,7 @@ mod test_utils { transaction, ); TestExecutor { - plan_arena: ExecArenaView::new(plan_arena), + plan_arena: ExecMetaArena::new(plan_arena), executor: Executor::new(arena, root, PlanKeeper::empty()), } } @@ -1040,7 +1036,7 @@ mod test { #[test] fn active_nodes_cannot_be_overwritten_or_relocated() { let table_arena = TableArenaCell::default(); - let mut plan_arena = PlanArena::new(&table_arena); + let mut plan_arena = ExecMetaArena::new(PlanArena::new(&table_arena)); let mut arena = ExecArena::<'_, MemoryTransaction>::with_capacity(0); arena.push(ExecNode::Dummy(Dummy::default())); let slot = diff --git a/src/expression/eq_col.rs b/src/expression/eq_col.rs index e1707421..11fd3be8 100644 --- a/src/expression/eq_col.rs +++ b/src/expression/eq_col.rs @@ -83,14 +83,34 @@ impl ExprVisitor for EqIgnoreColRefPosVisitor<'_, '_> { walk_expr(self, lhs, arena) } - fn visit_init( + fn visit_outer_param( + &mut self, + id: ScalarQueryRef, + ty: &LogicalType, + _arena: &(dyn MetaArena + '_), + ) -> Result<(), DatabaseError> { + self.equal = matches!(self.rhs(), ScalarExpression::OuterParam { id: rhs, ty: rhs_ty } if id == *rhs && ty == rhs_ty); + Ok(()) + } + + fn visit_outer_value( + &mut self, + id: ScalarQueryRef, + ty: &LogicalType, + _arena: &(dyn MetaArena + '_), + ) -> Result<(), DatabaseError> { + self.equal = matches!(self.rhs(), ScalarExpression::OuterValue { id: rhs, ty: rhs_ty } if id == *rhs && ty == rhs_ty); + Ok(()) + } + + fn visit_init_value( &mut self, id: ScalarQueryRef, ty: &LogicalType, _arena: &(dyn MetaArena + '_), ) -> Result<(), DatabaseError> { self.equal = match self.rhs() { - ScalarExpression::Init { + ScalarExpression::InitValue { id: rhs_id, ty: rhs_ty, } => id == *rhs_id && ty == rhs_ty, diff --git a/src/expression/evaluator.rs b/src/expression/evaluator.rs index 54d77aef..ee2aaf69 100644 --- a/src/expression/evaluator.rs +++ b/src/expression/evaluator.rs @@ -53,7 +53,9 @@ impl ScalarExpression { } val => Ok(Cow::Borrowed(val)), }, - ScalarExpression::Init { id, .. } => { + ScalarExpression::InitValue { id, .. } + | ScalarExpression::OuterValue { id, .. } + | ScalarExpression::OuterParam { id, .. } => { let value = arena.init_value(*id).ok_or_else(|| { DatabaseError::InvalidValue(format!("scalar query {id} is not initialized")) })?; @@ -415,7 +417,7 @@ fn trim_string(value: &str, trim_what: &str, trim_where: Option) mod tests { use super::*; use crate::planner::test::PlanArenaTestExt; - use crate::planner::{ExecArenaView, ExprRef, TableArenaCell}; + use crate::planner::{ExecMetaArena, ExprRef, TableArenaCell}; fn const_in( arena: &mut PlanArena<'_>, @@ -444,14 +446,14 @@ mod tests { let table_arena = TableArenaCell::default(); let mut arena = PlanArena::new(&table_arena); let reference = arena.alloc_scalar_query_ref(); - let mut init = arena.alloc_expression(ScalarExpression::Init { + let mut init = arena.alloc_expression(ScalarExpression::InitValue { id: reference, ty: LogicalType::Integer, }); PositionShift { delta: 5 }.visit(&mut init, &mut arena)?; assert!(matches!( arena.expression(init), - ScalarExpression::Init { id, ty: LogicalType::Integer } if *id == reference + ScalarExpression::InitValue { id, ty: LogicalType::Integer } if *id == reference )); assert!(!init.any_referenced_column(&arena, |_, _| true)?); let row = Tuple::new(None, vec![DataValue::Int32(99)]); @@ -459,7 +461,7 @@ mod tests { arena.expression(init).eval(&arena, Some(&row)), Err(DatabaseError::InvalidValue(message)) if message.contains("is not initialized") )); - let mut view = ExecArenaView::new(arena); + let mut view = ExecMetaArena::new(arena); assert!(view.expression(init).eval(&view, Some(&row)).is_err()); view.set_init_value(reference, DataValue::Null); let value = view.expression(init).eval(&view, Some(&row))?; diff --git a/src/expression/mod.rs b/src/expression/mod.rs index 08b1129f..e84a09a6 100644 --- a/src/expression/mod.rs +++ b/src/expression/mod.rs @@ -66,7 +66,15 @@ pub enum AliasType { #[derive(Debug, PartialEq, Eq, Clone, Hash, ReferenceSerialization)] pub enum ScalarExpression { Constant(DataValue), - Init { + OuterParam { + id: ScalarQueryRef, + ty: LogicalType, + }, + OuterValue { + id: ScalarQueryRef, + ty: LogicalType, + }, + InitValue { id: ScalarQueryRef, ty: LogicalType, }, @@ -485,7 +493,9 @@ impl TypeCast for ScalarExpression { } ScalarExpression::Position { .. } => Cow::Owned(LogicalType::Integer), ScalarExpression::Alias { expr, .. } => expr.return_type(arena), - ScalarExpression::Init { ty, .. } => Cow::Borrowed(ty), + ScalarExpression::InitValue { ty, .. } + | ScalarExpression::OuterValue { ty, .. } + | ScalarExpression::OuterParam { ty, .. } => Cow::Borrowed(ty), ScalarExpression::Empty | ScalarExpression::TableFunction(_) => unreachable!(), ScalarExpression::Tuple(exprs) => Cow::Owned(LogicalType::Tuple( exprs @@ -556,7 +566,9 @@ impl Explain for ExprRef { match arena.expression(*self) { ScalarExpression::Constant(value) => write!(f, "{value}"), - ScalarExpression::Init { id, .. } => write!(f, "Init({id})"), + ScalarExpression::InitValue { id, .. } => write!(f, "InitValue({id})"), + ScalarExpression::OuterValue { id, .. } => write!(f, "OuterValue({id})"), + ScalarExpression::OuterParam { id, .. } => write!(f, "OuterParam({id})"), ScalarExpression::ColumnRef { column, .. } => Explain::fmt(column, arena, f), ScalarExpression::Alias { alias, expr } => match alias { AliasType::Name(alias) => f.write_str(alias), diff --git a/src/expression/range_detacher.rs b/src/expression/range_detacher.rs index d132b71c..7ecb6e0b 100644 --- a/src/expression/range_detacher.rs +++ b/src/expression/range_detacher.rs @@ -530,7 +530,9 @@ impl<'a, M: RangeColumnMatcher, A: MetaArena + ?Sized> RangeDetacher<'a, M, A> { None } - ScalarExpression::Init { .. } + ScalarExpression::OuterParam { .. } + | ScalarExpression::OuterValue { .. } + | ScalarExpression::InitValue { .. } | ScalarExpression::Constant(_) | ScalarExpression::Alias { .. } | ScalarExpression::TypeCast { .. } @@ -554,7 +556,9 @@ impl<'a, M: RangeColumnMatcher, A: MetaArena + ?Sized> RangeDetacher<'a, M, A> { | ScalarExpression::TableFunction(_) | ScalarExpression::Empty => unreachable!(), }, - ScalarExpression::Init { .. } + ScalarExpression::OuterParam { .. } + | ScalarExpression::OuterValue { .. } + | ScalarExpression::InitValue { .. } | ScalarExpression::Constant(_) | ScalarExpression::ColumnRef { .. } => None, // FIXME: support [RangeDetacher::_detach] diff --git a/src/expression/visitor.rs b/src/expression/visitor.rs index 9e269d3e..e65c02ac 100644 --- a/src/expression/visitor.rs +++ b/src/expression/visitor.rs @@ -49,7 +49,25 @@ pub trait ExprVisitor: Sized { Ok(true) } - fn visit_init( + fn visit_init_value( + &mut self, + _id: ScalarQueryRef, + _ty: &LogicalType, + _arena: &A, + ) -> Result<(), DatabaseError> { + Ok(()) + } + + fn visit_outer_param( + &mut self, + _id: ScalarQueryRef, + _ty: &LogicalType, + _arena: &A, + ) -> Result<(), DatabaseError> { + Ok(()) + } + + fn visit_outer_value( &mut self, _id: ScalarQueryRef, _ty: &LogicalType, @@ -303,7 +321,9 @@ pub fn walk_expr>( ) -> Result<(), DatabaseError> { match arena.expression(expr) { ScalarExpression::Constant(value) => visitor.visit_constant(value), - ScalarExpression::Init { id, ty } => visitor.visit_init(*id, ty, arena), + ScalarExpression::OuterParam { id, ty } => visitor.visit_outer_param(*id, ty, arena), + ScalarExpression::OuterValue { id, ty } => visitor.visit_outer_value(*id, ty, arena), + ScalarExpression::InitValue { id, ty } => visitor.visit_init_value(*id, ty, arena), ScalarExpression::ColumnRef { column, .. } => visitor.visit_column_ref(column), ScalarExpression::Alias { expr, alias } => visitor.visit_alias(*expr, alias, arena), ScalarExpression::TypeCast { diff --git a/src/expression/visitor_mut.rs b/src/expression/visitor_mut.rs index 2903058f..41c0cb66 100644 --- a/src/expression/visitor_mut.rs +++ b/src/expression/visitor_mut.rs @@ -61,7 +61,25 @@ impl ExprVisitorMut for PositionShift { } pub trait ExprVisitorMut: Sized { - fn visit_init( + fn visit_outer_param( + &mut self, + _id: &mut ScalarQueryRef, + _ty: &mut LogicalType, + _arena: &mut dyn MetaArena, + ) -> Result<(), DatabaseError> { + Ok(()) + } + + fn visit_outer_value( + &mut self, + _id: &mut ScalarQueryRef, + _ty: &mut LogicalType, + _arena: &mut dyn MetaArena, + ) -> Result<(), DatabaseError> { + Ok(()) + } + + fn visit_init_value( &mut self, _id: &mut ScalarQueryRef, _ty: &mut LogicalType, @@ -404,7 +422,9 @@ pub fn walk_mut_expr( std::mem::replace(&mut *arena.expression_mut(*expr), ScalarExpression::Empty); let result = match &mut expression { ScalarExpression::Constant(value) => visitor.visit_constant(value, arena), - ScalarExpression::Init { id, ty } => visitor.visit_init(id, ty, arena), + ScalarExpression::OuterParam { id, ty } => visitor.visit_outer_param(id, ty, arena), + ScalarExpression::OuterValue { id, ty } => visitor.visit_outer_value(id, ty, arena), + ScalarExpression::InitValue { id, ty } => visitor.visit_init_value(id, ty, arena), ScalarExpression::ColumnRef { column, position } => { visitor.visit_column_ref(column, position, arena) } diff --git a/src/optimizer/rule/normalization/column_pruning.rs b/src/optimizer/rule/normalization/column_pruning.rs index 82bf63be..6653cbf9 100644 --- a/src/optimizer/rule/normalization/column_pruning.rs +++ b/src/optimizer/rule/normalization/column_pruning.rs @@ -531,7 +531,21 @@ impl ColumnPruning { let Childrens::Twins { left, right } = childrens else { unreachable!("scalar initialization requires two children"); }; + Self::extend_operator_referenced_columns( + operator, + &mut required_columns, + arena, + )?; + let left_start = outcome.removed_positions.len(); Self::_apply_appending(required_columns, all_referenced, left, outcome, arena)?; + if outcome.changed { + Self::remap_operator_after_child_change( + operator, + &outcome.removed_positions[left_start..], + &mut outcome.remapped_exprs, + arena, + )?; + } changed |= outcome.changed; let main_removed_end = outcome.removed_positions.len(); Self::_apply_appending( diff --git a/src/orm/mod.rs b/src/orm/mod.rs index 9ce6178a..dc1d9a80 100644 --- a/src/orm/mod.rs +++ b/src/orm/mod.rs @@ -1198,8 +1198,8 @@ where }; build(&mut context)? }; - if child_binder.context.has_outer_refs() { - self.binder.context.mark_outer_ref(); + if child_binder.context.has_join_outer_ref() { + self.binder.context.mark_join_outer_ref(); } Ok(plan) } diff --git a/src/planner/arena.rs b/src/planner/arena.rs index 8ca8fefa..397d0be4 100644 --- a/src/planner/arena.rs +++ b/src/planner/arena.rs @@ -46,7 +46,27 @@ impl ScalarQueryRelocator { } impl ExprVisitorMut for ScalarQueryRelocator { - fn visit_init( + fn visit_outer_param( + &mut self, + reference: &mut ScalarQueryRef, + _ty: &mut LogicalType, + _arena: &mut dyn MetaArena, + ) -> Result<(), DatabaseError> { + self.relocate(reference); + Ok(()) + } + + fn visit_outer_value( + &mut self, + reference: &mut ScalarQueryRef, + _ty: &mut LogicalType, + _arena: &mut dyn MetaArena, + ) -> Result<(), DatabaseError> { + self.relocate(reference); + Ok(()) + } + + fn visit_init_value( &mut self, reference: &mut ScalarQueryRef, _ty: &mut LogicalType, @@ -77,7 +97,9 @@ impl<'a> OperatorVisitorMut<'a> for ScalarQueryPlanRelocator<'_> { &mut self, op: &'a mut ScalarQueryInitOperator, ) -> Result<(), DatabaseError> { - self.references.relocate(&mut op.id); + for (reference, _) in &mut op.param_bindings { + self.references.relocate(reference); + } Ok(()) } } @@ -225,10 +247,6 @@ pub trait MetaArena { None } - fn set_init_value(&mut self, _reference: ScalarQueryRef, _value: DataValue) { - panic!("scalar query initialization requires an execution arena view") - } - fn alloc_dummy(&mut self, name: &str) -> ColumnRef { self.table_arena_cell().borrow().alloc_dummy(name) } @@ -294,9 +312,6 @@ impl MetaArena for Box { fn init_value(&self, reference: ScalarQueryRef) -> Option<&DataValue> { (**self).init_value(reference) } - fn set_init_value(&mut self, reference: ScalarQueryRef, value: DataValue) { - (**self).set_init_value(reference, value) - } fn alloc_column(&mut self, column: ColumnCatalog) -> ColumnRef { (**self).alloc_column(column) } @@ -327,12 +342,12 @@ impl MetaArena for Box { } /// Owns the metadata and scalar-query cache for one statement execution. -pub struct ExecArenaView { +pub struct ExecMetaArena { parent: A, init_values: HashMap>, } -impl ExecArenaView { +impl ExecMetaArena { pub(crate) fn new(parent: A) -> Self { Self { parent, @@ -340,12 +355,26 @@ impl ExecArenaView { } } + pub(crate) fn boxed<'a>(self) -> ExecMetaArena> + where + A: 'a, + { + ExecMetaArena { + parent: Box::new(self.parent), + init_values: self.init_values, + } + } + pub(crate) fn into_parent(self) -> A { self.parent } + + pub(crate) fn set_init_value(&mut self, reference: ScalarQueryRef, value: DataValue) { + self.init_values.insert(reference, Some(value)); + } } -impl MetaArena for ExecArenaView { +impl MetaArena for ExecMetaArena { fn arena_id(&self) -> usize { self.parent.arena_id() } @@ -370,9 +399,6 @@ impl MetaArena for ExecArenaView { fn bound_param(&self, id: usize) -> Option<&DataValue> { self.parent.bound_param(id) } - fn set_init_value(&mut self, reference: ScalarQueryRef, value: DataValue) { - self.init_values.insert(reference, Some(value)); - } fn expression_mut(&mut self, expr: ExprRef) -> ArenaExprMut<'_> { self.parent.expression_mut(expr) } @@ -1204,7 +1230,7 @@ mod tests { pos: 0 } ); - let mut view = ExecArenaView::new(params); + let mut view = ExecMetaArena::new(params); view.set_init_value(catalog, DataValue::Int32(11)); assert_eq!(view.arena_id(), 2); assert_eq!(view.init_value(catalog), Some(&DataValue::Int32(11))); diff --git a/src/planner/mod.rs b/src/planner/mod.rs index 090b6328..855277e2 100644 --- a/src/planner/mod.rs +++ b/src/planner/mod.rs @@ -30,7 +30,7 @@ use kite_sql_serde_macros::ReferenceSerialization; use std::fmt; use std::hash::{Hash, Hasher}; -pub(crate) use arena::{ExecArenaView, ParamArena, PlanRef}; +pub(crate) use arena::{ExecMetaArena, ParamArena, PlanRef}; pub use arena::{ExprRef, MetaArena, PlanArena, TableArena, TableArenaCell}; pub(crate) use plan_keeper::{PlanInput, PlanKeeper}; pub use scalar_query_ref::ScalarQueryRef; diff --git a/src/planner/operator/mod.rs b/src/planner/operator/mod.rs index fa129e89..fe80e28e 100644 --- a/src/planner/operator/mod.rs +++ b/src/planner/operator/mod.rs @@ -248,7 +248,6 @@ macro_rules! impl_display_explain { } impl_display_explain!( - ScalarQueryInitOperator, MarkApplyOperator, ScalarSubqueryOperator, FunctionScanOperator, @@ -367,6 +366,16 @@ impl Operator { Ok(()) } + fn visit_scalar_query_init( + &mut self, + op: &'operator ScalarQueryInitOperator, + ) -> Result<(), DatabaseError> { + for (_, expr) in &op.param_bindings { + ExprVisitor::visit(self, *expr, self.arena)?; + } + Ok(()) + } + fn visit_mark_apply( &mut self, op: &'operator MarkApplyOperator, @@ -1002,10 +1011,12 @@ mod tests { }); assert_eq!(referenced_columns(&delete, &mut arena)?, vec![a]); + let scalar = Operator::ScalarQueryInit(ScalarQueryInitOperator { + value: arena.alloc_expression(ScalarExpression::Constant(DataValue::Null)), + param_bindings: Vec::new(), + }); + assert!(referenced_columns(&scalar, &mut arena)?.is_empty()); let no_reference_operators = [ - Operator::ScalarQueryInit(ScalarQueryInitOperator { - id: arena.alloc_scalar_query_ref(), - }), Operator::ScalarSubquery(ScalarSubqueryOperator), Operator::Analyze(AnalyzeOperator { table_name: "users".into(), @@ -1390,11 +1401,15 @@ mod tests { let left = LogicalPlan::new(Operator::ShowTable, Childrens::None); let right = LogicalPlan::new(Operator::ShowView, Childrens::None); - let apply = - ScalarQueryInitOperator::build(left.clone(), right, arena.alloc_scalar_query_ref()); + let id = arena.alloc_scalar_query_ref(); + let value = arena.alloc_expression(ScalarExpression::InitValue { + id, + ty: LogicalType::Integer, + }); + let apply = ScalarQueryInitOperator::build(left.clone(), right, value, Vec::new()); assert_eq!( apply.operator.explain(&arena).to_string(), - "ScalarQueryInit #1:0" + "ScalarQueryInit InitValue(1:0)" ); assert!(matches!(*apply.childrens, Childrens::Twins { .. })); diff --git a/src/planner/operator/scalar_query_init.rs b/src/planner/operator/scalar_query_init.rs index 0f4bf462..1e3b22b8 100644 --- a/src/planner/operator/scalar_query_init.rs +++ b/src/planner/operator/scalar_query_init.rs @@ -13,30 +13,46 @@ // limitations under the License. use super::Operator; -use crate::planner::{Childrens, LogicalPlan, ScalarQueryRef}; +use crate::expression::ScalarExpression; +use crate::planner::{Childrens, Explain, ExprRef, LogicalPlan, MetaArena, ScalarQueryRef}; use kite_sql_serde_macros::ReferenceSerialization; use std::fmt; -use std::fmt::Formatter; #[derive(Debug, PartialEq, Eq, Clone, Hash, ReferenceSerialization)] pub struct ScalarQueryInitOperator { - pub id: ScalarQueryRef, + pub value: ExprRef, + pub param_bindings: Vec<(ScalarQueryRef, ExprRef)>, } impl ScalarQueryInitOperator { - pub fn build(left: LogicalPlan, right: LogicalPlan, id: ScalarQueryRef) -> LogicalPlan { + pub fn build( + left: LogicalPlan, + right: LogicalPlan, + value: ExprRef, + param_bindings: Vec<(ScalarQueryRef, ExprRef)>, + ) -> LogicalPlan { LogicalPlan::new( - Operator::ScalarQueryInit(ScalarQueryInitOperator { id }), + Operator::ScalarQueryInit(Self { + value, + param_bindings, + }), Childrens::Twins { left: Box::new(left), right: Box::new(right), }, ) } + + pub(crate) fn reference(&self, arena: &dyn MetaArena) -> ScalarQueryRef { + match arena.expression(self.value) { + ScalarExpression::InitValue { id, .. } | ScalarExpression::OuterValue { id, .. } => *id, + _ => unreachable!("scalar initializer requires a value marker"), + } + } } -impl fmt::Display for ScalarQueryInitOperator { - fn fmt(&self, f: &mut Formatter) -> fmt::Result { - write!(f, "ScalarQueryInit #{}", self.id) +impl Explain for ScalarQueryInitOperator { + fn fmt(&self, arena: &dyn MetaArena, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "ScalarQueryInit {}", self.value.explain(arena)) } } diff --git a/src/planner/operator/visitor.rs b/src/planner/operator/visitor.rs index 0cf604bc..b1e3b1ad 100644 --- a/src/planner/operator/visitor.rs +++ b/src/planner/operator/visitor.rs @@ -222,6 +222,17 @@ impl<'a, V: ExprVisitor, A: MetaArena + ?Sized> OperatorVisitor<'a> Ok(()) } + fn visit_scalar_query_init( + &mut self, + op: &'a ScalarQueryInitOperator, + ) -> Result<(), DatabaseError> { + ExprVisitor::visit(self.visitor, op.value, self.arena)?; + for (_, expr) in &op.param_bindings { + ExprVisitor::visit(self.visitor, *expr, self.arena)?; + } + Ok(()) + } + fn visit_mark_apply(&mut self, op: &'a MarkApplyOperator) -> Result<(), DatabaseError> { for expr in &op.predicates { ExprVisitor::visit(self.visitor, *expr, self.arena)?; @@ -452,7 +463,8 @@ pub(crate) mod tests { force_spill: false, }), Operator::ScalarQueryInit(ScalarQueryInitOperator { - id: arena.alloc_scalar_query_ref(), + param_bindings: Vec::new(), + value: expr(0), }), Operator::MarkApply(mark_apply), Operator::Filter(FilterOperator { @@ -660,7 +672,7 @@ pub(crate) mod tests { for operator in &operators { visitor.visit_operator(operator)?; } - assert_eq!(counter.0, 21); // Includes the Values row expression. + assert_eq!(counter.0, 22); // Includes the Values row expression. Ok(()) } diff --git a/src/planner/operator/visitor_mut.rs b/src/planner/operator/visitor_mut.rs index b7fd5825..0255a5c2 100644 --- a/src/planner/operator/visitor_mut.rs +++ b/src/planner/operator/visitor_mut.rs @@ -274,6 +274,17 @@ impl<'a, V: ExprVisitorMut> OperatorVisitorMut<'a> for OperatorExprVisitorMut<'_ Ok(()) } + fn visit_scalar_query_init( + &mut self, + op: &'a mut ScalarQueryInitOperator, + ) -> Result<(), DatabaseError> { + ExprVisitorMut::visit(self.visitor, &mut op.value, self.arena)?; + for (_, expr) in &mut op.param_bindings { + ExprVisitorMut::visit(self.visitor, expr, self.arena)?; + } + Ok(()) + } + fn visit_mark_apply(&mut self, op: &'a mut MarkApplyOperator) -> Result<(), DatabaseError> { for expr in &mut op.predicates { ExprVisitorMut::visit(self.visitor, expr, self.arena)?; @@ -523,7 +534,7 @@ mod tests { visitor.visit_operator(operator, None)?; } } - assert_eq!(counter.0, 21); // Includes the Values row expression. + assert_eq!(counter.0, 22); // Includes the Values row expression. Ok(()) } diff --git a/src/storage/mod.rs b/src/storage/mod.rs index 649d6ee2..5b756bc0 100644 --- a/src/storage/mod.rs +++ b/src/storage/mod.rs @@ -1240,6 +1240,7 @@ pub(crate) fn reuse_bound_as_excluded(bound: &mut Bound, key: &[u8]) { *bound = Bound::Excluded(bytes); } +#[cfg(any(test, all(not(target_arch = "wasm32"), feature = "lmdb")))] pub(crate) fn bounds_contain(min: Bound<&[u8]>, max: Bound<&[u8]>, key: &[u8]) -> bool { std::ops::RangeBounds::<[u8]>::contains(&(min, max), key) } diff --git a/tests/slt/subquery.slt b/tests/slt/subquery.slt index 872df2d1..279a487f 100644 --- a/tests/slt/subquery.slt +++ b/tests/slt/subquery.slt @@ -726,8 +726,12 @@ select (select v from init_rows) statement error select id from init_rows where v = (select v from init_rows) -statement error +query I rowsort select (select x.v from init_rows as x where x.id = init_rows.id) from init_rows +---- +10 +20 +30 statement ok create view init_max as select (select max(v) from init_rows) as m @@ -792,3 +796,216 @@ drop view init_min statement ok drop table init_rows + +# Correlated values use the execution arena; outer columns remain input dependencies. +statement ok +create table corr_rows(id int primary key, b int); + +statement ok +insert into corr_rows values (1,10),(2,20),(3,30),(4,null),(5,20); + +# WHERE and SELECT registrations are attached before their own consumers. +query T +explain select id,(select 7) from corr_rows where b<(select 25); +---- +Projection [corr_rows.id, InitValue(1:0)] [Project => (Sort Option: Follow)] ScalarQueryInit InitValue(1:0) Filter (corr_rows.b < InitValue(1:1)), Is Having: false [Filter => (Sort Option: Follow)] ScalarQueryInit InitValue(1:1) TableScan corr_rows -> [corr_rows.id, corr_rows.b] [SeqScan => (Sort Option: None)] ScalarSubquery [ScalarSubquery => (Sort Option: Follow)] Projection [25] [Project => (Sort Option: Follow)] Dummy [Dummy => (Sort Option: None)] ScalarSubquery [ScalarSubquery => (Sort Option: Follow)] Projection [7] [Project => (Sort Option: Follow)] Dummy [Dummy => (Sort Option: None)] + +query II rowsort +select id,(select 7) from corr_rows where b<(select 25); +---- +1 7 +2 7 +5 7 + +# b is needed solely as an outer parameter, not as a projected output column. +query I rowsort +select (select count(*) from corr_rows x where x.bcorr_rows.b) from corr_rows; +---- +1 0 3 +2 1 1 +3 3 0 +4 0 0 +5 1 1 + +query II rowsort +select o.id,(select count(*) from corr_rows x where x.b>o.b and x.id2) from corr_rows where id<4; +---- +1 10 +2 NULL +3 30 + +# Owned text values and NULL remain distinct across repeated scalar execution. +query IT rowsort +select id,(select cast(x.b as varchar(12)) from corr_rows x where x.id=corr_rows.id) from corr_rows; +---- +1 10 +2 20 +3 30 +4 NULL +5 20 + +statement error scalar subquery returned more than one row +select id,(select x.b from corr_rows x where x.b=corr_rows.b) from corr_rows; + +query II rowsort +select id,coalesce((select x.b from corr_rows x where x.id=corr_rows.id),-1)+(select 7) from corr_rows; +---- +1 17 +2 27 +3 37 +4 6 +5 27 + +statement error correlated scalar values across ORDER BY require a row-scoped execution context +select (select count(*) from corr_rows x where x.bl.id) from corr_rows o cross join corr_lookup l where o.id=3; +---- +3 1 2 +3 2 1 + +query II rowsort +select id,(select sum(x.b) from corr_rows x where x.b Date: Thu, 8 Oct 2026 04:14:40 +0800 Subject: [PATCH 11/16] feat: preserve correlated scalar values through sorting Mark row-scoped scalar references and move their values alongside cached sort rows, restoring them before projection. Cover memory sort, TopK and spilled merge runs without scanning plans or introducing separate value maps. Flatten memory-sort keys, compose spill codecs for references and pairs, and consume range combinations directly through callbacks. Extend scalar-subquery regressions including NULL, text and mixed constant/row-dependent results. --- src/binder/expr.rs | 4 +- src/binder/select.rs | 13 -- src/execution/dql/external_sort.rs | 41 ++++- src/execution/dql/scalar_query_init.rs | 2 +- src/execution/dql/sort.rs | 237 ++++++++++++++++++++----- src/execution/dql/top_k.rs | 85 +++++---- src/execution/spill/codec.rs | 69 ++++++- src/expression/evaluator.rs | 2 +- src/expression/range_detacher.rs | 51 ++++-- src/planner/arena.rs | 61 +++++-- src/planner/operator/mod.rs | 2 +- src/planner/scalar_query_ref.rs | 9 +- tests/slt/subquery.slt | 34 +++- 13 files changed, 465 insertions(+), 145 deletions(-) diff --git a/src/binder/expr.rs b/src/binder/expr.rs index f994a8d6..053dbecd 100644 --- a/src/binder/expr.rs +++ b/src/binder/expr.rs @@ -229,7 +229,7 @@ impl<'a, T: Transaction, A: AsRef<[(usize, LogicalType)]>> Binder<'a, '_, T, A> )); } let ty = arena.column(schema[0]).datatype().clone(); - let id = arena.alloc_scalar_query_ref(); + let id = arena.alloc_scalar_query_ref(!param_bindings.is_empty()); let value = arena.alloc_expression(if param_bindings.is_empty() { ScalarExpression::InitValue { id, ty: ty.clone() } } else { @@ -440,7 +440,7 @@ impl<'a, T: Transaction, A: AsRef<[(usize, LogicalType)]>> Binder<'a, '_, T, A> arena: &mut PlanArena, ) -> Result { let ty = expr.return_type(arena).into_owned(); - let id = arena.alloc_scalar_query_ref(); + let id = arena.alloc_scalar_query_ref(false); let source = arena.alloc_expression(expr); self.context.scalar_outer_bindings.push((id, source)); Ok(ScalarExpression::OuterParam { id, ty }) diff --git a/src/binder/select.rs b/src/binder/select.rs index fa20484b..9256df71 100644 --- a/src/binder/select.rs +++ b/src/binder/select.rs @@ -641,19 +641,6 @@ where } } } - if self.orderby.is_some() - && self - .binder - .context - .scalar_queries - .iter() - .any(|query| !query.param_bindings.is_empty()) - { - return Err(DatabaseError::UnsupportedStmt( - "correlated scalar values across ORDER BY require a row-scoped execution context" - .into(), - )); - } self.plan = self.binder.bind_scalar_queries( self.plan, &[QueryBindStep::Project, QueryBindStep::Sort], diff --git a/src/execution/dql/external_sort.rs b/src/execution/dql/external_sort.rs index 6fcbdf68..cd47fd57 100644 --- a/src/execution/dql/external_sort.rs +++ b/src/execution/dql/external_sort.rs @@ -80,6 +80,7 @@ impl<'a, T: Transaction + 'a> ExecutorNode<'a, T> for ExternalSort<'a> { arena.finish(); return Ok(()); }; + plan_arena.restore_outer_values(row.outer_values); arena.produce_tuple(row.tuple); return Ok(()); } @@ -94,7 +95,8 @@ impl<'a, T: Transaction + 'a> ExecutorNode<'a, T> for ExternalSort<'a> { let mut runs = Vec::new(); while arena.next_tuple(self.input, plan_arena)? { let tuple = arena.materialize_tuple(); - if let Some(segment) = rows.push(SortRow::new(sort_fields, tuple, plan_arena)?)? { + let row = SortRow::new(sort_fields, tuple, plan_arena)?; + if let Some(segment) = rows.push(row)? { runs.push(Run::new(segment, 1)); } } @@ -285,9 +287,10 @@ mod test { use crate::execution::spill::{SortRow, SpillVec}; use crate::expression::ScalarExpression; use crate::planner::operator::sort::SortField; + use crate::planner::{ExecMetaArena, MetaArena}; use crate::types::tuple::Tuple; - use crate::types::value::DataValue; - use crate::types::LogicalType; + use crate::types::value::{DataValue, Utf8Type}; + use crate::types::{CharLengthUnits, LogicalType}; use std::cmp::Ordering; #[test] @@ -308,6 +311,7 @@ mod test { nulls_first: false, }]; + let mut plan_arena = ExecMetaArena::new(plan_arena); let mut rows = SpillVec::new() .limit(4, usize::MAX) .on_flush(|rows| sort_segment(&sort_fields, rows)); @@ -315,7 +319,7 @@ mod test { let _ = rows.push(SortRow::new( &sort_fields, Tuple::new(None, vec![value]), - &plan_arena, + &mut plan_arena, )?)?; } @@ -358,6 +362,7 @@ mod test { DataValue::Int32(0), ]; + let mut plan_arena = ExecMetaArena::new(plan_arena); let mut rows = SpillVec::new() .limit(2, usize::MAX) .on_flush(|rows| sort_segment(&sort_fields, rows)); @@ -367,13 +372,34 @@ mod test { Some(DataValue::Int32(sequence as i32)), vec![value, DataValue::Int32(sequence as i32)], ); - if let Some(segment) = rows.push(SortRow::new(&sort_fields, tuple, &plan_arena)?)? { + let reference = plan_arena.alloc_scalar_query_ref(true); + plan_arena.set_init_value( + reference, + DataValue::Utf8 { + value: format!("row-{sequence}"), + ty: Utf8Type::Variable(None), + unit: CharLengthUnits::Characters, + }, + ); + let row = SortRow::new(&sort_fields, tuple, &mut plan_arena)?; + if let Some(segment) = rows.push(row)? { runs.push(Run::new(segment, 1)); } } let tuples = finish_sort(rows, runs, &sort_fields, 2)? - .map(|row| row.map(|row| row.tuple)) + .map(|row| { + row.map(|row| { + assert_eq!(row.outer_values.len(), 1); + let sequence = row.tuple.values[1].i32().unwrap(); + assert_eq!(row.outer_values[0].0.pos, sequence as usize); + assert_eq!( + row.outer_values[0].1.utf8(), + Some(format!("row-{sequence}").as_str()) + ); + row.tuple + }) + }) .collect::, _>>()?; let positions = tuples .iter() @@ -433,6 +459,7 @@ mod test { }, ]; + let mut plan_arena = ExecMetaArena::new(plan_arena); let mut rows = SpillVec::new().on_flush(|rows| sort_segment(&sort_fields, rows)); let mut runs = Vec::new(); for position in 0..ROW_COUNT { @@ -444,7 +471,7 @@ mod test { }; let sequence = DataValue::Int32(sequence as i32); let tuple = Tuple::new(Some(sequence.clone()), vec![key, sequence]); - if let Some(segment) = rows.push(SortRow::new(&sort_fields, tuple, &plan_arena)?)? { + if let Some(segment) = rows.push(SortRow::new(&sort_fields, tuple, &mut plan_arena)?)? { runs.push(Run::new(segment, 1)); } } diff --git a/src/execution/dql/scalar_query_init.rs b/src/execution/dql/scalar_query_init.rs index 78d3ac70..42bd61eb 100644 --- a/src/execution/dql/scalar_query_init.rs +++ b/src/execution/dql/scalar_query_init.rs @@ -204,7 +204,7 @@ mod tests { ] { let table_arena = TableArenaCell::default(); let mut metadata = PlanArena::new(&table_arena); - let reference = metadata.alloc_scalar_query_ref(); + let reference = metadata.alloc_scalar_query_ref(false); let column = metadata.alloc_column(ColumnCatalog::new( "v".into(), true, diff --git a/src/execution/dql/sort.rs b/src/execution/dql/sort.rs index 66db8873..402e8088 100644 --- a/src/execution/dql/sort.rs +++ b/src/execution/dql/sort.rs @@ -18,6 +18,7 @@ use crate::execution::{ }; use crate::planner::operator::sort::{SortField, SortOperator}; use crate::planner::LogicalPlan; +use crate::planner::ScalarQueryRef; use crate::planner::{ExecMetaArena, MetaArena}; use crate::storage::Transaction; use crate::types::tuple::Tuple; @@ -48,11 +49,6 @@ impl<'a, T> NullableVec<'a, T> { self.0.len() } - #[inline] - pub(crate) fn iter(&self) -> impl Iterator { - self.0.iter().map(|item| unsafe { item.assume_init_ref() }) - } - #[inline] pub(crate) fn pop(&mut self) -> Option { self.0.pop().map(|item| unsafe { item.assume_init() }) @@ -79,24 +75,25 @@ impl DerefMut for NullableVec<'_, T> { } } -pub(crate) fn sort_tuples( +pub(crate) fn sort_tuples( sort_fields: &[SortField], - tuples: &mut NullableVec<'_, (usize, Tuple)>, - plan_arena: &(dyn MetaArena + '_), + tuples: &mut NullableVec<'_, (usize, SortTuple)>, + plan_arena: &mut ExecMetaArena, ) -> Result<(), DatabaseError> { // Extract the results of calculating SortFields to avoid double calculation // of data during comparison. - let mut eval_values = vec![Vec::new(); sort_fields.len()]; - - for (x, SortField { expr, .. }) in sort_fields.iter().enumerate() { - for (_, tuple) in tuples.iter() { - eval_values[x].push( - plan_arena - .expression(*expr) - .eval(plan_arena, Some(tuple))? - .into_owned(), - ); + let width = sort_fields.len(); + let mut eval_values = Vec::with_capacity(tuples.len() * width); + + for (_, row) in tuples.iter_mut() { + plan_arena.restore_outer_values(std::mem::take(&mut row.outer_values)); + for SortField { expr, .. } in sort_fields { + let value = plan_arena + .expression(*expr) + .eval(plan_arena, Some(&row.tuple))?; + eval_values.push(value.into_owned()); } + row.outer_values = plan_arena.take_outer_values(); } tuples.0.sort_by(|tuple_1, tuple_2| { @@ -104,8 +101,8 @@ pub(crate) fn sort_tuples( let (i_2, _) = unsafe { tuple_2.assume_init_ref() }; compare_sort_keys( sort_fields, - eval_values.iter().map(|values| &values[*i_1]), - eval_values.iter().map(|values| &values[*i_2]), + eval_values[*i_1 * width..(*i_1 + 1) * width].iter(), + eval_values[*i_2 * width..(*i_2 + 1) * width].iter(), ) }); drop(eval_values); @@ -148,8 +145,13 @@ pub(crate) fn compare_sort_keys<'a>( Ordering::Equal } +pub(crate) struct SortTuple { + tuple: Tuple, + outer_values: Vec<(ScalarQueryRef, DataValue)>, +} + pub struct Sort<'a> { - rows: NullableVec<'static, (usize, Tuple)>, + rows: NullableVec<'static, (usize, SortTuple)>, _arena: Box, sort_fields: &'a [SortField], input: ExecId, @@ -168,7 +170,7 @@ impl<'a, T: Transaction + 'a> ReadExecutor<'a, T> for Sort<'a> { let input = build_read(arena, plan_arena, input, cache, transaction); let sort_arena = Box::::default(); let rows = unsafe { - transmute::, NullableVec<'static, (usize, Tuple)>>( + transmute::, NullableVec<'static, (usize, SortTuple)>>( NullableVec::new(&sort_arena), ) }; @@ -188,13 +190,20 @@ impl<'a, T: Transaction + 'a> ExecutorNode<'a, T> for Sort<'a> { plan_arena: &mut ExecMetaArena, ) -> Result<(), DatabaseError> { loop { - if let Some((_, tuple)) = self.rows.pop() { - arena.produce_tuple(tuple); + if let Some((_, row)) = self.rows.pop() { + plan_arena.restore_outer_values(row.outer_values); + arena.produce_tuple(row.tuple); return Ok(()); } while arena.next_tuple(self.input, plan_arena)? { let offset = self.rows.len(); - self.rows.put((offset, arena.materialize_tuple())); + self.rows.put(( + offset, + SortTuple { + tuple: arena.materialize_tuple(), + outer_values: plan_arena.take_outer_values(), + }, + )); } if self.rows.is_empty() { arena.finish(); @@ -210,15 +219,111 @@ impl<'a, T: Transaction + 'a> ExecutorNode<'a, T> for Sort<'a> { mod test { use crate::catalog::{ColumnCatalog, ColumnDesc}; use crate::errors::DatabaseError; - use crate::execution::dql::sort::{sort_tuples, NullableVec}; + use crate::execution::dql::sort::{sort_tuples, NullableVec, SortTuple}; use crate::expression::ScalarExpression; use crate::planner::operator::sort::SortField; + use crate::planner::{ExecMetaArena, PlanArena}; use crate::types::tuple::Tuple; use crate::types::value::DataValue; use crate::types::LogicalType; use bumpalo::Bump; use std::cell::Cell; + #[test] + fn memory_sort_restores_scalar_values_for_each_output_row() -> Result<(), DatabaseError> { + use super::Sort; + use crate::execution::{empty_context, ExecArena, ReadExecutor}; + use crate::planner::operator::scalar_query_init::ScalarQueryInitOperator; + use crate::planner::operator::scalar_subquery::ScalarSubqueryOperator; + use crate::planner::operator::values::ValuesOperator; + use crate::planner::operator::{sort::SortOperator, Operator}; + use crate::planner::{ + Childrens, ExecMetaArena, LogicalPlan, MetaArena, PlanArena, TableArenaCell, + }; + use crate::storage::memory::MemoryStorage; + use crate::storage::{StatisticsMetaCache, Storage, TableCache, ViewCache}; + + let storage = MemoryStorage::new(); + let transaction = storage.transaction()?; + let tables = TableCache::default(); + let views = ViewCache::default(); + let stats = StatisticsMetaCache::default(); + let cache = empty_context(&tables, &views, &stats); + let catalog = TableArenaCell::default(); + let mut metadata = PlanArena::new(&catalog); + let column = metadata.alloc_column(ColumnCatalog::new( + "id".into(), + false, + ColumnDesc::new(LogicalType::Integer, None, false, None)?, + )); + let input_expr = metadata.alloc_expression(ScalarExpression::column_expr(column, 0)); + let param = metadata.alloc_scalar_query_ref(false); + let result = metadata.alloc_scalar_query_ref(true); + let marker = metadata.alloc_expression(ScalarExpression::OuterValue { + id: result, + ty: LogicalType::Integer, + }); + let param_expr = metadata.alloc_expression(ScalarExpression::OuterParam { + id: param, + ty: LogicalType::Integer, + }); + let values = [3, 1, 2] + .into_iter() + .map(|value| { + metadata.alloc_expression(ScalarExpression::Constant(DataValue::Int32(value))) + }) + .collect(); + let input = LogicalPlan::new( + Operator::Values(ValuesOperator::new(values, 3, vec![column])), + Childrens::None, + ); + let query = ScalarSubqueryOperator::build(LogicalPlan::new( + Operator::Values(ValuesOperator::new(vec![param_expr], 1, vec![column])), + Childrens::None, + )); + let mut plan = + ScalarQueryInitOperator::build(input, query, marker, vec![(param, input_expr)]); + plan.populate_output_schema_recursive(&mut metadata); + let op = SortOperator { + sort_fields: vec![SortField { + expr: marker, + asc: true, + nulls_first: false, + }], + }; + let mut arena = ExecArena::with_capacity(0); + arena.init_context(cache, &transaction); + // Select the memory executor explicitly even when the spill feature is enabled. + let root = >::into_executor( + (&op, &plan), + &mut arena, + &mut metadata, + cache, + &transaction, + ); + let mut metadata = ExecMetaArena::new(metadata); + for expected in [1, 2, 3] { + assert!(arena.next_tuple(root, &mut metadata)?); + assert_eq!( + arena.result_tuple().values, + vec![DataValue::Int32(expected)] + ); + assert_eq!( + metadata.init_value(result), + Some(&DataValue::Int32(expected)) + ); + assert_eq!( + metadata + .expression(marker) + .eval(&metadata, Some(arena.result_tuple()))? + .as_ref(), + &DataValue::Int32(expected) + ); + } + assert!(!arena.next_tuple(root, &mut metadata)?); + Ok(()) + } + #[test] fn nullable_vec_drops_values() { struct DropValue<'a>(&'a Cell); @@ -241,13 +346,13 @@ mod test { fn sorted_rows<'a>( sort_fields: &[SortField], - mut tuples: NullableVec<'a, (usize, Tuple)>, - plan_arena: &crate::planner::PlanArena<'_>, + mut tuples: NullableVec<'a, (usize, SortTuple)>, + plan_arena: &mut ExecMetaArena>, ) -> Result + 'a, DatabaseError> { sort_tuples(sort_fields, &mut tuples, plan_arena)?; let mut rows = Vec::with_capacity(tuples.len()); - while let Some((_, tuple)) = tuples.pop() { - rows.push(tuple); + while let Some((_, row)) = tuples.pop() { + rows.push(row.tuple); } rows.reverse(); Ok(rows.into_iter()) @@ -279,12 +384,31 @@ mod test { ColumnDesc::new(LogicalType::Integer, None, false, None).unwrap(), ))]; + let mut plan_arena = ExecMetaArena::new(plan_arena); let arena = Bump::new(); let fn_tuples = || { let mut vec = NullableVec::new(&arena); - vec.put((0_usize, Tuple::new(None, vec![DataValue::Null]))); - vec.put((1_usize, Tuple::new(None, vec![DataValue::Int32(0)]))); - vec.put((2_usize, Tuple::new(None, vec![DataValue::Int32(1)]))); + vec.put(( + 0_usize, + SortTuple { + tuple: Tuple::new(None, vec![DataValue::Null]), + outer_values: Vec::new(), + }, + )); + vec.put(( + 1_usize, + SortTuple { + tuple: Tuple::new(None, vec![DataValue::Int32(0)]), + outer_values: Vec::new(), + }, + )); + vec.put(( + 2_usize, + SortTuple { + tuple: Tuple::new(None, vec![DataValue::Int32(1)]), + outer_values: Vec::new(), + }, + )); vec }; @@ -360,22 +484,22 @@ mod test { fn_asc_and_nulls_first_eq(Box::new(sorted_rows( &fn_sort_fields(true, true), fn_tuples(), - &plan_arena, + &mut plan_arena, )?)); fn_asc_and_nulls_last_eq(Box::new(sorted_rows( &fn_sort_fields(true, false), fn_tuples(), - &plan_arena, + &mut plan_arena, )?)); fn_desc_and_nulls_first_eq(Box::new(sorted_rows( &fn_sort_fields(false, true), fn_tuples(), - &plan_arena, + &mut plan_arena, )?)); fn_desc_and_nulls_last_eq(Box::new(sorted_rows( &fn_sort_fields(false, false), fn_tuples(), - &plan_arena, + &mut plan_arena, )?)); Ok(()) @@ -430,33 +554,52 @@ mod test { ColumnDesc::new(LogicalType::Integer, None, false, None).unwrap(), )), ]; + let mut plan_arena = ExecMetaArena::new(plan_arena); let arena = Bump::new(); let fn_tuples = || { let mut vec = NullableVec::new(&arena); vec.put(( 0_usize, - Tuple::new(None, vec![DataValue::Null, DataValue::Null]), + SortTuple { + tuple: Tuple::new(None, vec![DataValue::Null, DataValue::Null]), + outer_values: Vec::new(), + }, )); vec.put(( 1_usize, - Tuple::new(None, vec![DataValue::Int32(0), DataValue::Null]), + SortTuple { + tuple: Tuple::new(None, vec![DataValue::Int32(0), DataValue::Null]), + outer_values: Vec::new(), + }, )); vec.put(( 2_usize, - Tuple::new(None, vec![DataValue::Int32(1), DataValue::Null]), + SortTuple { + tuple: Tuple::new(None, vec![DataValue::Int32(1), DataValue::Null]), + outer_values: Vec::new(), + }, )); vec.put(( 3_usize, - Tuple::new(None, vec![DataValue::Null, DataValue::Int32(0)]), + SortTuple { + tuple: Tuple::new(None, vec![DataValue::Null, DataValue::Int32(0)]), + outer_values: Vec::new(), + }, )); vec.put(( 4_usize, - Tuple::new(None, vec![DataValue::Int32(0), DataValue::Int32(0)]), + SortTuple { + tuple: Tuple::new(None, vec![DataValue::Int32(0), DataValue::Int32(0)]), + outer_values: Vec::new(), + }, )); vec.put(( 5_usize, - Tuple::new(None, vec![DataValue::Int32(1), DataValue::Int32(0)]), + SortTuple { + tuple: Tuple::new(None, vec![DataValue::Int32(1), DataValue::Int32(0)]), + outer_values: Vec::new(), + }, )); vec }; @@ -596,22 +739,22 @@ mod test { fn_asc_1_and_nulls_first_1_and_asc_2_and_nulls_first_2_eq(Box::new(sorted_rows( &fn_sort_fields(true, true, true, true), fn_tuples(), - &plan_arena, + &mut plan_arena, )?)); fn_asc_1_and_nulls_last_1_and_asc_2_and_nulls_first_2_eq(Box::new(sorted_rows( &fn_sort_fields(true, false, true, true), fn_tuples(), - &plan_arena, + &mut plan_arena, )?)); fn_desc_1_and_nulls_first_1_and_asc_2_and_nulls_first_2_eq(Box::new(sorted_rows( &fn_sort_fields(false, true, true, true), fn_tuples(), - &plan_arena, + &mut plan_arena, )?)); fn_desc_1_and_nulls_last_1_and_asc_2_and_nulls_first_2_eq(Box::new(sorted_rows( &fn_sort_fields(false, false, true, true), fn_tuples(), - &plan_arena, + &mut plan_arena, )?)); Ok(()) diff --git a/src/execution/dql/top_k.rs b/src/execution/dql/top_k.rs index cdeaaed0..38ec59e5 100644 --- a/src/execution/dql/top_k.rs +++ b/src/execution/dql/top_k.rs @@ -19,11 +19,12 @@ use crate::execution::{ }; use crate::planner::operator::sort::SortField; use crate::planner::operator::top_k::TopKOperator; -use crate::planner::LogicalPlan; use crate::planner::{ExecMetaArena, MetaArena}; +use crate::planner::{LogicalPlan, ScalarQueryRef}; use crate::storage::table_codec::BumpBytes; use crate::storage::Transaction; use crate::types::tuple::Tuple; +use crate::types::value::DataValue; use bumpalo::Bump; use std::cmp::Ordering; use std::collections::{btree_set::IntoIter as BTreeSetIntoIter, BTreeSet}; @@ -34,6 +35,7 @@ struct CmpItem<'a> { key: BumpVec<'a, u8>, sequence: usize, tuple: Tuple, + outer_values: Vec<(ScalarQueryRef, DataValue)>, } impl PartialEq for CmpItem<'_> { @@ -66,7 +68,7 @@ fn top_sort<'a>( tuple: &mut Tuple, keep_count: usize, sequence: usize, - plan_arena: &(dyn MetaArena + '_), + plan_arena: &mut ExecMetaArena, ) -> Result<(), DatabaseError> { full_key.clear(); for SortField { @@ -92,12 +94,14 @@ fn top_sort<'a>( key: std::mem::replace(full_key, BumpBytes::new_in(full_key.bump())), sequence, tuple: std::mem::take(tuple), + outer_values: plan_arena.take_outer_values(), }); } else if let Some(mut cmp_item) = heap.pop_last() { if full_key.as_slice() < cmp_item.key.as_slice() { std::mem::swap(full_key, &mut cmp_item.key); cmp_item.sequence = sequence; cmp_item.tuple = std::mem::take(tuple); + cmp_item.outer_values = plan_arena.take_outer_values(); } heap.insert(cmp_item); } @@ -181,6 +185,7 @@ impl<'a, T: Transaction + 'a> ExecutorNode<'a, T> for TopK<'a> { } if let Some(item) = self.output.as_mut().and_then(std::iter::Iterator::next) { + plan_arena.restore_outer_values(item.outer_values); arena.produce_tuple(item.tuple); } else { arena.finish(); @@ -197,6 +202,7 @@ mod test { use crate::execution::dql::top_k::{top_sort, CmpItem}; use crate::expression::ScalarExpression; use crate::planner::operator::sort::SortField; + use crate::planner::ExecMetaArena; use crate::types::tuple::Tuple; use crate::types::value::DataValue; use crate::types::LogicalType; @@ -213,6 +219,7 @@ mod test { key, sequence, tuple: Tuple::new(None, vec![DataValue::Int32(value)]), + outer_values: Vec::new(), } }; let first = make_item(0, 10); @@ -252,6 +259,7 @@ mod test { nulls_first, }] }; + let mut plan_arena = ExecMetaArena::new(plan_arena); let arena = Bump::new(); let mut key_scratch = crate::storage::table_codec::BumpBytes::new_in(&arena); @@ -313,7 +321,7 @@ mod test { &mut Tuple::new(None, vec![DataValue::Null]), 2, 0, - &plan_arena, + &mut plan_arena, )?; top_sort( &mut key_scratch, @@ -322,7 +330,7 @@ mod test { &mut Tuple::new(None, vec![DataValue::Int32(0)]), 2, 1, - &plan_arena, + &mut plan_arena, )?; top_sort( &mut key_scratch, @@ -331,7 +339,7 @@ mod test { &mut Tuple::new(None, vec![DataValue::Int32(1)]), 2, 2, - &plan_arena, + &mut plan_arena, )?; fn_asc_and_nulls_first_eq(indices); @@ -344,7 +352,7 @@ mod test { &mut Tuple::new(None, vec![DataValue::Null]), 2, 3, - &plan_arena, + &mut plan_arena, )?; top_sort( &mut key_scratch, @@ -353,7 +361,7 @@ mod test { &mut Tuple::new(None, vec![DataValue::Int32(0)]), 2, 4, - &plan_arena, + &mut plan_arena, )?; top_sort( &mut key_scratch, @@ -362,7 +370,7 @@ mod test { &mut Tuple::new(None, vec![DataValue::Int32(1)]), 2, 5, - &plan_arena, + &mut plan_arena, )?; fn_asc_and_nulls_last_eq(indices); @@ -375,7 +383,7 @@ mod test { &mut Tuple::new(None, vec![DataValue::Null]), 2, 6, - &plan_arena, + &mut plan_arena, )?; top_sort( &mut key_scratch, @@ -384,7 +392,7 @@ mod test { &mut Tuple::new(None, vec![DataValue::Int32(0)]), 2, 7, - &plan_arena, + &mut plan_arena, )?; top_sort( &mut key_scratch, @@ -393,7 +401,7 @@ mod test { &mut Tuple::new(None, vec![DataValue::Int32(1)]), 2, 8, - &plan_arena, + &mut plan_arena, )?; fn_desc_and_nulls_first_eq(indices); @@ -406,7 +414,7 @@ mod test { &mut Tuple::new(None, vec![DataValue::Null]), 2, 9, - &plan_arena, + &mut plan_arena, )?; top_sort( &mut key_scratch, @@ -415,7 +423,7 @@ mod test { &mut Tuple::new(None, vec![DataValue::Int32(0)]), 2, 10, - &plan_arena, + &mut plan_arena, )?; top_sort( &mut key_scratch, @@ -424,7 +432,7 @@ mod test { &mut Tuple::new(None, vec![DataValue::Int32(1)]), 2, 11, - &plan_arena, + &mut plan_arena, )?; fn_desc_and_nulls_last_eq(indices); @@ -468,6 +476,7 @@ mod test { }, ] }; + let mut plan_arena = ExecMetaArena::new(plan_arena); let arena = Bump::new(); let mut key_scratch = crate::storage::table_codec::BumpBytes::new_in(&arena); @@ -615,7 +624,7 @@ mod test { &mut Tuple::new(None, vec![DataValue::Null, DataValue::Null]), 4, 12, - &plan_arena, + &mut plan_arena, )?; top_sort( &mut key_scratch, @@ -624,7 +633,7 @@ mod test { &mut Tuple::new(None, vec![DataValue::Int32(0), DataValue::Null]), 4, 13, - &plan_arena, + &mut plan_arena, )?; top_sort( &mut key_scratch, @@ -633,7 +642,7 @@ mod test { &mut Tuple::new(None, vec![DataValue::Int32(1), DataValue::Null]), 4, 14, - &plan_arena, + &mut plan_arena, )?; top_sort( &mut key_scratch, @@ -642,7 +651,7 @@ mod test { &mut Tuple::new(None, vec![DataValue::Null, DataValue::Int32(0)]), 4, 15, - &plan_arena, + &mut plan_arena, )?; top_sort( &mut key_scratch, @@ -651,7 +660,7 @@ mod test { &mut Tuple::new(None, vec![DataValue::Int32(0), DataValue::Int32(0)]), 4, 16, - &plan_arena, + &mut plan_arena, )?; top_sort( &mut key_scratch, @@ -660,7 +669,7 @@ mod test { &mut Tuple::new(None, vec![DataValue::Int32(1), DataValue::Int32(0)]), 4, 17, - &plan_arena, + &mut plan_arena, )?; fn_asc_1_and_nulls_first_1_and_asc_2_and_nulls_first_2_eq(indices); @@ -673,7 +682,7 @@ mod test { &mut Tuple::new(None, vec![DataValue::Null, DataValue::Null]), 4, 18, - &plan_arena, + &mut plan_arena, )?; top_sort( &mut key_scratch, @@ -682,7 +691,7 @@ mod test { &mut Tuple::new(None, vec![DataValue::Int32(0), DataValue::Null]), 4, 19, - &plan_arena, + &mut plan_arena, )?; top_sort( &mut key_scratch, @@ -691,7 +700,7 @@ mod test { &mut Tuple::new(None, vec![DataValue::Int32(1), DataValue::Null]), 4, 20, - &plan_arena, + &mut plan_arena, )?; top_sort( &mut key_scratch, @@ -700,7 +709,7 @@ mod test { &mut Tuple::new(None, vec![DataValue::Null, DataValue::Int32(0)]), 4, 21, - &plan_arena, + &mut plan_arena, )?; top_sort( &mut key_scratch, @@ -709,7 +718,7 @@ mod test { &mut Tuple::new(None, vec![DataValue::Int32(0), DataValue::Int32(0)]), 4, 22, - &plan_arena, + &mut plan_arena, )?; top_sort( &mut key_scratch, @@ -718,7 +727,7 @@ mod test { &mut Tuple::new(None, vec![DataValue::Int32(1), DataValue::Int32(0)]), 4, 23, - &plan_arena, + &mut plan_arena, )?; fn_asc_1_and_nulls_last_1_and_asc_2_and_nulls_first_2_eq(indices); @@ -731,7 +740,7 @@ mod test { &mut Tuple::new(None, vec![DataValue::Null, DataValue::Null]), 4, 24, - &plan_arena, + &mut plan_arena, )?; top_sort( &mut key_scratch, @@ -740,7 +749,7 @@ mod test { &mut Tuple::new(None, vec![DataValue::Int32(0), DataValue::Null]), 4, 25, - &plan_arena, + &mut plan_arena, )?; top_sort( &mut key_scratch, @@ -749,7 +758,7 @@ mod test { &mut Tuple::new(None, vec![DataValue::Int32(1), DataValue::Null]), 4, 26, - &plan_arena, + &mut plan_arena, )?; top_sort( &mut key_scratch, @@ -758,7 +767,7 @@ mod test { &mut Tuple::new(None, vec![DataValue::Null, DataValue::Int32(0)]), 4, 27, - &plan_arena, + &mut plan_arena, )?; top_sort( &mut key_scratch, @@ -767,7 +776,7 @@ mod test { &mut Tuple::new(None, vec![DataValue::Int32(0), DataValue::Int32(0)]), 4, 28, - &plan_arena, + &mut plan_arena, )?; top_sort( &mut key_scratch, @@ -776,7 +785,7 @@ mod test { &mut Tuple::new(None, vec![DataValue::Int32(1), DataValue::Int32(0)]), 4, 29, - &plan_arena, + &mut plan_arena, )?; fn_desc_1_and_nulls_first_1_and_asc_2_and_nulls_first_2_eq(indices); @@ -789,7 +798,7 @@ mod test { &mut Tuple::new(None, vec![DataValue::Null, DataValue::Null]), 4, 30, - &plan_arena, + &mut plan_arena, )?; top_sort( &mut key_scratch, @@ -798,7 +807,7 @@ mod test { &mut Tuple::new(None, vec![DataValue::Int32(0), DataValue::Null]), 4, 31, - &plan_arena, + &mut plan_arena, )?; top_sort( &mut key_scratch, @@ -807,7 +816,7 @@ mod test { &mut Tuple::new(None, vec![DataValue::Int32(1), DataValue::Null]), 4, 32, - &plan_arena, + &mut plan_arena, )?; top_sort( &mut key_scratch, @@ -816,7 +825,7 @@ mod test { &mut Tuple::new(None, vec![DataValue::Null, DataValue::Int32(0)]), 4, 33, - &plan_arena, + &mut plan_arena, )?; top_sort( &mut key_scratch, @@ -825,7 +834,7 @@ mod test { &mut Tuple::new(None, vec![DataValue::Int32(0), DataValue::Int32(0)]), 4, 34, - &plan_arena, + &mut plan_arena, )?; top_sort( &mut key_scratch, @@ -834,7 +843,7 @@ mod test { &mut Tuple::new(None, vec![DataValue::Int32(1), DataValue::Int32(0)]), 4, 35, - &plan_arena, + &mut plan_arena, )?; fn_desc_1_and_nulls_last_1_and_asc_2_and_nulls_first_2_eq(indices); diff --git a/src/execution/spill/codec.rs b/src/execution/spill/codec.rs index 15989af6..01e2f622 100644 --- a/src/execution/spill/codec.rs +++ b/src/execution/spill/codec.rs @@ -15,7 +15,7 @@ use super::SpillCodec; use crate::errors::DatabaseError; use crate::planner::operator::sort::SortField; -use crate::planner::MetaArena; +use crate::planner::{ExecMetaArena, MetaArena, ScalarQueryRef}; use crate::types::tuple::Tuple; use crate::types::value::DataValue; use std::io::{Read, Write}; @@ -24,13 +24,14 @@ use std::mem::size_of; pub(crate) struct SortRow { pub(crate) sort_values: Vec, pub(crate) tuple: Tuple, + pub(crate) outer_values: Vec<(ScalarQueryRef, DataValue)>, } impl SortRow { pub(crate) fn new( sort_fields: &[SortField], tuple: Tuple, - arena: &(dyn MetaArena + '_), + arena: &mut ExecMetaArena, ) -> Result { let sort_values = sort_fields .iter() @@ -41,25 +42,36 @@ impl SortRow { .map(|v| v.into_owned()) }) .collect::>()?; - Ok(Self { sort_values, tuple }) + Ok(Self { + sort_values, + tuple, + outer_values: arena.take_outer_values(), + }) } } impl SpillCodec for SortRow { fn encode(&self, writer: &mut W) -> Result<(), DatabaseError> { self.sort_values.encode(writer)?; - self.tuple.encode(writer) + self.tuple.encode(writer)?; + self.outer_values.encode(writer) } fn decode(reader: &mut R) -> Result { Ok(Self { sort_values: Vec::::decode(reader)?, tuple: Tuple::decode(reader)?, + outer_values: Vec::decode(reader)?, }) } fn estimated_size(&self) -> usize { size_of::() + .saturating_add( + self.outer_values + .estimated_size() + .saturating_sub(size_of::>()), + ) .saturating_add( self.sort_values .estimated_size() @@ -73,6 +85,55 @@ impl SpillCodec for SortRow { } } +impl SpillCodec for (A, B) { + fn encode(&self, writer: &mut W) -> Result<(), DatabaseError> { + self.0.encode(writer)?; + self.1.encode(writer) + } + + fn decode(reader: &mut R) -> Result { + Ok((A::decode(reader)?, B::decode(reader)?)) + } + + fn estimated_size(&self) -> usize { + size_of::() + .saturating_add(self.0.estimated_size().saturating_sub(size_of::())) + .saturating_add(self.1.estimated_size().saturating_sub(size_of::())) + } +} + +impl SpillCodec for ScalarQueryRef { + fn encode(&self, writer: &mut W) -> Result<(), DatabaseError> { + writer.write_all(&(self.arena_id as u64).to_le_bytes())?; + writer.write_all(&(self.pos as u64).to_le_bytes())?; + writer.write_all(&[u8::from(self.is_outer_value)])?; + Ok(()) + } + fn decode(reader: &mut R) -> Result { + let mut bytes = [0; 8]; + reader.read_exact(&mut bytes)?; + let arena_id = usize::try_from(u64::from_le_bytes(bytes))?; + reader.read_exact(&mut bytes)?; + let pos = usize::try_from(u64::from_le_bytes(bytes))?; + let mut marker = [0]; + reader.read_exact(&mut marker)?; + let is_outer_value = match marker[0] { + 0 => false, + 1 => true, + tag => { + return Err(DatabaseError::InvalidValue(format!( + "invalid scalar reference marker: {tag}" + ))) + } + }; + Ok(Self { arena_id, pos, is_outer_value }) + } + + fn estimated_size(&self) -> usize { + size_of::() + } +} + impl SpillCodec for DataValue { fn encode(&self, writer: &mut W) -> Result<(), DatabaseError> { self.encode_reference_value(writer) diff --git a/src/expression/evaluator.rs b/src/expression/evaluator.rs index ee2aaf69..7cffc4e0 100644 --- a/src/expression/evaluator.rs +++ b/src/expression/evaluator.rs @@ -445,7 +445,7 @@ mod tests { let table_arena = TableArenaCell::default(); let mut arena = PlanArena::new(&table_arena); - let reference = arena.alloc_scalar_query_ref(); + let reference = arena.alloc_scalar_query_ref(false); let mut init = arena.alloc_expression(ScalarExpression::InitValue { id: reference, ty: LogicalType::Integer, diff --git a/src/expression/range_detacher.rs b/src/expression/range_detacher.rs index 7ecb6e0b..3ac5f5fb 100644 --- a/src/expression/range_detacher.rs +++ b/src/expression/range_detacher.rs @@ -88,20 +88,18 @@ impl TreeNode { fn add_child(&mut self, child: TreeNode) { self.children.push(child); } -} -impl TreeNode { - fn enumeration(self, path: &mut Vec, combinations: &mut Vec>) { + fn enumeration(self, path: &mut Vec, consume: &mut impl FnMut(&[T])) { if self.value.is_none() && self.children.is_empty() { - combinations.push(path.clone()); + consume(path); } for mut child in self.children { if let Some(val) = child.value.take() { path.push(val); - Self::enumeration(child, path, combinations); + Self::enumeration(child, path, consume); let _ = path.pop(); } else { - Self::enumeration(child, path, combinations); + Self::enumeration(child, path, consume); } } } @@ -290,15 +288,11 @@ impl Range { } let node = build_tree(eqs, 0)?; - let mut combinations = Vec::new(); - - node.enumeration(&mut Vec::new(), &mut combinations); - + let mut path = Vec::with_capacity(eqs.len()); let mut ranges = Vec::new(); - - for tuple in combinations { - collect_tuple_range(&mut ranges, &tuple, self.clone()) - } + node.enumeration(&mut path, &mut |tuple| { + collect_tuple_range(&mut ranges, tuple, self.clone()); + }); Some(RangeDetacher::::ranges2range(ranges)) } } @@ -2837,6 +2831,35 @@ mod test { ]), ]; let combined = suffix.combining_eqs(&prefixes).unwrap(); + let eq = |value| Range::Eq(DataValue::Int32(value)); + assert_eq!( + eq(9).combining_eqs(&[ + Range::SortedRanges(vec![eq(1), eq(2)]), + Range::SortedRanges(vec![eq(3), eq(4)]), + ]), + Some(Range::SortedRanges(vec![ + Range::Eq(DataValue::Tuple(vec![ + DataValue::Int32(1), + DataValue::Int32(3), + DataValue::Int32(9) + ])), + Range::Eq(DataValue::Tuple(vec![ + DataValue::Int32(1), + DataValue::Int32(4), + DataValue::Int32(9) + ])), + Range::Eq(DataValue::Tuple(vec![ + DataValue::Int32(2), + DataValue::Int32(3), + DataValue::Int32(9) + ])), + Range::Eq(DataValue::Tuple(vec![ + DataValue::Int32(2), + DataValue::Int32(4), + DataValue::Int32(9) + ])), + ])) + ); let first = DataValue::Int32(1); let second = DataValue::Int32(2); let borrowed = [Range::Eq(&first), Range::Eq(&second)]; diff --git a/src/planner/arena.rs b/src/planner/arena.rs index 397d0be4..cf6716a7 100644 --- a/src/planner/arena.rs +++ b/src/planner/arena.rs @@ -239,7 +239,7 @@ pub trait MetaArena { fn arena_id(&self) -> usize; - fn alloc_scalar_query_ref(&mut self) -> ScalarQueryRef; + fn alloc_scalar_query_ref(&mut self, is_outer_value: bool) -> ScalarQueryRef; fn reserve_scalar_query_ref(&mut self, reference: ScalarQueryRef); @@ -303,8 +303,8 @@ impl MetaArena for Box { fn arena_id(&self) -> usize { (**self).arena_id() } - fn alloc_scalar_query_ref(&mut self) -> ScalarQueryRef { - (**self).alloc_scalar_query_ref() + fn alloc_scalar_query_ref(&mut self, is_outer_value: bool) -> ScalarQueryRef { + (**self).alloc_scalar_query_ref(is_outer_value) } fn reserve_scalar_query_ref(&mut self, reference: ScalarQueryRef) { (**self).reserve_scalar_query_ref(reference) @@ -372,14 +372,27 @@ impl ExecMetaArena { pub(crate) fn set_init_value(&mut self, reference: ScalarQueryRef, value: DataValue) { self.init_values.insert(reference, Some(value)); } + + pub(crate) fn take_outer_values(&mut self) -> Vec<(ScalarQueryRef, DataValue)> { + self.init_values + .extract_if(|reference, _| reference.is_outer_value) + .filter_map(|(reference, value)| value.map(|value| (reference, value))) + .collect() + } + + pub(crate) fn restore_outer_values(&mut self, values: Vec<(ScalarQueryRef, DataValue)>) { + for (reference, value) in values { + self.set_init_value(reference, value); + } + } } impl MetaArena for ExecMetaArena { fn arena_id(&self) -> usize { self.parent.arena_id() } - fn alloc_scalar_query_ref(&mut self) -> ScalarQueryRef { - self.parent.alloc_scalar_query_ref() + fn alloc_scalar_query_ref(&mut self, is_outer_value: bool) -> ScalarQueryRef { + self.parent.alloc_scalar_query_ref(is_outer_value) } fn reserve_scalar_query_ref(&mut self, reference: ScalarQueryRef) { self.parent.reserve_scalar_query_ref(reference) @@ -593,8 +606,9 @@ impl MetaArena for TableArena { fn arena_id(&self) -> usize { 0 } - fn alloc_scalar_query_ref(&mut self) -> ScalarQueryRef { + fn alloc_scalar_query_ref(&mut self, is_outer_value: bool) -> ScalarQueryRef { let reference = ScalarQueryRef { + is_outer_value, arena_id: 0, pos: self.scalar_query_count, }; @@ -949,8 +963,9 @@ impl MetaArena for PlanArena<'_> { fn arena_id(&self) -> usize { self.arena_id } - fn alloc_scalar_query_ref(&mut self) -> ScalarQueryRef { + fn alloc_scalar_query_ref(&mut self, is_outer_value: bool) -> ScalarQueryRef { let reference = ScalarQueryRef { + is_outer_value, arena_id: self.arena_id, pos: self.scalar_query_count, }; @@ -1110,8 +1125,9 @@ impl MetaArena for ParamArena<'_> { fn arena_id(&self) -> usize { self.parent.arena_id() + 1 } - fn alloc_scalar_query_ref(&mut self) -> ScalarQueryRef { + fn alloc_scalar_query_ref(&mut self, is_outer_value: bool) -> ScalarQueryRef { let reference = ScalarQueryRef { + is_outer_value, arena_id: self.arena_id(), pos: self.scalar_query_count, }; @@ -1199,22 +1215,44 @@ mod tests { use crate::types::LogicalType; use std::panic::{catch_unwind, AssertUnwindSafe}; + #[test] + fn sorting_moves_only_outer_results() { + let catalog = TableArenaCell::default(); + let mut plan = PlanArena::new(&catalog); + let init = plan.alloc_scalar_query_ref(false); + let param = plan.alloc_scalar_query_ref(false); + let result = plan.alloc_scalar_query_ref(true); + let mut arena = ExecMetaArena::new(plan); + arena.set_init_value(init, DataValue::Int32(7)); + arena.set_init_value(param, DataValue::Int32(3)); + arena.set_init_value(result, DataValue::Null); + let values = arena.take_outer_values(); + assert_eq!(values, vec![(result, DataValue::Null)]); + assert_eq!(arena.init_value(init), Some(&DataValue::Int32(7))); + assert_eq!(arena.init_value(param), Some(&DataValue::Int32(3))); + assert_eq!(arena.init_value(result), None); + arena.restore_outer_values(values); + assert_eq!(arena.init_value(result), Some(&DataValue::Null)); + } + #[test] fn scalar_query_namespaces_follow_arena_parent() -> Result<(), DatabaseError> { let root = TableArenaCell::default(); - let catalog = root.borrow_mut().alloc_scalar_query_ref(); + let catalog = root.borrow_mut().alloc_scalar_query_ref(false); assert_eq!( catalog, ScalarQueryRef { + is_outer_value: false, arena_id: 0, pos: 0 } ); let mut plan = PlanArena::new(&root); - let local = plan.alloc_scalar_query_ref(); + let local = plan.alloc_scalar_query_ref(false); assert_eq!( local, ScalarQueryRef { + is_outer_value: false, arena_id: 1, pos: 0 } @@ -1222,10 +1260,11 @@ mod tests { assert_ne!(catalog, local); let mut params = ParamArena::new(&plan, &[], &[])?; assert_eq!(params.arena_id(), 2); - let param_local = params.alloc_scalar_query_ref(); + let param_local = params.alloc_scalar_query_ref(false); assert_eq!( param_local, ScalarQueryRef { + is_outer_value: false, arena_id: 2, pos: 0 } diff --git a/src/planner/operator/mod.rs b/src/planner/operator/mod.rs index fe80e28e..cb8830e9 100644 --- a/src/planner/operator/mod.rs +++ b/src/planner/operator/mod.rs @@ -1401,7 +1401,7 @@ mod tests { let left = LogicalPlan::new(Operator::ShowTable, Childrens::None); let right = LogicalPlan::new(Operator::ShowView, Childrens::None); - let id = arena.alloc_scalar_query_ref(); + let id = arena.alloc_scalar_query_ref(false); let value = arena.alloc_expression(ScalarExpression::InitValue { id, ty: LogicalType::Integer, diff --git a/src/planner/scalar_query_ref.rs b/src/planner/scalar_query_ref.rs index debf2bcc..6b12f531 100644 --- a/src/planner/scalar_query_ref.rs +++ b/src/planner/scalar_query_ref.rs @@ -23,6 +23,7 @@ use std::io::{Read, Write}; pub struct ScalarQueryRef { pub(crate) arena_id: usize, pub(crate) pos: usize, + pub(crate) is_outer_value: bool, } impl std::fmt::Display for ScalarQueryRef { @@ -40,7 +41,8 @@ impl ReferenceSerialization for ScalarQueryRef { arena: &A, ) -> Result<(), DatabaseError> { self.arena_id.encode(writer, direct, tables, arena)?; - self.pos.encode(writer, direct, tables, arena) + self.pos.encode(writer, direct, tables, arena)?; + self.is_outer_value.encode(writer, direct, tables, arena) } fn decode( @@ -52,6 +54,7 @@ impl ReferenceSerialization for ScalarQueryRef { let reference = Self { arena_id: usize::decode(reader, context, tables, arena)?, pos: usize::decode(reader, context, tables, arena)?, + is_outer_value: bool::decode(reader, context, tables, arena)?, }; arena.reserve_scalar_query_ref(reference); Ok(reference) @@ -67,7 +70,7 @@ mod tests { #[test] fn decoding_preserves_source_reference() -> Result<(), DatabaseError> { let mut source = TableArena::default(); - let reference = source.alloc_scalar_query_ref(); + let reference = source.alloc_scalar_query_ref(true); let mut bytes = Vec::new(); let mut tables = ReferenceTables::new(); reference.encode(&mut bytes, false, &mut tables, &source)?; @@ -95,7 +98,7 @@ mod tests { &mut target, )?; assert_eq!(second, reference); - assert_ne!(target.alloc_scalar_query_ref(), reference); + assert_ne!(target.alloc_scalar_query_ref(false), reference); Ok(()) } } diff --git a/tests/slt/subquery.slt b/tests/slt/subquery.slt index 279a487f..e431f153 100644 --- a/tests/slt/subquery.slt +++ b/tests/slt/subquery.slt @@ -843,13 +843,31 @@ NULL 20 NULL 20 NULL 30 -# LIMIT is streaming, unlike ORDER BY; ordinary input rows still drive recomputation. +# LIMIT without sorting streams input; sorting preserves each row's scalar values. query I rowsort select (select count(*) from corr_rows x where x.bcorr_rows.b),(select 7) from corr_rows order by id desc limit 3 offset 1; +---- +4 0 0 7 +3 3 0 7 +2 1 1 7 + +# Null/text scalar results are paired with the original rows after ordering. +query IT +select id,(select cast(x.b as varchar(12)) from corr_rows x where x.id=corr_rows.id and x.id<>2) from corr_rows order by id desc; +---- +5 20 +4 NULL +3 30 +2 NULL +1 10 + query II rowsort select id,(select count(*) from corr_rows x where x.b Date: Thu, 8 Oct 2026 04:31:12 +0800 Subject: [PATCH 12/16] chore: fix formatting and preserve PR checklist structure --- AGENT.md | 6 +++++- src/execution/spill/codec.rs | 6 +++++- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/AGENT.md b/AGENT.md index 7e2cd95c..8073a5ff 100644 --- a/AGENT.md +++ b/AGENT.md @@ -151,7 +151,11 @@ KiteSQL aims to stay easy to build, easy to audit, and easy to understand. A valid PR should: -- Follow the repository pull request template +- Follow `.github/pull_request_template.md` exactly: preserve its headings, checklist labels, + ordering, and comments. In `Code changes` and `Check List`, only change checkbox states; + do not add, remove, or reword items, or insert explanatory paragraphs. +- Put test results, manual verification steps, compatibility notes, and side-effect explanations + in `Note for reviewer`, not inside the checklist. Keep the PR description concise. - Describe test coverage and verified behavior instead of listing commands that were run - Compile cleanly - Pass all tests via make diff --git a/src/execution/spill/codec.rs b/src/execution/spill/codec.rs index 01e2f622..6dc2b89d 100644 --- a/src/execution/spill/codec.rs +++ b/src/execution/spill/codec.rs @@ -126,7 +126,11 @@ impl SpillCodec for ScalarQueryRef { ))) } }; - Ok(Self { arena_id, pos, is_outer_value }) + Ok(Self { + arena_id, + pos, + is_outer_value, + }) } fn estimated_size(&self) -> usize { From 746719feba9e822b7b249aa9c175b58c7fa1c7a1 Mon Sep 17 00:00:00 2001 From: kould Date: Thu, 8 Oct 2026 08:14:41 +0800 Subject: [PATCH 13/16] fix: preserve shared CTE expressions during column pruning Keep shallow CTE plan references instead of cloning every expression at bind time. Remap positions through consumer-local expression references and reuse rewrites within each input schema. Cover repeated CTEs whose consumers require different columns. --- src/binder/select.rs | 2 +- .../rule/normalization/column_pruning.rs | 12 ++--- src/optimizer/rule/normalization/mod.rs | 54 +++++++++++++------ tests/slt/subquery.slt | 13 +++++ 4 files changed, 59 insertions(+), 22 deletions(-) diff --git a/src/binder/select.rs b/src/binder/select.rs index 9256df71..b74e9169 100644 --- a/src/binder/select.rs +++ b/src/binder/select.rs @@ -1188,7 +1188,7 @@ impl<'a: 'b, 'b, T: Transaction, A: AsRef<[(usize, LogicalType)]>> Binder<'a, 'b let table_alias = alias.as_ref().map(|alias| alias.name.clone()); if let Some(plan_ref) = self.context.cte(&table_name).map(|cte| cte.plan_ref) { - let mut plan = arena.plan(plan_ref).clone().clone_plan(arena)?; + let mut plan = arena.plan(plan_ref).clone(); if let Some(alias) = alias { plan = self.bind_alias( plan, diff --git a/src/optimizer/rule/normalization/column_pruning.rs b/src/optimizer/rule/normalization/column_pruning.rs index 6653cbf9..cda4b0da 100644 --- a/src/optimizer/rule/normalization/column_pruning.rs +++ b/src/optimizer/rule/normalization/column_pruning.rs @@ -30,7 +30,7 @@ use crate::planner::{Childrens, ExprRef, LogicalPlan, PlanArena}; use crate::types::value::{DataValue, Utf8Type}; use crate::types::CharLengthUnits; use crate::types::LogicalType; -use std::collections::HashSet; +use std::collections::HashMap; #[derive(Clone)] pub struct ColumnPruning; @@ -38,7 +38,7 @@ pub struct ColumnPruning; struct ApplyOutcome { changed: bool, removed_positions: Vec, - remapped_exprs: HashSet, + remapped_exprs: HashMap, } #[derive(Clone, Default)] @@ -115,7 +115,7 @@ impl ApplyOutcome { Self { changed: false, removed_positions: Vec::with_capacity(arena.allocated_columns_len()), - remapped_exprs: HashSet::new(), + remapped_exprs: HashMap::new(), } } } @@ -263,7 +263,7 @@ impl ColumnPruning { fn remap_operator_after_child_change( operator: &mut Operator, removed_positions: &[usize], - remapped_exprs: &mut HashSet, + remapped_exprs: &mut HashMap, arena: &mut PlanArena<'_>, ) -> Result<(), DatabaseError> { OperatorExprVisitorMut::new( @@ -276,7 +276,7 @@ impl ColumnPruning { fn remap_exprs_after_child_change<'a>( exprs: impl IntoIterator, removed_positions: &[usize], - remapped_exprs: &mut HashSet, + remapped_exprs: &mut HashMap, arena: &mut PlanArena<'_>, ) -> Result<(), DatabaseError> { if removed_positions.is_empty() { @@ -647,7 +647,7 @@ impl ColumnPruning { [left_removed_start..right_removed_end]; if !removed_positions.is_empty() { remap_expr_positions( - *filter, + filter, removed_positions, &mut outcome.remapped_exprs, arena, diff --git a/src/optimizer/rule/normalization/mod.rs b/src/optimizer/rule/normalization/mod.rs index 87eea173..5fee259d 100644 --- a/src/optimizer/rule/normalization/mod.rs +++ b/src/optimizer/rule/normalization/mod.rs @@ -13,7 +13,7 @@ // limitations under the License. use crate::errors::DatabaseError; -use crate::expression::visitor_mut::ExprVisitorMut; +use crate::expression::visitor_mut::{walk_mut_expr, ExprVisitorMut}; use crate::expression::{AliasType, ScalarExpression}; use crate::optimizer::core::rule::NormalizationRule; use crate::optimizer::rule::normalization::column_pruning::ColumnPruning; @@ -35,7 +35,7 @@ use crate::optimizer::rule::normalization::simplification::ConstantCalculation; use crate::optimizer::rule::normalization::simplification::SimplifyFilter; use crate::optimizer::rule::normalization::top_k::TopK; use crate::planner::{ExprRef, LogicalPlan}; -use std::collections::HashSet; +use std::collections::HashMap; mod column_pruning; mod combine_operators; mod compilation_in_advance; @@ -261,29 +261,49 @@ pub(crate) fn remap_position(position: &mut usize, removed_positions: &[usize]) struct PositionRemapper<'positions, 'visited> { removed_positions: &'positions [usize], - visited: &'visited mut HashSet, + remapped: &'visited mut HashMap, + changes: usize, } impl<'positions, 'visited> PositionRemapper<'positions, 'visited> { pub(super) fn new( removed_positions: &'positions [usize], - visited: &'visited mut HashSet, + remapped: &'visited mut HashMap, ) -> Self { - visited.clear(); + remapped.clear(); Self { removed_positions, - visited, + remapped, + changes: 0, } } } impl ExprVisitorMut for PositionRemapper<'_, '_> { - fn visit_expression_ref( + fn visit( &mut self, expr: &mut ExprRef, - _arena: &mut (dyn MetaArena + '_), - ) -> Result { - Ok(self.visited.insert(*expr)) + arena: &mut (dyn MetaArena + '_), + ) -> Result<(), DatabaseError> { + if let Some(remapped) = self.remapped.get(expr) { + if *expr != *remapped { + self.changes += 1; + *expr = *remapped; + } + return Ok(()); + } + let changes = self.changes; + let original = *expr; + let expression = arena.expression(original).clone(); + // Preserve shared definitions and rewrite only the current consumer's reference. + let mut rewritten = arena.alloc_expression(expression); + walk_mut_expr(self, &mut rewritten, arena)?; + if self.changes != changes { + *expr = rewritten; + } + self.remapped.insert(original, *expr); + self.remapped.insert(*expr, *expr); + Ok(()) } fn visit_column_ref( @@ -292,7 +312,11 @@ impl ExprVisitorMut for PositionRemapper<'_, '_> { position: &mut usize, _arena: &mut (dyn MetaArena + '_), ) -> Result<(), DatabaseError> { + let original = *position; remap_position(position, self.removed_positions); + if *position != original { + self.changes += 1; + } Ok(()) } @@ -310,21 +334,21 @@ impl ExprVisitorMut for PositionRemapper<'_, '_> { } pub(crate) fn remap_expr_positions( - mut expr: ExprRef, + expr: &mut ExprRef, removed_positions: &[usize], - visited: &mut HashSet, + remapped: &mut HashMap, arena: &mut crate::planner::PlanArena<'_>, ) -> Result<(), DatabaseError> { - PositionRemapper::new(removed_positions, visited).visit(&mut expr, arena) + PositionRemapper::new(removed_positions, remapped).visit(expr, arena) } pub(crate) fn remap_exprs_positions<'a>( exprs: impl IntoIterator, removed_positions: &[usize], - visited: &mut HashSet, + remapped: &mut HashMap, arena: &mut crate::planner::PlanArena<'_>, ) -> Result<(), DatabaseError> { - let mut remapper = PositionRemapper::new(removed_positions, visited); + let mut remapper = PositionRemapper::new(removed_positions, remapped); for expr in exprs { remapper.visit(expr, arena)?; } diff --git a/tests/slt/subquery.slt b/tests/slt/subquery.slt index e431f153..28303e70 100644 --- a/tests/slt/subquery.slt +++ b/tests/slt/subquery.slt @@ -749,6 +749,19 @@ with q as (select (select max(v) from init_rows) as m) select a.m, b.m from q as ---- 30 30 +# Repeated CTE inputs can prune different columns without changing shared definitions. +query II +with q as (select id,v,(select max(v) from init_rows) as m from init_rows) +select a.v,b.m from q a join q b on a.id=b.id where a.id=2; +---- +20 30 + +query II +with q as (select id,v,(select max(v) from init_rows) as m from init_rows) +select a.m,b.v from q a join q b on a.id=b.id where b.id=1; +---- +30 10 + query II select id, sum((select 2)) over (order by id) from init_rows order by id ---- From 3205c00461b7322ec61a4c6b92bf2a7736fb1acb Mon Sep 17 00:00:00 2001 From: kould Date: Thu, 8 Oct 2026 08:19:23 +0800 Subject: [PATCH 14/16] Revert "fix: preserve shared CTE expressions during column pruning" This reverts commit 746719feba9e822b7b249aa9c175b58c7fa1c7a1. --- src/binder/select.rs | 2 +- .../rule/normalization/column_pruning.rs | 12 ++--- src/optimizer/rule/normalization/mod.rs | 54 ++++++------------- tests/slt/subquery.slt | 13 ----- 4 files changed, 22 insertions(+), 59 deletions(-) diff --git a/src/binder/select.rs b/src/binder/select.rs index b74e9169..9256df71 100644 --- a/src/binder/select.rs +++ b/src/binder/select.rs @@ -1188,7 +1188,7 @@ impl<'a: 'b, 'b, T: Transaction, A: AsRef<[(usize, LogicalType)]>> Binder<'a, 'b let table_alias = alias.as_ref().map(|alias| alias.name.clone()); if let Some(plan_ref) = self.context.cte(&table_name).map(|cte| cte.plan_ref) { - let mut plan = arena.plan(plan_ref).clone(); + let mut plan = arena.plan(plan_ref).clone().clone_plan(arena)?; if let Some(alias) = alias { plan = self.bind_alias( plan, diff --git a/src/optimizer/rule/normalization/column_pruning.rs b/src/optimizer/rule/normalization/column_pruning.rs index cda4b0da..6653cbf9 100644 --- a/src/optimizer/rule/normalization/column_pruning.rs +++ b/src/optimizer/rule/normalization/column_pruning.rs @@ -30,7 +30,7 @@ use crate::planner::{Childrens, ExprRef, LogicalPlan, PlanArena}; use crate::types::value::{DataValue, Utf8Type}; use crate::types::CharLengthUnits; use crate::types::LogicalType; -use std::collections::HashMap; +use std::collections::HashSet; #[derive(Clone)] pub struct ColumnPruning; @@ -38,7 +38,7 @@ pub struct ColumnPruning; struct ApplyOutcome { changed: bool, removed_positions: Vec, - remapped_exprs: HashMap, + remapped_exprs: HashSet, } #[derive(Clone, Default)] @@ -115,7 +115,7 @@ impl ApplyOutcome { Self { changed: false, removed_positions: Vec::with_capacity(arena.allocated_columns_len()), - remapped_exprs: HashMap::new(), + remapped_exprs: HashSet::new(), } } } @@ -263,7 +263,7 @@ impl ColumnPruning { fn remap_operator_after_child_change( operator: &mut Operator, removed_positions: &[usize], - remapped_exprs: &mut HashMap, + remapped_exprs: &mut HashSet, arena: &mut PlanArena<'_>, ) -> Result<(), DatabaseError> { OperatorExprVisitorMut::new( @@ -276,7 +276,7 @@ impl ColumnPruning { fn remap_exprs_after_child_change<'a>( exprs: impl IntoIterator, removed_positions: &[usize], - remapped_exprs: &mut HashMap, + remapped_exprs: &mut HashSet, arena: &mut PlanArena<'_>, ) -> Result<(), DatabaseError> { if removed_positions.is_empty() { @@ -647,7 +647,7 @@ impl ColumnPruning { [left_removed_start..right_removed_end]; if !removed_positions.is_empty() { remap_expr_positions( - filter, + *filter, removed_positions, &mut outcome.remapped_exprs, arena, diff --git a/src/optimizer/rule/normalization/mod.rs b/src/optimizer/rule/normalization/mod.rs index 5fee259d..87eea173 100644 --- a/src/optimizer/rule/normalization/mod.rs +++ b/src/optimizer/rule/normalization/mod.rs @@ -13,7 +13,7 @@ // limitations under the License. use crate::errors::DatabaseError; -use crate::expression::visitor_mut::{walk_mut_expr, ExprVisitorMut}; +use crate::expression::visitor_mut::ExprVisitorMut; use crate::expression::{AliasType, ScalarExpression}; use crate::optimizer::core::rule::NormalizationRule; use crate::optimizer::rule::normalization::column_pruning::ColumnPruning; @@ -35,7 +35,7 @@ use crate::optimizer::rule::normalization::simplification::ConstantCalculation; use crate::optimizer::rule::normalization::simplification::SimplifyFilter; use crate::optimizer::rule::normalization::top_k::TopK; use crate::planner::{ExprRef, LogicalPlan}; -use std::collections::HashMap; +use std::collections::HashSet; mod column_pruning; mod combine_operators; mod compilation_in_advance; @@ -261,49 +261,29 @@ pub(crate) fn remap_position(position: &mut usize, removed_positions: &[usize]) struct PositionRemapper<'positions, 'visited> { removed_positions: &'positions [usize], - remapped: &'visited mut HashMap, - changes: usize, + visited: &'visited mut HashSet, } impl<'positions, 'visited> PositionRemapper<'positions, 'visited> { pub(super) fn new( removed_positions: &'positions [usize], - remapped: &'visited mut HashMap, + visited: &'visited mut HashSet, ) -> Self { - remapped.clear(); + visited.clear(); Self { removed_positions, - remapped, - changes: 0, + visited, } } } impl ExprVisitorMut for PositionRemapper<'_, '_> { - fn visit( + fn visit_expression_ref( &mut self, expr: &mut ExprRef, - arena: &mut (dyn MetaArena + '_), - ) -> Result<(), DatabaseError> { - if let Some(remapped) = self.remapped.get(expr) { - if *expr != *remapped { - self.changes += 1; - *expr = *remapped; - } - return Ok(()); - } - let changes = self.changes; - let original = *expr; - let expression = arena.expression(original).clone(); - // Preserve shared definitions and rewrite only the current consumer's reference. - let mut rewritten = arena.alloc_expression(expression); - walk_mut_expr(self, &mut rewritten, arena)?; - if self.changes != changes { - *expr = rewritten; - } - self.remapped.insert(original, *expr); - self.remapped.insert(*expr, *expr); - Ok(()) + _arena: &mut (dyn MetaArena + '_), + ) -> Result { + Ok(self.visited.insert(*expr)) } fn visit_column_ref( @@ -312,11 +292,7 @@ impl ExprVisitorMut for PositionRemapper<'_, '_> { position: &mut usize, _arena: &mut (dyn MetaArena + '_), ) -> Result<(), DatabaseError> { - let original = *position; remap_position(position, self.removed_positions); - if *position != original { - self.changes += 1; - } Ok(()) } @@ -334,21 +310,21 @@ impl ExprVisitorMut for PositionRemapper<'_, '_> { } pub(crate) fn remap_expr_positions( - expr: &mut ExprRef, + mut expr: ExprRef, removed_positions: &[usize], - remapped: &mut HashMap, + visited: &mut HashSet, arena: &mut crate::planner::PlanArena<'_>, ) -> Result<(), DatabaseError> { - PositionRemapper::new(removed_positions, remapped).visit(expr, arena) + PositionRemapper::new(removed_positions, visited).visit(&mut expr, arena) } pub(crate) fn remap_exprs_positions<'a>( exprs: impl IntoIterator, removed_positions: &[usize], - remapped: &mut HashMap, + visited: &mut HashSet, arena: &mut crate::planner::PlanArena<'_>, ) -> Result<(), DatabaseError> { - let mut remapper = PositionRemapper::new(removed_positions, remapped); + let mut remapper = PositionRemapper::new(removed_positions, visited); for expr in exprs { remapper.visit(expr, arena)?; } diff --git a/tests/slt/subquery.slt b/tests/slt/subquery.slt index 28303e70..e431f153 100644 --- a/tests/slt/subquery.slt +++ b/tests/slt/subquery.slt @@ -749,19 +749,6 @@ with q as (select (select max(v) from init_rows) as m) select a.m, b.m from q as ---- 30 30 -# Repeated CTE inputs can prune different columns without changing shared definitions. -query II -with q as (select id,v,(select max(v) from init_rows) as m from init_rows) -select a.v,b.m from q a join q b on a.id=b.id where a.id=2; ----- -20 30 - -query II -with q as (select id,v,(select max(v) from init_rows) as m from init_rows) -select a.m,b.v from q a join q b on a.id=b.id where b.id=1; ----- -30 10 - query II select id, sum((select 2)) over (order by id) from init_rows order by id ---- From 0ee2a40eb8d4eb1bebbeab71a5719b8ea9c8a60b Mon Sep 17 00:00:00 2001 From: kould Date: Thu, 8 Oct 2026 08:46:11 +0800 Subject: [PATCH 15/16] fix: clone CTE expressions without duplicating plan structures Clone the CTE plan structure once and recursively isolate its mutable expressions using ExprCloner. Preserve scalar initialization references and cover repeated CTE consumers with different pruning requirements. --- src/binder/select.rs | 6 ++++-- tests/slt/subquery.slt | 19 +++++++++++++++++++ 2 files changed, 23 insertions(+), 2 deletions(-) diff --git a/src/binder/select.rs b/src/binder/select.rs index 9256df71..4cb64dc1 100644 --- a/src/binder/select.rs +++ b/src/binder/select.rs @@ -33,7 +33,7 @@ use crate::catalog::{ColumnRef, ColumnRelation, TableName}; use crate::errors::DatabaseError; use crate::execution::dql::join::joins_nullable; use crate::expression::visitor::ExprVisitor; -use crate::expression::visitor_mut::{walk_mut_expr, ExprVisitorMut, PositionShift}; +use crate::expression::visitor_mut::{walk_mut_expr, ExprCloner, ExprVisitorMut, PositionShift}; use crate::expression::{AliasType, BinaryOperator, TypeCast}; use crate::iter_ext::Itertools; use crate::planner::operator::function_scan::FunctionScanOperator; @@ -42,6 +42,7 @@ use crate::planner::operator::join::JoinCondition; use crate::planner::operator::set_membership::{SetMembershipKind, SetMembershipOperator}; use crate::planner::operator::sort::{SortField, SortOperator}; use crate::planner::operator::union::UnionOperator; +use crate::planner::operator::visitor_mut::{OperatorExprVisitorMut, OperatorVisitorMut}; use crate::planner::{Childrens, ExprRef, LogicalPlan, PlanArena, ScalarQueryRef}; use crate::storage::Transaction; use crate::types::tuple::Schema; @@ -1188,7 +1189,8 @@ impl<'a: 'b, 'b, T: Transaction, A: AsRef<[(usize, LogicalType)]>> Binder<'a, 'b let table_alias = alias.as_ref().map(|alias| alias.name.clone()); if let Some(plan_ref) = self.context.cte(&table_name).map(|cte| cte.plan_ref) { - let mut plan = arena.plan(plan_ref).clone().clone_plan(arena)?; + let mut plan = arena.plan(plan_ref).clone(); + OperatorExprVisitorMut::new(&mut ExprCloner, arena).visit_plan(&mut plan)?; if let Some(alias) = alias { plan = self.bind_alias( plan, diff --git a/tests/slt/subquery.slt b/tests/slt/subquery.slt index e431f153..c2ed1b5c 100644 --- a/tests/slt/subquery.slt +++ b/tests/slt/subquery.slt @@ -749,6 +749,25 @@ with q as (select (select max(v) from init_rows) as m) select a.m, b.m from q as ---- 30 30 +# Repeated CTE inputs can prune different columns without changing shared definitions. +query II +with q as (select max(v) as m from init_rows) +select a.m,b.m from q a cross join q b; +---- +30 30 + +query II +with q as (select id,v,(select max(v) from init_rows) as m from init_rows) +select a.v,b.m from q a join q b on a.id=b.id where a.id=2; +---- +20 30 + +query II +with q as (select id,v,(select max(v) from init_rows) as m from init_rows) +select a.m,b.v from q a join q b on a.id=b.id where b.id=1; +---- +30 10 + query II select id, sum((select 2)) over (order by id) from init_rows order by id ---- From 0d528f8a5cc75b806fa7f22922423c622793b782 Mon Sep 17 00:00:00 2001 From: kould Date: Thu, 8 Oct 2026 09:05:09 +0800 Subject: [PATCH 16/16] refactor: remove unused tuple split boundary interface --- src/types/tuple.rs | 43 ------------------------------------------- 1 file changed, 43 deletions(-) diff --git a/src/types/tuple.rs b/src/types/tuple.rs index 9745cb31..da734600 100644 --- a/src/types/tuple.rs +++ b/src/types/tuple.rs @@ -84,12 +84,6 @@ pub trait TupleLike { fn value_at(&self, index: usize) -> &DataValue; - /// The boundary between the outermost split's left and right inputs. - #[inline] - fn split_point(&self) -> Option { - None - } - #[inline] fn as_slice(&self) -> Option<&[DataValue]> { None @@ -170,11 +164,6 @@ impl TupleLike for &T { (**self).value_at(index) } - #[inline] - fn split_point(&self) -> Option { - (**self).split_point() - } - #[inline] fn as_slice(&self) -> Option<&[DataValue]> { (**self).as_slice() @@ -187,11 +176,6 @@ impl TupleLike for SplitTupleRef<' self.left_len + self.right.len() } - #[inline] - fn split_point(&self) -> Option { - Some(self.left_len) - } - #[inline] fn value_at(&self, index: usize) -> &DataValue { if index < self.left_len { @@ -733,33 +717,6 @@ mod tests { ); } - #[test] - fn test_tuple_split_point() { - use super::{SplitTupleRef, TupleLike}; - - let left = Tuple::new(None, vec![DataValue::Int32(1)]); - let right = Tuple::new(None, vec![DataValue::Int32(2)]); - let tail = [DataValue::Int32(3)]; - assert_eq!(left.split_point(), None); - assert_eq!(tail.as_slice().split_point(), None); - assert_eq!(OneValueTupleLike(DataValue::Null).split_point(), None); - - let split = SplitTupleRef::new(&left, &right); - assert_eq!(split.split_point(), Some(1)); - let nested = SplitTupleRef::new(&split, tail.as_slice()); - assert_eq!(nested.split_point(), Some(2)); - assert_eq!(nested.value_at(nested.split_point().unwrap()), &tail[0]); - let dynamic: &dyn TupleLike = &nested; - assert_eq!(dynamic.split_point(), Some(2)); - assert_eq!(TupleLike::split_point(&dynamic), Some(2)); - assert_eq!(TupleLike::split_point(&&&nested), Some(2)); - - let empty: &[DataValue] = &[]; - assert_eq!(SplitTupleRef::new(empty, &left).split_point(), Some(0)); - assert_eq!(SplitTupleRef::new(&left, empty).split_point(), Some(1)); - assert_eq!(SplitTupleRef::new(empty, empty).split_point(), Some(0)); - } - #[test] fn test_tuple_deserialize_null_bits_and_primary_projection() { let serializers = [