diff --git a/AGENT.md b/AGENT.md index 7e2cd95c..8073a5ff 100644 --- a/AGENT.md +++ b/AGENT.md @@ -151,7 +151,11 @@ KiteSQL aims to stay easy to build, easy to audit, and easy to understand. A valid PR should: -- Follow the repository pull request template +- Follow `.github/pull_request_template.md` exactly: preserve its headings, checklist labels, + ordering, and comments. In `Code changes` and `Check List`, only change checkbox states; + do not add, remove, or reword items, or insert explanatory paragraphs. +- Put test results, manual verification steps, compatibility notes, and side-effect explanations + in `Note for reviewer`, not inside the checklist. Keep the PR description concise. - Describe test coverage and verified behavior instead of listing commands that were run - Compile cleanly - Pass all tests via make diff --git a/src/binder/aggregate.rs b/src/binder/aggregate.rs index e17dc898..5fb781ab 100644 --- a/src/binder/aggregate.rs +++ b/src/binder/aggregate.rs @@ -190,6 +190,15 @@ impl> Binder<'_, '_, T, A> { if expr.has_agg_call(arena)? { continue; } + if !expr.any_referenced_column(arena, |_, _| true)? { + if let Some(position) = unmatched_group_exprs + .iter() + .position(|group_expr| expr.eq_ignore_colref_pos(*group_expr, arena)) + { + unmatched_group_exprs.remove(position); + } + continue; + } let Some(position) = unmatched_group_exprs .iter() .position(|group_expr| expr.eq_ignore_colref_pos(*group_expr, arena)) @@ -202,6 +211,12 @@ impl> Binder<'_, '_, T, A> { unmatched_group_exprs.remove(position); } + unmatched_group_exprs.retain(|expr| { + !matches!( + arena.expression(expr.unpack_alias(arena)), + ScalarExpression::InitValue { .. } + ) + }); if !unmatched_group_exprs.is_empty() { return Err(DatabaseError::AggMiss( "in the GROUP BY clause the field must be in the select clause".to_string(), diff --git a/src/binder/create_view.rs b/src/binder/create_view.rs index 42a397aa..7f49753c 100644 --- a/src/binder/create_view.rs +++ b/src/binder/create_view.rs @@ -91,6 +91,7 @@ impl> Binder<'_, '_, T, A> { }; plan = self.bind_project(plan, exprs, arena)?; let schema = plan.output_schema(arena).clone(); + arena.materialize_scalar_queries(&mut plan)?; Ok(LogicalPlan::new( Operator::CreateView(CreateViewOperator { diff --git a/src/binder/expr.rs b/src/binder/expr.rs index 97b75fd4..053dbecd 100644 --- a/src/binder/expr.rs +++ b/src/binder/expr.rs @@ -18,14 +18,14 @@ use crate::expression; use crate::expression::agg::AggKind; use crate::iter_ext::Itertools; -use super::{Binder, BinderContext, QueryBindStep, SubQueryType}; +use super::{Binder, BinderContext, BoundScalarQuery, QueryBindStep, SubQueryType}; use crate::expression::function::scala::{ArcScalarFunctionImpl, ScalarFunction}; use crate::expression::function::table::TableFunction; use crate::expression::function::FunctionSummary; use crate::expression::{AliasType, ScalarExpression, TypeCast}; use crate::planner::operator::mark_apply::MarkApplyQuantifier; use crate::planner::operator::scalar_subquery::ScalarSubqueryOperator; -use crate::planner::{ExprRef, LogicalPlan, PlanArena}; +use crate::planner::{ExprRef, LogicalPlan, MetaArena, PlanArena}; use crate::storage::Transaction; use crate::types::value::{DataValue, Utf8Type}; use crate::types::{CharLengthUnits, LogicalType}; @@ -147,7 +147,7 @@ impl<'a, T: Transaction, A: AsRef<[(usize, LogicalType)]>> Binder<'a, '_, T, A> { let mut binder = Binder::new(self.context.fork_empty(), self.args, Some(&self.context)); let sub_query = build(&mut binder, arena)?; - let correlated = binder.context.has_outer_refs(); + let correlated = binder.context.has_join_outer_ref(); Ok((sub_query, correlated)) } @@ -206,23 +206,42 @@ impl<'a, T: Transaction, A: AsRef<[(usize, LogicalType)]>> Binder<'a, '_, T, A> &mut PlanArena<'arena>, ) -> Result, { - let (sub_query, column, correlated) = - self.bind_subquery_plan_with_output(None, arena, build)?; - let sub_query = ScalarSubqueryOperator::build(sub_query); - let (expr, sub_query) = match self.context.step_now() { - QueryBindStep::Where => (column, sub_query), - QueryBindStep::Project => self.bind_temp_table(column, sub_query, arena)?, - _ => { - return Err(DatabaseError::UnsupportedStmt( - "scalar subqueries can only appear in `WHERE` or SELECT list".to_string(), - )) - } - }; - self.context.sub_query(SubQueryType::SubQuery { - plan: sub_query, - correlated, + let mut child_context = self.context.fork_empty(); + child_context.capture_scalar_outer = true; + let mut binder = Binder::new(child_context, self.args, Some(&self.context)); + let mut sub_query = build(&mut binder, arena)?; + let param_bindings = binder.context.scalar_outer_bindings; + if !param_bindings.is_empty() && self.context.capture_scalar_outer { + return Err(DatabaseError::UnsupportedStmt( + "nested correlated scalar queries are not supported yet".into(), + )); + } + if !param_bindings.is_empty() && self.context.step_now() != QueryBindStep::Project { + return Err(DatabaseError::UnsupportedStmt( + "correlated scalar subqueries currently require the SELECT list".into(), + )); + } + let schema = sub_query.output_schema(arena); + if schema.len() != 1 { + return Err(DatabaseError::MisMatch( + "expects only one expression to be returned", + "the expression returned by the subquery", + )); + } + let ty = arena.column(schema[0]).datatype().clone(); + let id = arena.alloc_scalar_query_ref(!param_bindings.is_empty()); + let value = arena.alloc_expression(if param_bindings.is_empty() { + ScalarExpression::InitValue { id, ty: ty.clone() } + } else { + ScalarExpression::OuterValue { id, ty: ty.clone() } + }); + self.context.scalar_queries.push(BoundScalarQuery { + value, + step: self.context.step_now(), + plan: ScalarSubqueryOperator::build(sub_query), + param_bindings, }); - Ok(expr) + Ok(arena.expression(value).clone()) } pub(crate) fn bind_exists_subquery_plan<'arena, F>( @@ -330,6 +349,21 @@ impl<'a, T: Transaction, A: AsRef<[(usize, LogicalType)]>> Binder<'a, '_, T, A> arena: &mut PlanArena, ) -> Result { if table_name.is_none() { + // SELECT expressions and WHERE read input columns, not a same-named output alias. + if bind_table_name.is_none() + && matches!( + self.context.step_now(), + QueryBindStep::Project | QueryBindStep::Where + ) + { + let input = match self.context.using.get(column_name) { + Some(column) => Some(column.visible_expr(arena)?), + None => Self::find_column_in_scope(&self.context, arena, column_name)?, + }; + if let Some(input) = input { + return Ok(input); + } + } if let Some((_, expr)) = self .context .expr_aliases @@ -349,12 +383,14 @@ impl<'a, T: Transaction, A: AsRef<[(usize, LogicalType)]>> Binder<'a, '_, T, A> try_default!(&table_name, column_name); } if let Some(table) = table_name.or(bind_table_name) { + let mut is_outer = false; let (source, position_offset) = match Self::resolve_source_columns_in_scope(&self.context, table) { Ok(source) => source, Err(err) => { if let Some(parent) = self.parent { - self.context.mark_outer_ref(); + self.context.mark_join_outer_ref(); + is_outer = true; Self::resolve_source_columns_in_scope(parent, table).map_err(|_| err)? } else { return Err(err); @@ -365,10 +401,11 @@ impl<'a, T: Transaction, A: AsRef<[(usize, LogicalType)]>> Binder<'a, '_, T, A> Self::find_column_in_schema(source.schema().iter(), arena, column_name) .ok_or_else(|| DatabaseError::column_not_found(column_name.to_string()))?; - Ok(ScalarExpression::column_expr( - column, - position_offset + position, - )) + let expr = ScalarExpression::column_expr(column, position_offset + position); + if is_outer && self.context.capture_scalar_outer { + return self.capture_scalar_outer(expr, arena); + } + Ok(expr) } else { // handle col syntax let mut find_visible_column = @@ -381,8 +418,13 @@ impl<'a, T: Transaction, A: AsRef<[(usize, LogicalType)]>> Binder<'a, '_, T, A> let mut got_column = find_visible_column(&self.context)?; if got_column.is_none() { if let Some(parent) = self.parent { - self.context.mark_outer_ref(); + self.context.mark_join_outer_ref(); got_column = find_visible_column(parent)?; + if self.context.capture_scalar_outer { + if let Some(expr) = got_column { + return self.capture_scalar_outer(expr, arena); + } + } } } match got_column { @@ -392,6 +434,18 @@ impl<'a, T: Transaction, A: AsRef<[(usize, LogicalType)]>> Binder<'a, '_, T, A> } } + fn capture_scalar_outer( + &mut self, + expr: ScalarExpression, + arena: &mut PlanArena, + ) -> Result { + let ty = expr.return_type(arena).into_owned(); + let id = arena.alloc_scalar_query_ref(false); + let source = arena.alloc_expression(expr); + self.context.scalar_outer_bindings.push((id, source)); + Ok(ScalarExpression::OuterParam { id, ty }) + } + pub(crate) fn bind_binary_op_expr( &mut self, left_expr: ExprRef, @@ -409,10 +463,12 @@ impl<'a, T: Transaction, A: AsRef<[(usize, LogicalType)]>> Binder<'a, '_, T, A> LogicalType::max_logical_type(&left_ty, &right_ty)?.into_owned() } expression::BinaryOperator::Divide => { - if let LogicalType::Decimal(precision, scale) = - LogicalType::max_logical_type(&left_ty, &right_ty)?.into_owned() + let ty = LogicalType::max_logical_type(&left_ty, &right_ty)?.into_owned(); + if ty.is_signed_numeric() + || ty.is_unsigned_numeric() + || matches!(ty, LogicalType::Decimal(..)) { - LogicalType::Decimal(precision, scale) + ty } else { LogicalType::Double } @@ -448,6 +504,18 @@ impl<'a, T: Transaction, A: AsRef<[(usize, LogicalType)]>> Binder<'a, '_, T, A> op: expression::UnaryOperator, arena: &mut PlanArena, ) -> Result { + if op == expression::UnaryOperator::Plus { + let ty = expr.return_type(arena); + if ty.is_numeric() + || matches!( + ty.as_ref(), + LogicalType::SqlNull | LogicalType::Char(..) | LogicalType::Varchar(..) + ) + { + return Ok(arena.expression(expr).clone()); + } + return Err(DatabaseError::UnsupportedUnaryOperator(ty.into_owned(), op)); + } let ty = if let expression::UnaryOperator::Not = op { LogicalType::Boolean } else { diff --git a/src/binder/mod.rs b/src/binder/mod.rs index 8e61b81b..aae02188 100644 --- a/src/binder/mod.rs +++ b/src/binder/mod.rs @@ -68,7 +68,7 @@ use crate::errors::DatabaseError; use crate::expression::{ScalarExpression, TypeCast}; use crate::planner::operator::join::JoinType; use crate::planner::operator::mark_apply::MarkApplyQuantifier; -use crate::planner::{ExprRef, LogicalPlan, PlanArena, PlanRef}; +use crate::planner::{ExprRef, LogicalPlan, PlanArena, PlanRef, ScalarQueryRef}; use crate::storage::{TableCache, Transaction, ViewCache}; use crate::types::tuple::Schema; use crate::types::LogicalType; @@ -102,10 +102,6 @@ pub enum QueryBindStep { #[derive(Debug, Clone, Hash, Eq, PartialEq)] pub enum SubQueryType { - SubQuery { - plan: LogicalPlan, - correlated: bool, - }, ExistsSubQuery { plan: LogicalPlan, correlated: bool, @@ -151,8 +147,10 @@ pub(crate) struct CteCheckpoint { impl BoundSource<'_> { pub(crate) fn matches_name(&self, table_name: &str) -> bool { - self.table_name.as_ref() == table_name - || matches!(self.alias.as_ref(), Some(alias) if alias.as_ref() == table_name) + match &self.alias { + Some(alias) => alias.as_ref() == table_name, + None => self.table_name.as_ref() == table_name, + } } pub(crate) fn same_binding( @@ -237,6 +235,13 @@ impl UsingColumn { } } +pub(crate) struct BoundScalarQuery { + value: ExprRef, + step: QueryBindStep, + plan: LogicalPlan, + param_bindings: Vec<(ScalarQueryRef, ExprRef)>, +} + pub struct BinderContext<'a, T: Transaction> { pub(crate) scala_functions: &'a ScalaFunctions, pub(crate) table_functions: &'a TableFunctions, @@ -256,11 +261,14 @@ pub struct BinderContext<'a, T: Transaction> { pub(crate) agg_calls: Vec, // join using: HashMap, - - bind_step: QueryBindStep, + has_join_outer_ref: bool, + // subquery sub_queries: HashMap>, - has_outer_refs: bool, + scalar_queries: Vec, + capture_scalar_outer: bool, + scalar_outer_bindings: Vec<(ScalarQueryRef, ExprRef)>, + bind_step: QueryBindStep, pub(crate) allow_default: bool, } @@ -321,7 +329,10 @@ impl<'a, T: Transaction> BinderContext<'a, T> { using: Default::default(), bind_step: QueryBindStep::From, sub_queries: Default::default(), - has_outer_refs: false, + scalar_queries: Vec::new(), + capture_scalar_outer: false, + scalar_outer_bindings: Vec::new(), + has_join_outer_ref: false, allow_default: false, } } @@ -347,7 +358,10 @@ impl<'a, T: Transaction> BinderContext<'a, T> { using: self.using.clone(), bind_step: self.bind_step, sub_queries: Default::default(), - has_outer_refs: false, + scalar_queries: Vec::new(), + capture_scalar_outer: false, + scalar_outer_bindings: Vec::new(), + has_join_outer_ref: false, allow_default: self.allow_default, } } @@ -428,12 +442,12 @@ impl<'a, T: Transaction> BinderContext<'a, T> { self.sub_queries.remove(&self.bind_step) } - pub fn mark_outer_ref(&mut self) { - self.has_outer_refs = true; + pub fn mark_join_outer_ref(&mut self) { + self.has_join_outer_ref = true; } - pub fn has_outer_refs(&self) -> bool { - self.has_outer_refs + pub fn has_join_outer_ref(&self) -> bool { + self.has_join_outer_ref } pub fn table(&self, table_name: TableName) -> Result, DatabaseError> { diff --git a/src/binder/parser.rs b/src/binder/parser.rs index 5145ee9f..2dbafd27 100644 --- a/src/binder/parser.rs +++ b/src/binder/parser.rs @@ -164,18 +164,50 @@ impl Database { let sql = sql.as_ref(); let statements = prepare_all(sql).map_err(|err| err.with_sql_context(sql))?; - for statement in statements { - if !matches!(command_type(&statement)?, CommandType::DDL) { + for statement in &statements { + if !matches!(command_type(statement)?, CommandType::DDL) { return Err(DatabaseError::UnsupportedStmt( "`Database::ddl` only accepts DDL statements".to_string(), ) .with_sql_context(sql)); } + } + self.run_mut_statements(sql, statements, |_| Ok(())) + } + + /// Executes SQL with catalog mutations allowed, lending the last statement's results + /// to `consume`. On success, unread rows are drained before committing and publishing + /// catalog updates. A callback or execution error rolls the current statement back. + /// Earlier statements are completed and committed in order before the callback runs. + pub fn run_mut(&mut self, sql: impl AsRef, consume: C) -> Result + where + C: for<'a> FnOnce( + &mut TransactionIter<'a, S::TransactionType<'a>, PlanArena<'a>>, + ) -> Result, + { + let sql = sql.as_ref(); + let statements = prepare_all(sql).map_err(|err| err.with_sql_context(sql))?; + self.run_mut_statements(sql, statements, consume) + } + fn run_mut_statements( + &mut self, + sql: &str, + mut statements: Vec, + consume: C, + ) -> Result + where + C: for<'a> FnOnce( + &mut TransactionIter<'a, S::TransactionType<'a>, PlanArena<'a>>, + ) -> Result, + { + let last = statements + .pop() + .ok_or_else(|| DatabaseError::EmptyStatement.with_sql_context(sql))?; + for statement in statements { self.execute_mut(sql, &[], |binder, arena| binder.bind(&statement, arena))?; } - - Ok(()) + self.execute_mut_with(sql, &[], |binder, arena| binder.bind(&last, arena), consume) } /// Runs one or more SQL statements and returns an iterator for the final result set. @@ -245,7 +277,7 @@ impl Database { } else { let inner = Box::into_raw(Box::new(TransactionIter::new( schema, - Box::new(plan_arena) as Box, + plan_arena.boxed(), executor, transaction, ))); @@ -1224,6 +1256,12 @@ where Childrens::Only(Box::new(plan)), ); } + let steps = [ + QueryBindStep::From, + QueryBindStep::Where, + QueryBindStep::Project, + ]; + plan = self.binder.bind_scalar_queries(plan, &steps, self.arena)?; self.binder.bind_update(table_name, value_exprs, plan) } else { Err(DatabaseError::UnsupportedStmt(format!( @@ -2688,8 +2726,8 @@ impl<'a, 'parent, T: Transaction, A: AsRef<[(usize, LogicalType)]>> Binder<'a, ' let left_plan = { let mut left_binder = Binder::new(self.context.fork(), self.args, self.parent); let plan = left_binder.bind_set_expr(left, arena)?; - if left_binder.context.has_outer_refs() { - self.context.mark_outer_ref(); + if left_binder.context.has_join_outer_ref() { + self.context.mark_join_outer_ref(); } plan }; @@ -2697,8 +2735,8 @@ impl<'a, 'parent, T: Transaction, A: AsRef<[(usize, LogicalType)]>> Binder<'a, ' let right_plan = { let mut right_binder = Binder::new(self.context.fork(), self.args, self.parent); let plan = right_binder.bind_set_expr(right, arena)?; - if right_binder.context.has_outer_refs() { - self.context.mark_outer_ref(); + if right_binder.context.has_join_outer_ref() { + self.context.mark_join_outer_ref(); } plan }; @@ -3253,7 +3291,8 @@ impl<'a, 'parent, T: Transaction, A: AsRef<[(usize, LogicalType)]>> Binder<'a, ' stmt: &Statement, arena: &mut PlanArena, ) -> Result { - Ok(self.build_statement(arena).statement(stmt)?.finish()) + let plan = self.build_statement(arena).statement(stmt)?.finish(); + Ok(self.init_scalar_queries(plan)) } } diff --git a/src/binder/select.rs b/src/binder/select.rs index f7d4ebd4..4cb64dc1 100644 --- a/src/binder/select.rs +++ b/src/binder/select.rs @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +use crate::planner::operator::join::extract_join_keys; use crate::planner::MetaArena; use crate::{ expression::ScalarExpression, @@ -19,22 +20,20 @@ use crate::{ operator::{ filter::FilterOperator, join::JoinOperator as LJoinOperator, limit::LimitOperator, mark_apply::MarkApplyOperator, project::ProjectOperator, - scalar_apply::ScalarApplyOperator, Operator, + scalar_query_init::ScalarQueryInitOperator, Operator, }, operator::{join::JoinType, table_scan::TableScanOperator}, }, }; -use std::{ - borrow::Cow, - collections::{HashMap, HashSet}, -}; +use std::{borrow::Cow, collections::HashSet}; use super::{Binder, BinderContext, QueryBindStep, SetOperatorKind, Source, SubQueryType}; use crate::catalog::{ColumnRef, ColumnRelation, TableName}; use crate::errors::DatabaseError; use crate::execution::dql::join::joins_nullable; -use crate::expression::visitor_mut::{walk_mut_expr, ExprVisitorMut, PositionShift}; +use crate::expression::visitor::ExprVisitor; +use crate::expression::visitor_mut::{walk_mut_expr, ExprCloner, ExprVisitorMut, PositionShift}; use crate::expression::{AliasType, BinaryOperator, TypeCast}; use crate::iter_ext::Itertools; use crate::planner::operator::function_scan::FunctionScanOperator; @@ -43,7 +42,8 @@ use crate::planner::operator::join::JoinCondition; use crate::planner::operator::set_membership::{SetMembershipKind, SetMembershipOperator}; use crate::planner::operator::sort::{SortField, SortOperator}; use crate::planner::operator::union::UnionOperator; -use crate::planner::{Childrens, ExprRef, LogicalPlan, PlanArena}; +use crate::planner::operator::visitor_mut::{OperatorExprVisitorMut, OperatorVisitorMut}; +use crate::planner::{Childrens, ExprRef, LogicalPlan, PlanArena, ScalarQueryRef}; use crate::storage::Transaction; use crate::types::tuple::Schema; use crate::types::{ColumnId, LogicalType}; @@ -71,76 +71,12 @@ impl ExprVisitorMut for RightSidePositionGlobalizer<'_> { } } -/// Whether any of `exprs` reads the output of a SELECT-list scalar subquery. -/// Those subqueries are only joined in at the Project step, above DISTINCT / -/// GROUP BY / ORDER BY, so such a reference would read another column. -/// -/// Takes the subquery map rather than the binder so callers can pass -/// expressions borrowed from the same binder context. -// TODO(#386): evaluate uncorrelated scalar subqueries once, as constants -// (like PostgreSQL's InitPlan), so these clauses can use them. -fn references_select_list_sub_query( - sub_queries: &mut HashMap>, - exprs: impl IntoIterator, - arena: &mut PlanArena, -) -> Result { - let Some(sub_queries) = sub_queries.get_mut(&QueryBindStep::Project) else { - return Ok(false); - }; - // `exprs` can only be traversed once, so it is the outer loop. - for expr in exprs { - for sub_query in sub_queries.iter_mut() { - let SubQueryType::SubQuery { plan, .. } = sub_query else { - continue; - }; - let schema = plan.output_schema(arena); - if expr.any_referenced_column(arena, |arena, candidate| { - schema - .iter() - .any(|column| arena.same_column(*column, *candidate)) - })? { - return Ok(true); - } - } - } - Ok(false) -} - struct AppendedRightOutput { column: ColumnRef, child_position: usize, output_position: usize, } -struct SplitScopePositionRebinder<'a> { - left_schema: &'a Schema, - right_schema: &'a Schema, -} - -impl ExprVisitorMut for SplitScopePositionRebinder<'_> { - fn visit_column_ref( - &mut self, - column: &mut ColumnRef, - position: &mut usize, - arena: &mut (dyn MetaArena + '_), - ) -> Result<(), DatabaseError> { - if let Some(left_position) = self - .left_schema - .iter() - .position(|candidate| arena.same_column(*candidate, *column)) - { - *position = left_position; - } else if let Some(right_position) = self - .right_schema - .iter() - .position(|candidate| arena.same_column(*candidate, *column)) - { - *position = right_position; - } - Ok(()) - } -} - struct MarkerPositionGlobalizer<'a> { output_column: &'a ColumnRef, left_len: usize, @@ -205,7 +141,7 @@ where A: AsRef<[(usize, LogicalType)]>, { pub(crate) binder: &'s mut Binder<'a, 'b, T, A>, - pub(crate) arena: &'s mut crate::planner::PlanArena<'arena>, + pub(crate) arena: &'s mut PlanArena<'arena>, } pub struct BindPlanFrom<'s, 'a, 'b, 'arena, T, A, M = ()> @@ -214,7 +150,7 @@ where A: AsRef<[(usize, LogicalType)]>, { pub(crate) binder: &'s mut Binder<'a, 'b, T, A>, - pub(crate) arena: &'s mut crate::planner::PlanArena<'arena>, + pub(crate) arena: &'s mut PlanArena<'arena>, pub(crate) plan: LogicalPlan, pub(crate) _marker: std::marker::PhantomData, } @@ -225,7 +161,7 @@ where A: AsRef<[(usize, LogicalType)]>, { pub(crate) binder: &'s mut Binder<'a, 'b, T, A>, - pub(crate) arena: &'s mut crate::planner::PlanArena<'arena>, + pub(crate) arena: &'s mut PlanArena<'arena>, pub(super) plan: LogicalPlan, pub(super) select_list: Vec, pub(crate) _marker: std::marker::PhantomData, @@ -237,7 +173,7 @@ where A: AsRef<[(usize, LogicalType)]>, { pub(super) binder: &'s mut Binder<'a, 'b, T, A>, - pub(super) arena: &'s mut crate::planner::PlanArena<'arena>, + pub(super) arena: &'s mut PlanArena<'arena>, pub(super) plan: LogicalPlan, pub(super) select_list: Vec, } @@ -248,7 +184,7 @@ where A: AsRef<[(usize, LogicalType)]>, { binder: &'s mut Binder<'a, 'b, T, A>, - arena: &'s mut crate::planner::PlanArena<'arena>, + arena: &'s mut PlanArena<'arena>, plan: LogicalPlan, select_list: Vec, having: Option, @@ -261,7 +197,7 @@ where A: AsRef<[(usize, LogicalType)]>, { binder: &'s mut Binder<'a, 'b, T, A>, - arena: &'s mut crate::planner::PlanArena<'arena>, + arena: &'s mut PlanArena<'arena>, plan: LogicalPlan, select_list: Vec, orderby: Option>, @@ -273,7 +209,7 @@ where A: AsRef<[(usize, LogicalType)]>, { binder: &'s mut Binder<'a, 'b, T, A>, - arena: &'s mut crate::planner::PlanArena<'arena>, + arena: &'s mut PlanArena<'arena>, plan: LogicalPlan, select_list: Vec, orderby: Option>, @@ -285,7 +221,7 @@ where A: AsRef<[(usize, LogicalType)]>, { binder: &'s mut Binder<'a, 'b, T, A>, - arena: &'s mut crate::planner::PlanArena<'arena>, + arena: &'s mut PlanArena<'arena>, plan: LogicalPlan, select_list: Vec, orderby: Option>, @@ -297,7 +233,7 @@ where A: AsRef<[(usize, LogicalType)]>, { binder: &'s mut Binder<'a, 'b, T, A>, - arena: &'s mut crate::planner::PlanArena<'arena>, + arena: &'s mut PlanArena<'arena>, plan: LogicalPlan, select_list: Vec, } @@ -478,13 +414,26 @@ where #[cfg(feature = "orm")] pub fn finish(self) -> Result { + if self + .binder + .context + .scalar_queries + .iter() + .any(|query| !query.param_bindings.is_empty()) + { + return Err(DatabaseError::UnsupportedStmt( + "correlated scalar queries in ORM projections are not supported yet".into(), + )); + } for expr in &self.select_list { if expr.has_agg_call(self.arena)? || expr.has_window_call(self.arena)? { return self.aggregate_without_group()?.finish(); } } - self.binder - .bind_project(self.plan, self.select_list, self.arena) + let plan = self + .binder + .bind_project(self.plan, self.select_list, self.arena)?; + Ok(plan) } } @@ -543,7 +492,7 @@ where orderby: Option>, mut bind_sort_field: impl FnMut( &mut Binder<'a, 'b, T, A>, - &mut crate::planner::PlanArena<'arena>, + &mut PlanArena<'arena>, &[ExprRef], O, ) -> Result, @@ -553,6 +502,30 @@ where self.binder .extract_select_aggregate(&mut self.select_list, self.arena)?; + // Statement-constant scalar values needed by grouping/aggregate arguments are initialized + // before the aggregate, without turning them into input columns. + self.plan = + self.binder + .bind_scalar_queries(self.plan, &[QueryBindStep::Agg], self.arena)?; + if self + .binder + .context + .scalar_queries + .iter() + .any(|query| !query.param_bindings.is_empty()) + { + if !group_by.is_empty() || !self.binder.context.agg_calls.is_empty() { + return Err(DatabaseError::UnsupportedStmt( + "correlated scalar queries with outer aggregation are not supported yet".into(), + )); + } + } else if !group_by.is_empty() || !self.binder.context.agg_calls.is_empty() { + self.plan = self.binder.bind_scalar_queries( + self.plan, + &[QueryBindStep::Project], + self.arena, + )?; + } if !group_by.is_empty() { self.binder.extract_group_by_aggregate_exprs( &mut self.select_list, @@ -571,19 +544,21 @@ where self.arena, )?; } - let context = &mut self.binder.context; - if references_select_list_sub_query( - &mut context.sub_queries, - context.group_by_exprs.iter().copied(), - self.arena, - )? { - return Err(DatabaseError::UnsupportedStmt( - "GROUP BY over a scalar subquery in the SELECT list is not supported".to_string(), - )); - } + if !self.binder.context.agg_calls.is_empty() || !self.binder.context.group_by_exprs.is_empty() { + if self + .binder + .context + .scalar_queries + .iter() + .any(|q| !q.param_bindings.is_empty()) + { + return Err(DatabaseError::UnsupportedStmt( + "correlated scalar queries with outer aggregation are not supported yet".into(), + )); + } let agg_calls = std::mem::take(&mut self.binder.context.agg_calls); let group_by_exprs = std::mem::take(&mut self.binder.context.group_by_exprs); let output_exprs = self @@ -648,6 +623,30 @@ where pub(crate) fn window( mut self, ) -> Result, DatabaseError> { + if self + .binder + .context + .scalar_queries + .iter() + .any(|query| !query.param_bindings.is_empty()) + { + for expr in self.select_list.iter().chain( + self.orderby + .iter() + .flat_map(|fields| fields.iter().map(|field| &field.expr)), + ) { + if expr.has_window_call(self.arena)? { + return Err(DatabaseError::UnsupportedStmt( + "correlated scalar queries with outer windows are not supported yet".into(), + )); + } + } + } + self.plan = self.binder.bind_scalar_queries( + self.plan, + &[QueryBindStep::Project, QueryBindStep::Sort], + self.arena, + )?; self.plan = self.binder.bind_window( self.plan, &mut self.select_list, @@ -675,15 +674,24 @@ where distinct: bool, ) -> Result, DatabaseError> { if distinct { - if references_select_list_sub_query( - &mut self.binder.context.sub_queries, - self.select_list.iter().copied(), - self.arena, - )? { - return Err(DatabaseError::UnsupportedStmt( - "DISTINCT over a scalar subquery in the SELECT list is not supported" - .to_string(), - )); + struct OuterResult(bool); + impl ExprVisitor for OuterResult { + fn visit_outer_value( + &mut self, + _id: ScalarQueryRef, + _ty: &LogicalType, + _arena: &(dyn MetaArena + '_), + ) -> Result<(), DatabaseError> { + self.0 = true; + Ok(()) + } + } + let mut outer = OuterResult(false); + for expr in &self.select_list { + ExprVisitor::visit(&mut outer, *expr, self.arena)?; + } + if outer.0 { + return Err(DatabaseError::UnsupportedStmt("correlated scalar values across DISTINCT require a row-scoped execution context".into())); } let distinct_outputs = self.select_list.clone(); self.binder.bind_distinct_output_exprs( @@ -717,16 +725,6 @@ where mut self, ) -> Result, DatabaseError> { if let Some(orderby) = self.orderby { - if references_select_list_sub_query( - &mut self.binder.context.sub_queries, - orderby.iter().map(|field| field.expr), - self.arena, - )? { - return Err(DatabaseError::UnsupportedStmt( - "ORDER BY over a scalar subquery in the SELECT list is not supported" - .to_string(), - )); - } self.plan = self.binder.bind_sort(self.plan, orderby, self.arena)?; } @@ -791,9 +789,37 @@ impl BindPlanComplete { } impl<'a: 'b, 'b, T: Transaction, A: AsRef<[(usize, LogicalType)]>> Binder<'a, 'b, T, A> { + pub(crate) fn bind_scalar_queries( + &mut self, + mut plan: LogicalPlan, + on_steps: &[QueryBindStep], + _arena: &mut PlanArena, + ) -> Result { + for query in self + .context + .scalar_queries + .extract_if(.., |query| on_steps.contains(&query.step)) + { + plan = + ScalarQueryInitOperator::build(plan, query.plan, query.value, query.param_bindings); + } + Ok(plan) + } + + pub(crate) fn init_scalar_queries(&mut self, mut plan: LogicalPlan) -> LogicalPlan { + for query in std::mem::take(&mut self.context.scalar_queries) + .into_iter() + .rev() + { + plan = + ScalarQueryInitOperator::build(plan, query.plan, query.value, query.param_bindings); + } + plan + } + pub(crate) fn build_plan<'s, 'arena>( &'s mut self, - arena: &'s mut crate::planner::PlanArena<'arena>, + arena: &'s mut PlanArena<'arena>, ) -> BindPlanStart<'s, 'a, 'b, 'arena, T, A> { BindPlanStart { binder: self, @@ -801,7 +827,9 @@ impl<'a: 'b, 'b, T: Transaction, A: AsRef<[(usize, LogicalType)]>> Binder<'a, 'b } } - fn is_temp_alias_projection(exprs: &[ExprRef], arena: &crate::planner::PlanArena) -> bool { + /// Whether `exprs` only renames its input to an alias, as `bind_alias` builds for `FROM t AS x` + /// (`temp == false`) or for a temp table (`temp == true`). + fn is_alias_projection(exprs: &[ExprRef], temp: bool, arena: &PlanArena) -> bool { !exprs.is_empty() && exprs.iter().all(|expr| { matches!( @@ -814,7 +842,8 @@ impl<'a: 'b, 'b, T: Transaction, A: AsRef<[(usize, LogicalType)]>> Binder<'a, 'b ScalarExpression::ColumnRef { column, .. } if matches!( &arena.column(*column).summary().relation, - crate::catalog::ColumnRelation::Table { is_temp: true, .. } + crate::catalog::ColumnRelation::Table { is_temp, .. } + if *is_temp == temp ) ) ) @@ -824,7 +853,7 @@ impl<'a: 'b, 'b, T: Transaction, A: AsRef<[(usize, LogicalType)]>> Binder<'a, 'b pub(crate) fn is_joined_values_source( join_type: Option, source: &Source<'a>, - arena: &crate::planner::PlanArena, + arena: &PlanArena, ) -> bool { join_type.is_some() && matches!( @@ -876,31 +905,10 @@ impl<'a: 'b, 'b, T: Transaction, A: AsRef<[(usize, LogicalType)]>> Binder<'a, 'b Ok(()) } - fn globalize_join_filter_from_split_scope( - join_condition: &mut JoinCondition, - left_len: usize, - right_schema: &Schema, - arena: &mut crate::planner::PlanArena, - ) -> Result<(), DatabaseError> { - let JoinCondition::On { filter, .. } = join_condition else { - return Ok(()); - }; - - if let Some(expr) = filter { - RightSidePositionGlobalizer { - right_schema, - left_len, - } - .visit(expr, arena)?; - } - - Ok(()) - } - fn localize_appended_right_outputs<'expr>( exprs: impl Iterator, appended_outputs: &[AppendedRightOutput], - arena: &mut crate::planner::PlanArena, + arena: &mut PlanArena, ) -> Result<(), DatabaseError> { struct AppendedRightOutputBinder<'a> { appended_outputs: &'a [AppendedRightOutput], @@ -930,72 +938,11 @@ impl<'a: 'b, 'b, T: Transaction, A: AsRef<[(usize, LogicalType)]>> Binder<'a, 'b Ok(()) } - fn rebind_split_scope_positions( - mut expr: ExprRef, - left_schema: &Schema, - right_schema: &Schema, - arena: &mut crate::planner::PlanArena, - ) -> Result<(), DatabaseError> { - SplitScopePositionRebinder { - left_schema, - right_schema, - } - .visit(&mut expr, arena) - } - - fn build_join_from_split_scope_predicates( - &self, - mut children: LogicalPlan, - mut plan: LogicalPlan, - join_ty: JoinType, - predicates: impl IntoIterator, - rebind_positions: bool, - arena: &mut crate::planner::PlanArena, - ) -> Result { - let left_schema = children.output_schema(arena); - let right_schema = plan.output_schema(arena); - let mut on_keys = Vec::new(); - let mut filter = Vec::new(); - - for predicate in predicates { - if rebind_positions { - Self::rebind_split_scope_positions(predicate, left_schema, right_schema, arena)?; - } - Self::extract_join_keys( - predicate, - &mut on_keys, - &mut filter, - left_schema, - right_schema, - arena, - )?; - } - - let mut join_condition = JoinCondition::On { - on: on_keys, - filter: Self::combine_conjuncts(filter, arena), - }; - Self::globalize_join_filter_from_split_scope( - &mut join_condition, - left_schema.len(), - right_schema, - arena, - )?; - - Ok(LJoinOperator::build( - children, - plan, - join_condition, - join_ty, - self.force_nested_loop, - )) - } - fn bind_set_cast( &mut self, mut left_plan: LogicalPlan, mut right_plan: LogicalPlan, - arena: &mut crate::planner::PlanArena, + arena: &mut PlanArena, ) -> Result<(LogicalPlan, LogicalPlan), DatabaseError> { let mut left_cast = vec![]; let mut right_cast = vec![]; @@ -1044,7 +991,7 @@ impl<'a: 'b, 'b, T: Transaction, A: AsRef<[(usize, LogicalType)]>> Binder<'a, 'b is_all: bool, mut left_plan: LogicalPlan, mut right_plan: LogicalPlan, - arena: &mut crate::planner::PlanArena, + arena: &mut PlanArena, ) -> Result { let mut left_schema = left_plan.output_schema(arena); let mut right_schema = right_plan.output_schema(arena); @@ -1152,7 +1099,7 @@ impl<'a: 'b, 'b, T: Transaction, A: AsRef<[(usize, LogicalType)]>> Binder<'a, 'b alias_column: &[String], table_alias: TableName, table_name: TableName, - arena: &mut crate::planner::PlanArena, + arena: &mut PlanArena, ) -> Result { let input_schema = plan.output_schema(arena); let input_schema_len = input_schema.len(); @@ -1201,7 +1148,7 @@ impl<'a: 'b, 'b, T: Transaction, A: AsRef<[(usize, LogicalType)]>> Binder<'a, 'b &mut self, mut plan: LogicalPlan, source_name: TableName, - arena: &mut crate::planner::PlanArena, + arena: &mut PlanArena, ) -> LogicalPlan { let input_schema = plan.output_schema(arena); let input_schema_len = input_schema.len(); @@ -1237,12 +1184,13 @@ impl<'a: 'b, 'b, T: Transaction, A: AsRef<[(usize, LogicalType)]>> Binder<'a, 'b join_type: Option, table_name: TableName, alias: Option, - arena: &mut crate::planner::PlanArena, + arena: &mut PlanArena, ) -> Result { let table_alias = alias.as_ref().map(|alias| alias.name.clone()); if let Some(plan_ref) = self.context.cte(&table_name).map(|cte| cte.plan_ref) { let mut plan = arena.plan(plan_ref).clone(); + OperatorExprVisitorMut::new(&mut ExprCloner, arena).visit_plan(&mut plan)?; if let Some(alias) = alias { plan = self.bind_alias( plan, @@ -1307,7 +1255,7 @@ impl<'a: 'b, 'b, T: Transaction, A: AsRef<[(usize, LogicalType)]>> Binder<'a, 'b mut plan: LogicalPlan, alias: Option, joint_type: Option, - arena: &mut crate::planner::PlanArena, + arena: &mut PlanArena, ) -> Result { if let Some(alias) = alias { let source_name = arena.temp_table(); @@ -1362,7 +1310,7 @@ impl<'a: 'b, 'b, T: Transaction, A: AsRef<[(usize, LogicalType)]>> Binder<'a, 'b expr: ScalarExpression, alias: Option, joint_type: Option, - arena: &mut crate::planner::PlanArena, + arena: &mut PlanArena, ) -> Result { let ScalarExpression::TableFunction(function) = expr else { return Err(DatabaseError::UnsupportedStmt( @@ -1395,7 +1343,7 @@ impl<'a: 'b, 'b, T: Transaction, A: AsRef<[(usize, LogicalType)]>> Binder<'a, 'b #[allow(unused_assignments)] pub(crate) fn bind_table_column_refs( context: &BinderContext<'a, T>, - arena: &mut crate::planner::PlanArena, + arena: &mut PlanArena, exprs: &mut Vec, table_name: TableName, is_qualified_wildcard: bool, @@ -1403,7 +1351,7 @@ impl<'a: 'b, 'b, T: Transaction, A: AsRef<[(usize, LogicalType)]>> Binder<'a, 'b let (source, position_offset) = Self::resolve_source_columns_in_scope(context, table_name.as_ref())?; - let fn_not_on_using = |column: &ColumnRef, arena: &crate::planner::PlanArena<'_>| { + let fn_not_on_using = |column: &ColumnRef, arena: &PlanArena<'_>| { let column_catalog = arena.column(*column); if context.using.is_empty() { return Some(&table_name) == column_catalog.table_name(); @@ -1433,7 +1381,7 @@ impl<'a: 'b, 'b, T: Transaction, A: AsRef<[(usize, LogicalType)]>> Binder<'a, 'b fn wildcard_column_expr( context: &BinderContext<'a, T>, - arena: &mut crate::planner::PlanArena, + arena: &mut PlanArena, column: &ColumnRef, position: usize, is_qualified_wildcard: bool, @@ -1464,7 +1412,7 @@ impl<'a: 'b, 'b, T: Transaction, A: AsRef<[(usize, LogicalType)]>> Binder<'a, 'b mut right: LogicalPlan, join_type: JoinType, constraint: JoinConstraintInput, - arena: &mut crate::planner::PlanArena, + arena: &mut PlanArena, ) -> Result { let left_len = left.output_schema(arena).len(); right.output_schema(arena); @@ -1487,12 +1435,17 @@ impl<'a: 'b, 'b, T: Transaction, A: AsRef<[(usize, LogicalType)]>> Binder<'a, 'b &mut self, mut children: LogicalPlan, predicate: ExprRef, - arena: &mut crate::planner::PlanArena, + arena: &mut PlanArena, ) -> Result { + children = self.bind_scalar_queries(children, &[QueryBindStep::Where], arena)?; self.context.step(QueryBindStep::Where); + if predicate.has_agg_call(arena)? { + return Err(DatabaseError::AggMiss( + "aggregate functions are not allowed in WHERE".into(), + )); + } if let Some(sub_queries) = self.context.sub_queries_at_now() { - let mut uses_mark_apply = None; for sub_query in sub_queries { match sub_query { SubQueryType::ExistsSubQuery { @@ -1500,13 +1453,6 @@ impl<'a: 'b, 'b, T: Transaction, A: AsRef<[(usize, LogicalType)]>> Binder<'a, 'b correlated, output_column, } => { - if matches!(uses_mark_apply, Some(false)) { - return Err(DatabaseError::UnsupportedStmt( - "mixed EXISTS/IN with other WHERE subqueries is not supported yet" - .to_string(), - )); - } - uses_mark_apply = Some(true); let left_schema = children.output_schema(arena).clone(); let (plan, predicates) = Self::prepare_mark_apply( predicate, @@ -1533,13 +1479,6 @@ impl<'a: 'b, 'b, T: Transaction, A: AsRef<[(usize, LogicalType)]>> Binder<'a, 'b predicate: mut quantified_predicate, .. } => { - if matches!(uses_mark_apply, Some(false)) { - return Err(DatabaseError::UnsupportedStmt( - "mixed EXISTS/IN with other WHERE subqueries is not supported yet" - .to_string(), - )); - } - uses_mark_apply = Some(true); if correlated { quantified_predicate = Self::rewrite_correlated_quantified_predicate( quantified_predicate, @@ -1565,32 +1504,9 @@ impl<'a: 'b, 'b, T: Transaction, A: AsRef<[(usize, LogicalType)]>> Binder<'a, 'b predicates, ); } - SubQueryType::SubQuery { plan, correlated } => { - if matches!(uses_mark_apply, Some(true)) { - return Err(DatabaseError::UnsupportedStmt( - "mixed EXISTS/IN with other WHERE subqueries is not supported yet" - .to_string(), - )); - } - uses_mark_apply = Some(false); - if correlated { - return Err(DatabaseError::UnsupportedStmt( - "correlated scalar subqueries in WHERE are not supported" - .to_string(), - )); - } - children = self.build_join_from_split_scope_predicates( - children, - plan, - JoinType::Inner, - std::iter::once(predicate), - true, - arena, - )?; - } } } - if matches!(uses_mark_apply, Some(true)) { + { let passthrough_exprs = children .output_schema(arena) .iter() @@ -1608,7 +1524,6 @@ impl<'a: 'b, 'b, T: Transaction, A: AsRef<[(usize, LogicalType)]>> Binder<'a, 'b Childrens::Only(Box::new(filter)), )); } - return Ok(children); } Ok(FilterOperator::build(predicate, children, false)) } @@ -1616,7 +1531,7 @@ impl<'a: 'b, 'b, T: Transaction, A: AsRef<[(usize, LogicalType)]>> Binder<'a, 'b fn ensure_mark_apply_right_outputs( plan: &mut LogicalPlan, predicates: &[ExprRef], - arena: &mut crate::planner::PlanArena, + arena: &mut PlanArena, ) -> Result, DatabaseError> { let output_schema = plan.output_schema(arena).clone(); let output_len = output_schema.len(); @@ -1626,6 +1541,11 @@ impl<'a: 'b, 'b, T: Transaction, A: AsRef<[(usize, LogicalType)]>> Binder<'a, 'b .. } = plan { + // An alias projection already outputs every input column under the alias; its input + // columns may look like the outer query's (`t1` in `FROM t1 AS x`) but are not. + if Self::is_alias_projection(&op.exprs, false, arena) { + return Ok(Vec::new()); + } let Childrens::Only(child) = childrens.as_mut() else { return Ok(Vec::new()); }; @@ -1671,7 +1591,7 @@ impl<'a: 'b, 'b, T: Transaction, A: AsRef<[(usize, LogicalType)]>> Binder<'a, 'b correlated: bool, preserve_projection: bool, mut apply_predicates: Vec, - arena: &mut crate::planner::PlanArena, + arena: &mut PlanArena, ) -> Result<(LogicalPlan, Vec), DatabaseError> { let left_len = left_schema.len(); MarkerPositionGlobalizer { @@ -1743,7 +1663,7 @@ impl<'a: 'b, 'b, T: Transaction, A: AsRef<[(usize, LogicalType)]>> Binder<'a, 'b fn plan_has_correlated_refs( plan: &LogicalPlan, left_schema: &Schema, - arena: &mut crate::planner::PlanArena, + arena: &mut PlanArena, ) -> Result { if !plan .operator @@ -1772,7 +1692,7 @@ impl<'a: 'b, 'b, T: Transaction, A: AsRef<[(usize, LogicalType)]>> Binder<'a, 'b fn expr_has_correlated_refs( expr: ExprRef, left_schema: &Schema, - arena: &mut crate::planner::PlanArena, + arena: &mut PlanArena, ) -> Result { expr.any_referenced_column(arena, |arena, column| { left_schema @@ -1813,7 +1733,7 @@ impl<'a: 'b, 'b, T: Transaction, A: AsRef<[(usize, LogicalType)]>> Binder<'a, 'b plan: LogicalPlan, left_schema: &Schema, preserve_projection: bool, - arena: &mut crate::planner::PlanArena, + arena: &mut PlanArena, ) -> Result<(LogicalPlan, Vec), DatabaseError> { match plan.childrens.as_ref() { Childrens::Only(_) => {} @@ -1829,6 +1749,15 @@ impl<'a: 'b, 'b, T: Transaction, A: AsRef<[(usize, LogicalType)]>> Binder<'a, 'b } match plan { + // `FROM t AS x`: the alias gives the subquery's columns their own identity, which its + // predicates are bound to, so it is kept. It only reads its input, never the outer query. + plan if matches!( + &plan.operator, + Operator::Project(op) if Self::is_alias_projection(&op.exprs, false, arena) + ) => + { + Ok((plan, vec![])) + } LogicalPlan { operator: Operator::Filter(op), childrens, @@ -1871,7 +1800,7 @@ impl<'a: 'b, 'b, T: Transaction, A: AsRef<[(usize, LogicalType)]>> Binder<'a, 'b arena, )?; - if !preserve_projection || Self::is_temp_alias_projection(&op.exprs, arena) { + if !preserve_projection || Self::is_alias_projection(&op.exprs, true, arena) { Ok((child, correlated_filters)) } else { for expr in &op.exprs { @@ -1927,10 +1856,11 @@ impl<'a: 'b, 'b, T: Transaction, A: AsRef<[(usize, LogicalType)]>> Binder<'a, 'b fn bind_having( &mut self, - children: LogicalPlan, + mut children: LogicalPlan, mut having: ExprRef, - arena: &mut crate::planner::PlanArena, + arena: &mut PlanArena, ) -> Result { + children = self.bind_scalar_queries(children, &[QueryBindStep::Having], arena)?; self.context.step(QueryBindStep::Having); self.validate_having_orderby(having, arena)?; @@ -1950,44 +1880,12 @@ impl<'a: 'b, 'b, T: Transaction, A: AsRef<[(usize, LogicalType)]>> Binder<'a, 'b pub(crate) fn bind_project( &mut self, - mut children: LogicalPlan, - mut select_list: Vec, - arena: &mut crate::planner::PlanArena, + children: LogicalPlan, + select_list: Vec, + arena: &mut PlanArena, ) -> Result { + let children = self.bind_scalar_queries(children, &[QueryBindStep::Project], arena)?; self.context.step(QueryBindStep::Project); - - if let Some(sub_queries) = self.context.sub_queries_at_now() { - for sub_query in sub_queries { - let SubQueryType::SubQuery { - mut plan, - correlated, - } = sub_query - else { - return Err(DatabaseError::UnsupportedStmt( - "only scalar subqueries are supported in SELECT list".to_string(), - )); - }; - - if correlated { - return Err(DatabaseError::UnsupportedStmt( - "correlated scalar subqueries in SELECT list are not supported".to_string(), - )); - } - - let left_len = children.output_schema(arena).len(); - let right_schema = plan.output_schema(arena); - for expr in &mut select_list { - RightSidePositionGlobalizer { - right_schema, - left_len, - } - .visit(expr, arena)?; - } - - children = ScalarApplyOperator::build(children, plan); - } - } - Ok(Self::build_project_plan(children, select_list)) } @@ -1995,10 +1893,10 @@ impl<'a: 'b, 'b, T: Transaction, A: AsRef<[(usize, LogicalType)]>> Binder<'a, 'b &mut self, children: LogicalPlan, sort_fields: Vec, - _arena: &mut crate::planner::PlanArena, + arena: &mut PlanArena, ) -> Result { + let children = self.bind_scalar_queries(children, &[QueryBindStep::Sort], arena)?; self.context.step(QueryBindStep::Sort); - Ok(LogicalPlan::new( Operator::Sort(SortOperator { sort_fields }), Childrens::Only(Box::new(children)), @@ -2016,11 +1914,7 @@ impl<'a: 'b, 'b, T: Transaction, A: AsRef<[(usize, LogicalType)]>> Binder<'a, 'b Ok(LimitOperator::build(offset_value, limit_value, children)) } - pub fn extract_select_join( - &mut self, - select_items: &mut [ExprRef], - arena: &mut crate::planner::PlanArena, - ) { + pub fn extract_select_join(&mut self, select_items: &mut [ExprRef], arena: &mut PlanArena) { if self.context.bind_table.len() < 2 { return; } @@ -2075,7 +1969,7 @@ impl<'a: 'b, 'b, T: Transaction, A: AsRef<[(usize, LogicalType)]>> Binder<'a, 'b constraint: JoinConstraintInput, left_schema: &Schema, right_schema: &Schema, - arena: &mut crate::planner::PlanArena, + arena: &mut PlanArena, ) -> Result { match constraint { JoinConstraintInput::On(expr) => { @@ -2084,7 +1978,7 @@ impl<'a: 'b, 'b, T: Transaction, A: AsRef<[(usize, LogicalType)]>> Binder<'a, 'b // expression that didn't match equi-join pattern let mut filter = vec![]; - Self::extract_join_keys( + extract_join_keys( expr, &mut on_keys, &mut filter, @@ -2104,7 +1998,7 @@ impl<'a: 'b, 'b, T: Transaction, A: AsRef<[(usize, LogicalType)]>> Binder<'a, 'b fn find_column<'a>( schema: &'a Schema, name: &'a str, - arena: &crate::planner::PlanArena, + arena: &PlanArena, ) -> Option<(usize, &'a ColumnRef)> { schema .iter() @@ -2213,150 +2107,6 @@ impl<'a: 'b, 'b, T: Transaction, A: AsRef<[(usize, LogicalType)]>> Binder<'a, 'b } } } - - /// for sqlrs - /// original idea from datafusion planner.rs - /// Extracts equijoin ON condition be a single Eq or multiple conjunctive Eqs - /// Filters matching this pattern are added to `accum` - /// Filters that don't match this pattern are added to `accum_filter` - /// Examples: - /// ```text - /// foo = bar => accum=[(foo, bar)] accum_filter=[] - /// foo = bar AND bar = baz => accum=[(foo, bar), (bar, baz)] accum_filter=[] - /// foo = bar AND baz > 1 => accum=[(foo, bar)] accum_filter=[baz > 1] - /// ``` - fn extract_join_keys( - expr: ExprRef, - accum: &mut Vec<(ExprRef, ExprRef)>, - accum_filter: &mut Vec, - left_schema: &Schema, - right_schema: &Schema, - arena: &mut crate::planner::PlanArena, - ) -> Result<(), DatabaseError> { - let fn_contains = |schema: &Schema, column: ColumnRef| { - let summary = arena.column(column).summary(); - schema - .iter() - .any(|candidate| arena.column(*candidate).summary() == summary) - }; - let fn_or_contains = |column: ColumnRef| { - fn_contains(left_schema, column) || fn_contains(right_schema, column) - }; - - let expr = expr.unpack_alias(arena); - match arena.expression(expr) { - ScalarExpression::Binary { - left_expr, - right_expr, - op, - .. - } => { - match op { - BinaryOperator::Eq => { - match ( - left_expr.unpack_alias_ref(arena), - right_expr.unpack_alias_ref(arena), - ) { - // example: foo = bar - ( - ScalarExpression::ColumnRef { column: l, .. }, - ScalarExpression::ColumnRef { column: r, .. }, - ) => { - // reorder left and right joins keys to pattern: (left, right) - let key = if fn_contains(left_schema, *l) - && fn_contains(right_schema, *r) - { - Some((*left_expr, *right_expr)) - } else if fn_contains(left_schema, *r) - && fn_contains(right_schema, *l) - { - Some((*right_expr, *left_expr)) - } else { - if fn_or_contains(*l) || fn_or_contains(*r) { - accum_filter.push(expr); - } - None - }; - // Join keys are compared (and hashed) directly, so cast - // both to one type like `l = r` in a filter; otherwise - // e.g. `bigint = int` never matches. - if let Some((left, right)) = key { - let ty = LogicalType::max_logical_type( - &left.return_type(arena), - &right.return_type(arena), - )? - .into_owned(); - accum.push(( - left.type_cast(Cow::Borrowed(&ty), arena)?, - right.type_cast(Cow::Borrowed(&ty), arena)?, - )); - } - } - (ScalarExpression::ColumnRef { column, .. }, _) - | (_, ScalarExpression::ColumnRef { column, .. }) => { - if fn_or_contains(*column) { - accum_filter.push(expr); - } - } - _other => { - // example: baz > 1 - if left_expr.all_referenced_columns(arena, |_, column| { - fn_or_contains(*column) - })? && right_expr - .all_referenced_columns(arena, |_, column| { - fn_or_contains(*column) - })? - { - accum_filter.push(expr); - } - } - } - } - BinaryOperator::And => { - // example: foo = bar AND baz > 1 - let (left_expr, right_expr) = (*left_expr, *right_expr); - Self::extract_join_keys( - left_expr, - accum, - accum_filter, - left_schema, - right_schema, - arena, - )?; - Self::extract_join_keys( - right_expr, - accum, - accum_filter, - left_schema, - right_schema, - arena, - )?; - } - BinaryOperator::Or => { - accum_filter.push(expr); - } - _ => { - if left_expr - .all_referenced_columns(arena, |_, column| fn_or_contains(*column))? - && right_expr.all_referenced_columns(arena, |_, column| { - fn_or_contains(*column) - })? - { - accum_filter.push(expr); - } - } - } - } - _ => { - if expr.all_referenced_columns(arena, |_, column| fn_or_contains(*column))? { - // example: baz > 1 - accum_filter.push(expr); - } - } - } - - Ok(()) - } } #[cfg(all(test, not(target_arch = "wasm32")))] @@ -2372,7 +2122,7 @@ mod tests { MarkApplyKind, MarkApplyOperator, MarkApplyQuantifier, }; use crate::planner::operator::Operator; - use crate::planner::{Childrens, ExprRef, LogicalPlan, PlanArena}; + use crate::planner::{Childrens, ExprRef, LogicalPlan, PlanArena, TableArenaCell}; use crate::types::LogicalType; fn test_column(arena: &mut PlanArena, name: &str, position: usize) -> ExprRef { @@ -2417,7 +2167,7 @@ mod tests { #[test] fn test_right_side_position_globalizer_only_shifts_right_columns() -> Result<(), DatabaseError> { - let table_arena = crate::planner::TableArenaCell::default(); + let table_arena = TableArenaCell::default(); let mut arena = PlanArena::new(&table_arena); let left_column = arena.alloc_column(ColumnCatalog::new( "left".to_string(), @@ -2475,7 +2225,7 @@ mod tests { #[test] fn test_projection_output_binder_rewrites_to_project_slot() -> Result<(), DatabaseError> { - let table_arena = crate::planner::TableArenaCell::default(); + let table_arena = TableArenaCell::default(); let mut arena = PlanArena::new(&table_arena); let project_inner = test_column(&mut arena, "c1", 0); let project_output = arena.alloc_expression(ScalarExpression::Alias { @@ -2537,16 +2287,28 @@ mod tests { } #[test] - fn test_scalar_subquery_in_where_binds_as_inner_join() -> Result<(), DatabaseError> { + fn test_scalar_subquery_in_where_binds_as_init() -> Result<(), DatabaseError> { let table_states = build_t1_table()?; - let plan = table_states.plan("select * from t1 where c1 = (select max(c3) from t2)")?; - let Some((join_type, join_condition)) = find_join(&plan) else { - panic!("expected scalar subquery to introduce a join") + let mut arena = PlanArena::new(&table_states.table_arena); + let mut plan = table_states.plan_with_arena( + "select * from t1 where c1 = (select max(c3) from t2)", + &mut arena, + )?; + assert!(matches!(plan.operator, Operator::Project(_))); + assert!(find_join(&plan).is_none()); + let Childrens::Only(filter) = plan.childrens.as_mut() else { + panic!("expected project input") }; - - assert_eq!(*join_type, JoinType::Inner); - assert!(matches!(join_condition, JoinCondition::On { .. })); - + let Childrens::Only(init) = filter.childrens.as_mut() else { + panic!("expected filter input") + }; + assert!(matches!(init.operator, Operator::ScalarQueryInit(_))); + let Childrens::Twins { left, right } = init.childrens.as_mut() else { + panic!("expected init children") + }; + assert_eq!(left.output_schema(&mut arena).len(), 2); + assert_eq!(right.output_schema(&mut arena).len(), 1); + assert_eq!(plan.output_schema(&mut arena).len(), 2); Ok(()) } @@ -2616,63 +2378,26 @@ mod tests { Ok(()) } - fn find_top_join(plan: &LogicalPlan) -> Option<&LogicalPlan> { - if matches!(plan.operator, Operator::Join(_)) { - return Some(plan); - } - - match plan.childrens.as_ref() { - Childrens::Only(child) => find_top_join(child), - Childrens::Twins { .. } | Childrens::None => None, - } - } - - fn collect_column_positions(expr: ExprRef, arena: &PlanArena, positions: &mut Vec) { - match arena.expression(expr.unpack_alias(arena)) { - ScalarExpression::ColumnRef { position, .. } => positions.push(*position), - ScalarExpression::Binary { - left_expr, - right_expr, - .. - } => { - collect_column_positions(*left_expr, arena, positions); - collect_column_positions(*right_expr, arena, positions); - } - _ => {} - } - } - #[test] - fn test_multiple_scalar_subqueries_in_where_rebind_positions() -> Result<(), DatabaseError> { + fn test_multiple_scalar_subqueries_have_independent_slots() -> Result<(), DatabaseError> { let table_states = build_t1_table()?; let mut arena = PlanArena::new(&table_states.table_arena); let plan = table_states.plan_with_arena( "select * from t1 where c1 <= (select 4) and c1 > (select 1)", &mut arena, )?; - let outer_join = - find_top_join(&plan).expect("expected scalar subqueries to introduce a join"); - let Operator::Join(op) = &outer_join.operator else { - panic!("expected join plan") + let init = plan.childrens.only().childrens.only(); + let Operator::ScalarQueryInit(first) = &init.operator else { + panic!("expected scalar init") }; - let Childrens::Twins { left, .. } = outer_join.childrens.as_ref() else { - panic!("expected binary join") + let Childrens::Twins { left, .. } = init.childrens.as_ref() else { + panic!("expected init children") }; - let JoinCondition::On { - filter: Some(filter), - .. - } = &op.on - else { - panic!("expected join filter") + let Operator::ScalarQueryInit(second) = &left.operator else { + panic!("expected second scalar init") }; - let mut left_plan = left.as_ref().clone(); - let left_len = left_plan.output_schema(&mut arena).len(); - - let mut positions = Vec::new(); - collect_column_positions(*filter, &arena, &mut positions); - - assert_eq!(positions, vec![0, left_len - 1, 0, left_len]); - + assert_ne!(first.reference(&arena), second.reference(&arena)); + assert!(find_join(&plan).is_none()); Ok(()) } } diff --git a/src/catalog/column.rs b/src/catalog/column.rs index 04b20526..a3448b7c 100644 --- a/src/catalog/column.rs +++ b/src/catalog/column.rs @@ -183,7 +183,7 @@ impl ColumnCatalog { .map(|expr| { arena .expression(*expr) - .eval(arena, None) + .eval::<[DataValue]>(arena, None) .map(Cow::into_owned) }) .transpose() diff --git a/src/db.rs b/src/db.rs index cce05bb8..74cf4775 100644 --- a/src/db.rs +++ b/src/db.rs @@ -25,6 +25,7 @@ use crate::expression::function::table::{ ArcTableFunctionImpl, TableFunctionCatalog, TableFunctionImpl, }; use crate::expression::function::FunctionSummary; +use crate::function::abs::Abs; use crate::function::char_length::CharLength; #[cfg(feature = "time")] use crate::function::current_date::CurrentDate; @@ -42,7 +43,9 @@ use crate::optimizer::rule::normalization::NormalizationRuleImpl; #[cfg(feature = "orm")] use crate::orm::FromQueryRow; use crate::planner::operator::Operator; -use crate::planner::{LogicalPlan, MetaArena, PlanArena, PlanInput, PlanKeeper, TableArenaCell}; +use crate::planner::{ + ExecMetaArena, LogicalPlan, MetaArena, PlanArena, PlanInput, PlanKeeper, TableArenaCell, +}; #[cfg(all(not(target_arch = "wasm32"), feature = "lmdb"))] use crate::storage::lmdb::{LmdbConfig, LmdbStorage}; use crate::storage::memory::MemoryStorage; @@ -295,6 +298,9 @@ impl DataBaseBuilder { _p: Default::default(), }; + for ty in Abs::types() { + state.load_scalar_function(Abs::new(ty)); + } state.load_scalar_function(CharLength::new("char_length".to_lowercase())); state.load_scalar_function(CharLength::new("character_length".to_lowercase())); #[cfg(feature = "time")] @@ -333,7 +339,7 @@ fn optimizer_pipeline() -> HepOptimizerPipeline { ) .before_batch( "Predicate Pushdown".to_string(), - HepBatchStrategy::fix_point_topdown(10), + HepBatchStrategy::fix_point_topdown(30), vec![ NormalizationRuleImpl::PushPredicateThroughJoin, NormalizationRuleImpl::PushJoinPredicateIntoScan, @@ -582,7 +588,14 @@ impl State { transaction: &'a mut S::TransactionType<'txn>, plan: PlanInput<'a>, mut plan_arena: A, - ) -> Result<(Schema, A, Executor<'a, S::TransactionType<'txn>>), DatabaseError> + ) -> Result< + ( + Schema, + ExecMetaArena, + Executor<'a, S::TransactionType<'txn>>, + ), + DatabaseError, + > where S: 'txn, { @@ -599,6 +612,7 @@ impl State { &self.table_functions, ); let root = build_write(&mut arena, &mut plan_arena, plan, read_context, transaction); + let plan_arena = ExecMetaArena::new(plan_arena); let executor = Executor::new(arena, root, keeper); Ok((schema, plan_arena, executor)) @@ -612,7 +626,7 @@ impl State { ) -> Result< ( Schema, - PlanArena<'a>, + ExecMetaArena>, Executor<'a, S::TransactionType<'txn>>, ), DatabaseError, @@ -675,6 +689,7 @@ impl State { table_functions, ); let root = build_write(&mut arena, &mut plan_arena, plan, cache, transaction); + let plan_arena = ExecMetaArena::new(plan_arena); let executor = Executor::new(arena, root, keeper); Ok((schema, plan_arena, executor)) @@ -692,7 +707,7 @@ impl DDLApply { fn apply_to( self, state: &mut State, - plan_arena: &PlanArena, + plan_arena: &mut PlanArena, ) -> Result { let mut catalog_changed = false; match self { @@ -758,6 +773,26 @@ impl Database { &mut Binder<'bind, '_, S::TransactionType<'txn>, A>, &mut PlanArena<'a>, ) -> Result, + { + self.execute_mut_with(context, params, build, |_| Ok(())) + } + + pub(crate) fn execute_mut_with( + &mut self, + context: &str, + params: A, + build: F, + consume: C, + ) -> Result + where + A: AsRef<[(usize, LogicalType)]>, + F: for<'a, 'txn, 'bind> FnOnce( + &mut Binder<'bind, '_, S::TransactionType<'txn>, A>, + &mut PlanArena<'a>, + ) -> Result, + C: for<'a> FnOnce( + &mut TransactionIter<'a, S::TransactionType<'a>, PlanArena<'a>>, + ) -> Result, { let transaction = Box::into_raw(Box::new( self.storage @@ -772,16 +807,18 @@ impl Database { return Err(err.with_sql_context(context)); } }; - let (plan_arena, apply) = - match TransactionIter::new(schema, plan_arena, executor, transaction) - .done_with_ddl_apply() - { - Ok(apply) => apply, - Err(err) => { - unsafe { drop(Box::from_raw(transaction)) }; - return Err(err.with_sql_context(context)); - } - }; + let (result, mut plan_arena, apply) = match (|| { + let mut iter = TransactionIter::new(schema, plan_arena, executor, transaction); + let result = consume(&mut iter)?; + let (arena, apply) = iter.done_with_ddl_apply()?; + Ok::<_, DatabaseError>((result, arena, apply)) + })() { + Ok(result) => result, + Err(err) => { + unsafe { drop(Box::from_raw(transaction)) }; + return Err(err.with_sql_context(context)); + } + }; if let Err(err) = unsafe { Box::from_raw(transaction).commit() } { return Err(err.with_sql_context(context)); @@ -789,13 +826,13 @@ impl Database { let mut catalog_changed = false; for apply in apply { - catalog_changed |= unsafe { apply.apply_to(&mut *state, &plan_arena) } + catalog_changed |= unsafe { apply.apply_to(&mut *state, &mut plan_arena) } .map_err(|err| err.with_sql_context(context))?; } if catalog_changed { unsafe { (&mut *state).recycle_table_arena() }?; } - Ok(()) + Ok(result) } pub fn analyze(&mut self, table_name: impl AsRef) -> Result<(), DatabaseError> { @@ -926,7 +963,7 @@ impl<'a, S: Storage> BindSource<'a> for &'a Database { }; let inner = Box::into_raw(Box::new(TransactionIter::new( schema, - Box::new(arena) as Box, + arena.boxed(), executor, transaction, ))); @@ -1136,7 +1173,7 @@ impl<'a, 'txn, S: Storage> BindSource<'a> for &'a mut DBTransaction<'txn, S> { .execute(unsafe { &mut *transaction }, plan.into(), arena)?; Ok(TransactionIter::new( schema, - Box::new(arena) as Box, + arena.boxed(), executor, transaction, )) @@ -1157,53 +1194,72 @@ impl<'a, 'txn, S: Storage> BindSource<'a> for &'a mut DBTransaction<'txn, S> { } } -/// Raw result iterator returned by transaction execution APIs. -pub struct TransactionIter<'a, T: Transaction + 'a, A: MetaArena + 'a = Box> { +struct TransactionGuard<'a, T: Transaction + 'a> { executor: Option>, - plan_arena: Option, - schema: Schema, transaction: *mut T, statement_scope_active: bool, +} + +impl TransactionGuard<'_, T> { + #[inline] + fn finish_statement_scope( + &mut self, + ddl_apply: &mut Vec, + ) -> Result<(), DatabaseError> { + if !self.statement_scope_active { + return Ok(()); + } + + if let Some(mut executor) = self.executor.take() { + ddl_apply.extend(executor.take_ddl_apply()); + } + self.statement_scope_active = false; + unsafe { (*self.transaction).end_statement_scope() } + } +} + +impl Drop for TransactionGuard<'_, T> { + fn drop(&mut self) { + let _ = self.finish_statement_scope(&mut Vec::new()); + } +} + +/// Raw result iterator returned by transaction execution APIs. +pub struct TransactionIter<'a, T: Transaction + 'a, A: MetaArena + 'a = Box> { + // Drop execution before the metadata arena it borrows from. + guard: TransactionGuard<'a, T>, + plan_arena: ExecMetaArena, + schema: Schema, ddl_apply: Vec, } impl<'a, T: Transaction + 'a, A: MetaArena + 'a> TransactionIter<'a, T, A> { pub(crate) fn new( schema: Schema, - plan_arena: A, + plan_arena: ExecMetaArena, executor: Executor<'a, T>, transaction: *mut T, ) -> Self { Self { - executor: Some(executor), - plan_arena: Some(plan_arena), + guard: TransactionGuard { + executor: Some(executor), + transaction, + statement_scope_active: true, + }, + plan_arena, schema, - transaction, - statement_scope_active: true, ddl_apply: Vec::new(), } } #[inline] fn finish_statement_scope(&mut self) -> Result<(), DatabaseError> { - if !self.statement_scope_active { - return Ok(()); - } - - if let Some(mut executor) = self.executor.take() { - self.ddl_apply.extend(executor.take_ddl_apply()); - } - self.statement_scope_active = false; - unsafe { (*self.transaction).end_statement_scope() } + self.guard.finish_statement_scope(&mut self.ddl_apply) } #[inline] pub fn schema(&self, f: impl FnOnce(&SchemaView<'_, '_>) -> R) -> R { - let plan_arena = self - .plan_arena - .as_ref() - .expect("result iterator schema is unavailable after statement completion"); - let schema = SchemaView::new(&self.schema, plan_arena); + let schema = SchemaView::new(&self.schema, &self.plan_arena); f(&schema) } @@ -1212,14 +1268,11 @@ impl<'a, T: Transaction + 'a, A: MetaArena + 'a> TransactionIter<'a, T, A> { &mut self, f: impl FnOnce(&SchemaView<'_, '_>, &mut Tuple) -> R, ) -> Result, DatabaseError> { - let Some(executor) = self.executor.as_mut() else { + let Some(executor) = self.guard.executor.as_mut() else { return Ok(None); }; let executor_ptr = std::ptr::from_mut(executor); - let plan_arena = self - .plan_arena - .as_mut() - .expect("result iterator plan arena is unavailable after statement completion"); + let plan_arena = &mut self.plan_arena; match unsafe { (*executor_ptr).next_tuple(plan_arena) } { Ok(Some(tuple)) => { let schema = SchemaView::new(&self.schema, plan_arena); @@ -1246,18 +1299,14 @@ impl<'a, T: Transaction + 'a, A: MetaArena + 'a> TransactionIter<'a, T, A> { impl<'a, T: Transaction + 'a> TransactionIter<'a, T, PlanArena<'a>> { fn done_with_ddl_apply(mut self) -> Result<(PlanArena<'a>, Vec), DatabaseError> { while self.next_tuple(|_, _| ())?.is_some() {} - Ok(( - self.plan_arena - .take() - .expect("DDL apply plan arena is unavailable after statement completion"), - std::mem::take(&mut self.ddl_apply), - )) - } -} - -impl Drop for TransactionIter<'_, T, A> { - fn drop(&mut self) { - let _ = self.finish_statement_scope(); + let Self { + guard, + plan_arena, + ddl_apply, + .. + } = self; + drop(guard); + Ok((plan_arena.into_parent(), ddl_apply)) } } @@ -1282,13 +1331,12 @@ impl ResultIter for TransactionIter<'_, T, A> { pub(crate) mod test { use crate::binder::{Binder, BinderContext}; use crate::catalog::{ColumnCatalog, ColumnDesc}; - #[cfg(feature = "unsafe_txdb_checkpoint")] - use crate::db::CatalogKind; - use crate::db::{DataBaseBuilder, DatabaseError, ResultIter}; + use crate::db::{CatalogKind, DataBaseBuilder, DatabaseError, ResultIter}; use crate::expression::ScalarExpression; use crate::planner::operator::join::JoinCondition; use crate::planner::operator::Operator; use crate::planner::PlanArena; + use crate::storage::memory::MemoryStorage; use crate::storage::{ table_codec::TableCodec, Storage, TableCache, Transaction, TransactionIsolationLevel, }; @@ -1319,6 +1367,47 @@ pub(crate) mod test { assert_send_sync::>(); } + #[test] + fn scalar_query_view_references_survive_catalog_reload() -> Result<(), DatabaseError> { + let storage = MemoryStorage::new(); + { + let mut db = DataBaseBuilder::path(".").build_with_storage(storage.clone())?; + db.ddl("create view scalar_a as select (select 11) as v")?; + db.ddl("create view scalar_b as select (select 22) as v")?; + db.ddl("create view scalar_nested as select (select (select 33)) as v")?; + } + + let mut db = DataBaseBuilder::path(".").build_with_storage(storage)?; + // Loading in a different order must preserve the saved initialization references. + for name in ["scalar_b", "scalar_a", "scalar_nested"] { + db.load(CatalogKind::View(name.into()))?; + } + let sql = "select a.v, b.v, c.v, (select 44) from scalar_a a \ + cross join scalar_b b cross join scalar_nested c"; + let mut iter = db.run(sql)?; + assert_eq!( + iter.next_tuple(|_, row| row.values.clone())?, + Some(vec![ + DataValue::Int32(11), + DataValue::Int32(22), + DataValue::Int32(33), + DataValue::Int32(44), + ]) + ); + assert!(iter.next_tuple(|_, _| ())?.is_none()); + iter.done()?; + + // New definitions must not reuse references registered by the loaded views. + db.ddl("create view scalar_new as select (select 55) as v")?; + let mut iter = db.run("select a.v, b.v from scalar_a a cross join scalar_new b")?; + assert_eq!( + iter.next_tuple(|_, row| row.values.clone())?, + Some(vec![DataValue::Int32(11), DataValue::Int32(55)]) + ); + iter.done()?; + Ok(()) + } + pub(crate) fn build_table( table_cache: &mut TableCache, transaction: &mut T, diff --git a/src/db/prepared.rs b/src/db/prepared.rs index 380f61c1..2f61233c 100644 --- a/src/db/prepared.rs +++ b/src/db/prepared.rs @@ -155,18 +155,18 @@ mod tests { "select (($1 * 3 + 7) % 97) + ($1 / 2)", &[(1, LogicalType::Bigint)], )?; - for (value, expected) in [(7, 31.5), (23, 87.5)] { + for (value, expected) in [(7, 31_i64), (23, 87_i64)] { let mut iter = db.execute(&plan, [(1, DataValue::Int64(value))])?; iter.schema(|schema| { assert_eq!(schema.len(), 1); assert_eq!( schema.iter().next().unwrap().datatype(), - &LogicalType::Double + &LogicalType::Bigint ); }); assert_eq!( iter.next_tuple(|_, row| row.values.clone())?, - Some(vec![DataValue::Float64(expected.into())]) + Some(vec![DataValue::Int64(expected)]) ); assert!(iter.next_tuple(|_, _| ())?.is_none()); iter.done()?; @@ -175,7 +175,7 @@ mod tests { let mut iter = tx.execute(&plan, [(1, DataValue::Int64(7))])?; assert_eq!( iter.next_tuple(|_, row| row.values.clone())?, - Some(vec![DataValue::Float64(31.5.into())]) + Some(vec![DataValue::Int64(31)]) ); iter.done()?; tx.commit()?; diff --git a/src/execution/ddl/add_column.rs b/src/execution/ddl/add_column.rs index 878eff21..c0b78dcb 100644 --- a/src/execution/ddl/add_column.rs +++ b/src/execution/ddl/add_column.rs @@ -19,7 +19,7 @@ use crate::execution::{ }; use crate::iter_ext::Itertools; use crate::planner::operator::alter_table::add_column::AddColumnOperator; -use crate::planner::MetaArena; +use crate::planner::{ExecMetaArena, MetaArena}; use crate::storage::Transaction; use crate::types::index::{Index, IndexType}; use crate::types::value::DataValue; @@ -43,10 +43,10 @@ impl<'a, T: Transaction + 'a> WriteExecutor<'a, T> for AddColumn<'a> { } impl<'a, T: Transaction + 'a> ExecutorNode<'a, T> for AddColumn<'a> { - fn next_tuple( + fn next_tuple( &mut self, arena: &mut ExecArena<'a, T>, - plan_arena: &mut (dyn MetaArena + 'a), + plan_arena: &mut ExecMetaArena, ) -> Result<(), DatabaseError> { let table_cache = arena.table_cache(); let AddColumnOperator { diff --git a/src/execution/ddl/change_column.rs b/src/execution/ddl/change_column.rs index 7405111b..ccddade6 100644 --- a/src/execution/ddl/change_column.rs +++ b/src/execution/ddl/change_column.rs @@ -19,7 +19,7 @@ use crate::execution::{ }; use crate::iter_ext::Itertools; use crate::planner::operator::alter_table::change_column::{ChangeColumnOperator, NotNullChange}; -use crate::planner::MetaArena; +use crate::planner::{ExecMetaArena, MetaArena}; use crate::storage::Transaction; use crate::types::value::DataValue; @@ -42,10 +42,10 @@ impl<'a, T: Transaction + 'a> WriteExecutor<'a, T> for ChangeColumn<'a> { } impl<'a, T: Transaction + 'a> ExecutorNode<'a, T> for ChangeColumn<'a> { - fn next_tuple( + fn next_tuple( &mut self, arena: &mut ExecArena<'a, T>, - plan_arena: &mut (dyn MetaArena + 'a), + plan_arena: &mut ExecMetaArena, ) -> Result<(), DatabaseError> { let table_cache = arena.table_cache(); let ChangeColumnOperator { diff --git a/src/execution/ddl/create_index.rs b/src/execution/ddl/create_index.rs index 7705f1ec..2dd43683 100644 --- a/src/execution/ddl/create_index.rs +++ b/src/execution/ddl/create_index.rs @@ -20,10 +20,11 @@ use crate::execution::{ use crate::expression::ScalarExpression; use crate::planner::operator::create_index::CreateIndexOperator; use crate::planner::LogicalPlan; -use crate::planner::MetaArena; +use crate::planner::{ExecMetaArena, MetaArena}; use crate::storage::Transaction; use crate::types::index::Index; use crate::types::tuple::Schema; +use crate::types::value::DataValue; use crate::types::ColumnId; pub struct CreateIndex<'a> { @@ -53,10 +54,10 @@ impl<'a, T: Transaction + 'a> WriteExecutor<'a, T> for CreateIndex<'a> { } impl<'a, T: Transaction + 'a> ExecutorNode<'a, T> for CreateIndex<'a> { - fn next_tuple( + fn next_tuple( &mut self, arena: &mut ExecArena<'a, T>, - plan_arena: &mut (dyn MetaArena + 'a), + plan_arena: &mut ExecMetaArena, ) -> Result<(), DatabaseError> { let CreateIndexOperator { table_name, @@ -119,7 +120,7 @@ impl<'a, T: Transaction + 'a> ExecutorNode<'a, T> for CreateIndex<'a> { if arena.result_tuple().pk.is_none() { continue; } - arena.rewrite(&column_exprs, plan_arena, None)?; + arena.rewrite(&column_exprs, plan_arena, None::<&[DataValue]>)?; { let mut state = arena.local_state(plan_arena); let (tuple, transaction, table_codec) = state.tuple_transaction_codec_mut(); diff --git a/src/execution/ddl/create_table.rs b/src/execution/ddl/create_table.rs index 7633693c..9f398f8f 100644 --- a/src/execution/ddl/create_table.rs +++ b/src/execution/ddl/create_table.rs @@ -17,7 +17,7 @@ use crate::execution::{ DDLApply, ExecArena, ExecId, ExecNode, ExecutionContext, ExecutorNode, WriteExecutor, }; use crate::planner::operator::create_table::CreateTableOperator; -use crate::planner::MetaArena; +use crate::planner::{ExecMetaArena, MetaArena}; use crate::storage::Transaction; pub struct CreateTable<'a> { @@ -39,10 +39,10 @@ impl<'a, T: Transaction + 'a> WriteExecutor<'a, T> for CreateTable<'a> { } impl<'a, T: Transaction + 'a> ExecutorNode<'a, T> for CreateTable<'a> { - fn next_tuple( + fn next_tuple( &mut self, arena: &mut ExecArena<'a, T>, - plan_arena: &mut (dyn MetaArena + 'a), + plan_arena: &mut ExecMetaArena, ) -> Result<(), DatabaseError> { let CreateTableOperator { table_name, diff --git a/src/execution/ddl/create_view.rs b/src/execution/ddl/create_view.rs index 8933d153..f7184891 100644 --- a/src/execution/ddl/create_view.rs +++ b/src/execution/ddl/create_view.rs @@ -17,7 +17,7 @@ use crate::execution::{ DDLApply, ExecArena, ExecId, ExecNode, ExecutionContext, ExecutorNode, WriteExecutor, }; use crate::planner::operator::create_view::CreateViewOperator; -use crate::planner::MetaArena; +use crate::planner::{ExecMetaArena, MetaArena}; use crate::storage::Transaction; pub struct CreateView<'a> { @@ -39,10 +39,10 @@ impl<'a, T: Transaction + 'a> WriteExecutor<'a, T> for CreateView<'a> { } impl<'a, T: Transaction + 'a> ExecutorNode<'a, T> for CreateView<'a> { - fn next_tuple( + fn next_tuple( &mut self, arena: &mut ExecArena<'a, T>, - plan_arena: &mut (dyn MetaArena + 'a), + plan_arena: &mut ExecMetaArena, ) -> Result<(), DatabaseError> { let CreateViewOperator { view, or_replace } = self.op; let (transaction, table_codec) = arena.transaction_codec_mut(); diff --git a/src/execution/ddl/drop_column.rs b/src/execution/ddl/drop_column.rs index 31a981d3..db7c2c7e 100644 --- a/src/execution/ddl/drop_column.rs +++ b/src/execution/ddl/drop_column.rs @@ -19,7 +19,7 @@ use crate::execution::{ }; use crate::iter_ext::Itertools; use crate::planner::operator::alter_table::drop_column::DropColumnOperator; -use crate::planner::MetaArena; +use crate::planner::{ExecMetaArena, MetaArena}; use crate::storage::Transaction; pub struct DropColumn<'a> { @@ -41,10 +41,10 @@ impl<'a, T: Transaction + 'a> WriteExecutor<'a, T> for DropColumn<'a> { } impl<'a, T: Transaction + 'a> ExecutorNode<'a, T> for DropColumn<'a> { - fn next_tuple( + fn next_tuple( &mut self, arena: &mut ExecArena<'a, T>, - plan_arena: &mut (dyn MetaArena + 'a), + plan_arena: &mut ExecMetaArena, ) -> Result<(), DatabaseError> { let table_cache = arena.table_cache(); let DropColumnOperator { diff --git a/src/execution/ddl/drop_index.rs b/src/execution/ddl/drop_index.rs index 683fb42e..a4c53446 100644 --- a/src/execution/ddl/drop_index.rs +++ b/src/execution/ddl/drop_index.rs @@ -17,7 +17,7 @@ use crate::execution::{ DDLApply, ExecArena, ExecId, ExecNode, ExecutionContext, ExecutorNode, WriteExecutor, }; use crate::planner::operator::drop_index::DropIndexOperator; -use crate::planner::MetaArena; +use crate::planner::{ExecMetaArena, MetaArena}; use crate::storage::Transaction; pub struct DropIndex<'a> { @@ -39,10 +39,10 @@ impl<'a, T: Transaction + 'a> WriteExecutor<'a, T> for DropIndex<'a> { } impl<'a, T: Transaction + 'a> ExecutorNode<'a, T> for DropIndex<'a> { - fn next_tuple( + fn next_tuple( &mut self, arena: &mut ExecArena<'a, T>, - plan_arena: &mut (dyn MetaArena + 'a), + plan_arena: &mut ExecMetaArena, ) -> Result<(), DatabaseError> { let DropIndexOperator { table_name, diff --git a/src/execution/ddl/drop_table.rs b/src/execution/ddl/drop_table.rs index 9467418b..3df6c183 100644 --- a/src/execution/ddl/drop_table.rs +++ b/src/execution/ddl/drop_table.rs @@ -17,7 +17,7 @@ use crate::execution::{ DDLApply, ExecArena, ExecId, ExecNode, ExecutionContext, ExecutorNode, WriteExecutor, }; use crate::planner::operator::drop_table::DropTableOperator; -use crate::planner::MetaArena; +use crate::planner::{ExecMetaArena, MetaArena}; use crate::storage::Transaction; pub struct DropTable<'a> { @@ -39,10 +39,10 @@ impl<'a, T: Transaction + 'a> WriteExecutor<'a, T> for DropTable<'a> { } impl<'a, T: Transaction + 'a> ExecutorNode<'a, T> for DropTable<'a> { - fn next_tuple( + fn next_tuple( &mut self, arena: &mut ExecArena<'a, T>, - plan_arena: &mut (dyn MetaArena + 'a), + plan_arena: &mut ExecMetaArena, ) -> Result<(), DatabaseError> { let DropTableOperator { table_name, diff --git a/src/execution/ddl/drop_view.rs b/src/execution/ddl/drop_view.rs index 30785ad6..ce2b1277 100644 --- a/src/execution/ddl/drop_view.rs +++ b/src/execution/ddl/drop_view.rs @@ -17,7 +17,7 @@ use crate::execution::{ DDLApply, ExecArena, ExecId, ExecNode, ExecutionContext, ExecutorNode, WriteExecutor, }; use crate::planner::operator::drop_view::DropViewOperator; -use crate::planner::MetaArena; +use crate::planner::{ExecMetaArena, MetaArena}; use crate::storage::Transaction; pub struct DropView<'a> { @@ -39,10 +39,10 @@ impl<'a, T: Transaction + 'a> WriteExecutor<'a, T> for DropView<'a> { } impl<'a, T: Transaction + 'a> ExecutorNode<'a, T> for DropView<'a> { - fn next_tuple( + fn next_tuple( &mut self, arena: &mut ExecArena<'a, T>, - _: &mut (dyn MetaArena + 'a), + _: &mut ExecMetaArena, ) -> Result<(), DatabaseError> { let DropViewOperator { view_name, diff --git a/src/execution/ddl/truncate.rs b/src/execution/ddl/truncate.rs index 32a5dc4c..ffc54ee5 100644 --- a/src/execution/ddl/truncate.rs +++ b/src/execution/ddl/truncate.rs @@ -17,7 +17,7 @@ use crate::execution::{ ExecArena, ExecId, ExecNode, ExecutionContext, ExecutorNode, WriteExecutor, }; use crate::planner::operator::truncate::TruncateOperator; -use crate::planner::MetaArena; +use crate::planner::{ExecMetaArena, MetaArena}; use crate::storage::Transaction; pub struct Truncate<'a> { @@ -39,10 +39,10 @@ impl<'a, T: Transaction + 'a> WriteExecutor<'a, T> for Truncate<'a> { } impl<'a, T: Transaction + 'a> ExecutorNode<'a, T> for Truncate<'a> { - fn next_tuple( + fn next_tuple( &mut self, arena: &mut ExecArena<'a, T>, - plan_arena: &mut (dyn MetaArena + 'a), + plan_arena: &mut ExecMetaArena, ) -> Result<(), DatabaseError> { let TruncateOperator { table_name } = self.op; let mut state = arena.local_state(plan_arena); diff --git a/src/execution/dml/analyze.rs b/src/execution/dml/analyze.rs index 213d87a1..5b2bb92d 100644 --- a/src/execution/dml/analyze.rs +++ b/src/execution/dml/analyze.rs @@ -24,7 +24,7 @@ use crate::optimizer::core::histogram::{HistogramBuilder, ANALYZE_STATISTICS_REL use crate::optimizer::core::statistics_meta::StatisticsMeta; use crate::planner::operator::analyze::AnalyzeOperator; use crate::planner::LogicalPlan; -use crate::planner::MetaArena; +use crate::planner::{ExecMetaArena, MetaArena}; use crate::storage::{table_codec::TableCodec, Transaction}; use crate::types::index::IndexId; use crate::types::value::{DataValue, Utf8Type}; @@ -72,10 +72,10 @@ impl<'a, T: Transaction + 'a> WriteExecutor<'a, T> for Analyze<'a> { } impl<'a, T: Transaction + 'a> ExecutorNode<'a, T> for Analyze<'a> { - fn next_tuple( + fn next_tuple( &mut self, arena: &mut ExecArena<'a, T>, - plan_arena: &mut (dyn MetaArena + 'a), + plan_arena: &mut ExecMetaArena, ) -> Result<(), DatabaseError> { let Some(input) = self.input.take() else { arena.finish(); diff --git a/src/execution/dml/copy_from_file.rs b/src/execution/dml/copy_from_file.rs index c1d25aa8..697f00b5 100644 --- a/src/execution/dml/copy_from_file.rs +++ b/src/execution/dml/copy_from_file.rs @@ -19,7 +19,7 @@ use crate::execution::{ }; use crate::iter_ext::Itertools; use crate::planner::operator::copy_from_file::CopyFromFileOperator; -use crate::planner::MetaArena; +use crate::planner::{ExecMetaArena, MetaArena}; use crate::storage::Transaction; use crate::types::tuple::Tuple; use crate::types::tuple_builder::TupleBuilder; @@ -45,10 +45,10 @@ impl<'a, T: Transaction + 'a> WriteExecutor<'a, T> for CopyFromFile<'a> { } impl<'a, T: Transaction + 'a> ExecutorNode<'a, T> for CopyFromFile<'a> { - fn next_tuple( + fn next_tuple( &mut self, arena: &mut ExecArena<'a, T>, - plan_arena: &mut (dyn MetaArena + 'a), + plan_arena: &mut ExecMetaArena, ) -> Result<(), DatabaseError> { let op = self.op; let column_types = op diff --git a/src/execution/dml/copy_to_file.rs b/src/execution/dml/copy_to_file.rs index 07e02865..31e2ce0f 100644 --- a/src/execution/dml/copy_to_file.rs +++ b/src/execution/dml/copy_to_file.rs @@ -20,7 +20,7 @@ use crate::execution::{ use crate::iter_ext::Itertools; use crate::planner::operator::copy_to_file::CopyToFileOperator; use crate::planner::LogicalPlan; -use crate::planner::MetaArena; +use crate::planner::{ExecMetaArena, MetaArena}; use crate::storage::Transaction; use crate::types::tuple_builder::TupleBuilder; @@ -84,10 +84,10 @@ impl CopyToFile<'_> { } impl<'a, T: Transaction + 'a> ExecutorNode<'a, T> for CopyToFile<'a> { - fn next_tuple( + fn next_tuple( &mut self, arena: &mut ExecArena<'a, T>, - plan_arena: &mut (dyn MetaArena + 'a), + plan_arena: &mut ExecMetaArena, ) -> Result<(), DatabaseError> { let Some(input) = self.input.take() else { arena.finish(); diff --git a/src/execution/dml/delete.rs b/src/execution/dml/delete.rs index be80def3..301e6c98 100644 --- a/src/execution/dml/delete.rs +++ b/src/execution/dml/delete.rs @@ -19,7 +19,7 @@ use crate::execution::{ }; use crate::planner::operator::delete::DeleteOperator; use crate::planner::LogicalPlan; -use crate::planner::MetaArena; +use crate::planner::{ExecMetaArena, MetaArena}; use crate::storage::Transaction; use crate::types::index::{Index, IndexType}; use crate::types::tuple_builder::TupleBuilder; @@ -51,10 +51,10 @@ impl<'a, T: Transaction + 'a> WriteExecutor<'a, T> for Delete<'a> { } impl<'a, T: Transaction + 'a> ExecutorNode<'a, T> for Delete<'a> { - fn next_tuple( + fn next_tuple( &mut self, arena: &mut ExecArena<'a, T>, - plan_arena: &mut (dyn MetaArena + 'a), + plan_arena: &mut ExecMetaArena, ) -> Result<(), DatabaseError> { let Some(input) = self.input.take() else { arena.finish(); diff --git a/src/execution/dml/insert.rs b/src/execution/dml/insert.rs index edc9e84d..a2838170 100644 --- a/src/execution/dml/insert.rs +++ b/src/execution/dml/insert.rs @@ -20,7 +20,7 @@ use crate::execution::{ use crate::iter_ext::Itertools; use crate::planner::operator::insert::InsertOperator; use crate::planner::LogicalPlan; -use crate::planner::MetaArena; +use crate::planner::{ExecMetaArena, MetaArena}; use crate::storage::Transaction; use crate::types::index::Index; use crate::types::tuple::{Schema, Tuple}; @@ -92,10 +92,10 @@ impl Insert<'_> { } impl<'a, T: Transaction + 'a> ExecutorNode<'a, T> for Insert<'a> { - fn next_tuple( + fn next_tuple( &mut self, arena: &mut ExecArena<'a, T>, - plan_arena: &mut (dyn MetaArena + 'a), + plan_arena: &mut ExecMetaArena, ) -> Result<(), DatabaseError> { let Some(input) = self.input.take() else { arena.finish(); diff --git a/src/execution/dml/update.rs b/src/execution/dml/update.rs index 3b052a9e..2a722afa 100644 --- a/src/execution/dml/update.rs +++ b/src/execution/dml/update.rs @@ -21,7 +21,7 @@ use crate::iter_ext::Itertools; use crate::planner::operator::update::UpdateOperator; use crate::planner::ExprRef; use crate::planner::LogicalPlan; -use crate::planner::MetaArena; +use crate::planner::{ExecMetaArena, MetaArena}; use crate::storage::Transaction; use crate::types::index::{Index, IndexMeta, IndexType}; use crate::types::tuple::{Schema, Tuple}; @@ -88,10 +88,10 @@ impl Update<'_> { } impl<'a, T: Transaction + 'a> ExecutorNode<'a, T> for Update<'a> { - fn next_tuple( + fn next_tuple( &mut self, arena: &mut ExecArena<'a, T>, - plan_arena: &mut (dyn MetaArena + 'a), + plan_arena: &mut ExecMetaArena, ) -> Result<(), DatabaseError> { let Some(input) = self.input.take() else { arena.finish(); diff --git a/src/execution/dql/aggregate/avg.rs b/src/execution/dql/aggregate/avg.rs index c47461de..7e4b2702 100644 --- a/src/execution/dql/aggregate/avg.rs +++ b/src/execution/dql/aggregate/avg.rs @@ -18,6 +18,7 @@ use crate::execution::dql::aggregate::Accumulator; use crate::expression::BinaryOperator; use crate::types::evaluator::binary_create; use crate::types::value::DataValue; +use crate::types::LogicalType; use std::borrow::Cow; pub struct AvgAccumulator { @@ -60,24 +61,14 @@ impl Accumulator for AvgAccumulator { let Some(acc) = &self.inner else { return Ok(()); }; - let mut value = Cow::Borrowed(acc.result()); - let value_ty = value.logical_type(); - if self.count == 0 { return Ok(()); } - let quantity = if value_ty.is_signed_numeric() { - DataValue::Int64(self.count as i64) - } else { - DataValue::UInt32(self.count as u32) - }; - let quantity_ty = quantity.logical_type(); - - if value_ty != quantity_ty { - value = Cow::Owned(value.into_owned().cast(&quantity_ty)?) - } - let evaluator = binary_create(Cow::Owned(quantity_ty), BinaryOperator::Divide)?; - self.result = evaluator.binary_eval(value.as_ref(), &quantity)?; + // AVG returns Double even when ordinary integer division truncates. + let value = acc.result().clone().cast(&LogicalType::Double)?; + let quantity = DataValue::Float64((self.count as f64).into()); + let evaluator = binary_create(Cow::Owned(LogicalType::Double), BinaryOperator::Divide)?; + self.result = evaluator.binary_eval(&value, &quantity)?; Ok(()) } diff --git a/src/execution/dql/aggregate/hash_agg.rs b/src/execution/dql/aggregate/hash_agg.rs index 5028e343..7880d79a 100644 --- a/src/execution/dql/aggregate/hash_agg.rs +++ b/src/execution/dql/aggregate/hash_agg.rs @@ -22,7 +22,7 @@ use crate::execution::{ use crate::planner::operator::aggregate::AggregateOperator; use crate::planner::ExprRef; use crate::planner::LogicalPlan; -use crate::planner::MetaArena; +use crate::planner::{ExecMetaArena, MetaArena}; use crate::storage::Transaction; use crate::types::value::DataValue; use std::collections::hash_map::IntoIter as HashMapIntoIter; @@ -65,10 +65,10 @@ impl<'a, T: Transaction + 'a> ReadExecutor<'a, T> for HashAggExecutor<'a> { } impl<'a, T: Transaction + 'a> ExecutorNode<'a, T> for HashAggExecutor<'a> { - fn next_tuple( + fn next_tuple( &mut self, arena: &mut ExecArena<'a, T>, - plan_arena: &mut (dyn MetaArena + 'a), + plan_arena: &mut ExecMetaArena, ) -> Result<(), DatabaseError> { if self.output.is_none() { let mut group_hash_accs: HashMap, Vec>> = diff --git a/src/execution/dql/aggregate/mod.rs b/src/execution/dql/aggregate/mod.rs index 96ebdf52..2735f699 100644 --- a/src/execution/dql/aggregate/mod.rs +++ b/src/execution/dql/aggregate/mod.rs @@ -87,10 +87,10 @@ pub(crate) fn create_accumulators( .try_collect() } -pub(crate) fn update_accumulators( +pub(crate) fn update_accumulators( accs: &mut [Box], agg_calls: &[ExprRef], - tuple: &dyn TupleLike, + tuple: &T, arena: &(dyn MetaArena + '_), ) -> Result<(), DatabaseError> { for (acc, expr) in accs.iter_mut().zip(agg_calls.iter()) { diff --git a/src/execution/dql/aggregate/simple_agg.rs b/src/execution/dql/aggregate/simple_agg.rs index 7d0e889b..64e1fe2b 100644 --- a/src/execution/dql/aggregate/simple_agg.rs +++ b/src/execution/dql/aggregate/simple_agg.rs @@ -21,7 +21,7 @@ use crate::expression::ScalarExpression; use crate::planner::operator::aggregate::AggregateOperator; use crate::planner::ExprRef; use crate::planner::LogicalPlan; -use crate::planner::MetaArena; +use crate::planner::{ExecMetaArena, MetaArena}; use crate::storage::Transaction; pub struct SimpleAggExecutor<'a> { agg_calls: &'a [ExprRef], @@ -49,10 +49,10 @@ impl<'a, T: Transaction + 'a> ReadExecutor<'a, T> for SimpleAggExecutor<'a> { } impl<'a, T: Transaction + 'a> ExecutorNode<'a, T> for SimpleAggExecutor<'a> { - fn next_tuple( + fn next_tuple( &mut self, arena: &mut ExecArena<'a, T>, - plan_arena: &mut (dyn MetaArena + 'a), + plan_arena: &mut ExecMetaArena, ) -> Result<(), DatabaseError> { if self.returned { arena.finish(); diff --git a/src/execution/dql/aggregate/stream_agg.rs b/src/execution/dql/aggregate/stream_agg.rs index 2f238578..a9cb91af 100644 --- a/src/execution/dql/aggregate/stream_agg.rs +++ b/src/execution/dql/aggregate/stream_agg.rs @@ -22,7 +22,7 @@ use crate::execution::{ use crate::planner::operator::aggregate::AggregateOperator; use crate::planner::ExprRef; use crate::planner::LogicalPlan; -use crate::planner::MetaArena; +use crate::planner::{ExecMetaArena, MetaArena}; use crate::storage::Transaction; use crate::types::value::DataValue; use std::mem; @@ -65,10 +65,10 @@ impl<'a, T: Transaction + 'a> ReadExecutor<'a, T> for StreamAggExecutor<'a> { } impl<'a, T: Transaction + 'a> ExecutorNode<'a, T> for StreamAggExecutor<'a> { - fn next_tuple( + fn next_tuple( &mut self, arena: &mut ExecArena<'a, T>, - plan_arena: &mut (dyn MetaArena + 'a), + plan_arena: &mut ExecMetaArena, ) -> Result<(), DatabaseError> { loop { if !arena.next_tuple(self.input, plan_arena)? { diff --git a/src/execution/dql/aggregate/stream_distinct.rs b/src/execution/dql/aggregate/stream_distinct.rs index 87aab16f..1702fe10 100644 --- a/src/execution/dql/aggregate/stream_distinct.rs +++ b/src/execution/dql/aggregate/stream_distinct.rs @@ -19,9 +19,10 @@ use crate::execution::{ use crate::planner::operator::aggregate::AggregateOperator; use crate::planner::ExprRef; use crate::planner::LogicalPlan; -use crate::planner::MetaArena; +use crate::planner::{ExecMetaArena, MetaArena}; use crate::storage::Transaction; use crate::types::tuple::Tuple; +use crate::types::value::DataValue; pub struct StreamDistinctExecutor<'a> { groupby_exprs: &'a [ExprRef], @@ -49,10 +50,10 @@ impl<'a, T: Transaction + 'a> ReadExecutor<'a, T> for StreamDistinctExecutor<'a> } impl<'a, T: Transaction + 'a> ExecutorNode<'a, T> for StreamDistinctExecutor<'a> { - fn next_tuple( + fn next_tuple( &mut self, arena: &mut ExecArena<'a, T>, - plan_arena: &mut (dyn MetaArena + 'a), + plan_arena: &mut ExecMetaArena, ) -> Result<(), DatabaseError> { loop { if !arena.next_tuple(self.input, plan_arena)? { @@ -64,7 +65,7 @@ impl<'a, T: Transaction + 'a> ExecutorNode<'a, T> for StreamDistinctExecutor<'a> } return Ok(()); } - arena.rewrite(self.groupby_exprs, plan_arena, None)?; + arena.rewrite(self.groupby_exprs, plan_arena, None::<&[DataValue]>)?; if let Some(last_keys) = &mut self.last_keys { if last_keys.values == arena.result_tuple().values { diff --git a/src/execution/dql/describe.rs b/src/execution/dql/describe.rs index 98316a26..cbd18eff 100644 --- a/src/execution/dql/describe.rs +++ b/src/execution/dql/describe.rs @@ -16,7 +16,7 @@ use crate::catalog::{ColumnCatalog, ColumnRef, TableName}; use crate::errors::DatabaseError; use crate::execution::{ExecArena, ExecId, ExecNode, ExecutionContext, ExecutorNode, ReadExecutor}; use crate::planner::operator::describe::DescribeOperator; -use crate::planner::MetaArena; +use crate::planner::{ExecMetaArena, MetaArena}; use crate::storage::Transaction; use crate::types::value::{DataValue, Utf8Type}; use crate::types::CharLengthUnits; @@ -72,10 +72,10 @@ impl<'a, T: Transaction + 'a> ReadExecutor<'a, T> for Describe { } impl<'a, T: Transaction + 'a> ExecutorNode<'a, T> for Describe { - fn next_tuple( + fn next_tuple( &mut self, arena: &mut ExecArena<'a, T>, - plan_arena: &mut (dyn MetaArena + 'a), + plan_arena: &mut ExecMetaArena, ) -> Result<(), DatabaseError> { if self.columns.is_none() { let table = arena @@ -115,7 +115,7 @@ fn describe_default(column: &ColumnCatalog, arena: &(dyn MetaArena + '_)) -> Str .default .as_ref() .map(|expr| expr.output_name(arena)) - .unwrap_or_else(|| "null".to_string()) + .unwrap_or_else(|| "NULL".to_string()) } fn fill_describe_row(values: &mut Vec, column: &ColumnCatalog, default: String) { diff --git a/src/execution/dql/dummy.rs b/src/execution/dql/dummy.rs index ae0860ca..019053dc 100644 --- a/src/execution/dql/dummy.rs +++ b/src/execution/dql/dummy.rs @@ -14,7 +14,7 @@ use crate::errors::DatabaseError; use crate::execution::{ExecArena, ExecId, ExecNode, ExecutionContext, ExecutorNode, ReadExecutor}; -use crate::planner::MetaArena; +use crate::planner::{ExecMetaArena, MetaArena}; use crate::storage::Transaction; use crate::types::tuple::Tuple; @@ -46,10 +46,10 @@ impl<'a, T: Transaction + 'a> ReadExecutor<'a, T> for Dummy { } impl<'a, T: Transaction + 'a> ExecutorNode<'a, T> for Dummy { - fn next_tuple( + fn next_tuple( &mut self, arena: &mut ExecArena<'a, T>, - _: &mut (dyn MetaArena + 'a), + _: &mut ExecMetaArena, ) -> Result<(), DatabaseError> { let Some(row) = self.row.take() else { arena.finish(); diff --git a/src/execution/dql/explain.rs b/src/execution/dql/explain.rs index 05b113eb..dd999aba 100644 --- a/src/execution/dql/explain.rs +++ b/src/execution/dql/explain.rs @@ -15,7 +15,7 @@ use crate::errors::DatabaseError; use crate::execution::{ExecArena, ExecId, ExecNode, ExecutionContext, ExecutorNode, ReadExecutor}; use crate::planner::LogicalPlan; -use crate::planner::MetaArena; +use crate::planner::{ExecMetaArena, MetaArena}; use crate::storage::Transaction; use crate::types::value::{DataValue, Utf8Type}; use crate::types::CharLengthUnits; @@ -43,10 +43,10 @@ impl<'a, T: Transaction + 'a> ReadExecutor<'a, T> for Explain<'a> { } impl<'a, T: Transaction + 'a> ExecutorNode<'a, T> for Explain<'a> { - fn next_tuple( + fn next_tuple( &mut self, arena: &mut ExecArena<'a, T>, - plan_arena: &mut (dyn MetaArena + 'a), + plan_arena: &mut ExecMetaArena, ) -> Result<(), DatabaseError> { if self.emitted { arena.finish(); diff --git a/src/execution/dql/external_sort.rs b/src/execution/dql/external_sort.rs index 8a276fba..cd47fd57 100644 --- a/src/execution/dql/external_sort.rs +++ b/src/execution/dql/external_sort.rs @@ -20,7 +20,7 @@ use crate::execution::{ }; use crate::planner::operator::sort::{SortField, SortOperator}; use crate::planner::LogicalPlan; -use crate::planner::MetaArena; +use crate::planner::{ExecMetaArena, MetaArena}; use crate::storage::Transaction; use std::fs::File; use std::io::BufReader; @@ -69,10 +69,10 @@ impl<'a, T: Transaction + 'a> ReadExecutor<'a, T> for ExternalSort<'a> { } impl<'a, T: Transaction + 'a> ExecutorNode<'a, T> for ExternalSort<'a> { - fn next_tuple( + fn next_tuple( &mut self, arena: &mut ExecArena<'a, T>, - plan_arena: &mut (dyn MetaArena + 'a), + plan_arena: &mut ExecMetaArena, ) -> Result<(), DatabaseError> { loop { if let Some(rows) = &mut self.rows { @@ -80,6 +80,7 @@ impl<'a, T: Transaction + 'a> ExecutorNode<'a, T> for ExternalSort<'a> { arena.finish(); return Ok(()); }; + plan_arena.restore_outer_values(row.outer_values); arena.produce_tuple(row.tuple); return Ok(()); } @@ -94,7 +95,8 @@ impl<'a, T: Transaction + 'a> ExecutorNode<'a, T> for ExternalSort<'a> { let mut runs = Vec::new(); while arena.next_tuple(self.input, plan_arena)? { let tuple = arena.materialize_tuple(); - if let Some(segment) = rows.push(SortRow::new(sort_fields, tuple, plan_arena)?)? { + let row = SortRow::new(sort_fields, tuple, plan_arena)?; + if let Some(segment) = rows.push(row)? { runs.push(Run::new(segment, 1)); } } @@ -285,9 +287,10 @@ mod test { use crate::execution::spill::{SortRow, SpillVec}; use crate::expression::ScalarExpression; use crate::planner::operator::sort::SortField; + use crate::planner::{ExecMetaArena, MetaArena}; use crate::types::tuple::Tuple; - use crate::types::value::DataValue; - use crate::types::LogicalType; + use crate::types::value::{DataValue, Utf8Type}; + use crate::types::{CharLengthUnits, LogicalType}; use std::cmp::Ordering; #[test] @@ -308,6 +311,7 @@ mod test { nulls_first: false, }]; + let mut plan_arena = ExecMetaArena::new(plan_arena); let mut rows = SpillVec::new() .limit(4, usize::MAX) .on_flush(|rows| sort_segment(&sort_fields, rows)); @@ -315,7 +319,7 @@ mod test { let _ = rows.push(SortRow::new( &sort_fields, Tuple::new(None, vec![value]), - &plan_arena, + &mut plan_arena, )?)?; } @@ -358,6 +362,7 @@ mod test { DataValue::Int32(0), ]; + let mut plan_arena = ExecMetaArena::new(plan_arena); let mut rows = SpillVec::new() .limit(2, usize::MAX) .on_flush(|rows| sort_segment(&sort_fields, rows)); @@ -367,13 +372,34 @@ mod test { Some(DataValue::Int32(sequence as i32)), vec![value, DataValue::Int32(sequence as i32)], ); - if let Some(segment) = rows.push(SortRow::new(&sort_fields, tuple, &plan_arena)?)? { + let reference = plan_arena.alloc_scalar_query_ref(true); + plan_arena.set_init_value( + reference, + DataValue::Utf8 { + value: format!("row-{sequence}"), + ty: Utf8Type::Variable(None), + unit: CharLengthUnits::Characters, + }, + ); + let row = SortRow::new(&sort_fields, tuple, &mut plan_arena)?; + if let Some(segment) = rows.push(row)? { runs.push(Run::new(segment, 1)); } } let tuples = finish_sort(rows, runs, &sort_fields, 2)? - .map(|row| row.map(|row| row.tuple)) + .map(|row| { + row.map(|row| { + assert_eq!(row.outer_values.len(), 1); + let sequence = row.tuple.values[1].i32().unwrap(); + assert_eq!(row.outer_values[0].0.pos, sequence as usize); + assert_eq!( + row.outer_values[0].1.utf8(), + Some(format!("row-{sequence}").as_str()) + ); + row.tuple + }) + }) .collect::, _>>()?; let positions = tuples .iter() @@ -433,6 +459,7 @@ mod test { }, ]; + let mut plan_arena = ExecMetaArena::new(plan_arena); let mut rows = SpillVec::new().on_flush(|rows| sort_segment(&sort_fields, rows)); let mut runs = Vec::new(); for position in 0..ROW_COUNT { @@ -444,7 +471,7 @@ mod test { }; let sequence = DataValue::Int32(sequence as i32); let tuple = Tuple::new(Some(sequence.clone()), vec![key, sequence]); - if let Some(segment) = rows.push(SortRow::new(&sort_fields, tuple, &plan_arena)?)? { + if let Some(segment) = rows.push(SortRow::new(&sort_fields, tuple, &mut plan_arena)?)? { runs.push(Run::new(segment, 1)); } } diff --git a/src/execution/dql/filter.rs b/src/execution/dql/filter.rs index 3d6c0e51..757ed6ec 100644 --- a/src/execution/dql/filter.rs +++ b/src/execution/dql/filter.rs @@ -19,7 +19,7 @@ use crate::execution::{ use crate::planner::operator::filter::FilterOperator; use crate::planner::ExprRef; use crate::planner::LogicalPlan; -use crate::planner::MetaArena; +use crate::planner::{ExecMetaArena, MetaArena}; use crate::storage::Transaction; pub struct Filter { predicate: ExprRef, @@ -45,10 +45,10 @@ impl<'a, T: Transaction + 'a> ReadExecutor<'a, T> for Filter { } impl<'a, T: Transaction + 'a> ExecutorNode<'a, T> for Filter { - fn next_tuple( + fn next_tuple( &mut self, arena: &mut ExecArena<'a, T>, - plan_arena: &mut (dyn MetaArena + 'a), + plan_arena: &mut ExecMetaArena, ) -> Result<(), DatabaseError> { loop { if !arena.next_tuple(self.input, plan_arena)? { diff --git a/src/execution/dql/function_scan.rs b/src/execution/dql/function_scan.rs index e09455c3..e7b206c1 100644 --- a/src/execution/dql/function_scan.rs +++ b/src/execution/dql/function_scan.rs @@ -16,7 +16,7 @@ use crate::errors::DatabaseError; use crate::execution::{ExecArena, ExecId, ExecNode, ExecutionContext, ExecutorNode, ReadExecutor}; use crate::expression::function::table::TableFunction; use crate::planner::operator::function_scan::FunctionScanOperator; -use crate::planner::MetaArena; +use crate::planner::{ExecMetaArena, MetaArena}; use crate::storage::Transaction; use crate::types::tuple::Tuple; @@ -50,10 +50,10 @@ impl<'a, T: Transaction + 'a> ReadExecutor<'a, T> for FunctionScan<'a> { } impl<'a, T: Transaction + 'a> ExecutorNode<'a, T> for FunctionScan<'a> { - fn next_tuple( + fn next_tuple( &mut self, arena: &mut ExecArena<'a, T>, - plan_arena: &mut (dyn MetaArena + 'a), + plan_arena: &mut ExecMetaArena, ) -> Result<(), DatabaseError> { if self.iter.is_none() { let TableFunction { args, catalog } = self.table_function; diff --git a/src/execution/dql/index_scan.rs b/src/execution/dql/index_scan.rs index fcd38bd6..164fafa4 100644 --- a/src/execution/dql/index_scan.rs +++ b/src/execution/dql/index_scan.rs @@ -17,7 +17,7 @@ use crate::execution::{ExecArena, ExecId, ExecNode, ExecutionContext, ExecutorNo use crate::expression::range_detacher::{IndexRangeColumn, Range, RangeDetacher}; use crate::planner::operator::table_scan::TableScanOperator; use crate::planner::operator::SortOption; -use crate::planner::MetaArena; +use crate::planner::{ExecMetaArena, MetaArena}; use crate::storage::{IndexIter, IndexRanges, Iter, Transaction}; use crate::types::index::{IndexInfo, IndexLookup, RuntimeIndexProbe}; use std::borrow::Cow; @@ -99,10 +99,10 @@ impl<'a, T: Transaction + 'a> ReadExecutor<'a, T> for IndexScan<'a, T> { } impl<'a, T: Transaction + 'a> ExecutorNode<'a, T> for IndexScan<'a, T> { - fn next_tuple( + fn next_tuple( &mut self, arena: &mut ExecArena<'a, T>, - plan_arena: &mut (dyn MetaArena + 'a), + plan_arena: &mut ExecMetaArena, ) -> Result<(), DatabaseError> { let iter = match &mut self.iter { Some(iter) => iter, diff --git a/src/execution/dql/join/hash/mod.rs b/src/execution/dql/join/hash/mod.rs index 228cd36f..6f22e876 100644 --- a/src/execution/dql/join/hash/mod.rs +++ b/src/execution/dql/join/hash/mod.rs @@ -121,14 +121,14 @@ impl JoinProbeState for JoinProbeStateImpl { } } -pub(crate) fn filter( +pub(crate) fn filter( values: &T, filter_expr: &ExprRef, plan_arena: &(dyn MetaArena + '_), ) -> Result { match &*plan_arena .expression(*filter_expr) - .eval(plan_arena, Some(values as &dyn TupleLike))? + .eval(plan_arena, Some(values))? { DataValue::Boolean(false) | DataValue::Null => Ok(false), DataValue::Boolean(true) => Ok(true), diff --git a/src/execution/dql/join/hash_join.rs b/src/execution/dql/join/hash_join.rs index 52f05b9e..d2f42819 100644 --- a/src/execution/dql/join/hash_join.rs +++ b/src/execution/dql/join/hash_join.rs @@ -28,7 +28,7 @@ use crate::execution::{ use crate::planner::operator::join::{JoinCondition, JoinOperator, JoinType}; use crate::planner::ExprRef; use crate::planner::LogicalPlan; -use crate::planner::MetaArena; +use crate::planner::{ExecMetaArena, MetaArena}; use crate::storage::Transaction; use crate::types::tuple::Tuple; use crate::types::value::DataValue; @@ -143,10 +143,10 @@ impl HashJoin { Ok(()) } - fn initialize_build<'a, T: Transaction + 'a>( + fn initialize_build<'a, T: Transaction + 'a, A: MetaArena + 'a>( &mut self, arena: &mut ExecArena<'a, T>, - plan_arena: &mut (dyn MetaArena + 'a), + plan_arena: &mut ExecMetaArena, ) -> Result<(), DatabaseError> { if !matches!(self.state, HashJoinState::Build) { return Ok(()); @@ -251,10 +251,10 @@ impl<'a, T: Transaction + 'a> ReadExecutor<'a, T> for HashJoin { } impl<'a, T: Transaction + 'a> ExecutorNode<'a, T> for HashJoin { - fn next_tuple( + fn next_tuple( &mut self, arena: &mut ExecArena<'a, T>, - plan_arena: &mut (dyn MetaArena + 'a), + plan_arena: &mut ExecMetaArena, ) -> Result<(), DatabaseError> { if let Some(err) = self.init_error.take() { return Err(err); diff --git a/src/execution/dql/join/nested_loop_join.rs b/src/execution/dql/join/nested_loop_join.rs index ef5b94e5..72e4a9b0 100644 --- a/src/execution/dql/join/nested_loop_join.rs +++ b/src/execution/dql/join/nested_loop_join.rs @@ -16,7 +16,7 @@ //! [`JoinType::RightOuter`], [`JoinType::Cross`], [`JoinType::Full`]. use crate::planner::LogicalPlan; -use crate::planner::MetaArena; +use crate::planner::{ExecMetaArena, MetaArena}; use crate::errors::DatabaseError; use crate::execution::dql::join::RowBitmap; @@ -154,10 +154,10 @@ impl<'a, T: Transaction + 'a> ReadExecutor<'a, T> for NestedLoopJoin<'a> { } impl<'a, T: Transaction + 'a> ExecutorNode<'a, T> for NestedLoopJoin<'a> { - fn next_tuple( + fn next_tuple( &mut self, arena: &mut ExecArena<'a, T>, - plan_arena: &mut (dyn MetaArena + 'a), + plan_arena: &mut ExecMetaArena, ) -> Result<(), DatabaseError> { let mut state = std::mem::replace(&mut self.state, NestedLoopJoinState::End); @@ -579,6 +579,7 @@ mod test { let address = arena.nodes.items.as_ptr(); assert_eq!(count, 5); let mut rows = 0; + let mut plan_arena = ExecMetaArena::new(plan_arena); while arena.next_tuple(root, &mut plan_arena)? { rows += 1; assert_eq!(arena.result_tuple().values.len(), 9); diff --git a/src/execution/dql/limit.rs b/src/execution/dql/limit.rs index 9c9f6874..a49a0665 100644 --- a/src/execution/dql/limit.rs +++ b/src/execution/dql/limit.rs @@ -18,7 +18,7 @@ use crate::execution::{ }; use crate::planner::operator::limit::LimitOperator; use crate::planner::LogicalPlan; -use crate::planner::MetaArena; +use crate::planner::{ExecMetaArena, MetaArena}; use crate::storage::Transaction; pub struct Limit { offset: Option, @@ -50,10 +50,10 @@ impl<'a, T: Transaction + 'a> ReadExecutor<'a, T> for Limit { } impl<'a, T: Transaction + 'a> ExecutorNode<'a, T> for Limit { - fn next_tuple( + fn next_tuple( &mut self, arena: &mut ExecArena<'a, T>, - plan_arena: &mut (dyn MetaArena + 'a), + plan_arena: &mut ExecMetaArena, ) -> Result<(), DatabaseError> { let offset = self.offset.unwrap_or(0); let limit = self.limit.unwrap_or(usize::MAX); diff --git a/src/execution/dql/mark_apply.rs b/src/execution/dql/mark_apply.rs index cd17b9d4..4d9f6c6d 100644 --- a/src/execution/dql/mark_apply.rs +++ b/src/execution/dql/mark_apply.rs @@ -18,7 +18,7 @@ use crate::execution::{ }; use crate::planner::operator::mark_apply::{MarkApplyKind, MarkApplyOperator, MarkApplyQuantifier}; use crate::planner::LogicalPlan; -use crate::planner::MetaArena; +use crate::planner::{ExecMetaArena, MetaArena}; use crate::storage::Transaction; use crate::types::index::RuntimeIndexProbe; use crate::types::tuple::{SplitTupleRef, Tuple}; @@ -65,10 +65,10 @@ impl<'a, T: Transaction + 'a> ReadExecutor<'a, T> for MarkApply<'a> { } impl<'a, T: Transaction + 'a> ExecutorNode<'a, T> for MarkApply<'a> { - fn next_tuple( + fn next_tuple( &mut self, arena: &mut ExecArena<'a, T>, - plan_arena: &mut (dyn MetaArena + 'a), + plan_arena: &mut ExecMetaArena, ) -> Result<(), DatabaseError> { if matches!(self.op.kind, MarkApplyKind::InnerJoin) { return self.next_join_tuple(arena, plan_arena); @@ -88,10 +88,10 @@ impl<'a, T: Transaction + 'a> ExecutorNode<'a, T> for MarkApply<'a> { } impl<'a> MarkApply<'a> { - fn next_join_tuple( + fn next_join_tuple( &mut self, arena: &mut ExecArena<'a, T>, - plan_arena: &mut (dyn MetaArena + 'a), + plan_arena: &mut ExecMetaArena, ) -> Result<(), DatabaseError> { loop { if let Some((root, left)) = &mut self.join_input { @@ -174,10 +174,10 @@ impl<'a> MarkApply<'a> { .transpose() } - fn mark_value( + fn mark_value( &self, arena: &mut ExecArena<'a, T>, - plan_arena: &mut (dyn MetaArena + 'a), + plan_arena: &mut ExecMetaArena, left_tuple: &Tuple, ) -> Result { let probe = self.parameterized_probe_value(left_tuple, plan_arena)?; @@ -255,10 +255,10 @@ impl<'a> MarkApply<'a> { } } - fn scan_quantified_right_input( + fn scan_quantified_right_input( &self, arena: &mut ExecArena<'a, T>, - plan_arena: &mut (dyn MetaArena + 'a), + plan_arena: &mut ExecMetaArena, right_input: ExecId, quantifier: MarkApplyQuantifier, left_tuple: &Tuple, @@ -521,6 +521,7 @@ mod tests { let address = arena.nodes.items.as_ptr(); let count = arena.nodes.items.len(); assert_eq!(count, 3); + let mut plan_arena = ExecMetaArena::new(plan_arena); for _ in 0..4 { assert!(arena.next_tuple(root, &mut plan_arena)?); assert_eq!( @@ -611,6 +612,7 @@ mod tests { let address = arena.nodes.items.as_ptr(); assert_eq!(count, 3); let mut tuples = Vec::new(); + let mut plan_arena = ExecMetaArena::new(plan_arena); while arena.next_tuple(root, &mut plan_arena)? { tuples.push(arena.materialize_tuple()); assert_eq!(arena.nodes.items.len(), count); @@ -739,6 +741,7 @@ mod tests { }; let left_tuple = Tuple::new(None, vec![DataValue::Int32(2), DataValue::Int32(1)]); + let mut plan_arena = ExecMetaArena::new(plan_arena); assert_eq!( exec.mark_value(&mut arena, &mut plan_arena, &left_tuple)?, DataValue::Boolean(true) @@ -792,6 +795,7 @@ mod tests { }; let left_tuple = Tuple::new(None, vec![DataValue::Int32(2)]); + let mut plan_arena = ExecMetaArena::new(plan_arena); assert_eq!( exec.mark_value(&mut arena, &mut plan_arena, &left_tuple)?, DataValue::Boolean(true) @@ -845,6 +849,7 @@ mod tests { }; let left_tuple = Tuple::new(None, vec![DataValue::Null]); + let mut plan_arena = ExecMetaArena::new(plan_arena); assert_eq!( exec.mark_value(&mut arena, &mut plan_arena, &left_tuple)?, DataValue::Null diff --git a/src/execution/dql/mod.rs b/src/execution/dql/mod.rs index 03e8ac43..0bab43cd 100644 --- a/src/execution/dql/mod.rs +++ b/src/execution/dql/mod.rs @@ -26,7 +26,7 @@ pub(crate) mod limit; pub(crate) mod mark_apply; pub(crate) mod projection; pub(crate) mod recursive_cte; -pub(crate) mod scalar_apply; +pub(crate) mod scalar_query_init; pub(crate) mod scalar_subquery; pub(crate) mod seq_scan; pub(crate) mod set_membership; diff --git a/src/execution/dql/projection.rs b/src/execution/dql/projection.rs index 8a98d768..5c9f03a3 100644 --- a/src/execution/dql/projection.rs +++ b/src/execution/dql/projection.rs @@ -19,8 +19,9 @@ use crate::execution::{ use crate::planner::operator::project::ProjectOperator; use crate::planner::ExprRef; use crate::planner::LogicalPlan; -use crate::planner::MetaArena; +use crate::planner::{ExecMetaArena, MetaArena}; use crate::storage::Transaction; +use crate::types::value::DataValue; pub struct Projection<'a> { exprs: &'a [ExprRef], @@ -43,17 +44,17 @@ impl<'a, T: Transaction + 'a> ReadExecutor<'a, T> for Projection<'a> { } impl<'a, T: Transaction + 'a> ExecutorNode<'a, T> for Projection<'a> { - fn next_tuple( + fn next_tuple( &mut self, arena: &mut ExecArena<'a, T>, - plan_arena: &mut (dyn MetaArena + 'a), + plan_arena: &mut ExecMetaArena, ) -> Result<(), DatabaseError> { if !arena.next_tuple(self.input, plan_arena)? { arena.finish(); return Ok(()); } - arena.rewrite(self.exprs, plan_arena, None)?; + arena.rewrite(self.exprs, plan_arena, None::<&[DataValue]>)?; arena.resume(); Ok(()) } diff --git a/src/execution/dql/recursive_cte.rs b/src/execution/dql/recursive_cte.rs index 9563baef..654ec268 100644 --- a/src/execution/dql/recursive_cte.rs +++ b/src/execution/dql/recursive_cte.rs @@ -22,7 +22,7 @@ use crate::execution::{ }; use crate::planner::operator::recursive_cte::RecursiveScanOperator; use crate::planner::LogicalPlan; -use crate::planner::MetaArena; +use crate::planner::{ExecMetaArena, MetaArena}; use crate::storage::Transaction; use crate::types::tuple::Tuple; use std::mem; @@ -201,9 +201,9 @@ impl<'a, T: Transaction + 'a> RecursiveCte<'a, T> { } } - fn start_recursive( + fn start_recursive( &mut self, - plan_arena: &mut (dyn MetaArena + 'a), + plan_arena: &mut ExecMetaArena, ) -> Result { let Some(input) = mem::take(&mut self.working).into_input()? else { return Ok(false); @@ -246,10 +246,10 @@ impl<'a, T: Transaction + 'a> ReadExecutor<'a, T> for RecursiveCte<'a, T> { } impl<'a, T: Transaction + 'a> ExecutorNode<'a, T> for RecursiveCte<'a, T> { - fn next_tuple( + fn next_tuple( &mut self, arena: &mut ExecArena<'a, T>, - plan_arena: &mut (dyn MetaArena + 'a), + plan_arena: &mut ExecMetaArena, ) -> Result<(), DatabaseError> { loop { match self.phase { @@ -307,10 +307,10 @@ impl<'a, T: Transaction + 'a> ReadExecutor<'a, T> for RecursiveScan { } impl<'a, T: Transaction + 'a> ExecutorNode<'a, T> for RecursiveScan { - fn next_tuple( + fn next_tuple( &mut self, arena: &mut ExecArena<'a, T>, - _plan_arena: &mut (dyn MetaArena + 'a), + _plan_arena: &mut ExecMetaArena, ) -> Result<(), DatabaseError> { match self.input.next().transpose()? { Some(tuple) => arena.produce_tuple(tuple), diff --git a/src/execution/dql/scalar_apply.rs b/src/execution/dql/scalar_apply.rs deleted file mode 100644 index b506d68f..00000000 --- a/src/execution/dql/scalar_apply.rs +++ /dev/null @@ -1,280 +0,0 @@ -// Copyright 2024 KipData/KiteSQL -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use crate::errors::DatabaseError; -use crate::execution::{ - build_read, ExecArena, ExecId, ExecNode, ExecutionContext, ExecutorNode, ReadExecutor, -}; -use crate::planner::operator::scalar_apply::ScalarApplyOperator; -use crate::planner::LogicalPlan; -use crate::planner::MetaArena; -use crate::storage::Transaction; -use crate::types::tuple::Tuple; - -pub struct ScalarApply { - left_input: ExecId, - right_input: ExecId, - cached_right: Option, -} - -impl<'a, T: Transaction + 'a> ReadExecutor<'a, T> for ScalarApply { - type Input = (&'a ScalarApplyOperator, &'a LogicalPlan, &'a LogicalPlan); - - fn into_executor( - (_, left_input, right_input): Self::Input, - arena: &mut ExecArena<'a, T>, - plan_arena: &mut (dyn MetaArena + 'a), - cache: ExecutionContext<'_>, - transaction: &T, - ) -> ExecId { - let left_input = build_read(arena, plan_arena, left_input, cache, transaction); - let right_input = build_read(arena, plan_arena, right_input, cache, transaction); - arena.push(ExecNode::ScalarApply(Self { - left_input, - right_input, - cached_right: None, - })) - } -} - -impl<'a, T: Transaction + 'a> ExecutorNode<'a, T> for ScalarApply { - fn next_tuple( - &mut self, - arena: &mut ExecArena<'a, T>, - plan_arena: &mut (dyn MetaArena + 'a), - ) -> Result<(), DatabaseError> { - let right_tuple = - Self::load_right_once(&mut self.cached_right, self.right_input, arena, plan_arena)?; - if !arena.next_tuple(self.left_input, plan_arena)? { - arena.finish(); - return Ok(()); - } - arena - .result_tuple_mut() - .values - .extend(right_tuple.values.iter().cloned()); - arena.resume(); - Ok(()) - } -} - -impl ScalarApply { - fn load_right_once<'a, 'c, T: Transaction + 'a>( - cached_right: &'c mut Option, - right_input: ExecId, - arena: &mut ExecArena<'a, T>, - plan_arena: &mut (dyn MetaArena + 'a), - ) -> Result<&'c Tuple, DatabaseError> { - match cached_right { - Some(tuple) => Ok(tuple), - None => { - if !arena.next_tuple(right_input, plan_arena)? { - return Err(DatabaseError::InvalidValue( - "scalar apply right input returned no rows".to_string(), - )); - } - let first = arena.materialize_tuple(); - if arena.next_tuple(right_input, plan_arena)? { - return Err(DatabaseError::InvalidValue( - "scalar apply right input returned more than one row".to_string(), - )); - } - Ok(cached_right.insert(first)) - } - } - } -} - -#[cfg(all(test, not(target_arch = "wasm32")))] -mod tests { - use super::*; - use crate::catalog::{ColumnCatalog, ColumnDesc}; - use crate::execution::{execute_input, try_collect}; - use crate::planner::operator::scalar_subquery::ScalarSubqueryOperator; - use crate::planner::operator::values::ValuesOperator; - use crate::planner::operator::Operator; - use crate::planner::test::PlanArenaTestExt; - use crate::planner::{Childrens, LogicalPlan}; - use crate::storage::rocksdb::RocksStorage; - use crate::storage::{StatisticsMetaCache, Storage, TableCache, ViewCache}; - use crate::types::value::DataValue; - use crate::types::LogicalType; - use tempfile::TempDir; - - fn build_values( - arena: &mut crate::planner::PlanArena, - name: &str, - rows: Vec>, - ) -> LogicalPlan { - let desc = ColumnDesc::new(LogicalType::Integer, None, false, None).unwrap(); - let schema_ref = vec![arena.alloc_column(ColumnCatalog::new(name.to_string(), true, desc))]; - - LogicalPlan::new( - Operator::Values(ValuesOperator::new( - arena.alloc_expression_rows(&rows), - rows.len(), - schema_ref, - )), - Childrens::None, - ) - } - - fn build_test_storage() -> Result< - ( - TableCache, - ViewCache, - StatisticsMetaCache, - TempDir, - RocksStorage, - ), - DatabaseError, - > { - let meta_cache = crate::storage::StatisticsMetaCache::default(); - let view_cache = crate::storage::ViewCache::default(); - let table_cache = crate::storage::TableCache::default(); - - let temp_dir = TempDir::new().expect("unable to create temporary working directory"); - let storage = RocksStorage::new(temp_dir.path())?; - - Ok((table_cache, view_cache, meta_cache, temp_dir, storage)) - } - - #[test] - fn scalar_apply_repeats_scalar_result_for_each_left_row() -> Result<(), DatabaseError> { - let table_arena = crate::planner::TableArenaCell::default(); - let mut plan_arena = crate::planner::PlanArena::new(&table_arena); - let mut left = build_values( - &mut plan_arena, - "left_c1", - vec![ - vec![crate::types::value::DataValue::Int32(1)], - vec![crate::types::value::DataValue::Int32(2)], - ], - ); - let mut right = ScalarSubqueryOperator::build(build_values( - &mut plan_arena, - "right_c1", - vec![vec![crate::types::value::DataValue::Int32(7)]], - )); - - let (table_cache, view_cache, meta_cache, _temp_dir, storage) = build_test_storage()?; - let transaction = storage.transaction()?; - left.populate_output_schema_recursive(&mut plan_arena); - right.populate_output_schema_recursive(&mut plan_arena); - let tuples = try_collect(execute_input::<_, ScalarApply>( - (&ScalarApplyOperator, &left, &right), - crate::execution::empty_context(&table_cache, &view_cache, &meta_cache), - plan_arena, - &transaction, - ))?; - - let actual = tuples - .into_iter() - .flat_map(|tuple| tuple.values) - .collect::>(); - assert_eq!( - actual, - vec![ - DataValue::Int32(1), - DataValue::Int32(7), - DataValue::Int32(2), - DataValue::Int32(7), - ] - ); - - Ok(()) - } - - #[test] - fn scalar_subquery_checks_cardinality_on_second_call() -> Result<(), DatabaseError> { - for rows in [ - vec![], - vec![vec![DataValue::Int32(7)]], - vec![vec![DataValue::Int32(7)], vec![DataValue::Int32(8)]], - ] { - let row_count = rows.len(); - let table_arena = crate::planner::TableArenaCell::default(); - let mut plan_arena = crate::planner::PlanArena::new(&table_arena); - let mut input = build_values(&mut plan_arena, "right_c1", rows); - input.populate_output_schema_recursive(&mut plan_arena); - let (table_cache, view_cache, meta_cache, _temp_dir, storage) = build_test_storage()?; - let transaction = storage.transaction()?; - let mut executor = - execute_input::<_, crate::execution::dql::scalar_subquery::ScalarSubquery>( - (&ScalarSubqueryOperator, &input), - crate::execution::empty_context(&table_cache, &view_cache, &meta_cache), - plan_arena, - &transaction, - ); - let expected = if row_count == 0 { - DataValue::Null - } else { - DataValue::Int32(7) - }; - assert_eq!(executor.next_tuple()?.unwrap().values, vec![expected]); - if row_count > 1 { - assert!( - matches!(executor.next_tuple(), Err(DatabaseError::InvalidValue(message)) - if message == "scalar subquery returned more than one row") - ); - } else { - assert!(executor.next_tuple()?.is_none()); - assert!(executor.next_tuple()?.is_none()); - } - } - Ok(()) - } - - #[test] - fn scalar_apply_repeats_null_scalar_result_for_each_left_row() -> Result<(), DatabaseError> { - let table_arena = crate::planner::TableArenaCell::default(); - let mut plan_arena = crate::planner::PlanArena::new(&table_arena); - let mut left = build_values( - &mut plan_arena, - "left_c1", - vec![vec![DataValue::Int32(1)], vec![DataValue::Int32(2)]], - ); - let mut right = ScalarSubqueryOperator::build(build_values( - &mut plan_arena, - "right_c1", - vec![vec![DataValue::Null]], - )); - - let (table_cache, view_cache, meta_cache, _temp_dir, storage) = build_test_storage()?; - let transaction = storage.transaction()?; - left.populate_output_schema_recursive(&mut plan_arena); - right.populate_output_schema_recursive(&mut plan_arena); - let tuples = try_collect(execute_input::<_, ScalarApply>( - (&ScalarApplyOperator, &left, &right), - crate::execution::empty_context(&table_cache, &view_cache, &meta_cache), - plan_arena, - &transaction, - ))?; - - assert_eq!( - tuples - .into_iter() - .flat_map(|tuple| tuple.values) - .collect::>(), - vec![ - DataValue::Int32(1), - DataValue::Null, - DataValue::Int32(2), - DataValue::Null, - ] - ); - - Ok(()) - } -} diff --git a/src/execution/dql/scalar_query_init.rs b/src/execution/dql/scalar_query_init.rs new file mode 100644 index 00000000..42bd61eb --- /dev/null +++ b/src/execution/dql/scalar_query_init.rs @@ -0,0 +1,257 @@ +// Copyright 2024 KipData/KiteSQL +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use crate::errors::DatabaseError; +use crate::execution::{ + build_read, ExecArena, ExecId, ExecNode, ExecutionContext, ExecutorNode, ReadExecutor, +}; +use crate::expression::ScalarExpression; +use crate::planner::operator::scalar_query_init::ScalarQueryInitOperator; +use crate::planner::ExecMetaArena; +use crate::planner::{ExprRef, LogicalPlan, MetaArena, ScalarQueryRef}; +use crate::storage::Transaction; +use crate::types::tuple::Tuple; +use crate::types::value::DataValue; + +enum ScalarQueryInitState { + Initialize, + ReadInput, + ReadOuter, + EvaluateOuter, + Finished, +} + +pub struct ScalarQueryInit<'a> { + input: ExecId, + reference: ScalarQueryRef, + state: ScalarQueryInitState, + param_bindings: &'a [(ScalarQueryRef, ExprRef)], + scratch_tuple: Tuple, + + init_plan: &'a LogicalPlan, + init_pos: ExecId, + init: ExecId, +} + +impl<'a> ScalarQueryInit<'a> { + fn evaluate_init( + &self, + arena: &mut ExecArena<'a, T>, + plan_arena: &mut ExecMetaArena, + ) -> Result<(), DatabaseError> { + let mut value = DataValue::Null; + if arena.next_tuple(self.init, plan_arena)? { + std::mem::swap(&mut value, &mut arena.result_tuple_mut().values[0]); + } + if arena.next_tuple(self.init, plan_arena)? { + return Err(DatabaseError::InvalidValue( + "scalar subquery returned more than one row".into(), + )); + } + plan_arena.set_init_value(self.reference, value); + Ok(()) + } + + pub(crate) fn build( + op: &'a ScalarQueryInitOperator, + input: ExecId, + init_plan: &'a LogicalPlan, + arena: &mut ExecArena<'a, T>, + plan_arena: &mut (dyn MetaArena + 'a), + cache: ExecutionContext<'_>, + transaction: &T, + ) -> ExecId { + let init_pos = arena.nodes.position(); + let init = build_read(arena, plan_arena, init_plan, cache, transaction); + arena.push(ExecNode::ScalarQueryInit(Self { + input, + init, + reference: op.reference(plan_arena), + state: if matches!( + plan_arena.expression(op.value), + ScalarExpression::OuterValue { .. } + ) { + ScalarQueryInitState::ReadOuter + } else { + ScalarQueryInitState::Initialize + }, + param_bindings: &op.param_bindings, + scratch_tuple: Tuple::default(), + init_plan, + init_pos, + })) + } +} + +impl<'a, T: Transaction + 'a> ReadExecutor<'a, T> for ScalarQueryInit<'a> { + type Input = ( + &'a ScalarQueryInitOperator, + &'a LogicalPlan, + &'a LogicalPlan, + ); + + fn into_executor( + (op, input, init): Self::Input, + arena: &mut ExecArena<'a, T>, + plan_arena: &mut (dyn MetaArena + 'a), + cache: ExecutionContext<'_>, + transaction: &T, + ) -> ExecId { + let input = build_read(arena, plan_arena, input, cache, transaction); + Self::build(op, input, init, arena, plan_arena, cache, transaction) + } +} + +impl<'a, T: Transaction + 'a> ExecutorNode<'a, T> for ScalarQueryInit<'a> { + fn next_tuple( + &mut self, + arena: &mut ExecArena<'a, T>, + plan_arena: &mut ExecMetaArena, + ) -> Result<(), DatabaseError> { + loop { + match self.state { + ScalarQueryInitState::Initialize => { + if plan_arena.init_value(self.reference).is_none() { + self.evaluate_init(arena, plan_arena)?; + } + self.state = ScalarQueryInitState::ReadInput; + } + ScalarQueryInitState::ReadInput => { + if !arena.next_tuple(self.input, plan_arena)? { + self.state = ScalarQueryInitState::Finished; + } + return Ok(()); + } + ScalarQueryInitState::ReadOuter => { + if !arena.next_tuple(self.input, plan_arena)? { + self.state = ScalarQueryInitState::Finished; + return Ok(()); + } + // Preserve the outer row and lend the previous subquery buffer to the arena. + std::mem::swap(arena.result_tuple_mut(), &mut self.scratch_tuple); + for (reference, expr) in self.param_bindings { + let value = plan_arena + .expression(*expr) + .eval(plan_arena, Some(&self.scratch_tuple))? + .into_owned(); + plan_arena.set_init_value(*reference, value); + } + self.state = ScalarQueryInitState::EvaluateOuter; + } + ScalarQueryInitState::EvaluateOuter => { + let previous = arena.nodes.position(); + arena.nodes.seek(self.init_pos); + self.init = build_read( + arena, + plan_arena, + self.init_plan, + arena.context(), + arena.transaction(), + ); + arena.nodes.seek(previous); + self.evaluate_init(arena, plan_arena)?; + std::mem::swap(arena.result_tuple_mut(), &mut self.scratch_tuple); + self.state = ScalarQueryInitState::ReadOuter; + arena.resume(); + return Ok(()); + } + ScalarQueryInitState::Finished => { + arena.finish(); + return Ok(()); + } + } + } + } +} + +#[cfg(all(test, not(target_arch = "wasm32")))] +mod tests { + use super::*; + use crate::catalog::{ColumnCatalog, ColumnDesc}; + use crate::execution::empty_context; + use crate::expression::ScalarExpression; + use crate::planner::operator::scalar_subquery::ScalarSubqueryOperator; + use crate::planner::operator::values::ValuesOperator; + use crate::planner::operator::Operator; + use crate::planner::{Childrens, ExecMetaArena, PlanArena, TableArenaCell}; + use crate::storage::memory::MemoryStorage; + use crate::storage::{StatisticsMetaCache, Storage, TableCache, ViewCache}; + use crate::types::LogicalType; + + #[test] + fn scalar_init_shared_slot_caches_null_and_checks_cardinality() -> Result<(), DatabaseError> { + let storage = MemoryStorage::new(); + let transaction = storage.transaction()?; + let tables = TableCache::default(); + let views = ViewCache::default(); + let stats = StatisticsMetaCache::default(); + let cache = empty_context(&tables, &views, &stats); + for (value, row_count) in [ + (DataValue::Null, 1), + (DataValue::Int32(7), 1), + (DataValue::Int32(7), 2), + ] { + let table_arena = TableArenaCell::default(); + let mut metadata = PlanArena::new(&table_arena); + let reference = metadata.alloc_scalar_query_ref(false); + let column = metadata.alloc_column(ColumnCatalog::new( + "v".into(), + true, + ColumnDesc::new(LogicalType::Integer, None, false, None)?, + )); + let expr = metadata.alloc_expression(ScalarExpression::Constant(value.clone())); + let marker = metadata.alloc_expression(ScalarExpression::InitValue { + id: reference, + ty: LogicalType::Integer, + }); + let make_init = |rows| { + ScalarSubqueryOperator::build(LogicalPlan::new( + Operator::Values(ValuesOperator::new(vec![expr; rows], rows, vec![column])), + Childrens::None, + )) + }; + let mut first = ScalarQueryInitOperator::build( + LogicalPlan::new(Operator::Dummy, Childrens::None), + make_init(row_count), + marker, + Vec::new(), + ); + // This duplicate would error if executed, making cache reuse observable even for NULL. + let mut duplicate = ScalarQueryInitOperator::build( + LogicalPlan::new(Operator::Dummy, Childrens::None), + make_init(2), + marker, + Vec::new(), + ); + first.populate_output_schema_recursive(&mut metadata); + duplicate.populate_output_schema_recursive(&mut metadata); + let mut arena = ExecArena::with_capacity(0); + arena.init_context(cache, &transaction); + let first_root = build_read(&mut arena, &mut metadata, &first, cache, &transaction); + let duplicate_root = + build_read(&mut arena, &mut metadata, &duplicate, cache, &transaction); + let mut view = ExecMetaArena::new(metadata); + if row_count == 2 { + assert!(arena.next_tuple(first_root, &mut view).is_err()); + assert_eq!(view.init_value(reference), None); + } else { + assert!(arena.next_tuple(first_root, &mut view)?); + assert_eq!(view.init_value(reference), Some(&value)); + assert!(arena.next_tuple(duplicate_root, &mut view)?); + assert!(!arena.next_tuple(duplicate_root, &mut view)?); + } + } + Ok(()) + } +} diff --git a/src/execution/dql/scalar_subquery.rs b/src/execution/dql/scalar_subquery.rs index 822f81cb..98d060a4 100644 --- a/src/execution/dql/scalar_subquery.rs +++ b/src/execution/dql/scalar_subquery.rs @@ -18,7 +18,7 @@ use crate::execution::{ }; use crate::planner::operator::scalar_subquery::ScalarSubqueryOperator; use crate::planner::LogicalPlan; -use crate::planner::MetaArena; +use crate::planner::{ExecMetaArena, MetaArena}; use crate::storage::Transaction; use crate::types::value::DataValue; @@ -49,10 +49,10 @@ impl<'a, T: Transaction + 'a> ReadExecutor<'a, T> for ScalarSubquery { } impl<'a, T: Transaction + 'a> ExecutorNode<'a, T> for ScalarSubquery { - fn next_tuple( + fn next_tuple( &mut self, arena: &mut ExecArena<'a, T>, - plan_arena: &mut (dyn MetaArena + 'a), + plan_arena: &mut ExecMetaArena, ) -> Result<(), DatabaseError> { let has_next = arena.next_tuple(self.input, plan_arena)?; if self.returned { diff --git a/src/execution/dql/seq_scan.rs b/src/execution/dql/seq_scan.rs index 6effc8e3..2e237084 100644 --- a/src/execution/dql/seq_scan.rs +++ b/src/execution/dql/seq_scan.rs @@ -15,7 +15,7 @@ use crate::errors::DatabaseError; use crate::execution::{ExecArena, ExecId, ExecNode, ExecutionContext, ExecutorNode, ReadExecutor}; use crate::planner::operator::table_scan::TableScanOperator; -use crate::planner::MetaArena; +use crate::planner::{ExecMetaArena, MetaArena}; use crate::storage::{Iter, Transaction, TupleIter}; pub(crate) struct SeqScan<'a, T: Transaction + 'a> { @@ -38,10 +38,10 @@ impl<'a, T: Transaction + 'a> ReadExecutor<'a, T> for SeqScan<'a, T> { } impl<'a, T: Transaction + 'a> ExecutorNode<'a, T> for SeqScan<'a, T> { - fn next_tuple( + fn next_tuple( &mut self, arena: &mut ExecArena<'a, T>, - plan_arena: &mut (dyn MetaArena + 'a), + plan_arena: &mut ExecMetaArena, ) -> Result<(), DatabaseError> { let state = arena.local_state(plan_arena); let iter = match &mut self.iter { diff --git a/src/execution/dql/set_membership.rs b/src/execution/dql/set_membership.rs index 75c38726..a83a8679 100644 --- a/src/execution/dql/set_membership.rs +++ b/src/execution/dql/set_membership.rs @@ -18,7 +18,7 @@ use crate::execution::{ }; use crate::planner::operator::set_membership::SetMembershipKind; use crate::planner::LogicalPlan; -use crate::planner::MetaArena; +use crate::planner::{ExecMetaArena, MetaArena}; use crate::storage::Transaction; use crate::types::tuple::Tuple; use std::collections::HashMap; @@ -54,10 +54,10 @@ impl<'a, T: Transaction + 'a> ReadExecutor<'a, T> for SetMembership { } impl<'a, T: Transaction + 'a> ExecutorNode<'a, T> for SetMembership { - fn next_tuple( + fn next_tuple( &mut self, arena: &mut ExecArena<'a, T>, - plan_arena: &mut (dyn MetaArena + 'a), + plan_arena: &mut ExecMetaArena, ) -> Result<(), DatabaseError> { if !self.built { while arena.next_tuple(self.right_input, plan_arena)? { diff --git a/src/execution/dql/show_table.rs b/src/execution/dql/show_table.rs index 9d0a2b60..ebd617ad 100644 --- a/src/execution/dql/show_table.rs +++ b/src/execution/dql/show_table.rs @@ -14,7 +14,7 @@ use crate::errors::DatabaseError; use crate::execution::{ExecArena, ExecId, ExecNode, ExecutionContext, ExecutorNode, ReadExecutor}; -use crate::planner::MetaArena; +use crate::planner::{ExecMetaArena, MetaArena}; use crate::storage::{TableIter, Transaction}; use crate::types::value::{DataValue, Utf8Type}; use crate::types::CharLengthUnits; @@ -29,7 +29,7 @@ impl<'a, T: Transaction + 'a> ReadExecutor<'a, T> for ShowTables<'a, T> { fn into_executor( _: Self::Input, arena: &mut ExecArena<'a, T>, - _: &mut (dyn MetaArena + 'a), + _: &mut (dyn MetaArena + '_), _: ExecutionContext<'_>, _: &T, ) -> ExecId { @@ -38,10 +38,10 @@ impl<'a, T: Transaction + 'a> ReadExecutor<'a, T> for ShowTables<'a, T> { } impl<'a, T: Transaction + 'a> ExecutorNode<'a, T> for ShowTables<'a, T> { - fn next_tuple( + fn next_tuple( &mut self, arena: &mut ExecArena<'a, T>, - plan_arena: &mut (dyn MetaArena + 'a), + plan_arena: &mut ExecMetaArena, ) -> Result<(), DatabaseError> { if self.metas.is_none() { let mut state = arena.local_state(plan_arena); diff --git a/src/execution/dql/show_view.rs b/src/execution/dql/show_view.rs index 12f337a7..dfeef23d 100644 --- a/src/execution/dql/show_view.rs +++ b/src/execution/dql/show_view.rs @@ -14,7 +14,7 @@ use crate::errors::DatabaseError; use crate::execution::{ExecArena, ExecId, ExecNode, ExecutionContext, ExecutorNode, ReadExecutor}; -use crate::planner::MetaArena; +use crate::planner::{ExecMetaArena, MetaArena}; use crate::storage::{Transaction, ViewIter}; use crate::types::value::{DataValue, Utf8Type}; use crate::types::CharLengthUnits; @@ -38,10 +38,10 @@ impl<'a, T: Transaction + 'a> ReadExecutor<'a, T> for ShowViews<'a, T> { } impl<'a, T: Transaction + 'a> ExecutorNode<'a, T> for ShowViews<'a, T> { - fn next_tuple( + fn next_tuple( &mut self, arena: &mut ExecArena<'a, T>, - plan_arena: &mut (dyn MetaArena + 'a), + plan_arena: &mut ExecMetaArena, ) -> Result<(), DatabaseError> { if self.metas.is_none() { let context = arena.context(); diff --git a/src/execution/dql/sort.rs b/src/execution/dql/sort.rs index 85257185..402e8088 100644 --- a/src/execution/dql/sort.rs +++ b/src/execution/dql/sort.rs @@ -18,7 +18,8 @@ use crate::execution::{ }; use crate::planner::operator::sort::{SortField, SortOperator}; use crate::planner::LogicalPlan; -use crate::planner::MetaArena; +use crate::planner::ScalarQueryRef; +use crate::planner::{ExecMetaArena, MetaArena}; use crate::storage::Transaction; use crate::types::tuple::Tuple; use crate::types::value::DataValue; @@ -48,11 +49,6 @@ impl<'a, T> NullableVec<'a, T> { self.0.len() } - #[inline] - pub(crate) fn iter(&self) -> impl Iterator { - self.0.iter().map(|item| unsafe { item.assume_init_ref() }) - } - #[inline] pub(crate) fn pop(&mut self) -> Option { self.0.pop().map(|item| unsafe { item.assume_init() }) @@ -79,24 +75,25 @@ impl DerefMut for NullableVec<'_, T> { } } -pub(crate) fn sort_tuples( +pub(crate) fn sort_tuples( sort_fields: &[SortField], - tuples: &mut NullableVec<'_, (usize, Tuple)>, - plan_arena: &(dyn MetaArena + '_), + tuples: &mut NullableVec<'_, (usize, SortTuple)>, + plan_arena: &mut ExecMetaArena, ) -> Result<(), DatabaseError> { // Extract the results of calculating SortFields to avoid double calculation // of data during comparison. - let mut eval_values = vec![Vec::new(); sort_fields.len()]; - - for (x, SortField { expr, .. }) in sort_fields.iter().enumerate() { - for (_, tuple) in tuples.iter() { - eval_values[x].push( - plan_arena - .expression(*expr) - .eval(plan_arena, Some(tuple))? - .into_owned(), - ); + let width = sort_fields.len(); + let mut eval_values = Vec::with_capacity(tuples.len() * width); + + for (_, row) in tuples.iter_mut() { + plan_arena.restore_outer_values(std::mem::take(&mut row.outer_values)); + for SortField { expr, .. } in sort_fields { + let value = plan_arena + .expression(*expr) + .eval(plan_arena, Some(&row.tuple))?; + eval_values.push(value.into_owned()); } + row.outer_values = plan_arena.take_outer_values(); } tuples.0.sort_by(|tuple_1, tuple_2| { @@ -104,8 +101,8 @@ pub(crate) fn sort_tuples( let (i_2, _) = unsafe { tuple_2.assume_init_ref() }; compare_sort_keys( sort_fields, - eval_values.iter().map(|values| &values[*i_1]), - eval_values.iter().map(|values| &values[*i_2]), + eval_values[*i_1 * width..(*i_1 + 1) * width].iter(), + eval_values[*i_2 * width..(*i_2 + 1) * width].iter(), ) }); drop(eval_values); @@ -148,8 +145,13 @@ pub(crate) fn compare_sort_keys<'a>( Ordering::Equal } +pub(crate) struct SortTuple { + tuple: Tuple, + outer_values: Vec<(ScalarQueryRef, DataValue)>, +} + pub struct Sort<'a> { - rows: NullableVec<'static, (usize, Tuple)>, + rows: NullableVec<'static, (usize, SortTuple)>, _arena: Box, sort_fields: &'a [SortField], input: ExecId, @@ -168,7 +170,7 @@ impl<'a, T: Transaction + 'a> ReadExecutor<'a, T> for Sort<'a> { let input = build_read(arena, plan_arena, input, cache, transaction); let sort_arena = Box::::default(); let rows = unsafe { - transmute::, NullableVec<'static, (usize, Tuple)>>( + transmute::, NullableVec<'static, (usize, SortTuple)>>( NullableVec::new(&sort_arena), ) }; @@ -182,19 +184,26 @@ impl<'a, T: Transaction + 'a> ReadExecutor<'a, T> for Sort<'a> { } impl<'a, T: Transaction + 'a> ExecutorNode<'a, T> for Sort<'a> { - fn next_tuple( + fn next_tuple( &mut self, arena: &mut ExecArena<'a, T>, - plan_arena: &mut (dyn MetaArena + 'a), + plan_arena: &mut ExecMetaArena, ) -> Result<(), DatabaseError> { loop { - if let Some((_, tuple)) = self.rows.pop() { - arena.produce_tuple(tuple); + if let Some((_, row)) = self.rows.pop() { + plan_arena.restore_outer_values(row.outer_values); + arena.produce_tuple(row.tuple); return Ok(()); } while arena.next_tuple(self.input, plan_arena)? { let offset = self.rows.len(); - self.rows.put((offset, arena.materialize_tuple())); + self.rows.put(( + offset, + SortTuple { + tuple: arena.materialize_tuple(), + outer_values: plan_arena.take_outer_values(), + }, + )); } if self.rows.is_empty() { arena.finish(); @@ -210,15 +219,111 @@ impl<'a, T: Transaction + 'a> ExecutorNode<'a, T> for Sort<'a> { mod test { use crate::catalog::{ColumnCatalog, ColumnDesc}; use crate::errors::DatabaseError; - use crate::execution::dql::sort::{sort_tuples, NullableVec}; + use crate::execution::dql::sort::{sort_tuples, NullableVec, SortTuple}; use crate::expression::ScalarExpression; use crate::planner::operator::sort::SortField; + use crate::planner::{ExecMetaArena, PlanArena}; use crate::types::tuple::Tuple; use crate::types::value::DataValue; use crate::types::LogicalType; use bumpalo::Bump; use std::cell::Cell; + #[test] + fn memory_sort_restores_scalar_values_for_each_output_row() -> Result<(), DatabaseError> { + use super::Sort; + use crate::execution::{empty_context, ExecArena, ReadExecutor}; + use crate::planner::operator::scalar_query_init::ScalarQueryInitOperator; + use crate::planner::operator::scalar_subquery::ScalarSubqueryOperator; + use crate::planner::operator::values::ValuesOperator; + use crate::planner::operator::{sort::SortOperator, Operator}; + use crate::planner::{ + Childrens, ExecMetaArena, LogicalPlan, MetaArena, PlanArena, TableArenaCell, + }; + use crate::storage::memory::MemoryStorage; + use crate::storage::{StatisticsMetaCache, Storage, TableCache, ViewCache}; + + let storage = MemoryStorage::new(); + let transaction = storage.transaction()?; + let tables = TableCache::default(); + let views = ViewCache::default(); + let stats = StatisticsMetaCache::default(); + let cache = empty_context(&tables, &views, &stats); + let catalog = TableArenaCell::default(); + let mut metadata = PlanArena::new(&catalog); + let column = metadata.alloc_column(ColumnCatalog::new( + "id".into(), + false, + ColumnDesc::new(LogicalType::Integer, None, false, None)?, + )); + let input_expr = metadata.alloc_expression(ScalarExpression::column_expr(column, 0)); + let param = metadata.alloc_scalar_query_ref(false); + let result = metadata.alloc_scalar_query_ref(true); + let marker = metadata.alloc_expression(ScalarExpression::OuterValue { + id: result, + ty: LogicalType::Integer, + }); + let param_expr = metadata.alloc_expression(ScalarExpression::OuterParam { + id: param, + ty: LogicalType::Integer, + }); + let values = [3, 1, 2] + .into_iter() + .map(|value| { + metadata.alloc_expression(ScalarExpression::Constant(DataValue::Int32(value))) + }) + .collect(); + let input = LogicalPlan::new( + Operator::Values(ValuesOperator::new(values, 3, vec![column])), + Childrens::None, + ); + let query = ScalarSubqueryOperator::build(LogicalPlan::new( + Operator::Values(ValuesOperator::new(vec![param_expr], 1, vec![column])), + Childrens::None, + )); + let mut plan = + ScalarQueryInitOperator::build(input, query, marker, vec![(param, input_expr)]); + plan.populate_output_schema_recursive(&mut metadata); + let op = SortOperator { + sort_fields: vec![SortField { + expr: marker, + asc: true, + nulls_first: false, + }], + }; + let mut arena = ExecArena::with_capacity(0); + arena.init_context(cache, &transaction); + // Select the memory executor explicitly even when the spill feature is enabled. + let root = >::into_executor( + (&op, &plan), + &mut arena, + &mut metadata, + cache, + &transaction, + ); + let mut metadata = ExecMetaArena::new(metadata); + for expected in [1, 2, 3] { + assert!(arena.next_tuple(root, &mut metadata)?); + assert_eq!( + arena.result_tuple().values, + vec![DataValue::Int32(expected)] + ); + assert_eq!( + metadata.init_value(result), + Some(&DataValue::Int32(expected)) + ); + assert_eq!( + metadata + .expression(marker) + .eval(&metadata, Some(arena.result_tuple()))? + .as_ref(), + &DataValue::Int32(expected) + ); + } + assert!(!arena.next_tuple(root, &mut metadata)?); + Ok(()) + } + #[test] fn nullable_vec_drops_values() { struct DropValue<'a>(&'a Cell); @@ -241,13 +346,13 @@ mod test { fn sorted_rows<'a>( sort_fields: &[SortField], - mut tuples: NullableVec<'a, (usize, Tuple)>, - plan_arena: &crate::planner::PlanArena<'_>, + mut tuples: NullableVec<'a, (usize, SortTuple)>, + plan_arena: &mut ExecMetaArena>, ) -> Result + 'a, DatabaseError> { sort_tuples(sort_fields, &mut tuples, plan_arena)?; let mut rows = Vec::with_capacity(tuples.len()); - while let Some((_, tuple)) = tuples.pop() { - rows.push(tuple); + while let Some((_, row)) = tuples.pop() { + rows.push(row.tuple); } rows.reverse(); Ok(rows.into_iter()) @@ -279,12 +384,31 @@ mod test { ColumnDesc::new(LogicalType::Integer, None, false, None).unwrap(), ))]; + let mut plan_arena = ExecMetaArena::new(plan_arena); let arena = Bump::new(); let fn_tuples = || { let mut vec = NullableVec::new(&arena); - vec.put((0_usize, Tuple::new(None, vec![DataValue::Null]))); - vec.put((1_usize, Tuple::new(None, vec![DataValue::Int32(0)]))); - vec.put((2_usize, Tuple::new(None, vec![DataValue::Int32(1)]))); + vec.put(( + 0_usize, + SortTuple { + tuple: Tuple::new(None, vec![DataValue::Null]), + outer_values: Vec::new(), + }, + )); + vec.put(( + 1_usize, + SortTuple { + tuple: Tuple::new(None, vec![DataValue::Int32(0)]), + outer_values: Vec::new(), + }, + )); + vec.put(( + 2_usize, + SortTuple { + tuple: Tuple::new(None, vec![DataValue::Int32(1)]), + outer_values: Vec::new(), + }, + )); vec }; @@ -360,22 +484,22 @@ mod test { fn_asc_and_nulls_first_eq(Box::new(sorted_rows( &fn_sort_fields(true, true), fn_tuples(), - &plan_arena, + &mut plan_arena, )?)); fn_asc_and_nulls_last_eq(Box::new(sorted_rows( &fn_sort_fields(true, false), fn_tuples(), - &plan_arena, + &mut plan_arena, )?)); fn_desc_and_nulls_first_eq(Box::new(sorted_rows( &fn_sort_fields(false, true), fn_tuples(), - &plan_arena, + &mut plan_arena, )?)); fn_desc_and_nulls_last_eq(Box::new(sorted_rows( &fn_sort_fields(false, false), fn_tuples(), - &plan_arena, + &mut plan_arena, )?)); Ok(()) @@ -430,33 +554,52 @@ mod test { ColumnDesc::new(LogicalType::Integer, None, false, None).unwrap(), )), ]; + let mut plan_arena = ExecMetaArena::new(plan_arena); let arena = Bump::new(); let fn_tuples = || { let mut vec = NullableVec::new(&arena); vec.put(( 0_usize, - Tuple::new(None, vec![DataValue::Null, DataValue::Null]), + SortTuple { + tuple: Tuple::new(None, vec![DataValue::Null, DataValue::Null]), + outer_values: Vec::new(), + }, )); vec.put(( 1_usize, - Tuple::new(None, vec![DataValue::Int32(0), DataValue::Null]), + SortTuple { + tuple: Tuple::new(None, vec![DataValue::Int32(0), DataValue::Null]), + outer_values: Vec::new(), + }, )); vec.put(( 2_usize, - Tuple::new(None, vec![DataValue::Int32(1), DataValue::Null]), + SortTuple { + tuple: Tuple::new(None, vec![DataValue::Int32(1), DataValue::Null]), + outer_values: Vec::new(), + }, )); vec.put(( 3_usize, - Tuple::new(None, vec![DataValue::Null, DataValue::Int32(0)]), + SortTuple { + tuple: Tuple::new(None, vec![DataValue::Null, DataValue::Int32(0)]), + outer_values: Vec::new(), + }, )); vec.put(( 4_usize, - Tuple::new(None, vec![DataValue::Int32(0), DataValue::Int32(0)]), + SortTuple { + tuple: Tuple::new(None, vec![DataValue::Int32(0), DataValue::Int32(0)]), + outer_values: Vec::new(), + }, )); vec.put(( 5_usize, - Tuple::new(None, vec![DataValue::Int32(1), DataValue::Int32(0)]), + SortTuple { + tuple: Tuple::new(None, vec![DataValue::Int32(1), DataValue::Int32(0)]), + outer_values: Vec::new(), + }, )); vec }; @@ -596,22 +739,22 @@ mod test { fn_asc_1_and_nulls_first_1_and_asc_2_and_nulls_first_2_eq(Box::new(sorted_rows( &fn_sort_fields(true, true, true, true), fn_tuples(), - &plan_arena, + &mut plan_arena, )?)); fn_asc_1_and_nulls_last_1_and_asc_2_and_nulls_first_2_eq(Box::new(sorted_rows( &fn_sort_fields(true, false, true, true), fn_tuples(), - &plan_arena, + &mut plan_arena, )?)); fn_desc_1_and_nulls_first_1_and_asc_2_and_nulls_first_2_eq(Box::new(sorted_rows( &fn_sort_fields(false, true, true, true), fn_tuples(), - &plan_arena, + &mut plan_arena, )?)); fn_desc_1_and_nulls_last_1_and_asc_2_and_nulls_first_2_eq(Box::new(sorted_rows( &fn_sort_fields(false, false, true, true), fn_tuples(), - &plan_arena, + &mut plan_arena, )?)); Ok(()) diff --git a/src/execution/dql/top_k.rs b/src/execution/dql/top_k.rs index 108a772b..38ec59e5 100644 --- a/src/execution/dql/top_k.rs +++ b/src/execution/dql/top_k.rs @@ -19,11 +19,12 @@ use crate::execution::{ }; use crate::planner::operator::sort::SortField; use crate::planner::operator::top_k::TopKOperator; -use crate::planner::LogicalPlan; -use crate::planner::MetaArena; +use crate::planner::{ExecMetaArena, MetaArena}; +use crate::planner::{LogicalPlan, ScalarQueryRef}; use crate::storage::table_codec::BumpBytes; use crate::storage::Transaction; use crate::types::tuple::Tuple; +use crate::types::value::DataValue; use bumpalo::Bump; use std::cmp::Ordering; use std::collections::{btree_set::IntoIter as BTreeSetIntoIter, BTreeSet}; @@ -34,6 +35,7 @@ struct CmpItem<'a> { key: BumpVec<'a, u8>, sequence: usize, tuple: Tuple, + outer_values: Vec<(ScalarQueryRef, DataValue)>, } impl PartialEq for CmpItem<'_> { @@ -66,7 +68,7 @@ fn top_sort<'a>( tuple: &mut Tuple, keep_count: usize, sequence: usize, - plan_arena: &(dyn MetaArena + '_), + plan_arena: &mut ExecMetaArena, ) -> Result<(), DatabaseError> { full_key.clear(); for SortField { @@ -92,12 +94,14 @@ fn top_sort<'a>( key: std::mem::replace(full_key, BumpBytes::new_in(full_key.bump())), sequence, tuple: std::mem::take(tuple), + outer_values: plan_arena.take_outer_values(), }); } else if let Some(mut cmp_item) = heap.pop_last() { if full_key.as_slice() < cmp_item.key.as_slice() { std::mem::swap(full_key, &mut cmp_item.key); cmp_item.sequence = sequence; cmp_item.tuple = std::mem::take(tuple); + cmp_item.outer_values = plan_arena.take_outer_values(); } heap.insert(cmp_item); } @@ -143,10 +147,10 @@ impl<'a, T: Transaction + 'a> ReadExecutor<'a, T> for TopK<'a> { } impl<'a, T: Transaction + 'a> ExecutorNode<'a, T> for TopK<'a> { - fn next_tuple( + fn next_tuple( &mut self, arena: &mut ExecArena<'a, T>, - plan_arena: &mut (dyn MetaArena + 'a), + plan_arena: &mut ExecMetaArena, ) -> Result<(), DatabaseError> { if self.output.is_none() { let keep_count = self.offset.unwrap_or(0) + self.limit; @@ -181,6 +185,7 @@ impl<'a, T: Transaction + 'a> ExecutorNode<'a, T> for TopK<'a> { } if let Some(item) = self.output.as_mut().and_then(std::iter::Iterator::next) { + plan_arena.restore_outer_values(item.outer_values); arena.produce_tuple(item.tuple); } else { arena.finish(); @@ -197,6 +202,7 @@ mod test { use crate::execution::dql::top_k::{top_sort, CmpItem}; use crate::expression::ScalarExpression; use crate::planner::operator::sort::SortField; + use crate::planner::ExecMetaArena; use crate::types::tuple::Tuple; use crate::types::value::DataValue; use crate::types::LogicalType; @@ -213,6 +219,7 @@ mod test { key, sequence, tuple: Tuple::new(None, vec![DataValue::Int32(value)]), + outer_values: Vec::new(), } }; let first = make_item(0, 10); @@ -252,6 +259,7 @@ mod test { nulls_first, }] }; + let mut plan_arena = ExecMetaArena::new(plan_arena); let arena = Bump::new(); let mut key_scratch = crate::storage::table_codec::BumpBytes::new_in(&arena); @@ -313,7 +321,7 @@ mod test { &mut Tuple::new(None, vec![DataValue::Null]), 2, 0, - &plan_arena, + &mut plan_arena, )?; top_sort( &mut key_scratch, @@ -322,7 +330,7 @@ mod test { &mut Tuple::new(None, vec![DataValue::Int32(0)]), 2, 1, - &plan_arena, + &mut plan_arena, )?; top_sort( &mut key_scratch, @@ -331,7 +339,7 @@ mod test { &mut Tuple::new(None, vec![DataValue::Int32(1)]), 2, 2, - &plan_arena, + &mut plan_arena, )?; fn_asc_and_nulls_first_eq(indices); @@ -344,7 +352,7 @@ mod test { &mut Tuple::new(None, vec![DataValue::Null]), 2, 3, - &plan_arena, + &mut plan_arena, )?; top_sort( &mut key_scratch, @@ -353,7 +361,7 @@ mod test { &mut Tuple::new(None, vec![DataValue::Int32(0)]), 2, 4, - &plan_arena, + &mut plan_arena, )?; top_sort( &mut key_scratch, @@ -362,7 +370,7 @@ mod test { &mut Tuple::new(None, vec![DataValue::Int32(1)]), 2, 5, - &plan_arena, + &mut plan_arena, )?; fn_asc_and_nulls_last_eq(indices); @@ -375,7 +383,7 @@ mod test { &mut Tuple::new(None, vec![DataValue::Null]), 2, 6, - &plan_arena, + &mut plan_arena, )?; top_sort( &mut key_scratch, @@ -384,7 +392,7 @@ mod test { &mut Tuple::new(None, vec![DataValue::Int32(0)]), 2, 7, - &plan_arena, + &mut plan_arena, )?; top_sort( &mut key_scratch, @@ -393,7 +401,7 @@ mod test { &mut Tuple::new(None, vec![DataValue::Int32(1)]), 2, 8, - &plan_arena, + &mut plan_arena, )?; fn_desc_and_nulls_first_eq(indices); @@ -406,7 +414,7 @@ mod test { &mut Tuple::new(None, vec![DataValue::Null]), 2, 9, - &plan_arena, + &mut plan_arena, )?; top_sort( &mut key_scratch, @@ -415,7 +423,7 @@ mod test { &mut Tuple::new(None, vec![DataValue::Int32(0)]), 2, 10, - &plan_arena, + &mut plan_arena, )?; top_sort( &mut key_scratch, @@ -424,7 +432,7 @@ mod test { &mut Tuple::new(None, vec![DataValue::Int32(1)]), 2, 11, - &plan_arena, + &mut plan_arena, )?; fn_desc_and_nulls_last_eq(indices); @@ -468,6 +476,7 @@ mod test { }, ] }; + let mut plan_arena = ExecMetaArena::new(plan_arena); let arena = Bump::new(); let mut key_scratch = crate::storage::table_codec::BumpBytes::new_in(&arena); @@ -615,7 +624,7 @@ mod test { &mut Tuple::new(None, vec![DataValue::Null, DataValue::Null]), 4, 12, - &plan_arena, + &mut plan_arena, )?; top_sort( &mut key_scratch, @@ -624,7 +633,7 @@ mod test { &mut Tuple::new(None, vec![DataValue::Int32(0), DataValue::Null]), 4, 13, - &plan_arena, + &mut plan_arena, )?; top_sort( &mut key_scratch, @@ -633,7 +642,7 @@ mod test { &mut Tuple::new(None, vec![DataValue::Int32(1), DataValue::Null]), 4, 14, - &plan_arena, + &mut plan_arena, )?; top_sort( &mut key_scratch, @@ -642,7 +651,7 @@ mod test { &mut Tuple::new(None, vec![DataValue::Null, DataValue::Int32(0)]), 4, 15, - &plan_arena, + &mut plan_arena, )?; top_sort( &mut key_scratch, @@ -651,7 +660,7 @@ mod test { &mut Tuple::new(None, vec![DataValue::Int32(0), DataValue::Int32(0)]), 4, 16, - &plan_arena, + &mut plan_arena, )?; top_sort( &mut key_scratch, @@ -660,7 +669,7 @@ mod test { &mut Tuple::new(None, vec![DataValue::Int32(1), DataValue::Int32(0)]), 4, 17, - &plan_arena, + &mut plan_arena, )?; fn_asc_1_and_nulls_first_1_and_asc_2_and_nulls_first_2_eq(indices); @@ -673,7 +682,7 @@ mod test { &mut Tuple::new(None, vec![DataValue::Null, DataValue::Null]), 4, 18, - &plan_arena, + &mut plan_arena, )?; top_sort( &mut key_scratch, @@ -682,7 +691,7 @@ mod test { &mut Tuple::new(None, vec![DataValue::Int32(0), DataValue::Null]), 4, 19, - &plan_arena, + &mut plan_arena, )?; top_sort( &mut key_scratch, @@ -691,7 +700,7 @@ mod test { &mut Tuple::new(None, vec![DataValue::Int32(1), DataValue::Null]), 4, 20, - &plan_arena, + &mut plan_arena, )?; top_sort( &mut key_scratch, @@ -700,7 +709,7 @@ mod test { &mut Tuple::new(None, vec![DataValue::Null, DataValue::Int32(0)]), 4, 21, - &plan_arena, + &mut plan_arena, )?; top_sort( &mut key_scratch, @@ -709,7 +718,7 @@ mod test { &mut Tuple::new(None, vec![DataValue::Int32(0), DataValue::Int32(0)]), 4, 22, - &plan_arena, + &mut plan_arena, )?; top_sort( &mut key_scratch, @@ -718,7 +727,7 @@ mod test { &mut Tuple::new(None, vec![DataValue::Int32(1), DataValue::Int32(0)]), 4, 23, - &plan_arena, + &mut plan_arena, )?; fn_asc_1_and_nulls_last_1_and_asc_2_and_nulls_first_2_eq(indices); @@ -731,7 +740,7 @@ mod test { &mut Tuple::new(None, vec![DataValue::Null, DataValue::Null]), 4, 24, - &plan_arena, + &mut plan_arena, )?; top_sort( &mut key_scratch, @@ -740,7 +749,7 @@ mod test { &mut Tuple::new(None, vec![DataValue::Int32(0), DataValue::Null]), 4, 25, - &plan_arena, + &mut plan_arena, )?; top_sort( &mut key_scratch, @@ -749,7 +758,7 @@ mod test { &mut Tuple::new(None, vec![DataValue::Int32(1), DataValue::Null]), 4, 26, - &plan_arena, + &mut plan_arena, )?; top_sort( &mut key_scratch, @@ -758,7 +767,7 @@ mod test { &mut Tuple::new(None, vec![DataValue::Null, DataValue::Int32(0)]), 4, 27, - &plan_arena, + &mut plan_arena, )?; top_sort( &mut key_scratch, @@ -767,7 +776,7 @@ mod test { &mut Tuple::new(None, vec![DataValue::Int32(0), DataValue::Int32(0)]), 4, 28, - &plan_arena, + &mut plan_arena, )?; top_sort( &mut key_scratch, @@ -776,7 +785,7 @@ mod test { &mut Tuple::new(None, vec![DataValue::Int32(1), DataValue::Int32(0)]), 4, 29, - &plan_arena, + &mut plan_arena, )?; fn_desc_1_and_nulls_first_1_and_asc_2_and_nulls_first_2_eq(indices); @@ -789,7 +798,7 @@ mod test { &mut Tuple::new(None, vec![DataValue::Null, DataValue::Null]), 4, 30, - &plan_arena, + &mut plan_arena, )?; top_sort( &mut key_scratch, @@ -798,7 +807,7 @@ mod test { &mut Tuple::new(None, vec![DataValue::Int32(0), DataValue::Null]), 4, 31, - &plan_arena, + &mut plan_arena, )?; top_sort( &mut key_scratch, @@ -807,7 +816,7 @@ mod test { &mut Tuple::new(None, vec![DataValue::Int32(1), DataValue::Null]), 4, 32, - &plan_arena, + &mut plan_arena, )?; top_sort( &mut key_scratch, @@ -816,7 +825,7 @@ mod test { &mut Tuple::new(None, vec![DataValue::Null, DataValue::Int32(0)]), 4, 33, - &plan_arena, + &mut plan_arena, )?; top_sort( &mut key_scratch, @@ -825,7 +834,7 @@ mod test { &mut Tuple::new(None, vec![DataValue::Int32(0), DataValue::Int32(0)]), 4, 34, - &plan_arena, + &mut plan_arena, )?; top_sort( &mut key_scratch, @@ -834,7 +843,7 @@ mod test { &mut Tuple::new(None, vec![DataValue::Int32(1), DataValue::Int32(0)]), 4, 35, - &plan_arena, + &mut plan_arena, )?; fn_desc_1_and_nulls_last_1_and_asc_2_and_nulls_first_2_eq(indices); diff --git a/src/execution/dql/union.rs b/src/execution/dql/union.rs index 12374e68..1d409344 100644 --- a/src/execution/dql/union.rs +++ b/src/execution/dql/union.rs @@ -17,7 +17,7 @@ use crate::execution::{ build_read, ExecArena, ExecId, ExecNode, ExecutionContext, ExecutorNode, ReadExecutor, }; use crate::planner::LogicalPlan; -use crate::planner::MetaArena; +use crate::planner::{ExecMetaArena, MetaArena}; use crate::storage::Transaction; pub struct Union { left_input: ExecId, @@ -46,10 +46,10 @@ impl<'a, T: Transaction + 'a> ReadExecutor<'a, T> for Union { } impl<'a, T: Transaction + 'a> ExecutorNode<'a, T> for Union { - fn next_tuple( + fn next_tuple( &mut self, arena: &mut ExecArena<'a, T>, - plan_arena: &mut (dyn MetaArena + 'a), + plan_arena: &mut ExecMetaArena, ) -> Result<(), DatabaseError> { if self.reading_left { if arena.next_tuple(self.left_input, plan_arena)? { diff --git a/src/execution/dql/values.rs b/src/execution/dql/values.rs index d76df905..9d75aa43 100644 --- a/src/execution/dql/values.rs +++ b/src/execution/dql/values.rs @@ -16,9 +16,10 @@ use crate::errors::DatabaseError; use crate::execution::{ExecArena, ExecId, ExecNode, ExecutionContext, ExecutorNode, ReadExecutor}; use crate::planner::operator::values::ValuesOperator; use crate::planner::ExprRef; -use crate::planner::MetaArena; +use crate::planner::{ExecMetaArena, MetaArena}; use crate::storage::Transaction; use crate::types::tuple::Schema; +use crate::types::value::DataValue; pub struct Values<'a> { rows: std::slice::Iter<'a, ExprRef>, @@ -58,10 +59,10 @@ impl<'a, T: Transaction + 'a> ReadExecutor<'a, T> for Values<'a> { } impl<'a, T: Transaction + 'a> ExecutorNode<'a, T> for Values<'a> { - fn next_tuple( + fn next_tuple( &mut self, arena: &mut ExecArena<'a, T>, - plan_arena: &mut (dyn MetaArena + 'a), + plan_arena: &mut ExecMetaArena, ) -> Result<(), DatabaseError> { if self.remaining_rows == 0 { arena.finish(); @@ -78,7 +79,7 @@ impl<'a, T: Transaction + 'a> ExecutorNode<'a, T> for Values<'a> { output.values.push( plan_arena .expression(*expr) - .eval(plan_arena, None)? + .eval::<[DataValue]>(plan_arena, None)? .into_owned() .cast(ty)?, ); diff --git a/src/execution/dql/window.rs b/src/execution/dql/window.rs index cf5453da..32cc9aa1 100644 --- a/src/execution/dql/window.rs +++ b/src/execution/dql/window.rs @@ -20,7 +20,7 @@ use crate::expression::window::WindowFunctionKind; use crate::planner::operator::sort::SortField; use crate::planner::operator::window::WindowOperator; use crate::planner::LogicalPlan; -use crate::planner::MetaArena; +use crate::planner::{ExecMetaArena, MetaArena}; use crate::storage::Transaction; use crate::types::tuple::Tuple; use crate::types::value::DataValue; @@ -219,10 +219,10 @@ impl<'a> Window<'a> { } impl<'a, T: Transaction + 'a> ExecutorNode<'a, T> for Window<'a> { - fn next_tuple( + fn next_tuple( &mut self, arena: &mut ExecArena<'a, T>, - plan_arena: &mut (dyn MetaArena + 'a), + plan_arena: &mut ExecMetaArena, ) -> Result<(), DatabaseError> { let mut output_ready = true; loop { diff --git a/src/execution/mod.rs b/src/execution/mod.rs index f08780cd..a43b2421 100644 --- a/src/execution/mod.rs +++ b/src/execution/mod.rs @@ -27,7 +27,7 @@ use self::ddl::add_column::AddColumn; use self::ddl::change_column::ChangeColumn; use self::dql::join::nested_loop_join::NestedLoopJoin; use self::dql::mark_apply::MarkApply; -use self::dql::scalar_apply::ScalarApply; +use self::dql::scalar_query_init::ScalarQueryInit; use crate::db::{ScalaFunctions, TableFunctions}; use crate::errors::DatabaseError; use crate::execution::ddl::create_index::CreateIndex; @@ -76,7 +76,7 @@ use crate::expression::ScalarExpression; use crate::planner::operator::join::JoinCondition; use crate::planner::operator::{Operator, PhysicalOption, PlanImpl}; use crate::planner::MetaArena; -use crate::planner::{ExprRef, LogicalPlan, PlanKeeper}; +use crate::planner::{ExecMetaArena, ExprRef, LogicalPlan, PlanKeeper}; use crate::storage::table_codec::TableCodec; use crate::storage::{StatisticsMetaCache, TableCache, Transaction, ViewCache}; use crate::types::index::RuntimeIndexProbe; @@ -179,9 +179,9 @@ impl<'a, T: Transaction + 'a> Executor<'a, T> { } } - pub(crate) fn next_tuple( + pub(crate) fn next_tuple( &mut self, - plan_arena: &mut (dyn MetaArena + 'a), + plan_arena: &mut ExecMetaArena, ) -> Result, DatabaseError> { if !self.arena.next_tuple(self.root, plan_arena)? { return Ok(None); @@ -228,7 +228,7 @@ pub(crate) enum ExecNode<'a, T: Transaction + 'a> { Projection(Projection<'a>), RecursiveCte(RecursiveCte<'a, T>), RecursiveScan(RecursiveScan), - ScalarApply(ScalarApply), + ScalarQueryInit(ScalarQueryInit<'a>), ScalarSubquery(ScalarSubquery), SetMembership(SetMembership), SeqScan(SeqScan<'a, T>), @@ -247,18 +247,18 @@ pub(crate) enum ExecNode<'a, T: Transaction + 'a> { } pub(crate) trait ExecutorNode<'a, T: Transaction + 'a>: Sized { - fn next_tuple( + fn next_tuple( &mut self, arena: &mut ExecArena<'a, T>, - plan_arena: &mut (dyn MetaArena + 'a), + plan_arena: &mut ExecMetaArena, ) -> Result<(), DatabaseError>; } impl<'a, T: Transaction + 'a> ExecNode<'a, T> { - fn next_tuple( + fn next_tuple( &mut self, arena: &mut ExecArena<'a, T>, - plan_arena: &mut (dyn MetaArena + 'a), + plan_arena: &mut ExecMetaArena, ) -> Result<(), DatabaseError> { match self { ExecNode::AddColumn(exec) => { @@ -351,8 +351,8 @@ impl<'a, T: Transaction + 'a> ExecNode<'a, T> { ExecNode::RecursiveScan(exec) => { >::next_tuple(exec, arena, plan_arena) } - ExecNode::ScalarApply(exec) => { - >::next_tuple(exec, arena, plan_arena) + ExecNode::ScalarQueryInit(exec) => { + as ExecutorNode<'a, T>>::next_tuple(exec, arena, plan_arena) } ExecNode::ScalarSubquery(exec) => { >::next_tuple(exec, arena, plan_arena) @@ -638,11 +638,11 @@ impl<'a, T: Transaction + 'a> ExecArena<'a, T> { std::mem::take(&mut self.result.tuple) } - pub(crate) fn rewrite( + pub(crate) fn rewrite( &mut self, exprs: &[E], arena: &dyn MetaArena, - input: Option<&dyn TupleLike>, + input: Option<&TInput>, ) -> Result<(), DatabaseError> { let values = &mut self.result.tuple.values; let base = values.len(); @@ -651,10 +651,12 @@ impl<'a, T: Transaction + 'a> ExecArena<'a, T> { for expr in exprs { let value = { let input_values = &values[..base]; - let current: &dyn TupleLike = input.unwrap_or(&input_values); - expr.expression(arena) - .eval(arena, Some(current)) - .map(|value| value.into_owned()) + let expression = expr.expression(arena); + match input { + Some(input) => expression.eval(arena, Some(input)), + None => expression.eval(arena, Some(&input_values)), + } + .map(|value| value.into_owned()) }; match value { Ok(value) => values.push(value), @@ -686,10 +688,10 @@ impl<'a, T: Transaction + 'a> ExecArena<'a, T> { self.resume(); } - pub(crate) fn next_tuple( + pub(crate) fn next_tuple( &mut self, id: ExecId, - plan_arena: &mut (dyn MetaArena + 'a), + plan_arena: &mut ExecMetaArena, ) -> Result { self.result.status = None; let slot = &self.nodes.items[id] as *const std::cell::RefCell>; @@ -789,9 +791,9 @@ where } } Operator::Filter(op) => read!(Filter, (op, plan.childrens.only())), - Operator::ScalarApply(op) => { + Operator::ScalarQueryInit(op) => { let (left, right) = plan.childrens.twins(); - read!(ScalarApply, (op, left, right)) + read!(ScalarQueryInit<'a>, (op, left, right)) } Operator::MarkApply(op) => { let (left, right) = plan.childrens.twins(); @@ -869,7 +871,7 @@ pub(crate) fn build_write<'a, T>( plan_arena: &mut (dyn MetaArena + 'a), plan: &'a LogicalPlan, cache: ExecutionContext<'a>, - transaction: &'a mut T, + transaction: &'a T, ) -> ExecId where T: Transaction + 'a, @@ -889,6 +891,11 @@ where } match &plan.operator { + Operator::ScalarQueryInit(op) => { + let (input, init) = plan.childrens.twins(); + let input = build_write(arena, plan_arena, input, cache, transaction); + ScalarQueryInit::build(op, input, init, arena, plan_arena, cache, transaction) + } Operator::Insert(op) => write!(Insert<'a>, (op, plan.childrens.only())), Operator::Update(op) => write!(Update<'a>, (op, plan.childrens.only())), Operator::Delete(op) => write!(Delete<'a>, (op, plan.childrens.only())), @@ -942,7 +949,7 @@ mod test_utils { pub(crate) struct TestExecutor<'a, T: Transaction + 'a> { executor: Executor<'a, T>, - plan_arena: PlanArena<'a>, + plan_arena: ExecMetaArena>, } impl TestExecutor<'_, T> { @@ -971,8 +978,8 @@ mod test_utils { transaction, ); TestExecutor { + plan_arena: ExecMetaArena::new(plan_arena), executor: Executor::new(arena, root, PlanKeeper::empty()), - plan_arena, } } @@ -997,8 +1004,8 @@ mod test_utils { transaction, ); TestExecutor { + plan_arena: ExecMetaArena::new(plan_arena), executor: Executor::new(arena, root, PlanKeeper::empty()), - plan_arena, } } @@ -1022,13 +1029,14 @@ pub(crate) use test_utils::{empty_context, execute_input, execute_input_mut, try #[cfg(test)] mod test { use super::*; + use crate::planner::TableArenaCell; use crate::storage::memory::MemoryTransaction; use std::panic::{catch_unwind, AssertUnwindSafe}; #[test] fn active_nodes_cannot_be_overwritten_or_relocated() { - let table_arena = crate::planner::TableArenaCell::default(); - let mut plan_arena = crate::planner::PlanArena::new(&table_arena); + let table_arena = TableArenaCell::default(); + let mut plan_arena = ExecMetaArena::new(PlanArena::new(&table_arena)); let mut arena = ExecArena::<'_, MemoryTransaction>::with_capacity(0); arena.push(ExecNode::Dummy(Dummy::default())); let slot = diff --git a/src/execution/spill/codec.rs b/src/execution/spill/codec.rs index 15989af6..6dc2b89d 100644 --- a/src/execution/spill/codec.rs +++ b/src/execution/spill/codec.rs @@ -15,7 +15,7 @@ use super::SpillCodec; use crate::errors::DatabaseError; use crate::planner::operator::sort::SortField; -use crate::planner::MetaArena; +use crate::planner::{ExecMetaArena, MetaArena, ScalarQueryRef}; use crate::types::tuple::Tuple; use crate::types::value::DataValue; use std::io::{Read, Write}; @@ -24,13 +24,14 @@ use std::mem::size_of; pub(crate) struct SortRow { pub(crate) sort_values: Vec, pub(crate) tuple: Tuple, + pub(crate) outer_values: Vec<(ScalarQueryRef, DataValue)>, } impl SortRow { pub(crate) fn new( sort_fields: &[SortField], tuple: Tuple, - arena: &(dyn MetaArena + '_), + arena: &mut ExecMetaArena, ) -> Result { let sort_values = sort_fields .iter() @@ -41,25 +42,36 @@ impl SortRow { .map(|v| v.into_owned()) }) .collect::>()?; - Ok(Self { sort_values, tuple }) + Ok(Self { + sort_values, + tuple, + outer_values: arena.take_outer_values(), + }) } } impl SpillCodec for SortRow { fn encode(&self, writer: &mut W) -> Result<(), DatabaseError> { self.sort_values.encode(writer)?; - self.tuple.encode(writer) + self.tuple.encode(writer)?; + self.outer_values.encode(writer) } fn decode(reader: &mut R) -> Result { Ok(Self { sort_values: Vec::::decode(reader)?, tuple: Tuple::decode(reader)?, + outer_values: Vec::decode(reader)?, }) } fn estimated_size(&self) -> usize { size_of::() + .saturating_add( + self.outer_values + .estimated_size() + .saturating_sub(size_of::>()), + ) .saturating_add( self.sort_values .estimated_size() @@ -73,6 +85,59 @@ impl SpillCodec for SortRow { } } +impl SpillCodec for (A, B) { + fn encode(&self, writer: &mut W) -> Result<(), DatabaseError> { + self.0.encode(writer)?; + self.1.encode(writer) + } + + fn decode(reader: &mut R) -> Result { + Ok((A::decode(reader)?, B::decode(reader)?)) + } + + fn estimated_size(&self) -> usize { + size_of::() + .saturating_add(self.0.estimated_size().saturating_sub(size_of::())) + .saturating_add(self.1.estimated_size().saturating_sub(size_of::())) + } +} + +impl SpillCodec for ScalarQueryRef { + fn encode(&self, writer: &mut W) -> Result<(), DatabaseError> { + writer.write_all(&(self.arena_id as u64).to_le_bytes())?; + writer.write_all(&(self.pos as u64).to_le_bytes())?; + writer.write_all(&[u8::from(self.is_outer_value)])?; + Ok(()) + } + fn decode(reader: &mut R) -> Result { + let mut bytes = [0; 8]; + reader.read_exact(&mut bytes)?; + let arena_id = usize::try_from(u64::from_le_bytes(bytes))?; + reader.read_exact(&mut bytes)?; + let pos = usize::try_from(u64::from_le_bytes(bytes))?; + let mut marker = [0]; + reader.read_exact(&mut marker)?; + let is_outer_value = match marker[0] { + 0 => false, + 1 => true, + tag => { + return Err(DatabaseError::InvalidValue(format!( + "invalid scalar reference marker: {tag}" + ))) + } + }; + Ok(Self { + arena_id, + pos, + is_outer_value, + }) + } + + fn estimated_size(&self) -> usize { + size_of::() + } +} + impl SpillCodec for DataValue { fn encode(&self, writer: &mut W) -> Result<(), DatabaseError> { self.encode_reference_value(writer) diff --git a/src/expression/eq_col.rs b/src/expression/eq_col.rs index 73817db9..11fd3be8 100644 --- a/src/expression/eq_col.rs +++ b/src/expression/eq_col.rs @@ -22,8 +22,8 @@ use crate::expression::function::table::TableFunction; use crate::expression::visitor::{walk_expr, ExprVisitor}; use crate::expression::window::WindowCall; use crate::expression::{BinaryOperator, ScalarExpression, TrimWhereField, UnaryOperator}; -use crate::planner::ExprRef; use crate::planner::MetaArena; +use crate::planner::{ExprRef, ScalarQueryRef}; use crate::types::evaluator::{BinaryEvaluatorRef, CastEvaluatorRef, UnaryEvaluatorRef}; use crate::types::value::DataValue; use crate::types::LogicalType; @@ -83,6 +83,42 @@ impl ExprVisitor for EqIgnoreColRefPosVisitor<'_, '_> { walk_expr(self, lhs, arena) } + fn visit_outer_param( + &mut self, + id: ScalarQueryRef, + ty: &LogicalType, + _arena: &(dyn MetaArena + '_), + ) -> Result<(), DatabaseError> { + self.equal = matches!(self.rhs(), ScalarExpression::OuterParam { id: rhs, ty: rhs_ty } if id == *rhs && ty == rhs_ty); + Ok(()) + } + + fn visit_outer_value( + &mut self, + id: ScalarQueryRef, + ty: &LogicalType, + _arena: &(dyn MetaArena + '_), + ) -> Result<(), DatabaseError> { + self.equal = matches!(self.rhs(), ScalarExpression::OuterValue { id: rhs, ty: rhs_ty } if id == *rhs && ty == rhs_ty); + Ok(()) + } + + fn visit_init_value( + &mut self, + id: ScalarQueryRef, + ty: &LogicalType, + _arena: &(dyn MetaArena + '_), + ) -> Result<(), DatabaseError> { + self.equal = match self.rhs() { + ScalarExpression::InitValue { + id: rhs_id, + ty: rhs_ty, + } => id == *rhs_id && ty == rhs_ty, + _ => false, + }; + Ok(()) + } + fn visit_constant(&mut self, lhs: &DataValue) -> Result<(), DatabaseError> { self.equal = matches!(self.rhs(), ScalarExpression::Constant(rhs) if lhs == rhs); Ok(()) diff --git a/src/expression/evaluator.rs b/src/expression/evaluator.rs index b0be5fbd..7cffc4e0 100644 --- a/src/expression/evaluator.rs +++ b/src/expression/evaluator.rs @@ -41,10 +41,10 @@ macro_rules! eval_to_num { } impl ScalarExpression { - pub fn eval<'a>( + pub fn eval<'a, T: TupleLike + ?Sized>( &'a self, arena: &'a (dyn MetaArena + '_), - tuple: Option<&'a dyn TupleLike>, + tuple: Option<&'a T>, ) -> Result, DatabaseError> { match self { ScalarExpression::Constant(val) => match val { @@ -53,6 +53,14 @@ impl ScalarExpression { } val => Ok(Cow::Borrowed(val)), }, + ScalarExpression::InitValue { id, .. } + | ScalarExpression::OuterValue { id, .. } + | ScalarExpression::OuterParam { id, .. } => { + let value = arena.init_value(*id).ok_or_else(|| { + DatabaseError::InvalidValue(format!("scalar query {id} is not initialized")) + })?; + Ok(Cow::Borrowed(value)) + } ScalarExpression::ColumnRef { position, .. } => { let Some(tuple) = tuple else { return Ok(Cow::Owned(DataValue::Null)); @@ -80,12 +88,20 @@ impl ScalarExpression { } } ScalarExpression::Binary { + op, left_expr, right_expr, evaluator, .. } => { let left = arena.expression(*left_expr).eval(arena, tuple)?; + if matches!( + (op, left.as_ref()), + (BinaryOperator::And, DataValue::Boolean(false)) + | (BinaryOperator::Or, DataValue::Boolean(true)) + ) { + return Ok(left); + } let right = arena.expression(*right_expr).eval(arena, tuple)?; evaluator @@ -262,7 +278,7 @@ impl ScalarExpression { } ScalarExpression::ScalaFunction(ScalarFunction { inner, args, .. }) => { let value = match tuple { - Some(tuple) => inner.eval(args, arena, Some(tuple))?, + Some(tuple) => inner.eval(args, arena, Some(&tuple))?, None => inner.eval(args, arena, None)?, }; value.cast(inner.return_type()).map(Cow::Owned) @@ -401,7 +417,7 @@ fn trim_string(value: &str, trim_what: &str, trim_where: Option) mod tests { use super::*; use crate::planner::test::PlanArenaTestExt; - use crate::planner::ExprRef; + use crate::planner::{ExecMetaArena, ExprRef, TableArenaCell}; fn const_in( arena: &mut PlanArena<'_>, @@ -422,12 +438,45 @@ mod tests { }) } + #[test] + fn init_reads_execution_view_and_preserves_slot_position() -> Result<(), DatabaseError> { + use crate::expression::visitor_mut::{ExprVisitorMut, PositionShift}; + use crate::types::tuple::Tuple; + + let table_arena = TableArenaCell::default(); + let mut arena = PlanArena::new(&table_arena); + let reference = arena.alloc_scalar_query_ref(false); + let mut init = arena.alloc_expression(ScalarExpression::InitValue { + id: reference, + ty: LogicalType::Integer, + }); + PositionShift { delta: 5 }.visit(&mut init, &mut arena)?; + assert!(matches!( + arena.expression(init), + ScalarExpression::InitValue { id, ty: LogicalType::Integer } if *id == reference + )); + assert!(!init.any_referenced_column(&arena, |_, _| true)?); + let row = Tuple::new(None, vec![DataValue::Int32(99)]); + assert!(matches!( + arena.expression(init).eval(&arena, Some(&row)), + Err(DatabaseError::InvalidValue(message)) if message.contains("is not initialized") + )); + let mut view = ExecMetaArena::new(arena); + assert!(view.expression(init).eval(&view, Some(&row)).is_err()); + view.set_init_value(reference, DataValue::Null); + let value = view.expression(init).eval(&view, Some(&row))?; + assert!(matches!(value, Cow::Borrowed(DataValue::Null))); + let value = view.expression(init).eval::(&view, None)?; + assert!(matches!(value, Cow::Borrowed(DataValue::Null))); + Ok(()) + } + #[test] fn eval_borrows_leaf_values_and_owns_binary_results() -> Result<(), DatabaseError> { use crate::types::evaluator::binary_create; use crate::types::tuple::Tuple; - let table_arena = crate::planner::TableArenaCell::default(); + let table_arena = TableArenaCell::default(); let mut arena = PlanArena::new(&table_arena); let value = DataValue::Int64(42); let constant = arena.alloc_expression(ScalarExpression::Constant(value.clone())); @@ -472,7 +521,7 @@ mod tests { use crate::types::evaluator::cast_create; use crate::types::tuple::Tuple; - let table_arena = crate::planner::TableArenaCell::default(); + let table_arena = TableArenaCell::default(); let mut arena = PlanArena::new(&table_arena); let value = DataValue::Int32(7); let column_ref = arena.alloc_column(crate::catalog::ColumnCatalog::new( @@ -532,7 +581,7 @@ mod tests { #[test] fn in_eval_matches_even_if_null_appears_first() -> Result<(), DatabaseError> { - let table_arena = crate::planner::TableArenaCell::default(); + let table_arena = TableArenaCell::default(); let mut arena = PlanArena::new(&table_arena); let expr = const_in( &mut arena, @@ -542,7 +591,10 @@ mod tests { ); assert_eq!( - arena.expression(expr).eval(&arena, None)?.into_owned(), + arena + .expression(expr) + .eval::(&arena, None)? + .into_owned(), DataValue::Boolean(true) ); Ok(()) @@ -550,7 +602,7 @@ mod tests { #[test] fn in_eval_returns_null_when_only_null_blocks_non_match() -> Result<(), DatabaseError> { - let table_arena = crate::planner::TableArenaCell::default(); + let table_arena = TableArenaCell::default(); let mut arena = PlanArena::new(&table_arena); let expr = const_in( &mut arena, @@ -560,7 +612,10 @@ mod tests { ); assert_eq!( - arena.expression(expr).eval(&arena, None)?.into_owned(), + arena + .expression(expr) + .eval::(&arena, None)? + .into_owned(), DataValue::Null ); Ok(()) @@ -568,7 +623,7 @@ mod tests { #[test] fn not_in_eval_matches_even_if_null_appears_first() -> Result<(), DatabaseError> { - let table_arena = crate::planner::TableArenaCell::default(); + let table_arena = TableArenaCell::default(); let mut arena = PlanArena::new(&table_arena); let expr = const_in( &mut arena, @@ -578,7 +633,10 @@ mod tests { ); assert_eq!( - arena.expression(expr).eval(&arena, None)?.into_owned(), + arena + .expression(expr) + .eval::(&arena, None)? + .into_owned(), DataValue::Boolean(false) ); Ok(()) diff --git a/src/expression/mod.rs b/src/expression/mod.rs index f65f8ad5..e84a09a6 100644 --- a/src/expression/mod.rs +++ b/src/expression/mod.rs @@ -20,7 +20,7 @@ use crate::expression::function::table::TableFunction; use crate::expression::visitor::{walk_expr, ExprVisitor}; use crate::expression::visitor_mut::ExprVisitorMut; use crate::planner::operator::sort::SortField; -use crate::planner::{Explain, ExprRef, MetaArena}; +use crate::planner::{Explain, ExprRef, MetaArena, ScalarQueryRef}; use crate::types::evaluator::{ binary_create, cast_create, unary_create, BinaryEvaluatorRef, CastEvaluatorRef, UnaryEvaluatorRef, @@ -66,6 +66,18 @@ pub enum AliasType { #[derive(Debug, PartialEq, Eq, Clone, Hash, ReferenceSerialization)] pub enum ScalarExpression { Constant(DataValue), + OuterParam { + id: ScalarQueryRef, + ty: LogicalType, + }, + OuterValue { + id: ScalarQueryRef, + ty: LogicalType, + }, + InitValue { + id: ScalarQueryRef, + ty: LogicalType, + }, ColumnRef { column: ColumnRef, position: usize, @@ -481,6 +493,9 @@ impl TypeCast for ScalarExpression { } ScalarExpression::Position { .. } => Cow::Owned(LogicalType::Integer), ScalarExpression::Alias { expr, .. } => expr.return_type(arena), + ScalarExpression::InitValue { ty, .. } + | ScalarExpression::OuterValue { ty, .. } + | ScalarExpression::OuterParam { ty, .. } => Cow::Borrowed(ty), ScalarExpression::Empty | ScalarExpression::TableFunction(_) => unreachable!(), ScalarExpression::Tuple(exprs) => Cow::Owned(LogicalType::Tuple( exprs @@ -551,6 +566,9 @@ impl Explain for ExprRef { match arena.expression(*self) { ScalarExpression::Constant(value) => write!(f, "{value}"), + ScalarExpression::InitValue { id, .. } => write!(f, "InitValue({id})"), + ScalarExpression::OuterValue { id, .. } => write!(f, "OuterValue({id})"), + ScalarExpression::OuterParam { id, .. } => write!(f, "OuterParam({id})"), ScalarExpression::ColumnRef { column, .. } => Explain::fmt(column, arena, f), ScalarExpression::Alias { alias, expr } => match alias { AliasType::Name(alias) => f.write_str(alias), @@ -1084,6 +1102,7 @@ mod test { use crate::expression::{AliasType, BinaryOperator, ScalarExpression, UnaryOperator}; use crate::function::current_date::CurrentDate; use crate::function::numbers::Numbers; + use crate::planner::operator::sort::SortField; use crate::planner::{ExprRef, MetaArena, PlanArena, TableArenaCell}; use crate::serdes::{ReferenceDecodeContext, ReferenceSerialization, ReferenceTables}; use crate::storage::rocksdb::RocksStorage; @@ -1544,7 +1563,7 @@ mod test { }, spec: WindowSpec { partition_by: vec![two], - order_by: vec![crate::planner::operator::sort::SortField::from(three).desc()], + order_by: vec![SortField::from(three).desc()], }, }), Some(&context), diff --git a/src/expression/range_detacher.rs b/src/expression/range_detacher.rs index 7f0ebbde..3ac5f5fb 100644 --- a/src/expression/range_detacher.rs +++ b/src/expression/range_detacher.rs @@ -88,20 +88,18 @@ impl TreeNode { fn add_child(&mut self, child: TreeNode) { self.children.push(child); } -} -impl TreeNode { - fn enumeration(self, path: &mut Vec, combinations: &mut Vec>) { + fn enumeration(self, path: &mut Vec, consume: &mut impl FnMut(&[T])) { if self.value.is_none() && self.children.is_empty() { - combinations.push(path.clone()); + consume(path); } for mut child in self.children { if let Some(val) = child.value.take() { path.push(val); - Self::enumeration(child, path, combinations); + Self::enumeration(child, path, consume); let _ = path.pop(); } else { - Self::enumeration(child, path, combinations); + Self::enumeration(child, path, consume); } } } @@ -290,15 +288,11 @@ impl Range { } let node = build_tree(eqs, 0)?; - let mut combinations = Vec::new(); - - node.enumeration(&mut Vec::new(), &mut combinations); - + let mut path = Vec::with_capacity(eqs.len()); let mut ranges = Vec::new(); - - for tuple in combinations { - collect_tuple_range(&mut ranges, &tuple, self.clone()) - } + node.enumeration(&mut path, &mut |tuple| { + collect_tuple_range(&mut ranges, tuple, self.clone()); + }); Some(RangeDetacher::::ranges2range(ranges)) } } @@ -330,6 +324,12 @@ impl RangeColumnMatcher for IndexRangeColumn { } } +enum Placement { + Before, + After, + Overlap, +} + pub struct RangeDetacher< 'a, M: RangeColumnMatcher = IndexRangeColumn, @@ -524,7 +524,10 @@ impl<'a, M: RangeColumnMatcher, A: MetaArena + ?Sized> RangeDetacher<'a, M, A> { None } - ScalarExpression::Constant(_) + ScalarExpression::OuterParam { .. } + | ScalarExpression::OuterValue { .. } + | ScalarExpression::InitValue { .. } + | ScalarExpression::Constant(_) | ScalarExpression::Alias { .. } | ScalarExpression::TypeCast { .. } | ScalarExpression::IsNull { .. } @@ -547,7 +550,11 @@ impl<'a, M: RangeColumnMatcher, A: MetaArena + ?Sized> RangeDetacher<'a, M, A> { | ScalarExpression::TableFunction(_) | ScalarExpression::Empty => unreachable!(), }, - ScalarExpression::Constant(_) | ScalarExpression::ColumnRef { .. } => None, + ScalarExpression::OuterParam { .. } + | ScalarExpression::OuterValue { .. } + | ScalarExpression::InitValue { .. } + | ScalarExpression::Constant(_) + | ScalarExpression::ColumnRef { .. } => None, // FIXME: support [RangeDetacher::_detach] ScalarExpression::Unary { .. } | ScalarExpression::In { .. } @@ -687,11 +694,12 @@ impl<'a, M: RangeColumnMatcher, A: MetaArena + ?Sized> RangeDetacher<'a, M, A> { Ok(Range::Eq(unpack_bound(bound_eq))) } BinaryOperator::Or => { + // NULL sorts after every value, so only an unbounded `max` already covers it. if eq.is_null() { - return Ok(if matches!(min, Bound::Excluded(_)) { - Range::SortedRanges(vec![Range::Eq(eq), Range::Scope { min, max }]) - } else { + return Ok(if matches!(max, Bound::Unbounded) { Range::Scope { min, max } + } else { + Range::SortedRanges(vec![Range::Scope { min, max }, Range::Eq(eq)]) }); } let bound_eq = Bound::Excluded(eq); @@ -739,9 +747,7 @@ impl<'a, M: RangeColumnMatcher, A: MetaArena + ?Sized> RangeDetacher<'a, M, A> { if bounds_have_parameter(&[&min, &max]) || ranges.iter().any(Range::has_parameter) { return Err((Range::Scope { min, max }, Range::SortedRanges(ranges))); } - let merged_ranges = - Self::extract_merge_ranges(op, Some(Range::Scope { min, max }), ranges, &mut 0); - Ok(Self::ranges2range(merged_ranges)) + Self::merge_ranges(op, Range::Scope { min, max }, ranges) } // e.g. (c1 = 1 or c1 = 2) ? c1 > 1 (Range::SortedRanges(ranges), Range::Scope { min, max }) => { @@ -779,9 +785,7 @@ impl<'a, M: RangeColumnMatcher, A: MetaArena + ?Sized> RangeDetacher<'a, M, A> { if eq.has_parameter() || ranges.iter().any(Range::has_parameter) { return Err((Range::Eq(eq), Range::SortedRanges(ranges))); } - let merged_ranges = - Self::extract_merge_ranges(op, Some(Range::Eq(eq)), ranges, &mut 0); - Ok(Self::ranges2range(merged_ranges)) + Self::merge_ranges(op, Range::Eq(eq), ranges) } // e.g. (c1 = 1 or c1 = 2) ? c1 = 1 (Range::SortedRanges(ranges), Range::Eq(eq)) => { @@ -789,7 +793,7 @@ impl<'a, M: RangeColumnMatcher, A: MetaArena + ?Sized> RangeDetacher<'a, M, A> { .map_err(|(right, left)| (left, right)) } // e.g. (c1 = 1 or c1 = 2) ? (c1 = 1 or c1 = 2) - (Range::SortedRanges(left_ranges), Range::SortedRanges(mut right_ranges)) => { + (Range::SortedRanges(left_ranges), Range::SortedRanges(right_ranges)) => { if left_ranges.iter().any(Range::has_parameter) || right_ranges.iter().any(Range::has_parameter) { @@ -798,12 +802,7 @@ impl<'a, M: RangeColumnMatcher, A: MetaArena + ?Sized> RangeDetacher<'a, M, A> { Range::SortedRanges(right_ranges), )); } - let mut idx = 0; - for left_range in left_ranges { - right_ranges = - Self::extract_merge_ranges(op, Some(left_range), right_ranges, &mut idx) - } - Ok(Self::ranges2range(right_ranges)) + Self::merge_ranges(op, Range::SortedRanges(left_ranges), right_ranges) } } } @@ -818,132 +817,176 @@ impl<'a, M: RangeColumnMatcher, A: MetaArena + ?Sized> RangeDetacher<'a, M, A> { } } - #[allow(unreachable_code)] - fn extract_merge_ranges( + #[allow(clippy::result_large_err)] + fn merge_ranges( op: BinaryOperator, - mut binary: Option, + left: Range, mut ranges: Vec, - idx: &mut usize, - ) -> Vec { - // FIXME: Lots of duplicate code - while *idx < ranges.len() { - match (&binary, &ranges[*idx]) { - ( - Some(Range::Scope { - min: l_min, - max: l_max, - }), - Range::Scope { - min: r_min, - max: r_max, - }, - ) => { - if let Some(true) = - Self::bound_compared(l_max, r_min, true, false).map(Ordering::is_lt) - { - ranges.insert(*idx, binary.unwrap()); - return ranges; - } else if let Some(true) = - Self::bound_compared(l_min, r_max, false, true).map(Ordering::is_gt) - { - *idx += 1; - continue; - } else { - binary = Self::merge_binary(op, binary.unwrap(), ranges.remove(*idx)).ok(); - } - } - ( - Some(Range::Scope { - min: l_min, - max: l_max, - }), - Range::Eq(r_val), - ) => { - let r_bound = Bound::Included(r_val); - - if let Some(true) = - Self::bound_compared(l_max, &r_bound, true, false).map(Ordering::is_lt) - { - ranges.insert(*idx, binary.unwrap()); - return ranges; - } else if Self::bound_compared(l_min, &r_bound, false, true) - .map(Ordering::is_gt) - .unwrap_or_else(|| op == BinaryOperator::Or) - { - *idx += 1; - continue; - } else if r_val.is_null() { - let _ = ranges.remove(*idx); - } else { - binary = Self::merge_binary(op, binary.unwrap(), ranges.remove(*idx)).ok(); + ) -> Result { + match op { + BinaryOperator::And => Self::intersect_ranges(left, ranges), + BinaryOperator::Or => { + Self::union_ranges(left, &mut ranges); + Ok(Self::ranges2range(ranges)) + } + _ => Err((left, Range::SortedRanges(ranges))), + } + } + + fn placement(op: BinaryOperator, left: &Range, right: &Range) -> Option { + let placement = match (left, right) { + ( + Range::Scope { + min: l_min, + max: l_max, + }, + Range::Scope { + min: r_min, + max: r_max, + }, + ) => Self::bound_compared(l_max, r_min, true, false) + .is_some_and(Ordering::is_lt) + .then_some(Placement::Before) + .or_else(|| { + Self::bound_compared(l_min, r_max, false, true) + .is_some_and(Ordering::is_gt) + .then_some(Placement::After) + }), + ( + Range::Scope { + min: l_min, + max: l_max, + }, + Range::Eq(r_val), + ) => { + let r_bound = Bound::Included(r_val); + Self::bound_compared(l_max, &r_bound, true, false) + .is_some_and(Ordering::is_lt) + .then_some(Placement::Before) + .or_else(|| { + Self::bound_compared(l_min, &r_bound, false, true) + .map(Ordering::is_gt) + .unwrap_or_else(|| op == BinaryOperator::Or) + .then_some(Placement::After) + }) + } + (Range::Eq(l_val), Range::Eq(r_val)) => match Self::bound_compared( + &Bound::Included(l_val), + &Bound::Included(r_val), + false, + false, + ) { + Some(Ordering::Less) => Some(Placement::Before), + Some(Ordering::Greater) => Some(Placement::After), + _ => None, + }, + ( + Range::Eq(l_val), + Range::Scope { + min: r_min, + max: r_max, + }, + ) => { + let l_bound = Bound::Included(l_val); + Self::bound_compared(&l_bound, r_min, true, false) + .map(Ordering::is_lt) + .unwrap_or_else(|| op == BinaryOperator::Or) + .then_some(Placement::Before) + .or_else(|| { + Self::bound_compared(&l_bound, r_max, false, true) + .is_some_and(Ordering::is_gt) + .then_some(Placement::After) + }) + } + _ => return None, + }; + Some(placement.unwrap_or(Placement::Overlap)) + } + + #[allow(clippy::result_large_err)] + fn intersect_ranges(left: Range, mut ranges: Vec) -> Result { + let len = ranges.len(); + let l_ranges = match &left { + Range::SortedRanges(l_ranges) => l_ranges.as_slice(), + single => std::slice::from_ref(single), + }; + let mut start = 0; + for l_range in l_ranges { + let mut idx = start; + while idx < len { + let Some(placement) = Self::placement(BinaryOperator::And, l_range, &ranges[idx]) + else { + ranges.truncate(len); + return Err((left, Range::SortedRanges(ranges))); + }; + match placement { + Placement::After => { + idx += 1; + start = idx; } - } - (Some(Range::Eq(l_val)), Range::Eq(r_val)) => { - if let Some(true) = l_val.partial_cmp(r_val).map(Ordering::is_lt) { - ranges.insert(*idx, binary.unwrap()); - return ranges; - } else if let Some(true) = l_val.partial_cmp(r_val).map(Ordering::is_gt) { - *idx += 1; - continue; - } else { - binary = Self::merge_binary(op, binary.unwrap(), ranges.remove(*idx)).ok(); + Placement::Before => break, + Placement::Overlap => { + match Self::merge_binary( + BinaryOperator::And, + l_range.clone(), + ranges[idx].clone(), + ) { + Ok(Range::Dummy) => (), + Ok(range) => ranges.push(range), + Err(_) => { + ranges.truncate(len); + return Err((left, Range::SortedRanges(ranges))); + } + } + idx += 1; } } - ( - Some(Range::Eq(l_val)), - Range::Scope { - min: r_min, - max: r_max, - }, - ) => { - let l_bound = Bound::Included(l_val); - - if Self::bound_compared(&l_bound, r_min, true, false) - .map(Ordering::is_lt) - .unwrap_or_else(|| op == BinaryOperator::Or) - { - ranges.insert(*idx, binary.unwrap()); - return ranges; - } else if let Some(true) = - Self::bound_compared(&l_bound, r_max, false, true).map(Ordering::is_gt) - { - *idx += 1; - continue; - } else if l_val.is_null() { - binary = Some(ranges.remove(*idx)); - } else { - binary = Self::merge_binary(op, binary.unwrap(), ranges.remove(*idx)).ok(); - } + } + } + drop(ranges.drain(..len)); + Ok(Self::ranges2range(ranges)) + } + + fn union_ranges(binary: Range, ranges: &mut Vec) { + match binary { + Range::Dummy => (), + Range::SortedRanges(l_ranges) => { + for l_range in l_ranges { + Self::union_ranges(l_range, ranges); } - (Some(Range::Dummy), _) => { - binary = match op { - BinaryOperator::And => return vec![], - BinaryOperator::Or => Some(ranges.remove(*idx)), - _ => None, - }; + } + piece => Self::union_piece(piece, ranges), + } + } + + #[allow(unreachable_code)] + fn union_piece(mut binary: Range, ranges: &mut Vec) { + let mut idx = 0; + while idx < ranges.len() { + let Some(placement) = Self::placement(BinaryOperator::Or, &binary, &ranges[idx]) else { + #[cfg(debug_assertions)] + { + unreachable!(); } - (Some(Range::SortedRanges(l_ranges)), r_range) => { - return Self::extract_merge_ranges( - op, - Some(r_range.clone()), - l_ranges.clone(), - &mut 0, - ); + ranges.clear(); + return; + }; + match placement { + Placement::Before => { + ranges.insert(idx, binary); + return; } - (None, _) => break, - _ => { - #[cfg(debug_assertions)] - { - unreachable!(); + Placement::After => idx += 1, + Placement::Overlap => { + match Self::merge_binary(BinaryOperator::Or, binary, ranges.remove(idx)) { + Ok(range @ (Range::Scope { .. } | Range::Eq(_))) => binary = range, + Ok(range) => return Self::union_ranges(range, ranges), + Err(_) => return, } - return vec![]; } } } - if let Some(range) = binary { - ranges.push(range); - } - ranges + ranges.push(binary); } fn or_scope_merge( @@ -2788,6 +2831,35 @@ mod test { ]), ]; let combined = suffix.combining_eqs(&prefixes).unwrap(); + let eq = |value| Range::Eq(DataValue::Int32(value)); + assert_eq!( + eq(9).combining_eqs(&[ + Range::SortedRanges(vec![eq(1), eq(2)]), + Range::SortedRanges(vec![eq(3), eq(4)]), + ]), + Some(Range::SortedRanges(vec![ + Range::Eq(DataValue::Tuple(vec![ + DataValue::Int32(1), + DataValue::Int32(3), + DataValue::Int32(9) + ])), + Range::Eq(DataValue::Tuple(vec![ + DataValue::Int32(1), + DataValue::Int32(4), + DataValue::Int32(9) + ])), + Range::Eq(DataValue::Tuple(vec![ + DataValue::Int32(2), + DataValue::Int32(3), + DataValue::Int32(9) + ])), + Range::Eq(DataValue::Tuple(vec![ + DataValue::Int32(2), + DataValue::Int32(4), + DataValue::Int32(9) + ])), + ])) + ); let first = DataValue::Int32(1); let second = DataValue::Int32(2); let borrowed = [Range::Eq(&first), Range::Eq(&second)]; @@ -2865,4 +2937,56 @@ mod test { .is_none()); assert!(!suffix.only_eq()); } + + /// AND over a union must intersect every piece instead of keeping the disjoint ones. + #[test] + fn test_detach_and_over_or() -> Result<(), DatabaseError> { + let table_state = build_t1_table()?; + let mut plan_arena = crate::planner::PlanArena::new(&table_state.table_arena); + let eq = |v| Range::Eq(DataValue::Int32(v)); + let cases = [ + ("c1 > 10 and (c1 = 1 or c1 = 20)", eq(20)), + // `c1 > 3` must still be applied after it has been intersected with `c1 = 5`. + ( + "c1 > 3 and (c1 = 5 or c1 = 7)", + Range::SortedRanges(vec![eq(5), eq(7)]), + ), + ("(c1 = 1 or c1 = 20) and (c1 = 5 or c1 = 20)", eq(20)), + // One `rj` overlapping several `li` must be intersected with each of them. + ( + "(c1 = 5 or c1 = 7) and (c1 < 10 or c1 > 20)", + Range::SortedRanges(vec![eq(5), eq(7)]), + ), + ( + "(c1 < 10 or c1 > 20) and (c1 = 5 or c1 = 7 or c1 = 15 or c1 = 25)", + Range::SortedRanges(vec![eq(5), eq(7), eq(25)]), + ), + ( + "c1 > 3 and (c1 < 2 or c1 > 30)", + Range::Scope { + min: Bound::Excluded(DataValue::Int32(30)), + max: Bound::Unbounded, + }, + ), + ("c1 = 1 and (c1 > 10 or c1 < 3)", eq(1)), + ( + "(c1 > 5 or c1 < 0) and (c1 = -1 or c1 = 3 or c1 = 7)", + Range::SortedRanges(vec![eq(-1), eq(7)]), + ), + ("(c1 = 1 or c1 = 2) and (c1 = 3 or c1 = 4)", Range::Dummy), + ("c1 = 3 and (c1 between 5 and 2 or c1 > 9)", Range::Dummy), + ]; + for (predicate, expected) in cases { + let sql = format!("select * from t1 where {predicate}"); + let plan = table_state.plan_with_arena(&sql, &mut plan_arena)?; + let op = plan_filter(plan, &mut plan_arena)?.unwrap(); + let detached = + RangeDetacher::new("t1", table_state.column_id_by_name("c1"), &mut plan_arena) + .detach(op.predicate)? + .expect("c1 predicate should be consumed"); + assert_eq!(detached.range, expected, "{predicate}"); + assert_eq!(detached.residual, None, "{predicate}"); + } + Ok(()) + } } diff --git a/src/expression/visitor.rs b/src/expression/visitor.rs index 3f4f586f..e65c02ac 100644 --- a/src/expression/visitor.rs +++ b/src/expression/visitor.rs @@ -21,7 +21,7 @@ use crate::expression::window::WindowCall; use crate::expression::{ AliasType, BinaryOperator, ScalarExpression, TrimWhereField, UnaryOperator, }; -use crate::planner::{ExprRef, MetaArena}; +use crate::planner::{ExprRef, MetaArena, ScalarQueryRef}; use crate::types::evaluator::{BinaryEvaluatorRef, CastEvaluatorRef, UnaryEvaluatorRef}; use crate::types::value::DataValue; use crate::types::LogicalType; @@ -49,6 +49,33 @@ pub trait ExprVisitor: Sized { Ok(true) } + fn visit_init_value( + &mut self, + _id: ScalarQueryRef, + _ty: &LogicalType, + _arena: &A, + ) -> Result<(), DatabaseError> { + Ok(()) + } + + fn visit_outer_param( + &mut self, + _id: ScalarQueryRef, + _ty: &LogicalType, + _arena: &A, + ) -> Result<(), DatabaseError> { + Ok(()) + } + + fn visit_outer_value( + &mut self, + _id: ScalarQueryRef, + _ty: &LogicalType, + _arena: &A, + ) -> Result<(), DatabaseError> { + Ok(()) + } + fn visit_constant(&mut self, _value: &DataValue) -> Result<(), DatabaseError> { Ok(()) } @@ -294,6 +321,9 @@ pub fn walk_expr>( ) -> Result<(), DatabaseError> { match arena.expression(expr) { ScalarExpression::Constant(value) => visitor.visit_constant(value), + ScalarExpression::OuterParam { id, ty } => visitor.visit_outer_param(*id, ty, arena), + ScalarExpression::OuterValue { id, ty } => visitor.visit_outer_value(*id, ty, arena), + ScalarExpression::InitValue { id, ty } => visitor.visit_init_value(*id, ty, arena), ScalarExpression::ColumnRef { column, .. } => visitor.visit_column_ref(column), ScalarExpression::Alias { expr, alias } => visitor.visit_alias(*expr, alias, arena), ScalarExpression::TypeCast { diff --git a/src/expression/visitor_mut.rs b/src/expression/visitor_mut.rs index 439d9e4a..41c0cb66 100644 --- a/src/expression/visitor_mut.rs +++ b/src/expression/visitor_mut.rs @@ -21,8 +21,8 @@ use crate::expression::window::WindowCall; use crate::expression::{ AliasType, BinaryOperator, ScalarExpression, TrimWhereField, UnaryOperator, }; -use crate::planner::ExprRef; use crate::planner::MetaArena; +use crate::planner::{ExprRef, ScalarQueryRef}; use crate::types::evaluator::{BinaryEvaluatorRef, CastEvaluatorRef, UnaryEvaluatorRef}; use crate::types::value::DataValue; use crate::types::LogicalType; @@ -61,6 +61,33 @@ impl ExprVisitorMut for PositionShift { } pub trait ExprVisitorMut: Sized { + fn visit_outer_param( + &mut self, + _id: &mut ScalarQueryRef, + _ty: &mut LogicalType, + _arena: &mut dyn MetaArena, + ) -> Result<(), DatabaseError> { + Ok(()) + } + + fn visit_outer_value( + &mut self, + _id: &mut ScalarQueryRef, + _ty: &mut LogicalType, + _arena: &mut dyn MetaArena, + ) -> Result<(), DatabaseError> { + Ok(()) + } + + fn visit_init_value( + &mut self, + _id: &mut ScalarQueryRef, + _ty: &mut LogicalType, + _arena: &mut dyn MetaArena, + ) -> Result<(), DatabaseError> { + Ok(()) + } + fn visit( &mut self, expr: &mut ExprRef, @@ -395,6 +422,9 @@ pub fn walk_mut_expr( std::mem::replace(&mut *arena.expression_mut(*expr), ScalarExpression::Empty); let result = match &mut expression { ScalarExpression::Constant(value) => visitor.visit_constant(value, arena), + ScalarExpression::OuterParam { id, ty } => visitor.visit_outer_param(id, ty, arena), + ScalarExpression::OuterValue { id, ty } => visitor.visit_outer_value(id, ty, arena), + ScalarExpression::InitValue { id, ty } => visitor.visit_init_value(id, ty, arena), ScalarExpression::ColumnRef { column, position } => { visitor.visit_column_ref(column, position, arena) } diff --git a/src/function/abs.rs b/src/function/abs.rs new file mode 100644 index 00000000..9f5cd34c --- /dev/null +++ b/src/function/abs.rs @@ -0,0 +1,91 @@ +// Copyright 2024 KipData/KiteSQL +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use crate::errors::DatabaseError; +use crate::expression::function::scala::FuncMonotonicity; +use crate::expression::function::scala::ScalarFunctionImpl; +use crate::expression::function::FunctionSummary; +use crate::planner::ExprRef; +use crate::planner::MetaArena; +use crate::types::tuple::TupleLike; +use crate::types::value::DataValue; +use crate::types::LogicalType; +use ordered_float::OrderedFloat; +use std::sync::Arc; + +#[derive(Debug)] +pub(crate) struct Abs { + summary: FunctionSummary, + return_type: LogicalType, +} + +impl Abs { + pub(crate) fn new(ty: LogicalType) -> Arc { + Arc::new(Self { + summary: FunctionSummary { + name: "abs".into(), + arg_types: vec![ty.clone()], + }, + return_type: ty, + }) + } + + pub(crate) fn types() -> [LogicalType; 10] { + [ + LogicalType::Tinyint, + LogicalType::Smallint, + LogicalType::Integer, + LogicalType::Bigint, + LogicalType::UTinyint, + LogicalType::USmallint, + LogicalType::UInteger, + LogicalType::UBigint, + LogicalType::Float, + LogicalType::Double, + ] + } +} + +impl ScalarFunctionImpl for Abs { + fn eval( + &self, + exprs: &[ExprRef], + arena: &(dyn MetaArena + '_), + tuples: Option<&dyn TupleLike>, + ) -> Result { + let value = arena.expression(exprs[0]).eval(arena, tuples)?; + let overflow = || DatabaseError::OverFlow; + Ok(match value.as_ref() { + DataValue::Int8(v) => DataValue::Int8(v.checked_abs().ok_or_else(overflow)?), + DataValue::Int16(v) => DataValue::Int16(v.checked_abs().ok_or_else(overflow)?), + DataValue::Int32(v) => DataValue::Int32(v.checked_abs().ok_or_else(overflow)?), + DataValue::Int64(v) => DataValue::Int64(v.checked_abs().ok_or_else(overflow)?), + DataValue::Float32(v) => DataValue::Float32(OrderedFloat(v.0.abs())), + DataValue::Float64(v) => DataValue::Float64(OrderedFloat(v.0.abs())), + _ => value.into_owned(), + }) + } + + fn monotonicity(&self) -> Option { + None + } + + fn return_type(&self) -> &LogicalType { + &self.return_type + } + + fn summary(&self) -> &FunctionSummary { + &self.summary + } +} diff --git a/src/function/mod.rs b/src/function/mod.rs index 77982a03..9fe10775 100644 --- a/src/function/mod.rs +++ b/src/function/mod.rs @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +pub(crate) mod abs; pub(crate) mod char_length; #[cfg(feature = "time")] pub(crate) mod current_date; diff --git a/src/function/numbers.rs b/src/function/numbers.rs index f5c844e8..6613ad45 100644 --- a/src/function/numbers.rs +++ b/src/function/numbers.rs @@ -50,7 +50,10 @@ impl TableFunctionImpl for Numbers { args: &[ExprRef], arena: &(dyn MetaArena + '_), ) -> Result>>, DatabaseError> { - let mut value = arena.expression(args[0]).eval(arena, None)?.into_owned(); + let mut value = arena + .expression(args[0]) + .eval::<[DataValue]>(arena, None)? + .into_owned(); value = value.cast(&LogicalType::Integer)?; let num = value diff --git a/src/macros/mod.rs b/src/macros/mod.rs index 9db99307..18a87a1d 100644 --- a/src/macros/mod.rs +++ b/src/macros/mod.rs @@ -179,7 +179,7 @@ macro_rules! table_function { let mut _index = 0; $closure($({ - let mut value = arena.expression(args[_index]).eval(arena, None)?.into_owned(); + let mut value = arena.expression(args[_index]).eval::<[::kite_sql::types::value::DataValue]>(arena, None)?.into_owned(); _index += 1; value = value.cast(&$arg_ty)?; diff --git a/src/optimizer/heuristic/optimizer.rs b/src/optimizer/heuristic/optimizer.rs index a95778e9..b0b455c9 100644 --- a/src/optimizer/heuristic/optimizer.rs +++ b/src/optimizer/heuristic/optimizer.rs @@ -512,9 +512,14 @@ impl ImplementationRuleIndex { Operator::Project(_) if self.contains(ImplementationRuleImpl::Projection) => { Some(PhysicalOption::new(PlanImpl::Project, SortOption::Follow)) } - Operator::ScalarApply(_) if self.contains(ImplementationRuleImpl::ScalarApply) => Some( - PhysicalOption::new(PlanImpl::ScalarApply, SortOption::Follow), - ), + Operator::ScalarQueryInit(_) + if self.contains(ImplementationRuleImpl::ScalarQueryInit) => + { + Some(PhysicalOption::new( + PlanImpl::ScalarQueryInit, + SortOption::Follow, + )) + } Operator::ScalarSubquery(_) if self.contains(ImplementationRuleImpl::ScalarSubquery) => { diff --git a/src/optimizer/rule/implementation/dql/mod.rs b/src/optimizer/rule/implementation/dql/mod.rs index 9258b248..1d96ea47 100644 --- a/src/optimizer/rule/implementation/dql/mod.rs +++ b/src/optimizer/rule/implementation/dql/mod.rs @@ -20,7 +20,7 @@ pub(crate) mod join; pub(crate) mod limit; pub(crate) mod mark_apply; pub(crate) mod projection; -pub(crate) mod scalar_apply; +pub(crate) mod scalar_query_init; pub(crate) mod scalar_subquery; pub(crate) mod sort; pub(crate) mod table_scan; diff --git a/src/optimizer/rule/implementation/dql/scalar_apply.rs b/src/optimizer/rule/implementation/dql/scalar_query_init.rs similarity index 84% rename from src/optimizer/rule/implementation/dql/scalar_apply.rs rename to src/optimizer/rule/implementation/dql/scalar_query_init.rs index 55151508..9be962ba 100644 --- a/src/optimizer/rule/implementation/dql/scalar_apply.rs +++ b/src/optimizer/rule/implementation/dql/scalar_query_init.rs @@ -22,14 +22,14 @@ use crate::single_mapping; use std::sync::LazyLock; static SCALAR_APPLY_PATTERN: LazyLock = LazyLock::new(|| Pattern { - predicate: |op| matches!(op, Operator::ScalarApply(_)), + predicate: |op| matches!(op, Operator::ScalarQueryInit(_)), }); #[derive(Clone)] -pub struct ScalarApplyImplementation; +pub struct ScalarQueryInitImplementation; single_mapping!( - ScalarApplyImplementation, + ScalarQueryInitImplementation, SCALAR_APPLY_PATTERN, - PhysicalOption::new(PlanImpl::ScalarApply, SortOption::Follow) + PhysicalOption::new(PlanImpl::ScalarQueryInit, SortOption::Follow) ); diff --git a/src/optimizer/rule/implementation/mod.rs b/src/optimizer/rule/implementation/mod.rs index 6f9fc067..f31ae4cc 100644 --- a/src/optimizer/rule/implementation/mod.rs +++ b/src/optimizer/rule/implementation/mod.rs @@ -45,7 +45,7 @@ use crate::optimizer::rule::implementation::dql::join::JoinImplementation; use crate::optimizer::rule::implementation::dql::limit::LimitImplementation; use crate::optimizer::rule::implementation::dql::mark_apply::MarkApplyImplementation; use crate::optimizer::rule::implementation::dql::projection::ProjectionImplementation; -use crate::optimizer::rule::implementation::dql::scalar_apply::ScalarApplyImplementation; +use crate::optimizer::rule::implementation::dql::scalar_query_init::ScalarQueryInitImplementation; use crate::optimizer::rule::implementation::dql::scalar_subquery::ScalarSubqueryImplementation; use crate::optimizer::rule::implementation::dql::sort::SortImplementation; use crate::optimizer::rule::implementation::dql::table_scan::{ @@ -66,7 +66,7 @@ pub enum ImplementationRuleRootTag { Limit, MarkApply, Project, - ScalarApply, + ScalarQueryInit, ScalarSubquery, TableScan, FunctionScan, @@ -102,7 +102,7 @@ impl ImplementationRuleRootTag { Operator::Limit(_) => Some(Self::Limit), Operator::MarkApply(_) => Some(Self::MarkApply), Operator::Project(_) => Some(Self::Project), - Operator::ScalarApply(_) => Some(Self::ScalarApply), + Operator::ScalarQueryInit(_) => Some(Self::ScalarQueryInit), Operator::ScalarSubquery(_) => Some(Self::ScalarSubquery), Operator::TableScan(_) => Some(Self::TableScan), Operator::FunctionScan(_) => Some(Self::FunctionScan), @@ -150,7 +150,7 @@ pub enum ImplementationRuleImpl { Limit, MarkApply, Projection, - ScalarApply, + ScalarQueryInit, ScalarSubquery, SeqScan, FunctionScan, @@ -188,7 +188,7 @@ impl MatchPattern for ImplementationRuleImpl { ImplementationRuleImpl::Limit => LimitImplementation.pattern(), ImplementationRuleImpl::MarkApply => MarkApplyImplementation.pattern(), ImplementationRuleImpl::Projection => ProjectionImplementation.pattern(), - ImplementationRuleImpl::ScalarApply => ScalarApplyImplementation.pattern(), + ImplementationRuleImpl::ScalarQueryInit => ScalarQueryInitImplementation.pattern(), ImplementationRuleImpl::ScalarSubquery => ScalarSubqueryImplementation.pattern(), ImplementationRuleImpl::SeqScan => SeqScanImplementation.pattern(), ImplementationRuleImpl::IndexScan => IndexScanImplementation.pattern(), @@ -227,7 +227,7 @@ impl ImplementationRuleImpl { ImplementationRuleImpl::Limit => ImplementationRuleRootTag::Limit, ImplementationRuleImpl::MarkApply => ImplementationRuleRootTag::MarkApply, ImplementationRuleImpl::Projection => ImplementationRuleRootTag::Project, - ImplementationRuleImpl::ScalarApply => ImplementationRuleRootTag::ScalarApply, + ImplementationRuleImpl::ScalarQueryInit => ImplementationRuleRootTag::ScalarQueryInit, ImplementationRuleImpl::ScalarSubquery => ImplementationRuleRootTag::ScalarSubquery, ImplementationRuleImpl::SeqScan | ImplementationRuleImpl::IndexScan => { ImplementationRuleRootTag::TableScan @@ -278,7 +278,7 @@ impl ImplementationRule for ImplementationRuleImpl { ImplementationRuleImpl::Limit => update!(LimitImplementation), ImplementationRuleImpl::MarkApply => update!(MarkApplyImplementation), ImplementationRuleImpl::Projection => update!(ProjectionImplementation), - ImplementationRuleImpl::ScalarApply => update!(ScalarApplyImplementation), + ImplementationRuleImpl::ScalarQueryInit => update!(ScalarQueryInitImplementation), ImplementationRuleImpl::ScalarSubquery => update!(ScalarSubqueryImplementation), ImplementationRuleImpl::SeqScan => update!(SeqScanImplementation), ImplementationRuleImpl::IndexScan => update!(IndexScanImplementation), @@ -443,9 +443,9 @@ mod tests { )?; assert_sql_rule( "select (select c3 from t2 limit 1) from t1", - ImplementationRuleImpl::ScalarApply, - |op| matches!(op, Operator::ScalarApply(_)), - PlanImpl::ScalarApply, + ImplementationRuleImpl::ScalarQueryInit, + |op| matches!(op, Operator::ScalarQueryInit(_)), + PlanImpl::ScalarQueryInit, )?; assert_sql_rule( "select (select c3 from t2 limit 1) from t1", diff --git a/src/optimizer/rule/normalization/column_pruning.rs b/src/optimizer/rule/normalization/column_pruning.rs index a17a1007..6653cbf9 100644 --- a/src/optimizer/rule/normalization/column_pruning.rs +++ b/src/optimizer/rule/normalization/column_pruning.rs @@ -518,7 +518,7 @@ impl ColumnPruning { } Operator::Sort(_) | Operator::Limit(_) - | Operator::ScalarApply(_) + | Operator::ScalarQueryInit(_) | Operator::MarkApply(_) | Operator::ScalarSubquery(_) | Operator::Join(_) @@ -527,11 +527,44 @@ impl ColumnPruning { | Operator::SetMembership(_) | Operator::TopK(_) | Operator::Window(_) => { - if matches!(operator, Operator::ScalarApply(_) | Operator::MarkApply(_)) { - let mut child_required = required_columns; - Self::extend_operator_referenced_columns(operator, &mut child_required, arena)?; + if matches!(operator, Operator::ScalarQueryInit(_)) { + let Childrens::Twins { left, right } = childrens else { + unreachable!("scalar initialization requires two children"); + }; + Self::extend_operator_referenced_columns( + operator, + &mut required_columns, + arena, + )?; + let left_start = outcome.removed_positions.len(); + Self::_apply_appending(required_columns, all_referenced, left, outcome, arena)?; + if outcome.changed { + Self::remap_operator_after_child_change( + operator, + &outcome.removed_positions[left_start..], + &mut outcome.remapped_exprs, + arena, + )?; + } + changed |= outcome.changed; + let main_removed_end = outcome.removed_positions.len(); + Self::_apply_appending( + ReferencedColumns::default(), + true, + right, + outcome, + arena, + )?; + changed |= outcome.changed; + outcome.removed_positions.truncate(main_removed_end); + } else if matches!(operator, Operator::MarkApply(_)) { + Self::extend_operator_referenced_columns( + operator, + &mut required_columns, + arena, + )?; changed |= Self::apply_twins( - child_required, + required_columns, true, childrens, outcome, @@ -541,10 +574,9 @@ impl ColumnPruning { outcome.removed_positions.truncate(output_start); } else if matches!(operator, Operator::Join(_)) { let (old_left_outputs_len, left_removed_start, right_removed_start) = { - let mut child_required = required_columns; Self::extend_operator_referenced_columns( operator, - &mut child_required, + &mut required_columns, arena, )?; let old_left_outputs_len = match childrens { @@ -559,7 +591,7 @@ impl ColumnPruning { let left_removed_start = outcome.removed_positions.len(); Self::_apply_appending( - child_required.clone(), + required_columns.clone(), all_referenced, left.as_mut(), outcome, @@ -568,7 +600,7 @@ impl ColumnPruning { let left_changed = outcome.changed; let right_removed_start = outcome.removed_positions.len(); Self::_apply_appending( - child_required, + required_columns, all_referenced, right.as_mut(), outcome, @@ -657,10 +689,13 @@ impl ColumnPruning { outcome.removed_positions.truncate(output_start); } } else if matches!(operator, Operator::Union(_) | Operator::SetMembership(_)) { - let mut child_required = required_columns; - Self::extend_operator_referenced_columns(operator, &mut child_required, arena)?; + Self::extend_operator_referenced_columns( + operator, + &mut required_columns, + arena, + )?; changed |= Self::apply_twins( - child_required, + required_columns, all_referenced, childrens, outcome, @@ -671,14 +706,13 @@ impl ColumnPruning { } else { let child_start = outcome.removed_positions.len(); let child_changed = { - let mut child_required = required_columns; Self::extend_operator_referenced_columns( operator, - &mut child_required, + &mut required_columns, arena, )?; Self::apply_only_child( - child_required, + required_columns, all_referenced, childrens, outcome, diff --git a/src/optimizer/rule/normalization/compilation_in_advance.rs b/src/optimizer/rule/normalization/compilation_in_advance.rs index 7423f969..1fe28b46 100644 --- a/src/optimizer/rule/normalization/compilation_in_advance.rs +++ b/src/optimizer/rule/normalization/compilation_in_advance.rs @@ -38,7 +38,7 @@ impl EvaluatorBind { Self::_apply(left, arena)?; let bind_right = matches!( plan.operator, - Operator::ScalarApply(_) + Operator::ScalarQueryInit(_) | Operator::MarkApply(_) | Operator::Join(_) | Operator::Union(_) diff --git a/src/optimizer/rule/normalization/mod.rs b/src/optimizer/rule/normalization/mod.rs index f6ff5ad8..87eea173 100644 --- a/src/optimizer/rule/normalization/mod.rs +++ b/src/optimizer/rule/normalization/mod.rs @@ -108,7 +108,7 @@ impl NormalizationRuleRootTag { match operator { Operator::Aggregate(_) => Some(Self::Aggregate), Operator::MarkApply(_) => Some(Self::MarkApply), - Operator::ScalarApply(_) => Some(Self::Any), + Operator::ScalarQueryInit(_) => Some(Self::Any), Operator::Filter(_) => Some(Self::Filter), Operator::Join(_) => Some(Self::Join), Operator::Limit(_) => Some(Self::Limit), diff --git a/src/optimizer/rule/normalization/pushdown_predicates.rs b/src/optimizer/rule/normalization/pushdown_predicates.rs index 8eb9ade8..16fd7c0a 100644 --- a/src/optimizer/rule/normalization/pushdown_predicates.rs +++ b/src/optimizer/rule/normalization/pushdown_predicates.rs @@ -19,6 +19,7 @@ use crate::expression::{BinaryOperator, ScalarExpression}; use crate::optimizer::core::rule::NormalizationRule; use crate::optimizer::plan_utils::{replace_with_only_child, wrap_child_with}; use crate::planner::operator::filter::FilterOperator; +use crate::planner::operator::join::extract_join_keys; use crate::planner::operator::join::{JoinCondition, JoinType}; use crate::planner::operator::{Operator, SortOption}; use crate::planner::{Childrens, ExprRef, LogicalPlan, PlanArena}; @@ -159,17 +160,57 @@ impl NormalizationRule for PushPredicateThroughJoin { if !matches!( join_type, - JoinType::Inner | JoinType::LeftOuter | JoinType::RightOuter + JoinType::Inner | JoinType::Cross | JoinType::LeftOuter | JoinType::RightOuter ) { return Ok(false); } - let (left_filter, mut right_filter, common_filter, left_len) = + let (left_filter, mut right_filter, mut common_filter, left_len) = classify_join_filters(filter_op.predicate, join_plan.childrens.as_mut(), arena)?; + if join_type == JoinType::Cross { + if let Some(predicate) = common_filter { + let Childrens::Twins { left, right } = join_plan.childrens.as_mut() else { + return Ok(false); + }; + let left_schema = left.output_schema(arena).clone(); + let right_schema = right.output_schema(arena).clone(); + let mut keys = Vec::new(); + let mut residual = Vec::new(); + extract_join_keys( + predicate, + &mut keys, + &mut residual, + &left_schema, + &right_schema, + arena, + )?; + if !keys.is_empty() { + for (_, right) in &mut keys { + // WHERE uses joined positions; join keys use each input's local positions. + *right = right.clone_expression(arena)?; + PositionShift { + delta: -(left_len as isize), + } + .visit(right, arena)?; + } + if let Operator::Join(op) = &mut join_plan.operator { + op.join_type = JoinType::Inner; + op.on = JoinCondition::On { + on: keys, + filter: None, + }; + } + common_filter = reduce_filters(residual, filter_op.having, arena) + .map(|op| op.predicate); + applied = true; + } + } + } + let mut new_ops = (None, None, None); match join_type { - JoinType::Inner => { + JoinType::Inner | JoinType::Cross => { if let Some(left_filter_op) = reduce_filters(left_filter, filter_op.having, arena) { @@ -1036,6 +1077,39 @@ mod tests { Ok(()) } + #[test] + fn cross_join_without_conjunctive_keys_stays_cross() -> Result<(), DatabaseError> { + let tables = build_t1_table()?; + for predicate in ["c1 = c3 or c2 = c4", "c1 < c3", "c1 > 0 and c3 < 9"] { + let mut arena = PlanArena::new(&tables.table_arena); + let plan = tables.plan_with_arena( + &format!("select * from t1, t2 where {predicate}"), + &mut arena, + )?; + let plan = apply_pipeline( + plan, + HepOptimizerPipeline::builder().before_batch( + "cross join pushdown".into(), + HepBatchStrategy::once_topdown(), + vec![NormalizationRuleImpl::PushPredicateThroughJoin], + ), + &mut arena, + )?; + let child = plan.childrens.only(); + let join = if matches!(child.operator, Operator::Filter(_)) { + child.childrens.only() + } else { + child + }; + let Operator::Join(op) = &join.operator else { + panic!("expected join") + }; + assert_eq!(op.join_type, JoinType::Cross, "{predicate}"); + assert!(matches!(op.on, JoinCondition::None)); + } + Ok(()) + } + #[test] fn test_push_predicate_through_join_in_inner_join() -> Result<(), DatabaseError> { let table_state = build_t1_table()?; diff --git a/src/orm/mod.rs b/src/orm/mod.rs index da34d1e0..dc1d9a80 100644 --- a/src/orm/mod.rs +++ b/src/orm/mod.rs @@ -1198,8 +1198,8 @@ where }; build(&mut context)? }; - if child_binder.context.has_outer_refs() { - self.binder.context.mark_outer_ref(); + if child_binder.context.has_join_outer_ref() { + self.binder.context.mark_join_outer_ref(); } Ok(plan) } @@ -3709,7 +3709,7 @@ mod tests { len: "4".to_string(), nullable: true, key: "PRI".to_string(), - default: "null".to_string(), + default: "NULL".to_string(), } ); assert!(tuple diff --git a/src/planner/arena.rs b/src/planner/arena.rs index 4c801b85..cf6716a7 100644 --- a/src/planner/arena.rs +++ b/src/planner/arena.rs @@ -14,20 +14,102 @@ use crate::catalog::{ColumnCatalog, ColumnRef, TableName}; use crate::errors::DatabaseError; +use crate::expression::visitor_mut::ExprVisitorMut; use crate::expression::ScalarExpression; -use crate::planner::LogicalPlan; +use crate::planner::operator::scalar_query_init::ScalarQueryInitOperator; +use crate::planner::operator::visitor_mut::{ + walk_mut_operator, OperatorExprVisitorMut, OperatorVisitorMut, +}; +use crate::planner::operator::{Operator, PhysicalOption}; +use crate::planner::{LogicalPlan, ScalarQueryRef}; use crate::types::index::{IndexMeta, IndexMetaRef}; use crate::types::tuple::Schema; use crate::types::value::DataValue; +use crate::types::LogicalType; use std::cell::UnsafeCell; -use std::collections::HashSet; +use std::collections::{HashMap, HashSet}; use std::fmt; +struct ScalarQueryRelocator { + source_arena: usize, + target_arena: usize, + base: usize, +} + +impl ScalarQueryRelocator { + fn relocate(&self, reference: &mut ScalarQueryRef) { + if reference.arena_id == self.source_arena { + reference.arena_id = self.target_arena; + reference.pos += self.base; + } + } +} + +impl ExprVisitorMut for ScalarQueryRelocator { + fn visit_outer_param( + &mut self, + reference: &mut ScalarQueryRef, + _ty: &mut LogicalType, + _arena: &mut dyn MetaArena, + ) -> Result<(), DatabaseError> { + self.relocate(reference); + Ok(()) + } + + fn visit_outer_value( + &mut self, + reference: &mut ScalarQueryRef, + _ty: &mut LogicalType, + _arena: &mut dyn MetaArena, + ) -> Result<(), DatabaseError> { + self.relocate(reference); + Ok(()) + } + + fn visit_init_value( + &mut self, + reference: &mut ScalarQueryRef, + _ty: &mut LogicalType, + _arena: &mut dyn MetaArena, + ) -> Result<(), DatabaseError> { + self.relocate(reference); + Ok(()) + } +} + +struct ScalarQueryPlanRelocator<'a> { + references: ScalarQueryRelocator, + arena: &'a mut dyn MetaArena, +} + +impl<'a> OperatorVisitorMut<'a> for ScalarQueryPlanRelocator<'_> { + fn visit_operator( + &mut self, + operator: &'a mut Operator, + physical_option: Option<&'a mut PhysicalOption>, + ) -> Result<(), DatabaseError> { + OperatorExprVisitorMut::new(&mut self.references, self.arena) + .visit_operator(operator, physical_option)?; + walk_mut_operator(self, operator) + } + + fn visit_scalar_query_init( + &mut self, + op: &'a mut ScalarQueryInitOperator, + ) -> Result<(), DatabaseError> { + for (reference, _) in &mut op.param_bindings { + self.references.relocate(reference); + } + Ok(()) + } +} + pub struct TableArena { dummy_columns: [ColumnCatalog; DUMMY_COLUMN_COUNT], columns: Vec, indexes: Vec, expressions: Vec, + scalar_query_count: usize, version: usize, } @@ -62,6 +144,8 @@ pub struct PlanArena<'a> { table_arena_version: usize, allocated_columns_len: usize, temp_table_id: usize, + arena_id: usize, + scalar_query_count: usize, columns: Vec, indexes: Vec, expressions: Vec, @@ -153,6 +237,16 @@ pub trait MetaArena { None } + fn arena_id(&self) -> usize; + + fn alloc_scalar_query_ref(&mut self, is_outer_value: bool) -> ScalarQueryRef; + + fn reserve_scalar_query_ref(&mut self, reference: ScalarQueryRef); + + fn init_value(&self, _reference: ScalarQueryRef) -> Option<&DataValue> { + None + } + fn alloc_dummy(&mut self, name: &str) -> ColumnRef { self.table_arena_cell().borrow().alloc_dummy(name) } @@ -206,6 +300,18 @@ impl MetaArena for Box { fn bound_param(&self, id: usize) -> Option<&DataValue> { (**self).bound_param(id) } + fn arena_id(&self) -> usize { + (**self).arena_id() + } + fn alloc_scalar_query_ref(&mut self, is_outer_value: bool) -> ScalarQueryRef { + (**self).alloc_scalar_query_ref(is_outer_value) + } + fn reserve_scalar_query_ref(&mut self, reference: ScalarQueryRef) { + (**self).reserve_scalar_query_ref(reference) + } + fn init_value(&self, reference: ScalarQueryRef) -> Option<&DataValue> { + (**self).init_value(reference) + } fn alloc_column(&mut self, column: ColumnCatalog) -> ColumnRef { (**self).alloc_column(column) } @@ -235,6 +341,106 @@ impl MetaArena for Box { } } +/// Owns the metadata and scalar-query cache for one statement execution. +pub struct ExecMetaArena { + parent: A, + init_values: HashMap>, +} + +impl ExecMetaArena { + pub(crate) fn new(parent: A) -> Self { + Self { + parent, + init_values: HashMap::new(), + } + } + + pub(crate) fn boxed<'a>(self) -> ExecMetaArena> + where + A: 'a, + { + ExecMetaArena { + parent: Box::new(self.parent), + init_values: self.init_values, + } + } + + pub(crate) fn into_parent(self) -> A { + self.parent + } + + pub(crate) fn set_init_value(&mut self, reference: ScalarQueryRef, value: DataValue) { + self.init_values.insert(reference, Some(value)); + } + + pub(crate) fn take_outer_values(&mut self) -> Vec<(ScalarQueryRef, DataValue)> { + self.init_values + .extract_if(|reference, _| reference.is_outer_value) + .filter_map(|(reference, value)| value.map(|value| (reference, value))) + .collect() + } + + pub(crate) fn restore_outer_values(&mut self, values: Vec<(ScalarQueryRef, DataValue)>) { + for (reference, value) in values { + self.set_init_value(reference, value); + } + } +} + +impl MetaArena for ExecMetaArena { + fn arena_id(&self) -> usize { + self.parent.arena_id() + } + fn alloc_scalar_query_ref(&mut self, is_outer_value: bool) -> ScalarQueryRef { + self.parent.alloc_scalar_query_ref(is_outer_value) + } + fn reserve_scalar_query_ref(&mut self, reference: ScalarQueryRef) { + self.parent.reserve_scalar_query_ref(reference) + } + fn init_value(&self, reference: ScalarQueryRef) -> Option<&DataValue> { + self.init_values.get(&reference).and_then(Option::as_ref) + } + fn table_arena_cell<'a>(&self) -> &'a TableArenaCell + where + Self: 'a, + { + self.parent.table_arena_cell() + } + fn has_bound_params(&self) -> bool { + self.parent.has_bound_params() + } + fn bound_param(&self, id: usize) -> Option<&DataValue> { + self.parent.bound_param(id) + } + fn expression_mut(&mut self, expr: ExprRef) -> ArenaExprMut<'_> { + self.parent.expression_mut(expr) + } + fn alloc_column(&mut self, column: ColumnCatalog) -> ColumnRef { + self.parent.alloc_column(column) + } + fn alloc_index(&mut self, index: IndexMeta) -> IndexMetaRef { + self.parent.alloc_index(index) + } + fn alloc_expression(&mut self, expression: ScalarExpression) -> ExprRef { + self.parent.alloc_expression(expression) + } + fn column(&self, column: ColumnRef) -> &ColumnCatalog { + self.parent.column(column) + } + fn index(&self, index: IndexMetaRef) -> &IndexMeta { + self.parent.index(index) + } + fn expression(&self, expr: ExprRef) -> &ScalarExpression { + self.parent.expression(expr) + } + fn find_column(&self, column: &ColumnCatalog) -> Option { + self.parent.find_column(column) + } + fn find_index(&self, index: &IndexMeta) -> Option { + self.parent.find_index(index) + } +} + const DUMMY_COLUMN_NAMES: [&str; DUMMY_COLUMN_COUNT] = [ "TABLE", "VIEW", @@ -297,6 +503,7 @@ impl Default for TableArena { columns: Vec::new(), indexes: Vec::new(), expressions: Vec::new(), + scalar_query_count: 0, version: 0, } } @@ -396,6 +603,22 @@ impl fmt::Debug for TableArena { } impl MetaArena for TableArena { + fn arena_id(&self) -> usize { + 0 + } + fn alloc_scalar_query_ref(&mut self, is_outer_value: bool) -> ScalarQueryRef { + let reference = ScalarQueryRef { + is_outer_value, + arena_id: 0, + pos: self.scalar_query_count, + }; + self.scalar_query_count += 1; + reference + } + fn reserve_scalar_query_ref(&mut self, reference: ScalarQueryRef) { + assert_eq!(reference.arena_id, 0); + self.scalar_query_count = self.scalar_query_count.max(reference.pos + 1); + } fn alloc_column(&mut self, column: ColumnCatalog) -> ColumnRef { if let Some(column_ref) = self.find_column(&column) { return column_ref; @@ -526,6 +749,8 @@ impl<'a> PlanArena<'a> { table_arena_version, allocated_columns_len: 0, temp_table_id: 0, + arena_id: table_arena.borrow().arena_id() + 1, + scalar_query_count: 0, columns: Vec::new(), indexes: Vec::new(), expressions: Vec::new(), @@ -589,6 +814,26 @@ impl<'a> PlanArena<'a> { table_arena.increment_version(); } + /// Moves scalar definitions with a cached plan into the persistent namespace. + pub(crate) fn materialize_scalar_queries( + &mut self, + plan: &mut LogicalPlan, + ) -> Result<(), DatabaseError> { + self.assert_table_arena_unchanged(); + let target = self.table_arena.borrow_mut(); + let references = ScalarQueryRelocator { + source_arena: self.arena_id, + target_arena: target.arena_id(), + base: target.scalar_query_count, + }; + target.scalar_query_count += self.scalar_query_count; + ScalarQueryPlanRelocator { + references, + arena: self, + } + .visit_plan(plan) + } + pub(crate) fn materialize_into_table_arena(&self) { self.assert_table_arena_unchanged(); @@ -715,6 +960,26 @@ impl<'a> PlanArena<'a> { } impl MetaArena for PlanArena<'_> { + fn arena_id(&self) -> usize { + self.arena_id + } + fn alloc_scalar_query_ref(&mut self, is_outer_value: bool) -> ScalarQueryRef { + let reference = ScalarQueryRef { + is_outer_value, + arena_id: self.arena_id, + pos: self.scalar_query_count, + }; + self.scalar_query_count += 1; + reference + } + fn reserve_scalar_query_ref(&mut self, reference: ScalarQueryRef) { + if reference.arena_id == self.arena_id { + self.scalar_query_count = self.scalar_query_count.max(reference.pos + 1); + } else { + assert_eq!(reference.arena_id, 0); + assert!(reference.pos < self.table_arena.borrow().scalar_query_count); + } + } fn table_arena_cell<'a>(&self) -> &'a TableArenaCell where Self: 'a, @@ -820,6 +1085,7 @@ pub(crate) struct ParamArena<'a> { expressions: Vec<(Option, ArenaExpr)>, parameter_count: usize, parent_end: usize, + scalar_query_count: usize, } impl<'a> ParamArena<'a> { @@ -850,11 +1116,29 @@ impl<'a> ParamArena<'a> { expressions, parameter_count: parameter_expressions.len(), parent_end: parent.expression_end(), + scalar_query_count: 0, }) } } impl MetaArena for ParamArena<'_> { + fn arena_id(&self) -> usize { + self.parent.arena_id() + 1 + } + fn alloc_scalar_query_ref(&mut self, is_outer_value: bool) -> ScalarQueryRef { + let reference = ScalarQueryRef { + is_outer_value, + arena_id: self.arena_id(), + pos: self.scalar_query_count, + }; + self.scalar_query_count += 1; + reference + } + fn reserve_scalar_query_ref(&mut self, reference: ScalarQueryRef) { + if reference.arena_id == self.arena_id() { + self.scalar_query_count = self.scalar_query_count.max(reference.pos + 1); + } + } fn table_arena_cell<'a>(&self) -> &'a TableArenaCell where Self: 'a, @@ -931,6 +1215,74 @@ mod tests { use crate::types::LogicalType; use std::panic::{catch_unwind, AssertUnwindSafe}; + #[test] + fn sorting_moves_only_outer_results() { + let catalog = TableArenaCell::default(); + let mut plan = PlanArena::new(&catalog); + let init = plan.alloc_scalar_query_ref(false); + let param = plan.alloc_scalar_query_ref(false); + let result = plan.alloc_scalar_query_ref(true); + let mut arena = ExecMetaArena::new(plan); + arena.set_init_value(init, DataValue::Int32(7)); + arena.set_init_value(param, DataValue::Int32(3)); + arena.set_init_value(result, DataValue::Null); + let values = arena.take_outer_values(); + assert_eq!(values, vec![(result, DataValue::Null)]); + assert_eq!(arena.init_value(init), Some(&DataValue::Int32(7))); + assert_eq!(arena.init_value(param), Some(&DataValue::Int32(3))); + assert_eq!(arena.init_value(result), None); + arena.restore_outer_values(values); + assert_eq!(arena.init_value(result), Some(&DataValue::Null)); + } + + #[test] + fn scalar_query_namespaces_follow_arena_parent() -> Result<(), DatabaseError> { + let root = TableArenaCell::default(); + let catalog = root.borrow_mut().alloc_scalar_query_ref(false); + assert_eq!( + catalog, + ScalarQueryRef { + is_outer_value: false, + arena_id: 0, + pos: 0 + } + ); + let mut plan = PlanArena::new(&root); + let local = plan.alloc_scalar_query_ref(false); + assert_eq!( + local, + ScalarQueryRef { + is_outer_value: false, + arena_id: 1, + pos: 0 + } + ); + assert_ne!(catalog, local); + let mut params = ParamArena::new(&plan, &[], &[])?; + assert_eq!(params.arena_id(), 2); + let param_local = params.alloc_scalar_query_ref(false); + assert_eq!( + param_local, + ScalarQueryRef { + is_outer_value: false, + arena_id: 2, + pos: 0 + } + ); + let mut view = ExecMetaArena::new(params); + view.set_init_value(catalog, DataValue::Int32(11)); + assert_eq!(view.arena_id(), 2); + assert_eq!(view.init_value(catalog), Some(&DataValue::Int32(11))); + assert_eq!(view.init_value(local), None); + assert_eq!(view.init_value(param_local), None); + view.set_init_value(local, DataValue::Null); + assert_eq!(view.init_value(local), Some(&DataValue::Null)); + view.set_init_value(local, DataValue::Int32(33)); + assert_eq!(view.init_value(catalog), Some(&DataValue::Int32(11))); + assert_eq!(view.init_value(local), Some(&DataValue::Int32(33))); + Ok(()) + } + fn column(name: &str) -> ColumnCatalog { ColumnCatalog::new( name.to_string(), diff --git a/src/planner/mod.rs b/src/planner/mod.rs index d1ed25d2..855277e2 100644 --- a/src/planner/mod.rs +++ b/src/planner/mod.rs @@ -15,6 +15,7 @@ mod arena; pub mod operator; mod plan_keeper; +mod scalar_query_ref; use crate::catalog::TableName; use crate::errors::DatabaseError; @@ -29,9 +30,10 @@ use kite_sql_serde_macros::ReferenceSerialization; use std::fmt; use std::hash::{Hash, Hasher}; +pub(crate) use arena::{ExecMetaArena, ParamArena, PlanRef}; pub use arena::{ExprRef, MetaArena, PlanArena, TableArena, TableArenaCell}; -pub(crate) use arena::{ParamArena, PlanRef}; pub(crate) use plan_keeper::{PlanInput, PlanKeeper}; +pub use scalar_query_ref::ScalarQueryRef; pub(crate) trait Explain { fn fmt(&self, arena: &(dyn MetaArena + '_), f: &mut fmt::Formatter<'_>) -> fmt::Result; @@ -253,7 +255,11 @@ impl LogicalPlan { } _ => unreachable!(), }, - Operator::ScalarApply(_) | Operator::Join(_) => match childrens { + Operator::ScalarQueryInit(_) => match childrens { + Childrens::Twins { left, .. } => left.output_schema(arena).clone(), + _ => unreachable!(), + }, + Operator::Join(_) => match childrens { Childrens::Twins { left, right } => { let mut schema = left.output_schema(arena).clone(); schema.extend_from_slice(right.output_schema(arena)); diff --git a/src/planner/operator/join.rs b/src/planner/operator/join.rs index ea46b9d2..b2e52293 100644 --- a/src/planner/operator/join.rs +++ b/src/planner/operator/join.rs @@ -13,9 +13,15 @@ // limitations under the License. use super::{Operator, PlanImpl}; +use crate::catalog::ColumnRef; +use crate::errors::DatabaseError; +use crate::expression::{BinaryOperator, ScalarExpression, TypeCast}; use crate::planner::MetaArena; -use crate::planner::{Childrens, Explain, ExprRef, LogicalPlan}; +use crate::planner::{Childrens, Explain, ExprRef, LogicalPlan, PlanArena}; +use crate::types::tuple::Schema; +use crate::types::LogicalType; use kite_sql_serde_macros::ReferenceSerialization; +use std::borrow::Cow; use std::fmt; use std::fmt::Formatter; @@ -127,6 +133,145 @@ impl fmt::Display for JoinType { } } +/// for sqlrs +/// original idea from datafusion planner.rs +/// Extracts equijoin ON condition be a single Eq or multiple conjunctive Eqs +/// Filters matching this pattern are added to `accum` +/// Filters that don't match this pattern are added to `accum_filter` +/// Examples: +/// ```text +/// foo = bar => accum=[(foo, bar)] accum_filter=[] +/// foo = bar AND bar = baz => accum=[(foo, bar), (bar, baz)] accum_filter=[] +/// foo = bar AND baz > 1 => accum=[(foo, bar)] accum_filter=[baz > 1] +/// ``` +pub(crate) fn extract_join_keys( + expr: ExprRef, + accum: &mut Vec<(ExprRef, ExprRef)>, + accum_filter: &mut Vec, + left_schema: &Schema, + right_schema: &Schema, + arena: &mut PlanArena, +) -> Result<(), DatabaseError> { + let fn_contains = |schema: &Schema, column: ColumnRef| { + let summary = arena.column(column).summary(); + schema + .iter() + .any(|candidate| arena.column(*candidate).summary() == summary) + }; + let fn_or_contains = + |column: ColumnRef| fn_contains(left_schema, column) || fn_contains(right_schema, column); + + let expr = expr.unpack_alias(arena); + match arena.expression(expr) { + ScalarExpression::Binary { + left_expr, + right_expr, + op, + .. + } => { + match op { + BinaryOperator::Eq => { + match ( + left_expr.unpack_alias_ref(arena), + right_expr.unpack_alias_ref(arena), + ) { + // example: foo = bar + ( + ScalarExpression::ColumnRef { column: l, .. }, + ScalarExpression::ColumnRef { column: r, .. }, + ) => { + // reorder left and right joins keys to pattern: (left, right) + let key = if fn_contains(left_schema, *l) + && fn_contains(right_schema, *r) + { + Some((*left_expr, *right_expr)) + } else if fn_contains(left_schema, *r) && fn_contains(right_schema, *l) + { + Some((*right_expr, *left_expr)) + } else { + if fn_or_contains(*l) || fn_or_contains(*r) { + accum_filter.push(expr); + } + None + }; + // Join keys are compared (and hashed) directly, so cast + // both to one type like `l = r` in a filter; otherwise + // e.g. `bigint = int` never matches. + if let Some((left, right)) = key { + let ty = LogicalType::max_logical_type( + &left.return_type(arena), + &right.return_type(arena), + )? + .into_owned(); + accum.push(( + left.type_cast(Cow::Borrowed(&ty), arena)?, + right.type_cast(Cow::Borrowed(&ty), arena)?, + )); + } + } + (ScalarExpression::ColumnRef { column, .. }, _) + | (_, ScalarExpression::ColumnRef { column, .. }) => { + if fn_or_contains(*column) { + accum_filter.push(expr); + } + } + _other => { + // example: baz > 1 + if left_expr.all_referenced_columns(arena, |_, column| { + fn_or_contains(*column) + })? && right_expr.all_referenced_columns(arena, |_, column| { + fn_or_contains(*column) + })? { + accum_filter.push(expr); + } + } + } + } + BinaryOperator::And => { + // example: foo = bar AND baz > 1 + let (left_expr, right_expr) = (*left_expr, *right_expr); + extract_join_keys( + left_expr, + accum, + accum_filter, + left_schema, + right_schema, + arena, + )?; + extract_join_keys( + right_expr, + accum, + accum_filter, + left_schema, + right_schema, + arena, + )?; + } + BinaryOperator::Or => { + accum_filter.push(expr); + } + _ => { + if left_expr + .all_referenced_columns(arena, |_, column| fn_or_contains(*column))? + && right_expr + .all_referenced_columns(arena, |_, column| fn_or_contains(*column))? + { + accum_filter.push(expr); + } + } + } + } + _ => { + if expr.all_referenced_columns(arena, |_, column| fn_or_contains(*column))? { + // example: baz > 1 + accum_filter.push(expr); + } + } + } + + Ok(()) +} + #[cfg(test)] mod tests { use super::*; diff --git a/src/planner/operator/mod.rs b/src/planner/operator/mod.rs index beefe902..cb8830e9 100644 --- a/src/planner/operator/mod.rs +++ b/src/planner/operator/mod.rs @@ -37,7 +37,7 @@ pub mod limit; pub mod mark_apply; pub mod project; pub mod recursive_cte; -pub mod scalar_apply; +pub mod scalar_query_init; pub mod scalar_subquery; pub mod set_membership; pub mod sort; @@ -56,8 +56,8 @@ use self::{ aggregate::AggregateOperator, alter_table::add_column::AddColumnOperator, alter_table::change_column::ChangeColumnOperator, filter::FilterOperator, join::JoinOperator, limit::LimitOperator, mark_apply::MarkApplyOperator, project::ProjectOperator, - scalar_apply::ScalarApplyOperator, scalar_subquery::ScalarSubqueryOperator, sort::SortOperator, - table_scan::TableScanOperator, + scalar_query_init::ScalarQueryInitOperator, scalar_subquery::ScalarSubqueryOperator, + sort::SortOperator, table_scan::TableScanOperator, }; use crate::catalog::ColumnRef; use crate::errors::DatabaseError; @@ -97,7 +97,7 @@ pub enum Operator { // DQL Dummy, Aggregate(AggregateOperator), - ScalarApply(ScalarApplyOperator), + ScalarQueryInit(ScalarQueryInitOperator), MarkApply(MarkApplyOperator), Filter(FilterOperator), Join(JoinOperator), @@ -177,7 +177,7 @@ pub enum PlanImpl { HashAggregate, StreamAggregate, StreamDistinct, - ScalarApply, + ScalarQueryInit, MarkApply, Filter, HashJoin, @@ -248,7 +248,6 @@ macro_rules! impl_display_explain { } impl_display_explain!( - ScalarApplyOperator, MarkApplyOperator, ScalarSubqueryOperator, FunctionScanOperator, @@ -277,7 +276,7 @@ impl Explain for Operator { match self { Operator::Dummy => f.write_str("Dummy"), Operator::Aggregate(op) => Explain::fmt(op, arena, f), - Operator::ScalarApply(op) => Explain::fmt(op, arena, f), + Operator::ScalarQueryInit(op) => Explain::fmt(op, arena, f), Operator::MarkApply(op) => Explain::fmt(op, arena, f), Operator::Filter(op) => Explain::fmt(op, arena, f), Operator::Join(op) => Explain::fmt(op, arena, f), @@ -367,6 +366,16 @@ impl Operator { Ok(()) } + fn visit_scalar_query_init( + &mut self, + op: &'operator ScalarQueryInitOperator, + ) -> Result<(), DatabaseError> { + for (_, expr) in &op.param_bindings { + ExprVisitor::visit(self, *expr, self.arena)?; + } + Ok(()) + } + fn visit_mark_apply( &mut self, op: &'operator MarkApplyOperator, @@ -595,7 +604,7 @@ impl Explain for PlanImpl { PlanImpl::HashAggregate => f.write_str("HashAggregate"), PlanImpl::StreamAggregate => f.write_str("StreamAggregate"), PlanImpl::StreamDistinct => f.write_str("StreamDistinct"), - PlanImpl::ScalarApply => f.write_str("ScalarApply"), + PlanImpl::ScalarQueryInit => f.write_str("ScalarQueryInit"), PlanImpl::MarkApply => f.write_str("MarkApply"), PlanImpl::Filter => f.write_str("Filter"), PlanImpl::HashJoin => f.write_str("HashJoin"), @@ -767,7 +776,7 @@ mod tests { (PlanImpl::HashAggregate, "HashAggregate"), (PlanImpl::StreamAggregate, "StreamAggregate"), (PlanImpl::StreamDistinct, "StreamDistinct"), - (PlanImpl::ScalarApply, "ScalarApply"), + (PlanImpl::ScalarQueryInit, "ScalarQueryInit"), (PlanImpl::MarkApply, "MarkApply"), (PlanImpl::Filter, "Filter"), (PlanImpl::HashJoin, "HashJoin"), @@ -1002,8 +1011,12 @@ mod tests { }); assert_eq!(referenced_columns(&delete, &mut arena)?, vec![a]); + let scalar = Operator::ScalarQueryInit(ScalarQueryInitOperator { + value: arena.alloc_expression(ScalarExpression::Constant(DataValue::Null)), + param_bindings: Vec::new(), + }); + assert!(referenced_columns(&scalar, &mut arena)?.is_empty()); let no_reference_operators = [ - Operator::ScalarApply(ScalarApplyOperator), Operator::ScalarSubquery(ScalarSubqueryOperator), Operator::Analyze(AnalyzeOperator { table_name: "users".into(), @@ -1382,14 +1395,22 @@ mod tests { } #[test] - fn scalar_apply_and_subquery_build_expected_child_shapes() { + fn scalar_query_init_and_subquery_build_expected_child_shapes() { let table_arena = TableArenaCell::default(); - let arena = PlanArena::new(&table_arena); + let mut arena = PlanArena::new(&table_arena); let left = LogicalPlan::new(Operator::ShowTable, Childrens::None); let right = LogicalPlan::new(Operator::ShowView, Childrens::None); - let apply = ScalarApplyOperator::build(left.clone(), right); - assert_eq!(apply.operator.explain(&arena).to_string(), "ScalarApply"); + let id = arena.alloc_scalar_query_ref(false); + let value = arena.alloc_expression(ScalarExpression::InitValue { + id, + ty: LogicalType::Integer, + }); + let apply = ScalarQueryInitOperator::build(left.clone(), right, value, Vec::new()); + assert_eq!( + apply.operator.explain(&arena).to_string(), + "ScalarQueryInit InitValue(1:0)" + ); assert!(matches!(*apply.childrens, Childrens::Twins { .. })); let subquery = ScalarSubqueryOperator::build(left); diff --git a/src/planner/operator/scalar_apply.rs b/src/planner/operator/scalar_apply.rs deleted file mode 100644 index 99df8d84..00000000 --- a/src/planner/operator/scalar_apply.rs +++ /dev/null @@ -1,40 +0,0 @@ -// Copyright 2024 KipData/KiteSQL -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use super::Operator; -use crate::planner::{Childrens, LogicalPlan}; -use kite_sql_serde_macros::ReferenceSerialization; -use std::fmt; -use std::fmt::Formatter; - -#[derive(Debug, PartialEq, Eq, Clone, Hash, ReferenceSerialization)] -pub struct ScalarApplyOperator; - -impl ScalarApplyOperator { - pub fn build(left: LogicalPlan, right: LogicalPlan) -> LogicalPlan { - LogicalPlan::new( - Operator::ScalarApply(ScalarApplyOperator), - Childrens::Twins { - left: Box::new(left), - right: Box::new(right), - }, - ) - } -} - -impl fmt::Display for ScalarApplyOperator { - fn fmt(&self, f: &mut Formatter) -> fmt::Result { - write!(f, "ScalarApply") - } -} diff --git a/src/planner/operator/scalar_query_init.rs b/src/planner/operator/scalar_query_init.rs new file mode 100644 index 00000000..1e3b22b8 --- /dev/null +++ b/src/planner/operator/scalar_query_init.rs @@ -0,0 +1,58 @@ +// Copyright 2024 KipData/KiteSQL +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use super::Operator; +use crate::expression::ScalarExpression; +use crate::planner::{Childrens, Explain, ExprRef, LogicalPlan, MetaArena, ScalarQueryRef}; +use kite_sql_serde_macros::ReferenceSerialization; +use std::fmt; + +#[derive(Debug, PartialEq, Eq, Clone, Hash, ReferenceSerialization)] +pub struct ScalarQueryInitOperator { + pub value: ExprRef, + pub param_bindings: Vec<(ScalarQueryRef, ExprRef)>, +} + +impl ScalarQueryInitOperator { + pub fn build( + left: LogicalPlan, + right: LogicalPlan, + value: ExprRef, + param_bindings: Vec<(ScalarQueryRef, ExprRef)>, + ) -> LogicalPlan { + LogicalPlan::new( + Operator::ScalarQueryInit(Self { + value, + param_bindings, + }), + Childrens::Twins { + left: Box::new(left), + right: Box::new(right), + }, + ) + } + + pub(crate) fn reference(&self, arena: &dyn MetaArena) -> ScalarQueryRef { + match arena.expression(self.value) { + ScalarExpression::InitValue { id, .. } | ScalarExpression::OuterValue { id, .. } => *id, + _ => unreachable!("scalar initializer requires a value marker"), + } + } +} + +impl Explain for ScalarQueryInitOperator { + fn fmt(&self, arena: &dyn MetaArena, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "ScalarQueryInit {}", self.value.explain(arena)) + } +} diff --git a/src/planner/operator/visitor.rs b/src/planner/operator/visitor.rs index b1512982..b1e3b1ad 100644 --- a/src/planner/operator/visitor.rs +++ b/src/planner/operator/visitor.rs @@ -31,7 +31,10 @@ pub trait OperatorVisitor<'a>: Sized { Ok(()) } - fn visit_scalar_apply(&mut self, _op: &'a ScalarApplyOperator) -> Result<(), DatabaseError> { + fn visit_scalar_query_init( + &mut self, + _op: &'a ScalarQueryInitOperator, + ) -> Result<(), DatabaseError> { Ok(()) } @@ -219,6 +222,17 @@ impl<'a, V: ExprVisitor, A: MetaArena + ?Sized> OperatorVisitor<'a> Ok(()) } + fn visit_scalar_query_init( + &mut self, + op: &'a ScalarQueryInitOperator, + ) -> Result<(), DatabaseError> { + ExprVisitor::visit(self.visitor, op.value, self.arena)?; + for (_, expr) in &op.param_bindings { + ExprVisitor::visit(self.visitor, *expr, self.arena)?; + } + Ok(()) + } + fn visit_mark_apply(&mut self, op: &'a MarkApplyOperator) -> Result<(), DatabaseError> { for expr in &op.predicates { ExprVisitor::visit(self.visitor, *expr, self.arena)?; @@ -338,7 +352,7 @@ pub fn walk_operator<'a, V: OperatorVisitor<'a>>( match operator { Operator::Dummy => visitor.visit_dummy(), Operator::Aggregate(op) => visitor.visit_aggregate(op), - Operator::ScalarApply(op) => visitor.visit_scalar_apply(op), + Operator::ScalarQueryInit(op) => visitor.visit_scalar_query_init(op), Operator::MarkApply(op) => visitor.visit_mark_apply(op), Operator::Filter(op) => visitor.visit_filter(op), Operator::Join(op) => visitor.visit_join(op), @@ -448,7 +462,10 @@ pub(crate) mod tests { is_distinct: false, force_spill: false, }), - Operator::ScalarApply(ScalarApplyOperator), + Operator::ScalarQueryInit(ScalarQueryInitOperator { + param_bindings: Vec::new(), + value: expr(0), + }), Operator::MarkApply(mark_apply), Operator::Filter(FilterOperator { predicate: expr(5), @@ -655,7 +672,7 @@ pub(crate) mod tests { for operator in &operators { visitor.visit_operator(operator)?; } - assert_eq!(counter.0, 21); // Includes the Values row expression. + assert_eq!(counter.0, 22); // Includes the Values row expression. Ok(()) } diff --git a/src/planner/operator/visitor_mut.rs b/src/planner/operator/visitor_mut.rs index d1d101ee..0255a5c2 100644 --- a/src/planner/operator/visitor_mut.rs +++ b/src/planner/operator/visitor_mut.rs @@ -67,9 +67,9 @@ pub trait OperatorVisitorMut<'a>: Sized { Ok(()) } - fn visit_scalar_apply( + fn visit_scalar_query_init( &mut self, - _op: &'a mut ScalarApplyOperator, + _op: &'a mut ScalarQueryInitOperator, ) -> Result<(), DatabaseError> { Ok(()) } @@ -274,6 +274,17 @@ impl<'a, V: ExprVisitorMut> OperatorVisitorMut<'a> for OperatorExprVisitorMut<'_ Ok(()) } + fn visit_scalar_query_init( + &mut self, + op: &'a mut ScalarQueryInitOperator, + ) -> Result<(), DatabaseError> { + ExprVisitorMut::visit(self.visitor, &mut op.value, self.arena)?; + for (_, expr) in &mut op.param_bindings { + ExprVisitorMut::visit(self.visitor, expr, self.arena)?; + } + Ok(()) + } + fn visit_mark_apply(&mut self, op: &'a mut MarkApplyOperator) -> Result<(), DatabaseError> { for expr in &mut op.predicates { ExprVisitorMut::visit(self.visitor, expr, self.arena)?; @@ -403,7 +414,7 @@ pub fn walk_mut_operator<'a, V: OperatorVisitorMut<'a>>( match operator { Operator::Dummy => visitor.visit_dummy(), Operator::Aggregate(op) => visitor.visit_aggregate(op), - Operator::ScalarApply(op) => visitor.visit_scalar_apply(op), + Operator::ScalarQueryInit(op) => visitor.visit_scalar_query_init(op), Operator::MarkApply(op) => visitor.visit_mark_apply(op), Operator::Filter(op) => visitor.visit_filter(op), Operator::Join(op) => visitor.visit_join(op), @@ -523,7 +534,7 @@ mod tests { visitor.visit_operator(operator, None)?; } } - assert_eq!(counter.0, 21); // Includes the Values row expression. + assert_eq!(counter.0, 22); // Includes the Values row expression. Ok(()) } diff --git a/src/planner/scalar_query_ref.rs b/src/planner/scalar_query_ref.rs new file mode 100644 index 00000000..6b12f531 --- /dev/null +++ b/src/planner/scalar_query_ref.rs @@ -0,0 +1,104 @@ +// Copyright 2024 KipData/KiteSQL +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use crate::errors::DatabaseError; +use crate::planner::MetaArena; +use crate::serdes::{ReferenceDecodeContext, ReferenceSerialization, ReferenceTables}; +use crate::storage::Transaction; +use std::io::{Read, Write}; + +/// A definition's position in an Arena namespace, independent of execution-local slots. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub struct ScalarQueryRef { + pub(crate) arena_id: usize, + pub(crate) pos: usize, + pub(crate) is_outer_value: bool, +} + +impl std::fmt::Display for ScalarQueryRef { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{}:{}", self.arena_id, self.pos) + } +} + +impl ReferenceSerialization for ScalarQueryRef { + fn encode( + &self, + writer: &mut W, + direct: bool, + tables: &mut ReferenceTables, + arena: &A, + ) -> Result<(), DatabaseError> { + self.arena_id.encode(writer, direct, tables, arena)?; + self.pos.encode(writer, direct, tables, arena)?; + self.is_outer_value.encode(writer, direct, tables, arena) + } + + fn decode( + reader: &mut R, + context: Option<&ReferenceDecodeContext<'_, T>>, + tables: &ReferenceTables, + arena: &mut A, + ) -> Result { + let reference = Self { + arena_id: usize::decode(reader, context, tables, arena)?, + pos: usize::decode(reader, context, tables, arena)?, + is_outer_value: bool::decode(reader, context, tables, arena)?, + }; + arena.reserve_scalar_query_ref(reference); + Ok(reference) + } +} + +#[cfg(all(test, not(target_arch = "wasm32")))] +mod tests { + use super::*; + use crate::planner::TableArena; + use crate::storage::rocksdb::RocksTransaction; + + #[test] + fn decoding_preserves_source_reference() -> Result<(), DatabaseError> { + let mut source = TableArena::default(); + let reference = source.alloc_scalar_query_ref(true); + let mut bytes = Vec::new(); + let mut tables = ReferenceTables::new(); + reference.encode(&mut bytes, false, &mut tables, &source)?; + let mut target = TableArena::default(); + let context = ReferenceDecodeContext::::new(None); + let first = ScalarQueryRef::decode::( + &mut bytes.as_slice(), + Some(&context), + &tables, + &mut target, + )?; + let repeated = ScalarQueryRef::decode::( + &mut bytes.as_slice(), + Some(&context), + &tables, + &mut target, + )?; + assert_eq!(first, repeated); + assert_eq!(first, reference); + let next_view = ReferenceDecodeContext::::new(None); + let second = ScalarQueryRef::decode::( + &mut bytes.as_slice(), + Some(&next_view), + &tables, + &mut target, + )?; + assert_eq!(second, reference); + assert_ne!(target.alloc_scalar_query_ref(false), reference); + Ok(()) + } +} diff --git a/src/serdes/mod.rs b/src/serdes/mod.rs index a07c8341..93f068ee 100644 --- a/src/serdes/mod.rs +++ b/src/serdes/mod.rs @@ -180,6 +180,7 @@ impl ReferenceTables { mod tests { use crate::errors::DatabaseError; use crate::expression::range_detacher::Range; + use crate::planner::TableArena; use crate::serdes::{ReferenceSerialization, ReferenceTables}; use crate::storage::rocksdb::RocksTransaction; use crate::types::value::DataValue; @@ -195,7 +196,7 @@ mod tests { S: ReferenceSerialization + PartialEq + Debug, { let mut reference_tables = ReferenceTables::new(); - let mut arena = crate::planner::TableArena::default(); + let mut arena = TableArena::default(); let mut cursor = Cursor::new(Vec::new()); source.encode(&mut cursor, false, &mut reference_tables, &arena)?; diff --git a/src/storage/mod.rs b/src/storage/mod.rs index 649d6ee2..5b756bc0 100644 --- a/src/storage/mod.rs +++ b/src/storage/mod.rs @@ -1240,6 +1240,7 @@ pub(crate) fn reuse_bound_as_excluded(bound: &mut Bound, key: &[u8]) { *bound = Bound::Excluded(bytes); } +#[cfg(any(test, all(not(target_arch = "wasm32"), feature = "lmdb")))] pub(crate) fn bounds_contain(min: Bound<&[u8]>, max: Bound<&[u8]>, key: &[u8]) -> bool { std::ops::RangeBounds::<[u8]>::contains(&(min, max), key) } diff --git a/src/types/evaluator/binary.rs b/src/types/evaluator/binary.rs index 6a7a20d3..26cd8252 100644 --- a/src/types/evaluator/binary.rs +++ b/src/types/evaluator/binary.rs @@ -561,7 +561,12 @@ macro_rules! numeric_binary_evaluator_definition { right: &$crate::types::value::DataValue, ) -> Result<$crate::types::value::DataValue, $crate::errors::DatabaseError> { Ok(match (left, right) { - ($compute_type(v1), $compute_type(v2)) => $crate::types::value::DataValue::Float64(ordered_float::OrderedFloat(*v1 as f64 / *v2 as f64)), + ($compute_type(v1), $compute_type(v2)) => { + if *v2 == 0 { + return Err($crate::errors::DatabaseError::InvalidValue("division by zero".into())); + } + $compute_type(v1.checked_div(*v2).ok_or($crate::errors::DatabaseError::OverFlow)?) + }, ($compute_type(_), $crate::types::value::DataValue::Null) | ($crate::types::value::DataValue::Null, $compute_type(_)) | ($crate::types::value::DataValue::Null, $crate::types::value::DataValue::Null) => $crate::types::value::DataValue::Null, @@ -721,6 +726,72 @@ mod test { Ok(()) } + #[test] + fn integer_division_truncates_and_checks_errors() -> Result<(), DatabaseError> { + macro_rules! check_signed { + ($ty:expr, $variant:ident, $int:ty) => {{ + let divide = binary_create(Cow::Owned($ty), BinaryOperator::Divide)?; + for (left, right, expected) in [(5, 2, 2), (-5, 2, -2), (5, -2, -2), (-5, -2, 2)] { + assert_eq!( + divide + .binary_eval(&DataValue::$variant(left), &DataValue::$variant(right))?, + DataValue::$variant(expected) + ); + } + assert!(matches!( + divide.binary_eval(&DataValue::$variant(<$int>::MIN), &DataValue::$variant(-1)), + Err(DatabaseError::OverFlow) + )); + assert!(matches!( + divide.binary_eval(&DataValue::$variant(1), &DataValue::$variant(0)), + Err(DatabaseError::InvalidValue(_)) + )); + assert_eq!( + divide.binary_eval(&DataValue::Null, &DataValue::$variant(0))?, + DataValue::Null + ); + }}; + } + check_signed!(LogicalType::Tinyint, Int8, i8); + check_signed!(LogicalType::Smallint, Int16, i16); + check_signed!(LogicalType::Integer, Int32, i32); + check_signed!(LogicalType::Bigint, Int64, i64); + for (ty, left, right, expected) in [ + ( + LogicalType::UTinyint, + DataValue::UInt8(5), + DataValue::UInt8(2), + DataValue::UInt8(2), + ), + ( + LogicalType::USmallint, + DataValue::UInt16(5), + DataValue::UInt16(2), + DataValue::UInt16(2), + ), + ( + LogicalType::UInteger, + DataValue::UInt32(5), + DataValue::UInt32(2), + DataValue::UInt32(2), + ), + ( + LogicalType::UBigint, + DataValue::UInt64(9007199254740993), + DataValue::UInt64(1), + DataValue::UInt64(9007199254740993), + ), + ] { + let divide = binary_create(Cow::Owned(ty), BinaryOperator::Divide)?; + assert_eq!(divide.binary_eval(&left, &right)?, expected); + assert_eq!( + divide.binary_eval(&left, &DataValue::Null)?, + DataValue::Null + ); + } + Ok(()) + } + #[test] fn test_binary_evaluator_serialization() -> Result<(), DatabaseError> { let evaluator = create(LogicalType::Boolean, BinaryOperator::NotEq)?; diff --git a/src/types/evaluator/cast.rs b/src/types/evaluator/cast.rs index 60ec2a43..c7bd0262 100644 --- a/src/types/evaluator/cast.rs +++ b/src/types/evaluator/cast.rs @@ -178,10 +178,15 @@ macro_rules! float_to_int_cast { let float_value: $float_type = $float_value; if float_value.is_nan() { Ok(0) - } else if float_value <= 0.0 || float_value > <$int_type>::MAX as $float_type { - Err($crate::errors::DatabaseError::OverFlow) } else { - Ok(float_value as $int_type) + let truncated = float_value.trunc(); + // Use an exclusive upper bound: wide integer MAX rounds up to 2^N in floats. + let upper = <$int_type>::MAX as $float_type + 1.0; + if truncated < <$int_type>::MIN as $float_type || truncated >= upper { + Err($crate::errors::DatabaseError::OverFlow) + } else { + Ok(truncated as $int_type) + } } }}; } @@ -1163,6 +1168,85 @@ mod test { cast_create(&from, &to) } + #[test] + fn float_integer_cast_accepts_negative_and_zero_values() -> Result<(), DatabaseError> { + for (from, value) in [ + (LogicalType::Float, DataValue::Float32(OrderedFloat(-12.75))), + ( + LogicalType::Double, + DataValue::Float64(OrderedFloat(-12.75)), + ), + ] { + for (to, expected) in [ + (LogicalType::Tinyint, DataValue::Int8(-12)), + (LogicalType::Smallint, DataValue::Int16(-12)), + (LogicalType::Integer, DataValue::Int32(-12)), + (LogicalType::Bigint, DataValue::Int64(-12)), + ] { + assert_eq!(cast_eval(from.clone(), to, &value)?, expected); + } + } + for (from, zero) in [ + (LogicalType::Float, DataValue::Float32(OrderedFloat(0.0))), + (LogicalType::Double, DataValue::Float64(OrderedFloat(-0.0))), + ] { + assert_eq!( + cast_eval(from.clone(), LogicalType::Integer, &zero)?, + DataValue::Int32(0) + ); + assert_eq!( + cast_eval(from, LogicalType::UInteger, &zero)?, + DataValue::UInt32(0) + ); + } + Ok(()) + } + + #[test] + fn float_integer_cast_checks_target_bounds() { + macro_rules! check_bounds { + ($float:ty, $int:ty) => {{ + let min = <$int>::MIN as $float; + let upper = <$int>::MAX as $float + 1.0; + assert_eq!(float_to_int_cast!(min, $int, $float).unwrap(), <$int>::MIN); + assert!(matches!( + float_to_int_cast!(upper, $int, $float), + Err(DatabaseError::OverFlow) + )); + assert!(matches!( + float_to_int_cast!(min * 2.0 - 1.0, $int, $float), + Err(DatabaseError::OverFlow) + )); + assert!(matches!( + float_to_int_cast!(<$float>::INFINITY, $int, $float), + Err(DatabaseError::OverFlow) + )); + assert!(matches!( + float_to_int_cast!(<$float>::NEG_INFINITY, $int, $float), + Err(DatabaseError::OverFlow) + )); + }}; + } + check_bounds!(f32, i8); + check_bounds!(f32, i16); + check_bounds!(f32, i32); + check_bounds!(f32, i64); + check_bounds!(f64, i8); + check_bounds!(f64, i16); + check_bounds!(f64, i32); + check_bounds!(f64, i64); + check_bounds!(f32, u8); + check_bounds!(f32, u16); + check_bounds!(f32, u32); + check_bounds!(f32, u64); + check_bounds!(f64, u8); + check_bounds!(f64, u16); + check_bounds!(f64, u32); + check_bounds!(f64, u64); + assert_eq!(float_to_int_cast!(127.9_f64, i8, f64).unwrap(), 127); + assert_eq!(float_to_int_cast!(-128.9_f64, i8, f64).unwrap(), -128); + } + fn utf8(value: &str) -> DataValue { DataValue::Utf8 { value: value.to_string(), diff --git a/src/types/evaluator/float32.rs b/src/types/evaluator/float32.rs index 61ea730e..b7620cc5 100644 --- a/src/types/evaluator/float32.rs +++ b/src/types/evaluator/float32.rs @@ -19,9 +19,6 @@ use crate::types::LogicalType; #[cfg(feature = "decimal")] use rust_decimal::prelude::FromPrimitive; use std::hint; -pub fn float32_plus_unary_eval(value: &DataValue) -> DataValue { - value.clone() -} pub fn float32_minus_unary_eval(value: &DataValue) -> DataValue { match value { DataValue::Float32(value) => DataValue::Float32(-value), @@ -195,7 +192,6 @@ mod test { let left = DataValue::Float32(OrderedFloat(5.5)); let right = DataValue::Float32(OrderedFloat(2.0)); - assert_eq!(float32_plus_unary_eval(&left), left); assert_eq!( float32_minus_unary_eval(&left), DataValue::Float32(OrderedFloat(-5.5)) diff --git a/src/types/evaluator/float64.rs b/src/types/evaluator/float64.rs index 71e8679b..a1bc68d5 100644 --- a/src/types/evaluator/float64.rs +++ b/src/types/evaluator/float64.rs @@ -26,9 +26,6 @@ use rust_decimal::prelude::FromPrimitive; #[cfg(feature = "decimal")] use rust_decimal::Decimal; use std::hint; -pub fn float64_plus_unary_eval(value: &DataValue) -> DataValue { - value.clone() -} pub fn float64_minus_unary_eval(value: &DataValue) -> DataValue { match value { DataValue::Float64(value) => DataValue::Float64(-value), @@ -194,7 +191,6 @@ mod test { fn test_float64_binary_and_cast_evaluators() { let value = DataValue::Float64(ordered_float::OrderedFloat(1.5)); - assert_eq!(float64_plus_unary_eval(&value), value); assert_eq!( float64_minus_unary_eval(&value), DataValue::Float64(ordered_float::OrderedFloat(-1.5)) diff --git a/src/types/evaluator/unary.rs b/src/types/evaluator/unary.rs index 79e553ad..6c081616 100644 --- a/src/types/evaluator/unary.rs +++ b/src/types/evaluator/unary.rs @@ -25,58 +25,31 @@ use crate::types::evaluator::UnaryEvaluatorRef; use crate::types::LogicalType; use std::borrow::Cow; -const UNARY_INT8_PLUS: u16 = 0; -const UNARY_INT8_MINUS: u16 = 1; -const UNARY_INT16_PLUS: u16 = 2; -const UNARY_INT16_MINUS: u16 = 3; -const UNARY_INT32_PLUS: u16 = 4; -const UNARY_INT32_MINUS: u16 = 5; -const UNARY_INT64_PLUS: u16 = 6; -const UNARY_INT64_MINUS: u16 = 7; -const UNARY_BOOLEAN_NOT: u16 = 8; -const UNARY_FLOAT32_PLUS: u16 = 9; -const UNARY_FLOAT32_MINUS: u16 = 10; -const UNARY_FLOAT64_PLUS: u16 = 11; -const UNARY_FLOAT64_MINUS: u16 = 12; -// Any unary operator on the NULL literal yields NULL, as in PostgreSQL. -const UNARY_SQL_NULL: u16 = 13; - -// Evaluator positions are serialized ABI. Do not reorder or reuse existing -// positions; only append new positions at the end of the current layout. - -fn numeric_unary_ref( - plus: u16, - minus: u16, - ty: &LogicalType, - op: UnaryOperator, -) -> Result { - let pos = match op { - UnaryOperator::Plus => plus, - UnaryOperator::Minus => minus, - _ => return Err(DatabaseError::UnsupportedUnaryOperator(ty.clone(), op)), - }; - Ok(UnaryEvaluatorRef::new(pos)) -} +const UNARY_INT8_MINUS: u16 = 0; +const UNARY_INT16_MINUS: u16 = 1; +const UNARY_INT32_MINUS: u16 = 2; +const UNARY_INT64_MINUS: u16 = 3; +const UNARY_BOOLEAN_NOT: u16 = 4; +const UNARY_FLOAT32_MINUS: u16 = 5; +const UNARY_FLOAT64_MINUS: u16 = 6; +const UNARY_SQL_NULL: u16 = 7; pub fn unary_create( ty: Cow<'_, LogicalType>, op: UnaryOperator, ) -> Result { - let ty = ty.as_ref(); - match ty { - LogicalType::Tinyint => numeric_unary_ref(UNARY_INT8_PLUS, UNARY_INT8_MINUS, ty, op), - LogicalType::Smallint => numeric_unary_ref(UNARY_INT16_PLUS, UNARY_INT16_MINUS, ty, op), - LogicalType::Integer => numeric_unary_ref(UNARY_INT32_PLUS, UNARY_INT32_MINUS, ty, op), - LogicalType::Bigint => numeric_unary_ref(UNARY_INT64_PLUS, UNARY_INT64_MINUS, ty, op), - LogicalType::Boolean => match op { - UnaryOperator::Not => Ok(UnaryEvaluatorRef::new(UNARY_BOOLEAN_NOT)), - _ => Err(DatabaseError::UnsupportedUnaryOperator(ty.clone(), op)), - }, - LogicalType::Float => numeric_unary_ref(UNARY_FLOAT32_PLUS, UNARY_FLOAT32_MINUS, ty, op), - LogicalType::Double => numeric_unary_ref(UNARY_FLOAT64_PLUS, UNARY_FLOAT64_MINUS, ty, op), - LogicalType::SqlNull => Ok(UnaryEvaluatorRef::new(UNARY_SQL_NULL)), - _ => Err(DatabaseError::UnsupportedUnaryOperator(ty.clone(), op)), - } + let pos = match (ty.as_ref(), op) { + (LogicalType::Tinyint, UnaryOperator::Minus) => UNARY_INT8_MINUS, + (LogicalType::Smallint, UnaryOperator::Minus) => UNARY_INT16_MINUS, + (LogicalType::Integer, UnaryOperator::Minus) => UNARY_INT32_MINUS, + (LogicalType::Bigint, UnaryOperator::Minus) => UNARY_INT64_MINUS, + (LogicalType::Boolean, UnaryOperator::Not) => UNARY_BOOLEAN_NOT, + (LogicalType::Float, UnaryOperator::Minus) => UNARY_FLOAT32_MINUS, + (LogicalType::Double, UnaryOperator::Minus) => UNARY_FLOAT64_MINUS, + (LogicalType::SqlNull, UnaryOperator::Minus | UnaryOperator::Not) => UNARY_SQL_NULL, + _ => return Err(DatabaseError::UnsupportedUnaryOperator(ty.into_owned(), op)), + }; + Ok(UnaryEvaluatorRef::new(pos)) } pub(crate) fn eval_unary( @@ -84,18 +57,12 @@ pub(crate) fn eval_unary( value: &crate::types::value::DataValue, ) -> Result { Ok(match pos { - UNARY_INT8_PLUS => int8_plus_unary_eval(value), UNARY_INT8_MINUS => int8_minus_unary_eval(value)?, - UNARY_INT16_PLUS => int16_plus_unary_eval(value), UNARY_INT16_MINUS => int16_minus_unary_eval(value)?, - UNARY_INT32_PLUS => int32_plus_unary_eval(value), UNARY_INT32_MINUS => int32_minus_unary_eval(value)?, - UNARY_INT64_PLUS => int64_plus_unary_eval(value), UNARY_INT64_MINUS => int64_minus_unary_eval(value)?, UNARY_BOOLEAN_NOT => boolean_not_unary_eval(value), - UNARY_FLOAT32_PLUS => float32_plus_unary_eval(value), UNARY_FLOAT32_MINUS => float32_minus_unary_eval(value), - UNARY_FLOAT64_PLUS => float64_plus_unary_eval(value), UNARY_FLOAT64_MINUS => float64_minus_unary_eval(value), UNARY_SQL_NULL => crate::types::value::DataValue::Null, _ => unreachable!("unknown unary evaluator position {pos}"), @@ -106,11 +73,6 @@ pub(crate) fn eval_unary( macro_rules! numeric_unary_evaluator_definition { ($value_type:ident, $compute_type:path) => { paste::paste! { - pub fn [<$value_type:snake _plus_unary_eval>]( - value: &$crate::types::value::DataValue, - ) -> $crate::types::value::DataValue { - value.clone() - } // `-MIN` does not fit the type: report it like binary `+`/`-` do // instead of panicking (debug) or wrapping (release). pub fn [<$value_type:snake _minus_unary_eval>]( @@ -154,7 +116,7 @@ mod test { } #[test] - fn test_unary_evaluator_positions_are_stable() -> Result<(), DatabaseError> { + fn test_unary_evaluator_positions() -> Result<(), DatabaseError> { assert_eq!( create(LogicalType::Integer, UnaryOperator::Minus)?.pos, UNARY_INT32_MINUS @@ -164,8 +126,8 @@ mod test { UNARY_BOOLEAN_NOT ); assert_eq!( - create(LogicalType::Double, UnaryOperator::Plus)?.pos, - UNARY_FLOAT64_PLUS + create(LogicalType::Double, UnaryOperator::Minus)?.pos, + UNARY_FLOAT64_MINUS ); Ok(()) @@ -182,60 +144,30 @@ mod test { #[test] fn test_numeric_unary_evaluators() -> Result<(), DatabaseError> { let cases = vec![ - ( - LogicalType::Tinyint, - UnaryOperator::Plus, - DataValue::Int8(7), - DataValue::Int8(7), - ), ( LogicalType::Tinyint, UnaryOperator::Minus, DataValue::Int8(7), DataValue::Int8(-7), ), - ( - LogicalType::Smallint, - UnaryOperator::Plus, - DataValue::Int16(7), - DataValue::Int16(7), - ), ( LogicalType::Smallint, UnaryOperator::Minus, DataValue::Int16(7), DataValue::Int16(-7), ), - ( - LogicalType::Integer, - UnaryOperator::Plus, - DataValue::Int32(7), - DataValue::Int32(7), - ), ( LogicalType::Integer, UnaryOperator::Minus, DataValue::Int32(7), DataValue::Int32(-7), ), - ( - LogicalType::Bigint, - UnaryOperator::Plus, - DataValue::Int64(7), - DataValue::Int64(7), - ), ( LogicalType::Bigint, UnaryOperator::Minus, DataValue::Int64(7), DataValue::Int64(-7), ), - ( - LogicalType::Float, - UnaryOperator::Plus, - DataValue::Float32(OrderedFloat(1.5)), - DataValue::Float32(OrderedFloat(1.5)), - ), ( LogicalType::Double, UnaryOperator::Minus, diff --git a/src/types/tuple.rs b/src/types/tuple.rs index abbc9611..da734600 100644 --- a/src/types/tuple.rs +++ b/src/types/tuple.rs @@ -76,6 +76,12 @@ impl<'a, 'p> SchemaView<'a, 'p> { } pub trait TupleLike { + fn len(&self) -> usize; + + fn is_empty(&self) -> bool { + self.len() == 0 + } + fn value_at(&self, index: usize) -> &DataValue; #[inline] @@ -85,19 +91,15 @@ pub trait TupleLike { } #[derive(Clone, Copy)] -pub struct SplitTupleRef<'a> { - left: &'a [DataValue], - right: &'a [DataValue], +pub struct SplitTupleRef<'a, L: ?Sized, R: ?Sized> { + left: &'a L, + right: &'a R, left_len: usize, } -impl<'a> SplitTupleRef<'a> { - pub fn new(left: &'a Tuple, right: &'a Tuple) -> Self { - Self::from_slices(left.values.as_slice(), right.values.as_slice()) - } - - pub fn from_slices(left: &'a [DataValue], right: &'a [DataValue]) -> Self { - SplitTupleRef { +impl<'a, L: TupleLike + ?Sized, R: TupleLike + ?Sized> SplitTupleRef<'a, L, R> { + pub fn new(left: &'a L, right: &'a R) -> Self { + Self { left, right, left_len: left.len(), @@ -105,6 +107,12 @@ impl<'a> SplitTupleRef<'a> { } } +impl<'a> SplitTupleRef<'a, [DataValue], [DataValue]> { + pub fn from_slices(left: &'a [DataValue], right: &'a [DataValue]) -> Self { + Self::new(left, right) + } +} + #[derive(Clone, Debug, PartialEq, Eq, Hash, Default)] pub struct Tuple { pub pk: Option, @@ -112,6 +120,11 @@ pub struct Tuple { } impl TupleLike for Tuple { + #[inline] + fn len(&self) -> usize { + self.values.len() + } + #[inline] fn value_at(&self, index: usize) -> &DataValue { &self.values[index] @@ -124,6 +137,11 @@ impl TupleLike for Tuple { } impl TupleLike for [DataValue] { + #[inline] + fn len(&self) -> usize { + <[DataValue]>::len(self) + } + #[inline] fn value_at(&self, index: usize) -> &DataValue { &self[index] @@ -135,49 +153,35 @@ impl TupleLike for [DataValue] { } } -impl TupleLike for &Tuple { - #[inline] - fn value_at(&self, index: usize) -> &DataValue { - &self.values[index] - } - +impl TupleLike for &T { #[inline] - fn as_slice(&self) -> Option<&[DataValue]> { - Some(self.values.as_slice()) + fn len(&self) -> usize { + (**self).len() } -} -impl TupleLike for &[DataValue] { #[inline] fn value_at(&self, index: usize) -> &DataValue { - &self[index] + (**self).value_at(index) } #[inline] fn as_slice(&self) -> Option<&[DataValue]> { - Some(self) + (**self).as_slice() } } -impl TupleLike for &dyn TupleLike { +impl TupleLike for SplitTupleRef<'_, L, R> { #[inline] - fn value_at(&self, index: usize) -> &DataValue { - (*self).value_at(index) - } - - #[inline] - fn as_slice(&self) -> Option<&[DataValue]> { - (*self).as_slice() + fn len(&self) -> usize { + self.left_len + self.right.len() } -} -impl TupleLike for SplitTupleRef<'_> { #[inline] fn value_at(&self, index: usize) -> &DataValue { if index < self.left_len { - &self.left[index] + self.left.value_at(index) } else { - &self.right[index - self.left_len] + self.right.value_at(index - self.left_len) } } } @@ -299,6 +303,10 @@ mod tests { struct OneValueTupleLike(DataValue); impl super::TupleLike for OneValueTupleLike { + fn len(&self) -> usize { + 1 + } + fn value_at(&self, index: usize) -> &DataValue { assert_eq!(index, 0); &self.0 @@ -676,6 +684,27 @@ mod tests { assert_eq!(split.value_at(1), &DataValue::Int32(2)); let split = super::SplitTupleRef::from_slices(&left.values, &right.values); assert_eq!(split.value_at(1), &DataValue::Int32(2)); + assert_eq!(split.len(), 2); + assert!(!split.is_empty()); + assert_eq!(tuple_like.len(), 2); + assert_eq!(one.len(), 1); + let empty: &[DataValue] = &[]; + assert!(empty.is_empty()); + let tail = OneValueTupleLike(DataValue::Int32(9)); + let nested = super::SplitTupleRef::new(&split, &tail); + assert_eq!(nested.len(), 3); + assert_eq!(nested.value_at(0), &DataValue::Int32(1)); + assert_eq!(nested.value_at(1), &DataValue::Int32(2)); + assert_eq!(nested.value_at(2), &DataValue::Int32(9)); + assert_eq!(nested.as_slice(), None); + let empty_left = super::SplitTupleRef::new(empty, &tail); + assert_eq!(empty_left.len(), 1); + assert_eq!(empty_left.value_at(0), &DataValue::Int32(9)); + let empty_right = super::SplitTupleRef::new(&tail, empty); + assert_eq!(empty_right.len(), 1); + assert_eq!(empty_right.value_at(0), &DataValue::Int32(9)); + let empty_split = super::SplitTupleRef::new(empty, empty); + assert!(super::TupleLike::is_empty(&empty_split)); let tuple_slice: &[DataValue] = (&tuple).into(); assert_eq!(tuple_slice, tuple.values.as_slice()); assert_eq!( diff --git a/src/types/value.rs b/src/types/value.rs index a2bdc311..7564783c 100644 --- a/src/types/value.rs +++ b/src/types/value.rs @@ -1801,7 +1801,7 @@ impl fmt::Display for DataValue { DataValue::UInt32(e) => write!(f, "{e}")?, DataValue::UInt64(e) => write!(f, "{e}")?, DataValue::Utf8 { value: e, .. } => write!(f, "{e}")?, - DataValue::Null => write!(f, "null")?, + DataValue::Null => write!(f, "NULL")?, DataValue::Date32(e) => { #[cfg(feature = "time")] write!(f, "{}", DataValue::date_format(*e).unwrap())?; @@ -2361,7 +2361,7 @@ mod test { (DataValue::UInt32(3), "3", "UInt32(3)"), (DataValue::UInt64(4), "4", "UInt64(4)"), (utf8("kite"), "kite", "Utf8(\"kite\")"), - (DataValue::Null, "null", "null"), + (DataValue::Null, "NULL", "null"), #[cfg(feature = "decimal")] ( DataValue::Decimal(Decimal::new(123, 2)), diff --git a/tests/slt/abs.slt b/tests/slt/abs.slt new file mode 100644 index 00000000..30d5ef97 --- /dev/null +++ b/tests/slt/abs.slt @@ -0,0 +1,29 @@ +statement ok +create table t_abs(id int primary key, i int, b bigint, s smallint, d double, f float); + +statement ok +insert into t_abs values (0, -3, -9000000000, -7, -1.5, -2.5), (1, 4, 5, 6, 0.25, 1.5), (2, null, null, null, null, null); + +query IIIRR rowsort +select abs(i), abs(b), abs(s), abs(d), abs(f) from t_abs; +---- +3 9000000000 7 1.5 2.5 +4 5 6 0.25 1.5 +NULL NULL NULL NULL NULL + +query I rowsort +select abs(i - b) from t_abs where id < 2; +---- +1 +8999999997 + +query I +select id from t_abs where abs(i) > 3; +---- +1 + +statement error +select abs(-2147483648); + +statement ok +drop table t_abs; diff --git a/tests/slt/aggregation.slt b/tests/slt/aggregation.slt index 6afd2efb..70bd2efa 100644 --- a/tests/slt/aggregation.slt +++ b/tests/slt/aggregation.slt @@ -86,17 +86,40 @@ insert into sn values (1, 1), (2, 2); query I select sum(null) from sn; ---- -null +NULL query I select sum(distinct null) from sn; ---- -null +NULL query I select (select sum(null) from sn) from sn where id = 1; ---- -null +NULL + +# Aggregates, including SELECT aliases containing aggregates, cannot run in WHERE. +statement error agg miss: aggregate functions are not allowed in WHERE +select count(*) as n from sn where n is not null; + +statement error agg miss: aggregate functions are not allowed in WHERE +select + + 23 * + count(*) * - + 69 as n from sn where not n is null; + +statement error agg miss: aggregate functions are not allowed in WHERE +select count(*) as n from sn where coalesce(n, 0) > 0; + +statement error agg miss: aggregate functions are not allowed in WHERE +select id from sn where count(*) > 0; + +query I +select count(*) as n from sn having n > 0; +---- +2 + +query I +select count(*) from sn where a > 0; +---- +2 statement ok drop table sn diff --git a/tests/slt/alter_table.slt b/tests/slt/alter_table.slt index 0b87fadf..289a6ef0 100644 --- a/tests/slt/alter_table.slt +++ b/tests/slt/alter_table.slt @@ -19,10 +19,10 @@ alter table t1 add column v4 int not null query IIII rowsort select * from t1 ---- -1 1 null 0 -2 2 null 0 -3 3 null 0 -4 4 null 0 +1 1 NULL 0 +2 2 NULL 0 +3 3 NULL 0 +4 4 NULL 0 statement ok alter table t1 drop column if exists v1 @@ -38,10 +38,10 @@ alter table t1 drop column id query IIII rowsort select * from t1 ---- -1 null 0 -2 null 0 -3 null 0 -4 null 0 +1 NULL 0 +2 NULL 0 +3 NULL 0 +4 NULL 0 statement ok create table t2(id int primary key, v1 int) @@ -72,8 +72,8 @@ insert into t3 values (1, 10), (2, 20) query TTTTTT describe t3 ---- -id Integer 4 false PRIMARY null -v Integer 4 true EMPTY null +id Integer 4 false PRIMARY NULL +v Integer 4 true EMPTY NULL statement ok insert into t3 values (3, 10) @@ -107,8 +107,8 @@ select * from t3 query TTTTTT describe t3 ---- -id Integer 4 false PRIMARY null -v Integer 4 true EMPTY null +id Integer 4 false PRIMARY NULL +v Integer 4 true EMPTY NULL statement error alter table t3 modify column v int first diff --git a/tests/slt/basic_test.slt b/tests/slt/basic_test.slt index f05fd01d..48ef1f7e 100644 --- a/tests/slt/basic_test.slt +++ b/tests/slt/basic_test.slt @@ -3,6 +3,32 @@ select 1 ---- 1 +query IIIII +select 5/2, -5/2, 5/-2, -5/-2, 66 + 15/-71; +---- +2 -2 -2 2 66 + +query I +select cast(9007199254740993 as bigint)/1; +---- +9007199254740993 + +query RRR +select 5.0/2, 5/cast(2 as double), cast(5 as float)/cast(2 as float) + cast(1 as float); +---- +2.5 2.5 3.5 + +query II +select null/2, 2/null; +---- +NULL NULL + +statement error division by zero +select 1/0; + +statement error over flow +select cast(-2147483648 as int)/-1; + query R select 10000.00::FLOAT + 234.567::FLOAT ---- @@ -54,7 +80,7 @@ insert into t values(0, null) query T select * from t ---- -0 null +0 NULL statement ok drop table t diff --git a/tests/slt/change_column.slt b/tests/slt/change_column.slt index c25c441a..aad116af 100644 --- a/tests/slt/change_column.slt +++ b/tests/slt/change_column.slt @@ -25,9 +25,9 @@ alter table alter_users alter column v2 type bigint query TTTTTT describe alter_users ---- -id Integer 4 false PRIMARY null -value1 Integer 4 true UNIQUE null -v2 Bigint 8 true EMPTY null +id Integer 4 false PRIMARY NULL +value1 Integer 4 true UNIQUE NULL +v2 Bigint 8 true EMPTY NULL query I select v2 from alter_users order by id @@ -41,9 +41,9 @@ alter table alter_users change column v2 value2 bigint query TTTTTT describe alter_users ---- -id Integer 4 false PRIMARY null -value1 Integer 4 true UNIQUE null -value2 Bigint 8 true EMPTY null +id Integer 4 false PRIMARY NULL +value1 Integer 4 true UNIQUE NULL +value2 Bigint 8 true EMPTY NULL statement ok alter table alter_users alter column value2 set default 999 @@ -65,7 +65,7 @@ insert into alter_users (id, value1) values (4, 44) query T select value2 from alter_users where id = 4 ---- -null +NULL statement error alter table alter_users alter column value2 set not null @@ -79,9 +79,9 @@ alter table alter_users alter column value2 set not null query TTTTTT describe alter_users ---- -id Integer 4 false PRIMARY null -value1 Integer 4 true UNIQUE null -value2 Bigint 8 false EMPTY null +id Integer 4 false PRIMARY NULL +value1 Integer 4 true UNIQUE NULL +value2 Bigint 8 false EMPTY NULL statement error insert into alter_users (id, value1) values (5, 55) @@ -95,7 +95,7 @@ insert into alter_users (id, value1) values (6, 66) query T select value2 from alter_users where id = 6 ---- -null +NULL statement ok alter table alter_users modify column value2 int @@ -103,9 +103,9 @@ alter table alter_users modify column value2 int query TTTTTT describe alter_users ---- -id Integer 4 false PRIMARY null -value1 Integer 4 true UNIQUE null -value2 Integer 4 true EMPTY null +id Integer 4 false PRIMARY NULL +value1 Integer 4 true UNIQUE NULL +value2 Integer 4 true EMPTY NULL query I select value2 from alter_users where value2 is not null order by id @@ -124,9 +124,9 @@ alter table alter_users change column value2 value2 int not null query TTTTTT describe alter_users ---- -id Integer 4 false PRIMARY null -value1 Integer 4 true UNIQUE null -value2 Integer 4 false EMPTY null +id Integer 4 false PRIMARY NULL +value1 Integer 4 true UNIQUE NULL +value2 Integer 4 false EMPTY NULL statement error insert into alter_users (id, value1) values (7, 77) @@ -145,8 +145,8 @@ select value2 from alter_users where id = 7 query TTTTTT describe alter_users ---- -id Integer 4 false PRIMARY null -value1 Integer 4 true UNIQUE null +id Integer 4 false PRIMARY NULL +value1 Integer 4 true UNIQUE NULL value2 Integer 4 false EMPTY 707 statement ok @@ -163,8 +163,8 @@ select value2 from alter_users where id = 8 query TTTTTT describe alter_users ---- -id Integer 4 false PRIMARY null -value1 Integer 4 true UNIQUE null +id Integer 4 false PRIMARY NULL +value1 Integer 4 true UNIQUE NULL value2 Integer 4 true EMPTY 707 statement ok @@ -173,8 +173,8 @@ alter table alter_users modify column value2 int not null query TTTTTT describe alter_users ---- -id Integer 4 false PRIMARY null -value1 Integer 4 true UNIQUE null +id Integer 4 false PRIMARY NULL +value1 Integer 4 true UNIQUE NULL value2 Integer 4 false EMPTY 707 statement ok @@ -191,8 +191,8 @@ select value2 from alter_users where id = 9 query TTTTTT describe alter_users ---- -id Integer 4 false PRIMARY null -value1 Integer 4 true UNIQUE null +id Integer 4 false PRIMARY NULL +value1 Integer 4 true UNIQUE NULL value2 Integer 4 false EMPTY 808 statement ok diff --git a/tests/slt/char.slt b/tests/slt/char.slt index e054634e..12bbe883 100644 --- a/tests/slt/char.slt +++ b/tests/slt/char.slt @@ -17,7 +17,7 @@ query ITT select * from t1; ---- 0 🪁 🪁🪁🪁🪁🪁 🪁 -1 null null null +1 NULL NULL NULL statement ok create table t2(id int primary key, v1 varchar, v2 varchar(5 characters), v3 varchar(5 octets)) @@ -38,7 +38,32 @@ query ITT select * from t2; ---- 0 🪁 🪁🪁🪁🪁🪁 🪁 -1 null null null +1 NULL NULL NULL + +query TTT +select +v1, ++v2, +v3 from t1 order by id; +---- +🪁 🪁🪁🪁🪁🪁 🪁 +NULL NULL NULL + +query TTT +select +v1, ++v2, +v3 from t2 order by id; +---- +🪁 🪁🪁🪁🪁🪁 🪁 +NULL NULL NULL + +query TTT +select +'abc', +'12', +cast('abc' as char(3)); +---- +abc 12 abc + +query I +select id from t2 where +v1 is null; +---- +1 + +statement error +select -v1 from t2; statement ok drop table t1; diff --git a/tests/slt/crdb/and_or.slt b/tests/slt/crdb/and_or.slt index e418691f..32ed8e0d 100644 --- a/tests/slt/crdb/and_or.slt +++ b/tests/slt/crdb/and_or.slt @@ -7,12 +7,22 @@ CREATE TABLE t (k INT PRIMARY KEY null, a INT null, b INT null) statement ok INSERT INTO t VALUES (1, NULL, NULL), (2, NULL, 1), (3, 1, NULL), (4, 2, 0), (5, 3, 3) +statement error division by zero +SELECT a = 2 AND 3 / b = 1 FROM t WHERE k = 4; + +statement error division by zero +SELECT a <> 2 OR 3 / b = 1 FROM t WHERE k = 4; + +statement error division by zero +SELECT NULL AND 3 / b = 1 FROM t WHERE k = 4; + +# The zero divisor on k=4 is skipped for a decisive AND/OR left operand. query T SELECT a <> 2 AND 3 / b = 1 FROM t ORDER BY k ---- -null +NULL false -null +NULL false true @@ -24,9 +34,9 @@ SELECT a FROM t WHERE a <> 2 AND 3 / b = 1 ORDER BY k query T SELECT a = 2 OR 3 / b = 1 FROM t ORDER BY k ---- -null -null -null +NULL +NULL +NULL true true @@ -45,9 +55,9 @@ INSERT INTO t VALUES (1, NULL, NULL), (2, NULL, 1), (3, 1, NULL), (4, 2, 1), (5, query T SELECT a <> 2 AND 3 / b = 1 FROM t ORDER BY k ---- -null +NULL false -null +NULL false true @@ -59,9 +69,9 @@ SELECT a FROM t WHERE a <> 2 AND 3 / b = 1 ORDER BY k query T SELECT a = 2 OR 3 / b = 1 FROM t ORDER BY k ---- -null -null -null +NULL +NULL +NULL true true diff --git a/tests/slt/crdb/condition.slt b/tests/slt/crdb/condition.slt index 4b104c91..c23e4e15 100644 --- a/tests/slt/crdb/condition.slt +++ b/tests/slt/crdb/condition.slt @@ -1,12 +1,12 @@ query IT SELECT IF(1 = 2, NULL, 1), IF(2 = 2, NULL, 2) ---- -1 null +1 NULL query ITT SELECT NULLIF(1, 2), NULLIF(2, 2), NULLIF(NULL, NULL) ---- -1 null null +1 NULL NULL query IIII SELECT IFNULL(1, 2), IFNULL(NULL, 2), COALESCE(1, 2), COALESCE(NULL, 2) @@ -40,13 +40,13 @@ query III SELECT a, NULLIF(a, 2), IF(a = 2, NULL, a) FROM t ORDER BY a ---- 1 1 1 -2 null null +2 NULL NULL 3 3 3 query TTTT SELECT CASE WHEN false THEN 'one' WHEN true THEN 'two' ELSE 'three' END, CASE 1 WHEN 2 THEN 'two' WHEN 1 THEN 'one' ELSE 'three' END, CASE WHEN false THEN 'one' ELSE 'three' END, CASE WHEN false THEN 'one' END ---- -two one three null +two one three NULL query TTTTT SELECT CASE WHEN 1 = 1 THEN 'one' END, CASE false WHEN 0 = 1 THEN 'one' END, CASE 1 WHEN 2 THEN 'one' ELSE 'three' END, CASE NULL WHEN true THEN 'one' WHEN false THEN 'two' WHEN NULL THEN 'three' ELSE 'four' END, CASE WHEN false THEN 'one' WHEN true THEN 'two' END diff --git a/tests/slt/crdb/join.slt b/tests/slt/crdb/join.slt index 62debc74..b867d43e 100644 --- a/tests/slt/crdb/join.slt +++ b/tests/slt/crdb/join.slt @@ -11,14 +11,14 @@ query II SELECT * FROM onecolumn AS a(aid, x) CROSS JOIN onecolumn AS b(bid, y) order by x ---- 2 42 0 44 -2 42 1 null +2 42 1 NULL 2 42 2 42 0 44 0 44 -0 44 1 null +0 44 1 NULL 0 44 2 42 -1 null 0 44 -1 null 1 null -1 null 2 42 +1 NULL 0 44 +1 NULL 1 NULL +1 NULL 2 42 statement error SELECT x FROM onecolumn AS a, onecolumn AS b; @@ -46,14 +46,14 @@ SELECT * FROM onecolumn AS a(aid, x) LEFT OUTER JOIN onecolumn AS b(bid, y) ON a ---- 2 42 2 42 0 44 0 44 -1 null null null +1 NULL NULL NULL query I SELECT * FROM onecolumn AS a LEFT OUTER JOIN onecolumn AS b USING(x) ORDER BY x ---- 2 42 2 0 44 0 -1 null null +1 NULL NULL statement error SELECT * FROM onecolumn AS a, onecolumn AS b ORDER BY x @@ -63,28 +63,28 @@ SELECT * FROM onecolumn AS a NATURAL LEFT OUTER JOIN onecolumn AS b order by a.x ---- 2 42 0 44 -1 null +1 NULL query II SELECT * FROM onecolumn AS a(aid, x) RIGHT OUTER JOIN onecolumn AS b(bid, y) ON a.x = b.y order by x ---- 2 42 2 42 0 44 0 44 -null null 1 null +NULL NULL 1 NULL query I SELECT * FROM onecolumn AS a RIGHT OUTER JOIN onecolumn AS b USING(x) ORDER BY x ---- 2 2 42 0 0 44 -null 1 null +NULL 1 NULL query I SELECT * FROM onecolumn AS a NATURAL RIGHT OUTER JOIN onecolumn AS b order by x ---- 2 42 0 44 -1 null +1 NULL statement ok drop table if exists onecolumn_w @@ -100,8 +100,8 @@ SELECT * FROM onecolumn AS a NATURAL JOIN onecolumn_w as b ---- 0 44 0 42 0 44 1 43 -1 null 0 42 -1 null 1 43 +1 NULL 0 42 +1 NULL 1 43 2 42 0 42 2 42 1 43 @@ -118,38 +118,38 @@ query II SELECT * FROM onecolumn AS a FULL OUTER JOIN othercolumn AS b ON a.x = b.x ORDER BY a.x,b.x ---- 2 42 1 42 -0 44 null null -null null 2 16 -null null 0 43 -1 null null null +0 44 NULL NULL +NULL NULL 2 16 +NULL NULL 0 43 +1 NULL NULL NULL query II SELECT * FROM onecolumn AS a full OUTER JOIN othercolumn AS b ON a.x = b.x and a.x > 16 order by a.x ---- 2 42 1 42 -0 44 null null -null null 0 43 -null null 2 16 -1 null null null +0 44 NULL NULL +NULL NULL 0 43 +NULL NULL 2 16 +1 NULL NULL NULL query II SELECT * FROM onecolumn AS a full OUTER JOIN othercolumn AS b ON a.x = b.x and b.x > 16 order by b.x,a.x ---- -null null 2 16 +NULL NULL 2 16 2 42 1 42 -null null 0 43 -0 44 null null -1 null null null +NULL NULL 0 43 +0 44 NULL NULL +1 NULL NULL NULL query II SELECT a.x, b.x FROM onecolumn AS a full OUTER JOIN othercolumn AS b ON false order by a.x, b.x ---- -42 null -44 null -null 16 -null 42 -null 43 -null null +42 NULL +44 NULL +NULL 16 +NULL 42 +NULL 43 +NULL NULL query II SELECT a.x, b.x FROM onecolumn AS a full OUTER JOIN othercolumn AS b ON true order by a.x, b.x @@ -160,9 +160,9 @@ SELECT a.x, b.x FROM onecolumn AS a full OUTER JOIN othercolumn AS b ON true ord 44 16 44 42 44 43 -null 16 -null 42 -null 43 +NULL 16 +NULL 42 +NULL 43 # Tips: This case will make x take the value of both sides # query @@ -212,16 +212,16 @@ SELECT * FROM empty AS a JOIN onecolumn AS b USING(x) query IT SELECT * FROM onecolumn AS a(aid, x) LEFT OUTER JOIN empty AS b(bid, y) ON a.x = b.y ORDER BY a.x ---- -2 42 null null -0 44 null null -1 null null null +2 42 NULL NULL +0 44 NULL NULL +1 NULL NULL NULL query I rowsort SELECT * FROM onecolumn AS a LEFT OUTER JOIN empty AS b USING(x) ORDER BY x ---- -0 44 null -1 null null -2 42 null +0 44 NULL +1 NULL NULL +2 42 NULL query I SELECT * FROM empty AS a(aid, x) LEFT OUTER JOIN onecolumn AS b(bid, y) ON a.x = b.y @@ -242,9 +242,9 @@ SELECT * FROM onecolumn AS a RIGHT OUTER JOIN empty AS b USING(x) query II SELECT * FROM empty AS a(aid, x) FULL OUTER JOIN onecolumn AS b(bid, y) ON a.x = b.y ORDER BY b.y ---- -null null 2 42 -null null 0 44 -null null 1 null +NULL NULL 2 42 +NULL NULL 0 44 +NULL NULL 1 NULL statement ok SELECT * FROM empty AS a FULL OUTER JOIN onecolumn AS b USING(x) ORDER BY x @@ -252,23 +252,23 @@ SELECT * FROM empty AS a FULL OUTER JOIN onecolumn AS b USING(x) ORDER BY x query IIII SELECT * FROM onecolumn AS a(aid, x) FULL OUTER JOIN empty AS b(bid, y) ON a.x = b.y ORDER BY a.x ---- -2 42 null null -0 44 null null -1 null null null +2 42 NULL NULL +0 44 NULL NULL +1 NULL NULL NULL query III rowsort SELECT * FROM onecolumn AS a FULL OUTER JOIN empty AS b USING(x) ORDER BY x ---- -0 44 null -1 null null -2 42 null +0 44 NULL +1 NULL NULL +2 42 NULL query II SELECT * FROM empty AS a(aid, x) FULL OUTER JOIN onecolumn AS b(bid, y) ON a.x = b.y ORDER BY b.y ---- -null null 2 42 -null null 0 44 -null null 1 null +NULL NULL 2 42 +NULL NULL 0 44 +NULL NULL 1 NULL # query # SELECT * FROM empty AS a FULL OUTER JOIN onecolumn AS b USING(x) ORDER BY x @@ -292,7 +292,7 @@ query IIII SELECT * FROM twocolumn AS a JOIN twocolumn AS b ON a.x = a.y order by a.x ---- 3 45 45 0 44 51 -3 45 45 1 null 52 +3 45 45 1 NULL 52 3 45 45 2 42 53 3 45 45 3 45 45 @@ -307,22 +307,22 @@ query IT SELECT o.x, t.y FROM onecolumn o LEFT OUTER JOIN twocolumn t ON (o.x=t.x AND t.y=53) order by o.x ---- 42 53 -44 null -null null +44 NULL +NULL NULL query II SELECT o.x, t.y FROM onecolumn o LEFT OUTER JOIN twocolumn t ON (o.x=t.x AND o.x=44) order by o.x ---- -42 null +42 NULL 44 51 -null null +NULL NULL query II SELECT o.x, t.y FROM onecolumn o LEFT OUTER JOIN twocolumn t ON (o.x=t.x AND t.x=44) order by o.x ---- -42 null +42 NULL 44 51 -null null +NULL NULL # query # SELECT * FROM (SELECT x, 2 two FROM onecolumn) NATURAL FULL JOIN (SELECT x, y+1 plus1 FROM twocolumn) @@ -356,31 +356,31 @@ SELECT * FROM a LEFT OUTER JOIN b ON a.i = b.i ---- 1 2 0 2 true 2 3 1 3 true -0 1 null null null +0 1 NULL NULL NULL query III SELECT * FROM a RIGHT OUTER JOIN b ON a.i = b.i order by b ---- -null null 2 4 false +NULL NULL 2 4 false 1 2 0 2 true 2 3 1 3 true query III SELECT * FROM a FULL OUTER JOIN b ON a.i = b.i order by b ---- -null null 2 4 false +NULL NULL 2 4 false 1 2 0 2 true 2 3 1 3 true -0 1 null null null +0 1 NULL NULL NULL query III SELECT * FROM a FULL OUTER JOIN b ON (a.i = b.i and a.i>2) ORDER BY a.i, b.i ---- -0 1 null null null -1 2 null null null +0 1 NULL NULL NULL +1 2 NULL NULL NULL 2 3 1 3 true -null null 0 2 true -null null 2 4 false +NULL NULL 0 2 true +NULL NULL 2 4 false statement ok INSERT INTO b VALUES (3, 3, false) @@ -391,7 +391,7 @@ SELECT * FROM a RIGHT OUTER JOIN b ON a.i=b.i ORDER BY b.i, b.b 1 2 0 2 true 2 3 3 3 false 2 3 1 3 true -null null 2 4 false +NULL NULL 2 4 false query III SELECT * FROM a FULL OUTER JOIN b ON a.i=b.i ORDER BY b.i, b.b @@ -399,8 +399,8 @@ SELECT * FROM a FULL OUTER JOIN b ON a.i=b.i ORDER BY b.i, b.b 1 2 0 2 true 2 3 3 3 false 2 3 1 3 true -null null 2 4 false -0 1 null null null +NULL NULL 2 4 false +0 1 NULL NULL NULL # TODO # query IIIIII @@ -541,25 +541,25 @@ SELECT * FROM pairs, square WHERE pairs.a + pairs.b = square.sq query III SELECT a, b, n, sq FROM pairs FULL OUTER JOIN square ON pairs.a + pairs.b = square.sq order by a ---- -1 1 null null -1 2 null null +1 1 NULL NULL +1 2 NULL NULL 1 3 2 4 -1 4 null null -1 5 null null -1 6 null null -2 3 null null -2 4 null null -2 5 null null -2 6 null null -3 4 null null -3 5 null null +1 4 NULL NULL +1 5 NULL NULL +1 6 NULL NULL +2 3 NULL NULL +2 4 NULL NULL +2 5 NULL NULL +2 6 NULL NULL +3 4 NULL NULL +3 5 NULL NULL 3 6 3 9 4 5 3 9 -4 6 null null -null null 1 1 -null null 4 16 -null null 5 25 -null null 6 36 +4 6 NULL NULL +NULL NULL 1 1 +NULL NULL 4 16 +NULL NULL 5 25 +NULL NULL 6 36 query IIII SELECT pairs.a, pairs.b, square.* FROM pairs FULL OUTER JOIN square ON pairs.a + pairs.b = square.sq WHERE pairs.b%2 <> square.sq%2 order by a @@ -570,29 +570,29 @@ SELECT pairs.a, pairs.b, square.* FROM pairs FULL OUTER JOIN square ON pairs.a + query IITT rowsort SELECT * FROM (SELECT * FROM pairs LEFT JOIN square ON b = sq AND a > 1 AND n < 6) WHERE b > 1 AND (n IS NULL OR n > 1) AND (n IS NULL OR a < sq) ---- -1 1 2 null null +1 1 2 NULL NULL 10 3 4 2 4 -11 3 5 null null -12 3 6 null null -13 4 5 null null -14 4 6 null null -2 1 3 null null -3 1 4 null null -4 1 5 null null -5 1 6 null null -6 2 3 null null +11 3 5 NULL NULL +12 3 6 NULL NULL +13 4 5 NULL NULL +14 4 6 NULL NULL +2 1 3 NULL NULL +3 1 4 NULL NULL +4 1 5 NULL NULL +5 1 6 NULL NULL +6 2 3 NULL NULL 7 2 4 2 4 -8 2 5 null null -9 2 6 null null +8 2 5 NULL NULL +9 2 6 NULL NULL query IIII SELECT * FROM (SELECT * FROM pairs RIGHT JOIN square ON b = sq AND a > 1 AND n < 6) WHERE (a IS NULL OR a > 2) AND n > 1 AND (a IS NULL OR a < sq) order by n ---- 10 3 4 2 4 -null null null 3 9 -null null null 4 16 -null null null 5 25 -null null null 6 36 +NULL NULL NULL 3 9 +NULL NULL NULL 4 16 +NULL NULL NULL 5 25 +NULL NULL NULL 6 36 statement ok drop table if exists t1 @@ -697,25 +697,25 @@ query TTT SELECT s, str1.s, str2.s FROM str1 LEFT OUTER JOIN str2 USING(s) order by str1.s ---- A A A -D D null -a a null -c c null +D D NULL +a a NULL +c c NULL query TTT SELECT s, str1.s, str2.s FROM str1 RIGHT OUTER JOIN str2 USING(s) order by str2.s ---- A A A -B null B -C null C -E null E +B NULL B +C NULL C +E NULL E query ITIT SELECT * FROM str1 LEFT OUTER JOIN str2 ON str1.s = str2.s order by str1.a ---- -0 1 a null null null +0 1 a NULL NULL NULL 1 2 A 0 1 A -2 3 c null null null -3 4 D null null null +2 3 c NULL NULL NULL +3 4 D NULL NULL NULL statement ok INSERT INTO str1 VALUES (4, 1, 'a' ), (5, 2, 'A'), (6, 3, 'c'), (7, 4, 'D') @@ -725,17 +725,17 @@ select * from str1 right join str2 on str1.s = str2.s order by str2.a ---- 1 2 A 0 1 A 5 2 A 0 1 A -null null null 1 2 B -null null null 2 3 C -null null null 3 4 E +NULL NULL NULL 1 2 B +NULL NULL NULL 2 3 C +NULL NULL NULL 3 4 E query ITIT select * from str1 right join str2 on false order by str2.a ---- -null null null 0 1 A -null null null 1 2 B -null null null 2 3 C -null null null 3 4 E +NULL NULL NULL 0 1 A +NULL NULL NULL 1 2 B +NULL NULL NULL 2 3 C +NULL NULL NULL 3 4 E # query # SELECT s, str1.s, str2.s FROM str1 FULL OUTER JOIN str2 USING(s) @@ -767,8 +767,8 @@ query IIII SELECT * FROM xyu LEFT OUTER JOIN xyv USING(x, y) WHERE x > 2 order by xyu.uid ---- 2 3 1 31 7 31 -3 3 2 32 null null -4 4 4 44 null null +3 3 2 32 NULL NULL +4 4 4 44 NULL NULL # TODO: x > 2 means x on both sides # query IIII @@ -780,10 +780,10 @@ query IIIIII rowsort SELECT * FROM xyu FULL OUTER JOIN xyv USING(x, y) WHERE x > 2 ---- 2 3 1 31 7 31 -3 3 2 32 null null -4 4 4 44 null null -null 3 3 null 8 33 -null 5 5 null 9 55 +3 3 2 32 NULL NULL +4 4 4 44 NULL NULL +NULL 3 3 NULL 8 33 +NULL 5 5 NULL 9 55 query IIIIII SELECT * FROM xyu INNER JOIN xyv ON xyu.x = xyv.x AND xyu.y = xyv.y WHERE xyu.x = 1 AND xyu.y < 10 @@ -798,27 +798,27 @@ SELECT * FROM xyu INNER JOIN xyv ON xyu.x = xyv.x AND xyu.y = xyv.y AND xyu.x = query IIITTT SELECT * FROM xyu LEFT OUTER JOIN xyv ON xyu.x = xyv.x AND xyu.y = xyv.y AND xyu.x = 1 AND xyu.y < 10 order by xyu.uid ---- -0 0 0 0 null null null null +0 0 0 0 NULL NULL NULL NULL 1 1 1 1 5 1 1 1 -2 3 1 31 null null null null -3 3 2 32 null null null null -4 4 4 44 null null null null +2 3 1 31 NULL NULL NULL NULL +3 3 2 32 NULL NULL NULL NULL +4 4 4 44 NULL NULL NULL NULL query IIIIII SELECT * FROM xyu RIGHT OUTER JOIN xyv ON xyu.x = xyv.x AND xyu.y = xyv.y AND xyu.x = 1 AND xyu.y < 10 order by v ---- 1 1 1 1 5 1 1 1 -null null null null 6 2 2 2 -null null null null 7 3 1 31 -null null null null 8 3 3 33 -null null null null 9 5 5 55 +NULL NULL NULL NULL 6 2 2 2 +NULL NULL NULL NULL 7 3 1 31 +NULL NULL NULL NULL 8 3 3 33 +NULL NULL NULL NULL 9 5 5 55 query IIII rowsort SELECT * FROM (SELECT * FROM xyu ORDER BY x, y) AS xyu LEFT OUTER JOIN (SELECT * FROM xyv ORDER BY x, y) AS xyv USING(x, y) WHERE x > 2 ---- 2 3 1 31 7 31 -3 3 2 32 null null -4 4 4 44 null null +3 3 2 32 NULL NULL +4 4 4 44 NULL NULL # TODO: x > 2 means x on both sides # query IIII @@ -835,20 +835,20 @@ SELECT * FROM (SELECT * FROM xyu ORDER BY x, y) AS xyu LEFT OUTER JOIN (SELECT * query IIITTT rowsort SELECT * FROM (SELECT * FROM xyu ORDER BY x, y) AS xyu LEFT OUTER JOIN (SELECT * FROM xyv ORDER BY x, y) AS xyv ON xyu.x = xyv.x AND xyu.y = xyv.y AND xyu.x = 1 AND xyu.y < 10 ---- -0 0 0 0 null null null null +0 0 0 0 NULL NULL NULL NULL 1 1 1 1 5 1 1 1 -2 3 1 31 null null null null -3 3 2 32 null null null null -4 4 4 44 null null null null +2 3 1 31 NULL NULL NULL NULL +3 3 2 32 NULL NULL NULL NULL +4 4 4 44 NULL NULL NULL NULL query IIIIII SELECT * FROM xyu RIGHT OUTER JOIN (SELECT * FROM xyv ORDER BY x, y) AS xyv ON xyu.x = xyv.x AND xyu.y = xyv.y AND xyu.x = 1 AND xyu.y < 10 ORDER BY v ---- 1 1 1 1 5 1 1 1 -null null null null 6 2 2 2 -null null null null 7 3 1 31 -null null null null 8 3 3 33 -null null null null 9 5 5 55 +NULL NULL NULL NULL 6 2 2 2 +NULL NULL NULL NULL 7 3 1 31 +NULL NULL NULL NULL 8 3 3 33 +NULL NULL NULL NULL 9 5 5 55 statement ok drop table if exists l @@ -876,27 +876,27 @@ SELECT a, l.a, r.a FROM l INNER JOIN r USING(a) WHERE a = 2 query III SELECT a, l.a, r.a FROM l LEFT OUTER JOIN r USING(a) WHERE a = 1 ---- -1 1 null +1 1 NULL query III SELECT a, l.a, r.a FROM l RIGHT OUTER JOIN r USING(a) WHERE a = 4 ---- -4 null 4 +4 NULL 4 query III SELECT a, l.a, r.a FROM l FULL OUTER JOIN r USING(a) WHERE a = 1 ---- -1 1 null +1 1 NULL query III SELECT a, l.a, r.a FROM l FULL OUTER JOIN r USING(a) WHERE a = 4 ---- -4 null 4 +4 NULL 4 query III SELECT * FROM l LEFT OUTER JOIN r USING(a) WHERE a = 1 ---- -1 1 null +1 1 NULL query III SELECT * FROM l LEFT OUTER JOIN r USING(a) WHERE a = 2 @@ -911,7 +911,7 @@ SELECT * FROM l RIGHT OUTER JOIN r USING(a) WHERE a = 3 query III SELECT * FROM l RIGHT OUTER JOIN r USING(a) WHERE a = 4 ---- -null 4 1 +NULL 4 1 statement ok drop table if exists foo @@ -998,10 +998,10 @@ drop table if exists bar query TII SELECT * FROM onecolumn AS a(aid, x) RIGHT JOIN twocolumn ON false order by y ---- -null null 3 45 45 -null null 0 44 51 -null null 1 null 52 -null null 2 42 53 +NULL NULL 3 45 45 +NULL NULL 0 44 51 +NULL NULL 1 NULL 52 +NULL NULL 2 42 53 statement ok SELECT * FROM onecolumn AS a(aid, x) RIGHT JOIN twocolumn ON true where false order by y @@ -1187,3 +1187,84 @@ drop table onecolumn_w statement ok drop table str1 + +# Comma/CROSS joins must recognize conjunctive equality keys without losing residual filters. +statement ok +create table ij_a(id int primary key, k int, v int); + +statement ok +create table ij_b(id int primary key, k bigint, v int); + +statement ok +create table ij_c(id int primary key, k int); + +statement ok +insert into ij_a values (1,1,10),(2,2,20),(3,null,30),(4,2,25); + +statement ok +insert into ij_b values (1,2,100),(2,1,200),(3,2,300),(4,null,400); + +statement ok +insert into ij_c values (1,1),(2,2); + +# Both side filters are pushed down, while the cross-side inequality stays above the equijoin. +query T +explain select a.id,b.id from ij_a a,ij_b b where b.k=a.k and a.v>15 and b.v<250 and a.v (Sort Option: Follow)] Filter (a.v < b.v), Is Having: false [Filter => (Sort Option: Follow)] Inner Join On cast (a.k as Bigint) = b.k [HashJoin => (Sort Option: None)] Filter ((ij_a.v) as (a.v) > 15), Is Having: false [Filter => (Sort Option: Follow)] TableScan ij_a -> [ij_a.id, ij_a.k, ij_a.v] [SeqScan => (Sort Option: None)] Filter ((ij_b.v) as (b.v) < 250), Is Having: false [Filter => (Sort Option: Follow)] TableScan ij_b -> [ij_b.id, ij_b.k, ij_b.v] [SeqScan => (Sort Option: None)] + +query II +select a.id,b.id from ij_a a,ij_b b where b.k=a.k and a.v>15 and b.v<250 and a.v320 order by a.id,b.id; +---- +4 3 + +query II +select a.id,b.id from ij_a a,ij_b b where a.k=b.k or a.id=b.id order by a.id,b.id; +---- +1 1 +1 2 +2 1 +2 2 +2 3 +3 3 +4 1 +4 3 +4 4 + +query II +select a.id,b.id from ij_a a,ij_b b where a.id=2 and b.id=1; +---- +2 1 + +query III +select a.id,b.id,c.id from ij_a a,ij_b b,ij_c c where c.k=b.k and b.k=a.k and a.v>15 and c.id=2 order by a.id,b.id,c.id; +---- +2 1 2 +2 3 2 +4 1 2 +4 3 2 + +query II +select a.id,b.id from ij_a a,ij_b b where a.v>b.v order by a.id,b.id; +---- + +query II +select a.id,b.id from ij_a a,ij_a b where b.k=a.k and a.id=2 order by b.id; +---- +2 2 +2 4 + +statement ok +drop table ij_c; + +statement ok +drop table ij_b; + +statement ok +drop table ij_a; diff --git a/tests/slt/crdb/natural_join.slt b/tests/slt/crdb/natural_join.slt index 505e9be1..b3cd1f4c 100644 --- a/tests/slt/crdb/natural_join.slt +++ b/tests/slt/crdb/natural_join.slt @@ -81,9 +81,9 @@ query II SELECT k, x FROM t1 NATURAL JOIN t2 ---- 0 0 -0 null -null 0 -null null +0 NULL +NULL 0 +NULL NULL query II SELECT k, x FROM t1 JOIN t2 ON k = x diff --git a/tests/slt/crdb/order_by.slt b/tests/slt/crdb/order_by.slt index f93ed8d8..734b9f58 100644 --- a/tests/slt/crdb/order_by.slt +++ b/tests/slt/crdb/order_by.slt @@ -12,21 +12,21 @@ SELECT c FROM t ORDER BY c ---- false true -null +NULL query I SELECT c FROM t ORDER BY c ---- false true -null +NULL query I SELECT c FROM t ORDER BY c DESC ---- true false -null +NULL query II SELECT a, b FROM t ORDER BY b @@ -166,18 +166,18 @@ SELECT * FROM t ORDER BY true,a ---- 1 9 true 2 8 false -3 7 null -4 7 null -5 7 null +3 7 NULL +4 7 NULL +5 7 NULL query III SELECT * FROM t ORDER BY "a" ---- 1 9 true 2 8 false -3 7 null -4 7 null -5 7 null +3 7 NULL +4 7 NULL +5 7 NULL query error SELECT * FROM t ORDER BY 2.5,a @@ -232,8 +232,8 @@ INSERT INTO bar VALUES (0, NULL), (1, NULL) query IT SELECT * FROM bar ORDER BY baz, id ---- -0 null -1 null +0 NULL +1 NULL statement ok drop table if exists abcd @@ -272,8 +272,8 @@ INSERT INTO store VALUES (0, NULL, 10), (1, NULL, 5) query ITI SELECT * FROM store ORDER BY baz, extra ---- -1 null 5 -0 null 10 +1 NULL 5 +0 NULL 10 statement ok drop table if exists xy @@ -287,42 +287,42 @@ INSERT INTO xy VALUES (0, 2, NULL), (1, NULL, 6), (2, 2, 5), (3, 4, 8) query IT SELECT x, y FROM xy ORDER BY y NULLS FIRST ---- -2 null +2 NULL 2 5 -null 6 +NULL 6 4 8 query II SELECT x, y FROM xy ORDER BY y NULLS LAST ---- 2 5 -null 6 +NULL 6 4 8 -2 null +2 NULL query IT SELECT x, y FROM xy ORDER BY y DESC NULLS FIRST ---- -2 null +2 NULL 4 8 -null 6 +NULL 6 2 5 query II SELECT x, y FROM xy ORDER BY y DESC NULLS LAST ---- 4 8 -null 6 +NULL 6 2 5 -2 null +2 NULL query II SELECT x, y FROM xy ORDER BY y NULLS LAST ---- 2 5 -null 6 +NULL 6 4 8 -2 null +2 NULL statement ok INSERT INTO xy VALUES (4, NULL, NULL) @@ -330,72 +330,72 @@ INSERT INTO xy VALUES (4, NULL, NULL) query TI SELECT x, y FROM xy ORDER BY x NULLS FIRST, y NULLS LAST ---- -null 6 -null null +NULL 6 +NULL NULL 2 5 -2 null +2 NULL 4 8 query IT SELECT x, y FROM xy ORDER BY x NULLS LAST, y DESC NULLS FIRST ---- -2 null +2 NULL 2 5 4 8 -null null -null 6 +NULL NULL +NULL 6 query II SELECT x, y FROM xy ORDER BY x, y ---- 2 5 -2 null +2 NULL 4 8 -null 6 -null null +NULL 6 +NULL NULL query IT SELECT x, y FROM xy ORDER BY x, y DESC NULLS FIRST ---- -2 null +2 NULL 2 5 4 8 -null null -null 6 +NULL NULL +NULL 6 query IT SELECT x, y FROM xy ORDER BY x NULLS LAST, y DESC NULLS FIRST ---- -2 null +2 NULL 2 5 4 8 -null null -null 6 +NULL NULL +NULL 6 query TT SELECT x, y FROM xy ORDER BY x NULLS FIRST, y DESC NULLS LAST ---- -null 6 -null null +NULL 6 +NULL NULL 2 5 -2 null +2 NULL 4 8 query TT SELECT x, y FROM xy ORDER BY x NULLS FIRST, y DESC ---- -null 6 -null null +NULL 6 +NULL NULL 2 5 -2 null +2 NULL 4 8 query TI SELECT x, y FROM xy ORDER BY x NULLS FIRST, y DESC NULLS FIRST ---- -null null -null 6 -2 null +NULL NULL +NULL 6 +2 NULL 2 5 4 8 diff --git a/tests/slt/crdb/select.slt b/tests/slt/crdb/select.slt index b4eba0be..be782049 100644 --- a/tests/slt/crdb/select.slt +++ b/tests/slt/crdb/select.slt @@ -25,7 +25,7 @@ SELECT * FROM abc query TIII SELECT NULL AS z, * FROM abc ---- -null 1 2 3 +NULL 1 2 3 statement ok SELECT * FROM abc WHERE NULL @@ -79,17 +79,17 @@ INSERT INTO kv (k, v) VALUES ('a', null) query TT SELECT * FROM kv ---- -a null +a NULL query TT SELECT k,v FROM kv ---- -a null +a NULL query T SELECT v||'foo' FROM kv ---- -null +NULL # TODO: lower # query T @@ -105,12 +105,12 @@ a query TT SELECT kv.K,KV.v FROM kv ---- -a null +a NULL query TT SELECT kv.* FROM kv ---- -a null +a NULL query T SELECT FOO.k FROM kv AS foo WHERE foo.k = 'a' @@ -120,7 +120,7 @@ a query T SELECT foo.v FROM kv AS foo WHERE foo.k = 'a' ---- -null +NULL onlyif mysql statement ok @@ -232,7 +232,7 @@ INSERT INTO boolean_table (id, value) VALUES (1, NULL) query T SELECT value FROM boolean_table ---- -null +NULL query I SELECT CASE WHEN NULL THEN 1 ELSE 2 END @@ -267,7 +267,7 @@ true query T SELECT NULL IN (1, 2) ---- -null +NULL query I SELECT 1 IN (1, NULL) @@ -277,7 +277,7 @@ true query I SELECT 1 IN (NULL, 2) ---- -null +NULL # statement error 1105 # SELECT NULL IN ((1, 1)) @@ -530,7 +530,7 @@ INSERT INTO wide(id, n) VALUES(0, 10) query IITITTITTTTIRRI SELECT * FROM wide ---- -0 null null null null null null null null null null null null null 10 +0 NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL 10 statement ok drop table xyzw diff --git a/tests/slt/crdb/where.slt b/tests/slt/crdb/where.slt index 48cd417f..fc6b216f 100644 --- a/tests/slt/crdb/where.slt +++ b/tests/slt/crdb/where.slt @@ -93,14 +93,14 @@ SELECT a, b FROM ab WHERE a IN (1, 3, 4) ---- 1 10 3 30 -4 null +4 NULL query II SELECT a, b FROM ab WHERE a IN (1, 3, 4, NULL) ---- 1 10 3 30 -4 null +4 NULL query SELECT a, b FROM ab WHERE (a, b) IN ((1, 10), (3, 30), (4, 40)) diff --git a/tests/slt/describe.slt b/tests/slt/describe.slt index e327103e..f3377433 100644 --- a/tests/slt/describe.slt +++ b/tests/slt/describe.slt @@ -4,9 +4,9 @@ create table t9 (c1 int primary key, c2 int default 0, c3 varchar unique); query TTTTTT describe t9; ---- -c1 Integer 4 false PRIMARY null +c1 Integer 4 false PRIMARY NULL c2 Integer 4 true EMPTY 0 -c3 Varchar(None, CHARACTERS) variable true UNIQUE null +c3 Varchar(None, CHARACTERS) variable true UNIQUE NULL statement ok drop table t9; @@ -17,9 +17,9 @@ create table t9_m (c1 int primary key, c2 int primary key, c3 varchar unique); query TTTTTT describe t9_m; ---- -c1 Integer 4 false PRIMARY null -c2 Integer 4 false PRIMARY null -c3 Varchar(None, CHARACTERS) variable true UNIQUE null +c1 Integer 4 false PRIMARY NULL +c2 Integer 4 false PRIMARY NULL +c3 Varchar(None, CHARACTERS) variable true UNIQUE NULL statement ok drop table t9_m; diff --git a/tests/slt/distinct.slt b/tests/slt/distinct.slt index 0ce6bab9..4606171f 100644 --- a/tests/slt/distinct.slt +++ b/tests/slt/distinct.slt @@ -53,9 +53,9 @@ insert into distinct_alias values (0, 1, 2, 'a', 1.5), (1, 2, null, null, null), query TT rowsort select distinct lower(c) as x, c from distinct_alias; ---- +NULL NULL a A a a -null null query T rowsort select distinct (a = 1) as x from distinct_alias; @@ -73,7 +73,7 @@ query T rowsort select distinct upper(c) as x from distinct_alias order by x; ---- A -null +NULL statement ok drop table distinct_alias diff --git a/tests/slt/dummy.slt b/tests/slt/dummy.slt index 4dcb3eba..cbdbf778 100644 --- a/tests/slt/dummy.slt +++ b/tests/slt/dummy.slt @@ -45,7 +45,7 @@ false query B SELECT NULL=NULL ---- -null +NULL query B SELECT NOT(1=1) @@ -95,12 +95,12 @@ SELECT NULLIF(1, 2) query I SELECT NULLIF(NULL, 2) ---- -null +NULL query I SELECT NULLIF(NULL, NULL) ---- -null +NULL query I SELECT IFNULL(1, NULL) @@ -115,7 +115,7 @@ SELECT IFNULL(NULL, 1) query I SELECT IFNULL(NULL, NULL) ---- -null +NULL query I SELECT COALESCE(1) @@ -140,12 +140,12 @@ SELECT COALESCE(NULL, 2, NULL) query I SELECT COALESCE(NULL, NULL) ---- -null +NULL query I SELECT COALESCE() ---- -null +NULL query I SELECT CASE 1 WHEN 0 THEN 0 WHEN 1 THEN 1 WHEN NULL THEN 9 ELSE 2 END @@ -160,7 +160,7 @@ SELECT CASE 3 WHEN 0 THEN 0 WHEN 1 THEN 1 WHEN NULL THEN 9 ELSE 2 END query I SELECT CASE 3 WHEN 0 THEN 0 WHEN 1 THEN 1 WHEN NULL THEN 9 END ---- -null +NULL query I SELECT CASE 3 WHEN 0 THEN 0 WHEN 1 THEN 1 WHEN NULL THEN 9 ELSE 2 END @@ -180,7 +180,7 @@ SELECT CASE WHEN FALSE THEN 0 WHEN FALSE THEN 1 WHEN FALSE THEN 9 ELSE 2 END query I SELECT CASE WHEN FALSE THEN 0 WHEN FALSE THEN 1 WHEN FALSE THEN 9 END ---- -null +NULL query I SELECT CASE WHEN FALSE THEN 0 WHEN 1=1 THEN 1 WHEN FALSE THEN 9 ELSE 2 END @@ -227,12 +227,12 @@ true query I select - (null) from case_ty; ---- -null +NULL query T select 3 = - null from case_ty; ---- -null +NULL statement ok drop table case_ty diff --git a/tests/slt/exists.slt b/tests/slt/exists.slt index 80cadb32..496370bd 100644 --- a/tests/slt/exists.slt +++ b/tests/slt/exists.slt @@ -38,3 +38,42 @@ drop table t1 statement ok drop table t2 + +# A table alias hides the table name, and a correlated EXISTS keeps the alias identity of its +# columns, so non-equality correlations compare inner and outer rows. +statement ok +create table t_corr(pk int primary key, b int) + +statement ok +insert into t_corr values (0, 10), (1, 20), (2, 30), (3, null) + +statement error +select t_corr.b from t_corr as x + +query I rowsort +select b from t_corr where exists(select 1 from t_corr as x where x.b < t_corr.b) +---- +20 +30 + +query I rowsort +select b from t_corr where exists(select 1 from t_corr as x where x.b = t_corr.b) +---- +10 +20 +30 + +query I rowsort +select y.b from t_corr as y where exists(select 1 from t_corr as x where x.b > y.b) +---- +10 +20 + +query I rowsort +select b from t_corr where not exists(select 1 from t_corr as x where x.b < t_corr.b) +---- +10 +NULL + +statement ok +drop table t_corr diff --git a/tests/slt/filter_null.slt b/tests/slt/filter_null.slt index 01d3bde9..4eb4a1c6 100644 --- a/tests/slt/filter_null.slt +++ b/tests/slt/filter_null.slt @@ -35,7 +35,7 @@ select * from t where v1 > 1 query III select * from t where v1 is null ---- -1 null 3 +1 NULL 3 query III select * from t where v1 is not null diff --git a/tests/slt/insert.slt b/tests/slt/insert.slt index 1f7d2424..ca48be81 100644 --- a/tests/slt/insert.slt +++ b/tests/slt/insert.slt @@ -42,17 +42,17 @@ select * from t ---- 0 1 10 100 1 9 9 9 -10 null 1 null -11 null 9 null +10 NULL 1 NULL +11 NULL 9 NULL 2 2 20 200 20 1 10 100 21 9 9 9 3 3 30 300 4 4 40 400 5 1 10 100 -6 1 10 null -7 10 1 null -8 null null null +6 1 10 NULL +7 10 1 NULL +8 NULL NULL NULL statement ok drop table t; diff --git a/tests/slt/insert_multiple_primary_keys.slt b/tests/slt/insert_multiple_primary_keys.slt index d20662d3..0e1be266 100644 --- a/tests/slt/insert_multiple_primary_keys.slt +++ b/tests/slt/insert_multiple_primary_keys.slt @@ -40,9 +40,9 @@ select * from t 3 3 3 30 300 4 4 4 40 400 5 5 1 10 100 -6 6 1 10 null -7 7 10 1 null -8 8 null null null +6 6 1 10 NULL +7 7 10 1 NULL +8 8 NULL NULL NULL statement ok drop table t; diff --git a/tests/slt/join.slt b/tests/slt/join.slt index 40c5bcc2..c65945a9 100644 --- a/tests/slt/join.slt +++ b/tests/slt/join.slt @@ -59,9 +59,9 @@ insert into a values (0, 1, 1), (1, 2, 2), (2, 3, 3); query IIII rowsort select v1, v2, v3, v4 from a left join b on v1 = v3; ---- -1 1 null null -2 2 null null -3 3 null null +1 1 NULL NULL +2 2 NULL NULL +3 3 NULL NULL statement ok insert into b values (0, 1, 100), (1, 3, 300), (2, 4, 400); @@ -71,22 +71,22 @@ select v1, v2, v3, v4 from a left join b on v1 = v3; ---- 1 1 1 100 3 3 3 300 -2 2 null null +2 2 NULL NULL query IIII rowsort select v1, v2, v3, v4 from a right join b on v1 = v3; ---- 1 1 1 100 3 3 3 300 -null null 4 400 +NULL NULL 4 400 query IIII rowsort select v1, v2, v3, v4 from a full join b on v1 = v3; ---- 1 1 1 100 -2 2 null null +2 2 NULL NULL 3 3 3 300 -null null 4 400 +NULL NULL 4 400 statement ok drop table a; @@ -164,7 +164,7 @@ select a.*, b.* from a left join b on v1 < v3 or v2 < v4 0 1 1 1 2 2 2 0 1 1 2 3 3 4 1 2 2 2 3 3 4 -2 3 3 null null null null +2 3 3 NULL NULL NULL NULL query IIIIIII rowsort select a.*, b.* from a right join b on v1 <> v3 and v2 < v4 @@ -172,8 +172,8 @@ select a.*, b.* from a right join b on v1 <> v3 and v2 < v4 0 1 1 1 2 2 2 0 1 1 2 3 3 4 1 2 2 2 3 3 4 -null null null 0 1 1 1 -null null null 3 1 1 5 +NULL NULL NULL 0 1 1 1 +NULL NULL NULL 3 1 1 5 query IIIIIII rowsort select a.*, b.* from a cross join b @@ -312,8 +312,8 @@ left join lookup_inner i on o.k=i.k order by o.id,i.id; 1 1 1 2 2 3 -3 null -4 null +3 NULL +4 NULL 5 1 5 2 @@ -564,16 +564,16 @@ insert into full_using_r values (5, 2, 200), (6, 3, 300) query IIIII rowsort select * from full_using_l as l (lid, x, lv) full join full_using_r using(x) ---- -0 1 10 null null +0 1 10 NULL NULL 1 2 20 5 200 -null 3 null 6 300 +NULL 3 NULL 6 300 query IIIII rowsort select * from full_using_l full join full_using_r using(x) ---- -0 1 10 null null +0 1 10 NULL NULL 1 2 20 5 200 -null 3 null 6 300 +NULL 3 NULL 6 300 statement ok drop table full_using_l diff --git a/tests/slt/join_left_inner.slt b/tests/slt/join_left_inner.slt index 7df11dc0..db879aa6 100644 --- a/tests/slt/join_left_inner.slt +++ b/tests/slt/join_left_inner.slt @@ -40,9 +40,9 @@ insert into a values (0, 1, 1), (1, 2, 2), (2, 3, 3); query IIII rowsort select v1, v2, v3, v4 from a left join b on v1 = v3; ---- -1 1 null null -2 2 null null -3 3 null null +1 1 NULL NULL +2 2 NULL NULL +3 3 NULL NULL statement ok insert into b values (0, 1, 100), (1, 3, 300), (2, 4, 400); @@ -51,7 +51,7 @@ query IIII rowsort select v1, v2, v3, v4 from a left join b on v1 = v3; ---- 1 1 1 100 -2 2 null null +2 2 NULL NULL 3 3 3 300 statement ok diff --git a/tests/slt/order_by.slt b/tests/slt/order_by.slt index 2ac16bea..cdd049dd 100644 --- a/tests/slt/order_by.slt +++ b/tests/slt/order_by.slt @@ -72,18 +72,18 @@ select v1, v2 from t order by v1 asc, v2 asc ---- 1 0 2 2 -2 null -null 5 -null null +2 NULL +NULL 5 +NULL NULL query II select v1, v2 from t order by v1 asc nulls last, v2 asc ---- 1 0 2 2 -2 null -null 5 -null null +2 NULL +NULL 5 +NULL NULL statement ok drop table t diff --git a/tests/slt/set_operation.slt b/tests/slt/set_operation.slt index 38a1ffcb..4b2eccf9 100644 --- a/tests/slt/set_operation.slt +++ b/tests/slt/set_operation.slt @@ -18,7 +18,7 @@ select v from set_left union select v from set_right 3 4 5 -null +NULL query I rowsort select v from set_left union all select v from set_right @@ -33,9 +33,9 @@ select v from set_left union all select v from set_right 3 4 5 -null -null -null +NULL +NULL +NULL # Edge case: tuple values should be compared as whole set values across set # operators, rather than flattening their fields or comparing only one slot. @@ -75,14 +75,14 @@ select v from set_left except all select v from set_right ---- 2 5 -null +NULL query I rowsort select v from set_left intersect select v from set_right ---- 1 3 -null +NULL # Edge case: INTERSECT ALL emits min(left_count, right_count), so value 1 # appears twice even though the right side has three copies. @@ -92,14 +92,14 @@ select v from set_left intersect all select v from set_right 1 1 3 -null +NULL # Edge case: set membership treats NULL as a set value for duplicate # elimination/matching, unlike ordinary NULL = NULL predicate evaluation. query I rowsort select v from set_left where v is null intersect select v from set_right where v is null ---- -null +NULL query I rowsort select v from set_left where v is null except select v from set_right where v is null @@ -110,7 +110,7 @@ select v from set_left intersect select v from set_right order by v desc ---- 3 1 -null +NULL # Regression: branch table names from a set operation must not leak to the # top-level ORDER BY scope. diff --git a/tests/slt/sql_2016/E091_01.slt b/tests/slt/sql_2016/E091_01.slt index 81415d59..8b8d1333 100644 --- a/tests/slt/sql_2016/E091_01.slt +++ b/tests/slt/sql_2016/E091_01.slt @@ -6,7 +6,7 @@ CREATE TABLE TABLE_E091_01_01_01 ( ID INT PRIMARY KEY, A FLOAT ); query F SELECT AVG ( A ) FROM TABLE_E091_01_01_01 ---- -null +NULL statement ok drop table table_e091_01_01_01 diff --git a/tests/slt/sql_2016/E091_03.slt b/tests/slt/sql_2016/E091_03.slt index 77cc285b..aba93b58 100644 --- a/tests/slt/sql_2016/E091_03.slt +++ b/tests/slt/sql_2016/E091_03.slt @@ -6,7 +6,7 @@ CREATE TABLE TABLE_E091_03_01_01 ( ID INT PRIMARY KEY, A FLOAT ); query F SELECT MAX ( A ) FROM TABLE_E091_03_01_01 ---- -null +NULL statement ok drop table table_e091_03_01_01 diff --git a/tests/slt/sql_2016/E091_04.slt b/tests/slt/sql_2016/E091_04.slt index 8bce7520..b836ae0c 100644 --- a/tests/slt/sql_2016/E091_04.slt +++ b/tests/slt/sql_2016/E091_04.slt @@ -6,7 +6,7 @@ CREATE TABLE TABLE_E091_04_01_01 ( ID INT PRIMARY KEY, A FLOAT ); query F SELECT MIN ( A ) FROM TABLE_E091_04_01_01 ---- -null +NULL statement ok drop table table_e091_04_01_01 diff --git a/tests/slt/sql_2016/E091_05.slt b/tests/slt/sql_2016/E091_05.slt index 7408593a..30a475e6 100644 --- a/tests/slt/sql_2016/E091_05.slt +++ b/tests/slt/sql_2016/E091_05.slt @@ -6,7 +6,7 @@ CREATE TABLE TABLE_E091_05_01_01 ( ID INT PRIMARY KEY, A FLOAT ); query F SELECT SUM ( A ) FROM TABLE_E091_05_01_01 ---- -null +NULL statement ok drop table table_e091_05_01_01 diff --git a/tests/slt/sql_2016/E091_06.slt b/tests/slt/sql_2016/E091_06.slt index 4aadf607..6945e065 100644 --- a/tests/slt/sql_2016/E091_06.slt +++ b/tests/slt/sql_2016/E091_06.slt @@ -6,7 +6,7 @@ CREATE TABLE TABLE_E091_06_01_01 ( ID INT PRIMARY KEY, A FLOAT ); query F SELECT AVG ( ALL A ) FROM TABLE_E091_06_01_01 ---- -null +NULL statement ok CREATE TABLE TABLE_E091_06_02_01 ( ID INT PRIMARY KEY, A FLOAT ); @@ -22,7 +22,7 @@ CREATE TABLE TABLE_E091_06_03_01 ( ID INT PRIMARY KEY, A FLOAT ); query F SELECT MAX ( ALL A ) FROM TABLE_E091_06_03_01 ---- -null +NULL statement ok CREATE TABLE TABLE_E091_06_04_01 ( ID INT PRIMARY KEY, A FLOAT ); @@ -30,7 +30,7 @@ CREATE TABLE TABLE_E091_06_04_01 ( ID INT PRIMARY KEY, A FLOAT ); query F SELECT MIN ( ALL A ) FROM TABLE_E091_06_04_01 ---- -null +NULL statement ok CREATE TABLE TABLE_E091_06_05_01 ( ID INT PRIMARY KEY, A FLOAT ); @@ -38,7 +38,7 @@ CREATE TABLE TABLE_E091_06_05_01 ( ID INT PRIMARY KEY, A FLOAT ); query F SELECT SUM ( ALL A ) FROM TABLE_E091_06_05_01 ---- -null +NULL statement ok drop table table_e091_06_05_01 diff --git a/tests/slt/sql_2016/E091_07.slt b/tests/slt/sql_2016/E091_07.slt index 3bb91f12..241c5b34 100644 --- a/tests/slt/sql_2016/E091_07.slt +++ b/tests/slt/sql_2016/E091_07.slt @@ -6,7 +6,7 @@ CREATE TABLE TABLE_E091_07_01_01 ( ID INT PRIMARY KEY, A FLOAT ); query F SELECT AVG ( DISTINCT A ) FROM TABLE_E091_07_01_01 ---- -null +NULL statement ok CREATE TABLE TABLE_E091_07_01_02 ( ID INT PRIMARY KEY, A FLOAT ); @@ -22,7 +22,7 @@ CREATE TABLE TABLE_E091_07_01_03 ( ID INT PRIMARY KEY, A FLOAT ); query F SELECT MAX ( DISTINCT A ) FROM TABLE_E091_07_01_03 ---- -null +NULL statement ok CREATE TABLE TABLE_E091_07_01_04 ( ID INT PRIMARY KEY, A FLOAT ); @@ -30,7 +30,7 @@ CREATE TABLE TABLE_E091_07_01_04 ( ID INT PRIMARY KEY, A FLOAT ); query F SELECT MIN ( DISTINCT A ) FROM TABLE_E091_07_01_04 ---- -null +NULL statement ok CREATE TABLE TABLE_E091_07_01_05 ( ID INT PRIMARY KEY, A FLOAT ); @@ -38,7 +38,7 @@ CREATE TABLE TABLE_E091_07_01_05 ( ID INT PRIMARY KEY, A FLOAT ); query F SELECT SUM ( DISTINCT A ) FROM TABLE_E091_07_01_05 ---- -null +NULL statement ok drop table table_e091_07_01_01 diff --git a/tests/slt/sql_2016/E131.slt b/tests/slt/sql_2016/E131.slt index cadc5ac4..59884344 100644 --- a/tests/slt/sql_2016/E131.slt +++ b/tests/slt/sql_2016/E131.slt @@ -3,4 +3,4 @@ query T SELECT NULL ---- -null +NULL diff --git a/tests/slt/sql_2016/F131_03.slt b/tests/slt/sql_2016/F131_03.slt index 0c302c08..d733b64a 100644 --- a/tests/slt/sql_2016/F131_03.slt +++ b/tests/slt/sql_2016/F131_03.slt @@ -11,7 +11,7 @@ CREATE VIEW VIEW_F131_03_01_01 AS SELECT A, MIN ( B ) AS C FROM TABLE_F131_03_01 query I SELECT SUM ( C ) FROM VIEW_F131_03_01_01 ---- -null +NULL statement ok CREATE TABLE TABLE_F131_03_01_021 ( ID INT PRIMARY KEY, A INTEGER, B INTEGER ); diff --git a/tests/slt/sql_2016/F261_01.slt b/tests/slt/sql_2016/F261_01.slt index a0ba2577..3cb6bf54 100644 --- a/tests/slt/sql_2016/F261_01.slt +++ b/tests/slt/sql_2016/F261_01.slt @@ -8,12 +8,12 @@ SELECT CASE 0 WHEN 2 THEN 1 ELSE 1 END query I SELECT CASE 0 WHEN 2 THEN 1 ELSE NULL END ---- -null +NULL query I SELECT CASE 0 WHEN 2 THEN 1 END ---- -null +NULL query I SELECT CASE 0 WHEN 2 THEN NULL ELSE 1 END @@ -23,12 +23,12 @@ SELECT CASE 0 WHEN 2 THEN NULL ELSE 1 END query I SELECT CASE 0 WHEN 2 THEN NULL ELSE NULL END ---- -null +NULL query I SELECT CASE 0 WHEN 2 THEN NULL END ---- -null +NULL query I SELECT CASE 0 WHEN 2 THEN 1 ELSE 1 END @@ -38,12 +38,12 @@ SELECT CASE 0 WHEN 2 THEN 1 ELSE 1 END query I SELECT CASE 0 WHEN 2 THEN 1 ELSE NULL END ---- -null +NULL query I SELECT CASE 0 WHEN 2 THEN 1 END ---- -null +NULL query I SELECT CASE 0 WHEN 2 THEN NULL ELSE 1 END @@ -53,9 +53,9 @@ SELECT CASE 0 WHEN 2 THEN NULL ELSE 1 END query T SELECT CASE 0 WHEN 2 THEN NULL ELSE NULL END ---- -null +NULL query T SELECT CASE 0 WHEN 2 THEN NULL END ---- -null +NULL diff --git a/tests/slt/sql_2016/F261_02.slt b/tests/slt/sql_2016/F261_02.slt index 46a5057d..f36088d4 100644 --- a/tests/slt/sql_2016/F261_02.slt +++ b/tests/slt/sql_2016/F261_02.slt @@ -8,12 +8,12 @@ SELECT CASE WHEN 0 = 1 THEN 1 ELSE 1 END query I SELECT CASE WHEN 0 = 1 THEN 1 ELSE NULL END ---- -null +NULL query I SELECT CASE WHEN 0 = 1 THEN 1 END ---- -null +NULL query I SELECT CASE WHEN 0 = 1 THEN NULL ELSE 1 END @@ -23,9 +23,9 @@ SELECT CASE WHEN 0 = 1 THEN NULL ELSE 1 END query T SELECT CASE WHEN 0 = 1 THEN NULL ELSE NULL END ---- -null +NULL query T SELECT CASE WHEN 0 = 1 THEN NULL END ---- -null +NULL diff --git a/tests/slt/sql_2016/F261_03.slt b/tests/slt/sql_2016/F261_03.slt index d0a2a165..ea2cf040 100644 --- a/tests/slt/sql_2016/F261_03.slt +++ b/tests/slt/sql_2016/F261_03.slt @@ -3,4 +3,4 @@ query I SELECT NULLIF ( 1 , 1 ) ---- -null +NULL diff --git a/tests/slt/string_length.slt b/tests/slt/string_length.slt index 8d12547c..73f7bd93 100644 --- a/tests/slt/string_length.slt +++ b/tests/slt/string_length.slt @@ -29,7 +29,7 @@ select id from s where char_length(c) + 1 > 3; query I select char_length(c) from s where id = 3; ---- -null +NULL statement ok drop table s diff --git a/tests/slt/subquery.slt b/tests/slt/subquery.slt index 75b65510..c2ed1b5c 100644 --- a/tests/slt/subquery.slt +++ b/tests/slt/subquery.slt @@ -398,8 +398,8 @@ insert into scalar_outer values (1), (2); query IT select id, (select v from scalar_inner where v = -1) from scalar_outer order by id; ---- -1 null -2 null +1 NULL +2 NULL statement ok drop table scalar_outer; @@ -488,11 +488,13 @@ statement error select * from users where id = (select user_id from orders where amount = age); -# Mark-apply and scalar-join subqueries cannot share one WHERE scope yet. -statement error +# Scalar initialization and EXISTS can share a WHERE scope. +query IT select * from users where exists(select 1 from orders) and id = (select max(user_id) from orders); +---- +2 30 statement ok drop table users; @@ -500,7 +502,7 @@ drop table users; statement ok drop table orders; -# ScalarApply must preserve its cached first result while validating scalar cardinality. +# ScalarQueryInit validates cardinality before publishing the cached result. statement ok create table scalar_eof (id int primary key, v int, keep_row boolean); @@ -539,9 +541,9 @@ query II select outer_row.id, (select v from scalar_eof where id < 0) from scalar_eof outer_row order by outer_row.id; ---- -1 null -2 null -3 null +1 NULL +2 NULL +3 NULL # Validate before accepting the cached scalar, including a single outer-row limit. statement error @@ -551,26 +553,36 @@ from scalar_eof outer_row order by outer_row.id limit 1; statement ok drop table scalar_eof; -# SELECT-list scalar subqueries are joined in above DISTINCT / GROUP BY / -# ORDER BY, so those clauses cannot read them yet: reject instead of reading -# another column (which crashed, or silently returned wrong results). +# Scalar results are initialized before sorting, grouping and DISTINCT. statement ok create table sq_ref(id int primary key, c varchar); statement ok insert into sq_ref values (0, 'a'), (1, 'b'), (2, 'a'); -statement error +query II select id, (select count(*) from sq_ref) - id as x from sq_ref order by x; +---- +2 1 +1 2 +0 3 -statement error +query I select (select max(id) from sq_ref) + 1 as x from sq_ref order by x; +---- +3 +3 +3 -statement error +query I select distinct (select max(id) from sq_ref) from sq_ref; +---- +2 -statement error +query I select (select max(id) from sq_ref) as x from sq_ref group by x; +---- +2 # Not referenced by those clauses: still supported. query II @@ -636,3 +648,411 @@ drop table sq_outer; statement ok drop table sq_inner; + +# Uncorrelated scalar initialization across query clauses and DML. +statement ok +create table init_rows(id int primary key, v int) + +statement ok +insert into init_rows values (1, 10), (2, 20), (3, 30) + +query II +select id, v + (select max(v) from init_rows) as x from init_rows order by x desc +---- +3 60 +2 50 +1 40 + +query II +select id, v + (select max(v) from init_rows) as x from init_rows order by x desc limit 2 +---- +3 60 +2 50 + +query I +select (select (select max(v) from init_rows)) + (select min(v) from init_rows) +---- +40 + +query I +select distinct (select max(v) from init_rows) from init_rows +---- +30 + +query II +select (select max(v) from init_rows), count(*) from init_rows group by (select max(v) from init_rows) +---- +30 3 + +query I +select sum((select max(v) from init_rows)) from init_rows +---- +90 + +query I +select count(*) from init_rows having count(*) > (select 2) +---- +3 + +query I +select id from init_rows where v < (select max(v) from init_rows) order by id +---- +1 +2 + +query I +select id from init_rows where id in (select id from init_rows) and v = (select max(v) from init_rows) +---- +3 + +query I +select id from init_rows where exists(select 1 from init_rows) and v = (select max(v) from init_rows) +---- +3 + +query I +select coalesce((select 9), 0) +---- +9 + +query I +select coalesce((select v from init_rows where id < 0), -1) +---- +-1 + +statement error +select (select v from init_rows) + +statement error +select id from init_rows where v = (select v from init_rows) + +query I rowsort +select (select x.v from init_rows as x where x.id = init_rows.id) from init_rows +---- +10 +20 +30 + +statement ok +create view init_max as select (select max(v) from init_rows) as m + +statement ok +create view init_min as select (select min(v) from init_rows) as m + +query II +select a.m, b.m from init_max as a cross join init_min as b +---- +30 10 + +query II +with q as (select (select max(v) from init_rows) as m) select a.m, b.m from q as a cross join q as b +---- +30 30 + +# Repeated CTE inputs can prune different columns without changing shared definitions. +query II +with q as (select max(v) as m from init_rows) +select a.m,b.m from q a cross join q b; +---- +30 30 + +query II +with q as (select id,v,(select max(v) from init_rows) as m from init_rows) +select a.v,b.m from q a join q b on a.id=b.id where a.id=2; +---- +20 30 + +query II +with q as (select id,v,(select max(v) from init_rows) as m from init_rows) +select a.m,b.v from q a join q b on a.id=b.id where b.id=1; +---- +30 10 + +query II +select id, sum((select 2)) over (order by id) from init_rows order by id +---- +1 2 +2 4 +3 6 + +statement ok +update init_rows set v = 40 where id = (select max(id) from init_rows) + +query I +select (select max(v) from init_rows) +---- +40 + +statement ok +update init_rows set v = (select max(v) from init_rows) + 1 where id = 1 + +query I +select v from init_rows where id = 1 +---- +41 + +statement ok +insert into init_rows values (4, (select min(v) from init_rows)) + +query I +select v from init_rows where id = 4 +---- +20 + +statement ok +delete from init_rows where id = (select max(id) from init_rows) + +query I +select count(*) from init_rows +---- +3 + +statement ok +drop view init_max + +statement ok +drop view init_min + +statement ok +drop table init_rows + +# Correlated values use the execution arena; outer columns remain input dependencies. +statement ok +create table corr_rows(id int primary key, b int); + +statement ok +insert into corr_rows values (1,10),(2,20),(3,30),(4,null),(5,20); + +# WHERE and SELECT registrations are attached before their own consumers. +query T +explain select id,(select 7) from corr_rows where b<(select 25); +---- +Projection [corr_rows.id, InitValue(1:0)] [Project => (Sort Option: Follow)] ScalarQueryInit InitValue(1:0) Filter (corr_rows.b < InitValue(1:1)), Is Having: false [Filter => (Sort Option: Follow)] ScalarQueryInit InitValue(1:1) TableScan corr_rows -> [corr_rows.id, corr_rows.b] [SeqScan => (Sort Option: None)] ScalarSubquery [ScalarSubquery => (Sort Option: Follow)] Projection [25] [Project => (Sort Option: Follow)] Dummy [Dummy => (Sort Option: None)] ScalarSubquery [ScalarSubquery => (Sort Option: Follow)] Projection [7] [Project => (Sort Option: Follow)] Dummy [Dummy => (Sort Option: None)] + +query II rowsort +select id,(select 7) from corr_rows where b<(select 25); +---- +1 7 +2 7 +5 7 + +# b is needed solely as an outer parameter, not as a projected output column. +query I rowsort +select (select count(*) from corr_rows x where x.bcorr_rows.b),(select 7) from corr_rows order by id desc limit 3 offset 1; +---- +4 0 0 7 +3 3 0 7 +2 1 1 7 + +# Null/text scalar results are paired with the original rows after ordering. +query IT +select id,(select cast(x.b as varchar(12)) from corr_rows x where x.id=corr_rows.id and x.id<>2) from corr_rows order by id desc; +---- +5 20 +4 NULL +3 30 +2 NULL +1 10 + +query II rowsort +select id,(select count(*) from corr_rows x where x.bcorr_rows.b) from corr_rows; +---- +1 0 3 +2 1 1 +3 3 0 +4 0 0 +5 1 1 + +query II rowsort +select o.id,(select count(*) from corr_rows x where x.b>o.b and x.id2) from corr_rows where id<4; +---- +1 10 +2 NULL +3 30 + +# Owned text values and NULL remain distinct across repeated scalar execution. +query IT rowsort +select id,(select cast(x.b as varchar(12)) from corr_rows x where x.id=corr_rows.id) from corr_rows; +---- +1 10 +2 20 +3 30 +4 NULL +5 20 + +statement error scalar subquery returned more than one row +select id,(select x.b from corr_rows x where x.b=corr_rows.b) from corr_rows; + +query II rowsort +select id,coalesce((select x.b from corr_rows x where x.id=corr_rows.id),-1)+(select 7) from corr_rows; +---- +1 17 +2 27 +3 37 +4 6 +5 27 + +query I +select (select count(*) from corr_rows x where x.bl.id) from corr_rows o cross join corr_lookup l where o.id=3; +---- +3 1 2 +3 2 1 + +query II rowsort +select id,(select sum(x.b) from corr_rows x where x.b (1, 10); statement ok drop table tuple_in + +# A SELECT alias must not hide a same-named input column in SELECT or WHERE. +statement ok +create table alias_input(id int primary key, v int); + +statement ok +insert into alias_input values (1, 10), (2, 20), (3, null); + +query II +select id, -v as v from alias_input where v > 15 order by id; +---- +2 -20 + +query II +select id, cast(null as int) as v from alias_input where v is null; +---- +3 NULL + +query II +select id, 57 as v from alias_input where v < 15 order by id; +---- +1 57 + +query II +select 57 as v, v from alias_input order by id; +---- +57 10 +57 20 +57 NULL + +# ORDER BY still resolves the output alias before the input column. +query II +select id, -v as v from alias_input where id < 3 order by v; +---- +2 -20 +1 -10 + +# Retain existing alias fallback when there is no input column of that name. +query II +select id, v + 1 as x from alias_input where x = 11; +---- +1 11 + +query II +select id, -v as v from alias_input where exists(select 1 where alias_input.v > 15); +---- +2 -20 + +statement ok +create table alias_other(id int primary key, v int); + +statement error is ambiguous +select a.id as v from alias_input a cross join alias_other b where v > 0; + +statement ok +drop table alias_other; + +query IIII +select cast(-12.75 as int), cast(cast(-12.75 as float) as int), cast(0.0 as int), cast(-0.0 as int); +---- +-12 -12 0 0 + +query I +select id from alias_input where cast(-v as int) = -20; +---- +2 + +statement error over flow +select cast(2147483648.0 as int); + +statement error over flow +select cast(-2147483649.0 as int); + +statement ok +drop table alias_input; diff --git a/tests/slt/where_by_index.slt b/tests/slt/where_by_index.slt index b87056cf..02d78e9b 100644 --- a/tests/slt/where_by_index.slt +++ b/tests/slt/where_by_index.slt @@ -46,6 +46,26 @@ query IIT select * from t1 where id = 0 and id != 0; ---- +# AND over an OR must intersect every OR branch, not union them +query IIT +select * from t1 where id > 10 and (id = 3 or id = 21); +---- +21 22 23 + +query IIT +select * from t1 where (id = 3 or id = 21) and (id = 21 or id = 30); +---- +21 22 23 + +query IIT +select * from t1 where (id = 3 or id = 21) and (id = 6 or id = 9); +---- + +query IIT +select * from t1 where id > 10 and id < 30 and (id < 5 or id > 25); +---- +27 28 29 + query IIT select * from t1 where id = 0 or id != 0 limit 10; ---- @@ -182,7 +202,7 @@ select * from t1 where (c1 = 7 or c1 = 10) and c2 < 9; query IIT select * from t1 where c1 is null and c2 is null; ---- -100000000 null null +100000000 NULL NULL query IIT select * from t1 where c1 > 0 and c1 < 8; @@ -488,3 +508,59 @@ select item,w,qty from stock_bounds where w=1 and qty<2 order by item; statement ok drop table stock_bounds; + +# OR / AND with IS NULL over an index: NULL sorts after every value +statement ok +create table t_null_or(id int primary key, c1 int); + +statement ok +create index t_null_or_c1 on t_null_or (c1); + +statement ok +insert into t_null_or values (0, null), (1, 1), (2, 2), (3, 3), (4, 4), (5, 5), (6, 6), (7, 7), (8, 8), (9, 9), (10, 0), (11, 1), (12, 2), (13, 3), (14, 4), (15, 5), (16, 6), (17, 7), (18, 8), (19, 9), (20, null), (21, 1), (22, 2), (23, 3), (24, 4), (25, 5), (26, 6), (27, 7), (28, 8), (29, 9), (30, 0), (31, 1), (32, 2), (33, 3), (34, 4), (35, 5), (36, 6), (37, 7), (38, 8), (39, 9), (40, null), (41, 1), (42, 2), (43, 3), (44, 4), (45, 5), (46, 6), (47, 7), (48, 8), (49, 9), (50, 0), (51, 1), (52, 2), (53, 3), (54, 4), (55, 5), (56, 6), (57, 7), (58, 8), (59, 9), (60, null), (61, 1), (62, 2), (63, 3), (64, 4), (65, 5), (66, 6), (67, 7), (68, 8), (69, 9), (70, 0), (71, 1), (72, 2), (73, 3), (74, 4), (75, 5), (76, 6), (77, 7), (78, 8), (79, 9), (80, null), (81, 1), (82, 2), (83, 3), (84, 4), (85, 5), (86, 6), (87, 7), (88, 8), (89, 9), (90, 0), (91, 1), (92, 2), (93, 3), (94, 4), (95, 5), (96, 6), (97, 7), (98, 8), (99, 9), (100, null), (101, 1), (102, 2), (103, 3), (104, 4), (105, 5), (106, 6), (107, 7), (108, 8), (109, 9), (110, 0), (111, 1), (112, 2), (113, 3), (114, 4), (115, 5), (116, 6), (117, 7), (118, 8), (119, 9), (120, null), (121, 1), (122, 2), (123, 3), (124, 4), (125, 5), (126, 6), (127, 7), (128, 8), (129, 9), (130, 0), (131, 1), (132, 2), (133, 3), (134, 4), (135, 5), (136, 6), (137, 7), (138, 8), (139, 9), (140, null), (141, 1), (142, 2), (143, 3), (144, 4), (145, 5), (146, 6), (147, 7), (148, 8), (149, 9), (150, 0), (151, 1), (152, 2), (153, 3), (154, 4), (155, 5), (156, 6), (157, 7), (158, 8), (159, 9), (160, null), (161, 1), (162, 2), (163, 3), (164, 4), (165, 5), (166, 6), (167, 7), (168, 8), (169, 9), (170, 0), (171, 1), (172, 2), (173, 3), (174, 4), (175, 5), (176, 6), (177, 7), (178, 8), (179, 9), (180, null), (181, 1), (182, 2), (183, 3), (184, 4), (185, 5), (186, 6), (187, 7), (188, 8), (189, 9), (190, 0), (191, 1), (192, 2), (193, 3), (194, 4), (195, 5), (196, 6), (197, 7), (198, 8), (199, 9); + +statement ok +analyze table t_null_or; + +query I +select count(*) from t_null_or where c1 = 1 or c1 = 2 or c1 is null; +---- +50 + +query I +select count(*) from t_null_or where c1 > 3 or c1 is null; +---- +130 + +query I +select count(*) from t_null_or where c1 >= 3 or c1 is null; +---- +150 + +query I +select count(*) from t_null_or where c1 < 3 or c1 is null; +---- +60 + +query I +select count(*) from t_null_or where c1 is not null or c1 is null; +---- +200 + +query I +select count(*) from t_null_or where c1 = 1 or c1 > 3 or c1 is null; +---- +150 + +query I +select count(*) from t_null_or where c1 = 5 and (c1 is null or c1 > 3); +---- +20 + +query I +select count(*) from t_null_or where (c1 = 1 or c1 is null) and (c1 is null or c1 > 3); +---- +10 + +statement ok +drop table t_null_or; diff --git a/tests/slt/where_by_index_explain.slt b/tests/slt/where_by_index_explain.slt index f37d43c0..b24ada38 100644 --- a/tests/slt/where_by_index_explain.slt +++ b/tests/slt/where_by_index_explain.slt @@ -138,7 +138,7 @@ Projection [t1.id, t1.c1, t1.c2] [Project => (Sort Option: Follow)] Filter (t1.c query T explain select * from t1 where c1 is null and c2 is null; ---- -Projection [t1.id, t1.c1, t1.c2] [Project => (Sort Option: Follow)] Filter t1.c2 is null, Is Having: false [Filter => (Sort Option: Follow)] TableScan t1 -> [t1.id, t1.c1, t1.c2] [IndexScan By u_c1_index => null => (Sort Option: OrderBy: (t1.c1 Asc Nulls Last) ignore_prefix_len: 0)] +Projection [t1.id, t1.c1, t1.c2] [Project => (Sort Option: Follow)] Filter t1.c2 is null, Is Having: false [Filter => (Sort Option: Follow)] TableScan t1 -> [t1.id, t1.c1, t1.c2] [IndexScan By u_c1_index => NULL => (Sort Option: OrderBy: (t1.c1 Asc Nulls Last) ignore_prefix_len: 0)] query T explain select * from t1 where c1 > 0 and c1 < 8; diff --git a/tests/sqllogictest/Cargo.toml b/tests/sqllogictest/Cargo.toml index 9ee0f0c6..1c22a89d 100644 --- a/tests/sqllogictest/Cargo.toml +++ b/tests/sqllogictest/Cargo.toml @@ -2,6 +2,7 @@ name = "sqllogictest-test" version = "0.4.0" edition = "2021" +default-run = "sqllogictest-test" [dependencies] clap = { version = "4" } diff --git a/tests/sqllogictest/src/lib.rs b/tests/sqllogictest/src/lib.rs index 72f681f7..dc125001 100644 --- a/tests/sqllogictest/src/lib.rs +++ b/tests/sqllogictest/src/lib.rs @@ -12,10 +12,10 @@ // See the License for the specific language governing permissions and // limitations under the License. -use kite_sql::binder::{command_type, CommandType}; -use kite_sql::db::{prepare_all, Database, DatabaseIter, Statement}; +use kite_sql::db::Database; use kite_sql::errors::DatabaseError; use kite_sql::storage::lmdb::LmdbStorage; +use kite_sql::types::value::DataValue; use sqllogictest::{DBOutput, DefaultColumnType, DB}; use std::time::Instant; @@ -30,73 +30,37 @@ impl DB for SQLBase { fn run(&mut self, sql: &str) -> Result, Self::Error> { let start = Instant::now(); println!("|— Input SQL: {}", sql); - let mut statements = prepare_all(sql)?.into_iter().peekable(); - - let output = loop { - let Some(statement) = statements.next() else { - break DBOutput::StatementComplete(0); - }; - let is_last = statements.peek().is_none(); - match command_type(&statement)? { - CommandType::DDL => { - self.db.ddl(statement.to_string())?; - if is_last { - break DBOutput::StatementComplete(0); - } - } - CommandType::Analyze => { - execute_analyze_statement(&mut self.db, &statement)?; - if is_last { - break DBOutput::StatementComplete(0); - } - } - _ => { - let iter = self.db.run(statement.to_string())?; - if is_last { - break collect_output(iter)?; - } - iter.done()?; - } - } - }; - + let output = run_sql(&mut self.db, sql, |_, value| value.to_string())?; println!(" |— time spent: {:?}", start.elapsed()); Ok(output) } } -fn collect_output( - mut iter: DatabaseIter<'_, LmdbStorage>, -) -> Result, DatabaseError> { - let types = vec![DefaultColumnType::Any; iter.schema(|schema| schema.len())]; - let mut rows = Vec::new(); - - while let Some(row) = iter.next_tuple(|_, tuple| { - tuple - .values - .iter() - .map(|value| format!("{}", value)) - .collect() - })? { - rows.push(row); - } - iter.done()?; - if rows.is_empty() { - return Ok(DBOutput::StatementComplete(0)); - } - Ok(DBOutput::Rows { types, rows }) -} - -fn execute_analyze_statement( +/// Runs every statement in `sql` and returns the output of the last one, rendering each value of +/// column `i` with `format(i, value)`. +pub fn run_sql( db: &mut Database, - statement: &Statement, -) -> Result<(), DatabaseError> { - let Statement::Analyze(analyze) = statement else { - unreachable!("execute_analyze_statement only accepts ANALYZE") - }; - let table_name = analyze - .table_name - .as_ref() - .ok_or_else(|| DatabaseError::UnsupportedStmt("ANALYZE requires table name".to_string()))?; - db.analyze(table_name.to_string()) + sql: &str, + mut format: impl FnMut(usize, &DataValue) -> String, +) -> Result, DatabaseError> { + db.run_mut(sql, |iter| { + let width = iter.schema(|schema| schema.len()); + let types = vec![DefaultColumnType::Any; width]; + let mut rows = Vec::new(); + while let Some(row) = iter.next_tuple(|_, tuple| { + tuple + .values + .iter() + .enumerate() + .map(|(i, value)| format(i, value)) + .collect() + })? { + rows.push(row); + } + if rows.is_empty() { + Ok(DBOutput::StatementComplete(0)) + } else { + Ok(DBOutput::Rows { types, rows }) + } + }) }