diff --git a/Cargo.lock b/Cargo.lock index 6fd56360c54f2..58a4f97b139ce 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6104,9 +6104,8 @@ dependencies = [ [[package]] name = "sqlparser" -version = "0.63.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3679862809bd1f92e563cf6fd820e28f39135303668001c10331d5027183e8b4" +version = "0.64.0" +source = "git+https://github.com/alamb/sqlparser-rs?rev=02097a8cc593e26976d8da587f3eaa74c9bde1e8#02097a8cc593e26976d8da587f3eaa74c9bde1e8" dependencies = [ "log", "recursive", @@ -6115,9 +6114,8 @@ dependencies = [ [[package]] name = "sqlparser_derive" -version = "0.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2905ae8e8148d8f7f5748ea95d1e949fa4fc1d7f5161547b758ad118b2e4b0aa" +version = "0.7.0" +source = "git+https://github.com/alamb/sqlparser-rs?rev=02097a8cc593e26976d8da587f3eaa74c9bde1e8#02097a8cc593e26976d8da587f3eaa74c9bde1e8" dependencies = [ "proc-macro2", "quote", diff --git a/Cargo.toml b/Cargo.toml index a96d3cbf37e58..877cf971e0a14 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -187,7 +187,7 @@ rstest = "0.28.0" serde = { version = "1", features = ["derive"] } serde_json = "1" sha2 = "^0.11.0" -sqlparser = { version = "0.63.0", default-features = false, features = ["std", "visitor"] } +sqlparser = { git = "https://github.com/alamb/sqlparser-rs", rev = "02097a8cc593e26976d8da587f3eaa74c9bde1e8", default-features = false, features = ["std", "visitor"] } stacker = "0.1.24" strum = "0.28.0" strum_macros = "0.28.0" diff --git a/datafusion-examples/examples/relation_planner/table_sample.rs b/datafusion-examples/examples/relation_planner/table_sample.rs index 13801a5fffc66..302f1be1ba6f4 100644 --- a/datafusion-examples/examples/relation_planner/table_sample.rs +++ b/datafusion-examples/examples/relation_planner/table_sample.rs @@ -320,7 +320,8 @@ impl RelationPlanner for TableSamplePlanner { with_ordinality, partitions, json_path, - index_hints, + index_directives, + inheritance, } = relation else { return Ok(RelationPlanning::Original(Box::new(relation))); @@ -371,7 +372,8 @@ impl RelationPlanner for TableSamplePlanner { with_ordinality, partitions, json_path, - index_hints, + index_directives, + inheritance, }; let input = context.plan(base_relation)?; diff --git a/datafusion/sql/src/expr/mod.rs b/datafusion/sql/src/expr/mod.rs index f5730573e1ed3..0dd277e109e18 100644 --- a/datafusion/sql/src/expr/mod.rs +++ b/datafusion/sql/src/expr/mod.rs @@ -12,8 +12,8 @@ use datafusion_expr::planner::{ use sqlparser::ast::{ AccessExpr, BinaryOperator, CastFormat, CastKind, CeilFloorKind, DataType as SQLDataType, DateTimeField, DictionaryField, Expr as SQLExpr, - ExprWithAlias as SQLExprWithAlias, JsonPath, JsonPathElem, MapEntry, Spanned, - StructField, Subscript, TrimWhereField, TypedString, Value, ValueWithSpan, + ExprWithAlias as SQLExprWithAlias, InTarget, JsonPath, JsonPathElem, MapEntry, + Spanned, StructField, Subscript, TrimWhereField, TypedString, Value, ValueWithSpan, }; use sqlparser::ast::{Query, Visit, Visitor}; @@ -580,11 +580,36 @@ impl SqlToRel<'_, S> { )?), ))), - SQLExpr::InList { + SQLExpr::In { expr, - list, + target, negated, - } => self.sql_in_list_to_expr(*expr, list, negated, schema, planner_context), + global, + } => { + if global { + return not_impl_err!("GLOBAL IN is not supported"); + } + match target { + InTarget::List(list) => self.sql_in_list_to_expr( + *expr, + list, + negated, + schema, + planner_context, + ), + InTarget::Subquery(subquery) => self.parse_in_subquery( + *expr, + *subquery, + negated, + schema, + planner_context, + ), + InTarget::Unnest(_) => not_impl_err!("IN UNNEST is not supported"), + InTarget::Table { .. } => { + not_impl_err!("IN is not supported") + } + } + } SQLExpr::Like { negated, @@ -728,13 +753,6 @@ impl SqlToRel<'_, S> { SQLExpr::Exists { subquery, negated } => { self.parse_exists_subquery(*subquery, negated, schema, planner_context) } - SQLExpr::InSubquery { - expr, - subquery, - negated, - } => { - self.parse_in_subquery(*expr, *subquery, negated, schema, planner_context) - } SQLExpr::Subquery(subquery) => { self.parse_scalar_subquery(*subquery, schema, planner_context) } diff --git a/datafusion/sql/src/planner.rs b/datafusion/sql/src/planner.rs index b51d1ab777595..e50332ace56dd 100644 --- a/datafusion/sql/src/planner.rs +++ b/datafusion/sql/src/planner.rs @@ -898,6 +898,9 @@ impl<'a, S: ContextProvider> SqlToRel<'a, S> { | SQLDataType::FixedString(_) | SQLDataType::Tuple(_) | SQLDataType::Nested(_) + | SQLDataType::Object(_) + | SQLDataType::Zerofill(_) + | SQLDataType::Collate(_, _) | SQLDataType::Union(_) | SQLDataType::Nullable(_) | SQLDataType::LowCardinality(_) diff --git a/datafusion/sql/src/relation/join.rs b/datafusion/sql/src/relation/join.rs index f7cf2b6e296b9..db679693fb012 100644 --- a/datafusion/sql/src/relation/join.rs +++ b/datafusion/sql/src/relation/join.rs @@ -159,6 +159,9 @@ impl SqlToRel<'_, S> { .asof_join_on(right, None, match_condition)? .build(), JoinConstraint::Natural => plan_err!("NATURAL ASOF JOIN is not supported"), + JoinConstraint::UsingAlias(_, _) => { + not_impl_err!("USING (...) AS alias is not supported") + } } } @@ -218,6 +221,9 @@ impl SqlToRel<'_, S> { .join_using(right, join_type, keys)? .build() } + JoinConstraint::UsingAlias(_, _) => { + not_impl_err!("USING (...) AS alias is not supported") + } JoinConstraint::Natural => { let left_cols: HashSet<&String> = left.schema().fields().iter().map(|f| f.name()).collect(); diff --git a/datafusion/sql/src/statement.rs b/datafusion/sql/src/statement.rs index e3701bc1a3814..625b654864888 100644 --- a/datafusion/sql/src/statement.rs +++ b/datafusion/sql/src/statement.rs @@ -199,6 +199,7 @@ fn calc_inline_constraints_from_columns(columns: &[ColumnDef]) -> Vec SqlToRel<'_, S> { constraints, if_not_exists, or_replace, - without_rowid, + sqlite_table_options, like, clone, comment, @@ -322,7 +323,6 @@ impl SqlToRel<'_, S> { partition_by, cluster_by, clustered_by, - strict, copy_grants, enable_schema_evolution, change_tracking, @@ -400,7 +400,10 @@ impl SqlToRel<'_, S> { if location.is_some() { return not_impl_err!("Location not supported"); } - if without_rowid { + if sqlite_table_options + .options + .contains(&ast::SqliteTableOption::WithoutRowid) + { return not_impl_err!("Without rowid not supported"); } if like.is_some() { @@ -433,7 +436,10 @@ impl SqlToRel<'_, S> { if clustered_by.is_some() { return not_impl_err!("Clustered by not supported"); } - if strict { + if sqlite_table_options + .options + .contains(&ast::SqliteTableOption::Strict) + { return not_impl_err!("Strict not supported"); } if copy_grants { @@ -1056,6 +1062,8 @@ impl SqlToRel<'_, S> { source, partitioned, after_columns, + by_name, + overriding, table, on, returning, @@ -1101,6 +1109,12 @@ impl SqlToRel<'_, S> { if !after_columns.is_empty() { plan_err!("After-columns clause not supported")?; } + if by_name { + plan_err!("INSERT BY NAME not supported")?; + } + if overriding.is_some() { + plan_err!("Insert-overriding clause not supported")?; + } if on.is_some() { plan_err!("Insert-on clause not supported")?; } @@ -1165,7 +1179,15 @@ impl SqlToRel<'_, S> { optimizer_hints, output, order_by, + on_cluster, + settings, }) => { + if on_cluster.is_some() { + plan_err!("UPDATE ON CLUSTER not supported")?; + } + if settings.is_some() { + plan_err!("Update with settings not supported")?; + } let from_clauses = from.map(|update_table_from_kind| match update_table_from_kind { UpdateTableFromKind::BeforeSet(from_clauses) => from_clauses, @@ -1217,10 +1239,18 @@ impl SqlToRel<'_, S> { delete_token: _, optimizer_hints, output, + on_cluster, + settings, }) => { if !tables.is_empty() { plan_err!("DELETE
not supported")?; } + if on_cluster.is_some() { + plan_err!("DELETE ON CLUSTER not supported")?; + } + if settings.is_some() { + plan_err!("Delete with settings not supported")?; + } if using.is_some() { plan_err!("Using clause not supported")?; @@ -1468,7 +1498,8 @@ impl SqlToRel<'_, S> { ast::CreateFunctionBody::AsBeforeOptions{body: expr, link_symbol: _link_symbol} => expr, ast::CreateFunctionBody::AsAfterOptions(expr) => expr, ast::CreateFunctionBody::Return(expr) => expr, - ast::CreateFunctionBody::AsBeginEnd(_) => { + ast::CreateFunctionBody::AsBeginEnd(_) + | ast::CreateFunctionBody::BeginAtomic(_) => { return not_impl_err!( "BEGIN/END enclosed function body syntax is not supported" )?; @@ -2466,7 +2497,8 @@ impl SqlToRel<'_, S> { partitions, json_path, sample, - index_hints, + index_directives, + inheritance, } => { if alias .as_ref() @@ -2483,7 +2515,8 @@ impl SqlToRel<'_, S> { || !partitions.is_empty() || json_path.is_some() || sample.is_some() - || !index_hints.is_empty() + || !index_directives.is_empty() + || inheritance.is_some() { return not_impl_err!( "MERGE target table modifiers are not supported" diff --git a/datafusion/sql/src/unparser/ast.rs b/datafusion/sql/src/unparser/ast.rs index 51c38b032a64e..851d4b07f504f 100644 --- a/datafusion/sql/src/unparser/ast.rs +++ b/datafusion/sql/src/unparser/ast.rs @@ -590,7 +590,7 @@ pub struct TableRelationBuilder { with_hints: Vec, version: Option, partitions: Vec, - index_hints: Vec, + index_directives: Vec, } impl TableRelationBuilder { @@ -618,8 +618,11 @@ impl TableRelationBuilder { self.partitions = value; self } - pub fn index_hints(&mut self, value: Vec) -> &mut Self { - self.index_hints = value; + pub fn index_directives( + &mut self, + value: Vec, + ) -> &mut Self { + self.index_directives = value; self } pub fn build(&self) -> Result { @@ -639,7 +642,8 @@ impl TableRelationBuilder { with_ordinality: false, json_path: None, sample: None, - index_hints: self.index_hints.clone(), + index_directives: self.index_directives.clone(), + inheritance: None, }) } fn create_empty() -> Self { @@ -650,7 +654,7 @@ impl TableRelationBuilder { with_hints: Default::default(), version: Default::default(), partitions: Default::default(), - index_hints: Default::default(), + index_directives: Default::default(), } } } diff --git a/datafusion/sql/src/unparser/expr.rs b/datafusion/sql/src/unparser/expr.rs index 053b1beb63e98..9f89903d118e4 100644 --- a/datafusion/sql/src/unparser/expr.rs +++ b/datafusion/sql/src/unparser/expr.rs @@ -158,10 +158,11 @@ impl Unparser<'_> { .iter() .map(|e| self.expr_to_sql_inner(e)) .collect::>>()?; - Ok(ast::Expr::InList { + Ok(ast::Expr::In { expr: Box::new(self.expr_to_sql_inner(expr)?), - list: list_expr, + target: ast::InTarget::List(list_expr), negated: *negated, + global: false, }) } Expr::ScalarFunction(ScalarFunction { func, args }) => { @@ -462,10 +463,11 @@ impl Unparser<'_> { "Subquery must be a Query, but found {sub_statement:?}" ); }; - Ok(ast::Expr::InSubquery { + Ok(ast::Expr::In { expr: inexpr, - subquery: sub_query, + target: ast::InTarget::Subquery(sub_query), negated: insubq.negated, + global: false, }) } Expr::SetComparison(set_cmp) => { @@ -3926,7 +3928,7 @@ mod tests { match tz { Some(tz) => ast::DataType::Custom( ObjectName::from(vec![Ident::new("TIMESTAMPTZ")]), - vec![tz.to_string()], + vec![ast::TypeModifier::Identifier(Ident::new(tz.to_string()))], ), None => ast::DataType::Timestamp(None, TimezoneInfo::None), } diff --git a/datafusion/sql/src/unparser/plan.rs b/datafusion/sql/src/unparser/plan.rs index 564de3a2816f6..ab2ef983992fe 100644 --- a/datafusion/sql/src/unparser/plan.rs +++ b/datafusion/sql/src/unparser/plan.rs @@ -2763,6 +2763,7 @@ impl Unparser<'_> { JoinType::Inner => match &constraint { ast::JoinConstraint::On(_) | ast::JoinConstraint::Using(_) + | ast::JoinConstraint::UsingAlias(_, _) | ast::JoinConstraint::Natural => ast::JoinOperator::Inner(constraint), ast::JoinConstraint::None => { // Inner joins with no conditions or filters are not valid SQL in most systems,