chore: test sqlparser-rs 0.64.0 release branch before release

Cargo.toml

@@ -187,7 +187,7 @@ rstest = "0.28.0"
 serde = { version = "1", features = ["derive"] }
 serde_json = "1"
 sha2 = "^0.11.0"
-sqlparser = { version = "0.63.0", default-features = false, features = ["std", "visitor"] }
+sqlparser = { git = "https://github.com/alamb/sqlparser-rs", rev = "02097a8cc593e26976d8da587f3eaa74c9bde1e8", default-features = false, features = ["std", "visitor"] }
 stacker = "0.1.24"
 strum = "0.28.0"
 strum_macros = "0.28.0"

datafusion-examples/examples/relation_planner/table_sample.rs

@@ -320,7 +320,8 @@ impl RelationPlanner for TableSamplePlanner {
             with_ordinality,
             partitions,
             json_path,
-            index_hints,
+            index_directives,
+            inheritance,
         } = relation
         else {
             return Ok(RelationPlanning::Original(Box::new(relation)));
@@ -371,7 +372,8 @@ impl RelationPlanner for TableSamplePlanner {
             with_ordinality,
             partitions,
             json_path,
-            index_hints,
+            index_directives,
+            inheritance,
         };
         let input = context.plan(base_relation)?;
 

datafusion/sql/src/expr/mod.rs

@@ -12,8 +12,8 @@ use datafusion_expr::planner::{
 use sqlparser::ast::{
     AccessExpr, BinaryOperator, CastFormat, CastKind, CeilFloorKind,
     DataType as SQLDataType, DateTimeField, DictionaryField, Expr as SQLExpr,
-    ExprWithAlias as SQLExprWithAlias, JsonPath, JsonPathElem, MapEntry, Spanned,
-    StructField, Subscript, TrimWhereField, TypedString, Value, ValueWithSpan,
+    ExprWithAlias as SQLExprWithAlias, InTarget, JsonPath, JsonPathElem, MapEntry,
+    Spanned, StructField, Subscript, TrimWhereField, TypedString, Value, ValueWithSpan,
 };
 use sqlparser::ast::{Query, Visit, Visitor};
 
@@ -580,11 +580,36 @@ impl<S: ContextProvider> SqlToRel<'_, S> {
                 )?),
             ))),
 
-            SQLExpr::InList {
+            SQLExpr::In {
                 expr,
-                list,
+                target,
                 negated,
-            } => self.sql_in_list_to_expr(*expr, list, negated, schema, planner_context),
+                global,
+            } => {
+                if global {
+                    return not_impl_err!("GLOBAL IN is not supported");
+                }
+                match target {
+                    InTarget::List(list) => self.sql_in_list_to_expr(
+                        *expr,
+                        list,
+                        negated,
+                        schema,
+                        planner_context,
+                    ),
+                    InTarget::Subquery(subquery) => self.parse_in_subquery(
+                        *expr,
+                        *subquery,
+                        negated,
+                        schema,
+                        planner_context,
+                    ),
+                    InTarget::Unnest(_) => not_impl_err!("IN UNNEST is not supported"),
+                    InTarget::Table { .. } => {
+                        not_impl_err!("IN <table name> is not supported")
+                    }
+                }
+            }
 
             SQLExpr::Like {
                 negated,
@@ -728,13 +753,6 @@ impl<S: ContextProvider> SqlToRel<'_, S> {
             SQLExpr::Exists { subquery, negated } => {
                 self.parse_exists_subquery(*subquery, negated, schema, planner_context)
             }
-            SQLExpr::InSubquery {
-                expr,
-                subquery,
-                negated,
-            } => {
-                self.parse_in_subquery(*expr, *subquery, negated, schema, planner_context)
-            }
             SQLExpr::Subquery(subquery) => {
                 self.parse_scalar_subquery(*subquery, schema, planner_context)
             }

datafusion/sql/src/planner.rs

@@ -898,6 +898,9 @@ impl<'a, S: ContextProvider> SqlToRel<'a, S> {
             | SQLDataType::FixedString(_)
             | SQLDataType::Tuple(_)
             | SQLDataType::Nested(_)
+            | SQLDataType::Object(_)
+            | SQLDataType::Zerofill(_)
+            | SQLDataType::Collate(_, _)
             | SQLDataType::Union(_)
             | SQLDataType::Nullable(_)
             | SQLDataType::LowCardinality(_)

datafusion/sql/src/relation/join.rs

@@ -159,6 +159,9 @@ impl<S: ContextProvider> SqlToRel<'_, S> {
                 .asof_join_on(right, None, match_condition)?
                 .build(),
             JoinConstraint::Natural => plan_err!("NATURAL ASOF JOIN is not supported"),
+            JoinConstraint::UsingAlias(_, _) => {
+                not_impl_err!("USING (...) AS alias is not supported")
+            }
         }
     }
 
@@ -218,6 +221,9 @@ impl<S: ContextProvider> SqlToRel<'_, S> {
                     .join_using(right, join_type, keys)?
                     .build()
             }
+            JoinConstraint::UsingAlias(_, _) => {
+                not_impl_err!("USING (...) AS alias is not supported")
+            }
             JoinConstraint::Natural => {
                 let left_cols: HashSet<&String> =
                     left.schema().fields().iter().map(|f| f.name()).collect();

datafusion/sql/src/statement.rs

@@ -199,6 +199,7 @@ fn calc_inline_constraints_from_columns(columns: &[ColumnDef]) -> Vec<TableConst
                 | ast::ColumnOption::Generated { .. }
                 | ast::ColumnOption::Comment(_)
                 | ast::ColumnOption::Options(_)
+                | ast::ColumnOption::GenericOptions(_)
                 | ast::ColumnOption::OnUpdate(_)
                 | ast::ColumnOption::Materialized(_)
                 | ast::ColumnOption::Ephemeral(_)
@@ -311,7 +312,7 @@ impl<S: ContextProvider> SqlToRel<'_, S> {
                 constraints,
                 if_not_exists,
                 or_replace,
-                without_rowid,
+                sqlite_table_options,
                 like,
                 clone,
                 comment,
@@ -322,7 +323,6 @@ impl<S: ContextProvider> SqlToRel<'_, S> {
                 partition_by,
                 cluster_by,
                 clustered_by,
-                strict,
                 copy_grants,
                 enable_schema_evolution,
                 change_tracking,
@@ -400,7 +400,10 @@ impl<S: ContextProvider> SqlToRel<'_, S> {
                 if location.is_some() {
                     return not_impl_err!("Location not supported");
                 }
-                if without_rowid {
+                if sqlite_table_options
+                    .options
+                    .contains(&ast::SqliteTableOption::WithoutRowid)
+                {
                     return not_impl_err!("Without rowid not supported");
                 }
                 if like.is_some() {
@@ -433,7 +436,10 @@ impl<S: ContextProvider> SqlToRel<'_, S> {
                 if clustered_by.is_some() {
                     return not_impl_err!("Clustered by not supported");
                 }
-                if strict {
+                if sqlite_table_options
+                    .options
+                    .contains(&ast::SqliteTableOption::Strict)
+                {
                     return not_impl_err!("Strict not supported");
                 }
                 if copy_grants {
@@ -1056,6 +1062,8 @@ impl<S: ContextProvider> SqlToRel<'_, S> {
                 source,
                 partitioned,
                 after_columns,
+                by_name,
+                overriding,
                 table,
                 on,
                 returning,
@@ -1101,6 +1109,12 @@ impl<S: ContextProvider> SqlToRel<'_, S> {
                 if !after_columns.is_empty() {
                     plan_err!("After-columns clause not supported")?;
                 }
+                if by_name {
+                    plan_err!("INSERT BY NAME not supported")?;
+                }
+                if overriding.is_some() {
+                    plan_err!("Insert-overriding clause not supported")?;
+                }
                 if on.is_some() {
                     plan_err!("Insert-on clause not supported")?;
                 }
@@ -1165,7 +1179,15 @@ impl<S: ContextProvider> SqlToRel<'_, S> {
                 optimizer_hints,
                 output,
                 order_by,
+                on_cluster,
+                settings,
             }) => {
+                if on_cluster.is_some() {
+                    plan_err!("UPDATE ON CLUSTER not supported")?;
+                }
+                if settings.is_some() {
+                    plan_err!("Update with settings not supported")?;
+                }
                 let from_clauses =
                     from.map(|update_table_from_kind| match update_table_from_kind {
                         UpdateTableFromKind::BeforeSet(from_clauses) => from_clauses,
@@ -1217,10 +1239,18 @@ impl<S: ContextProvider> SqlToRel<'_, S> {
                 delete_token: _,
                 optimizer_hints,
                 output,
+                on_cluster,
+                settings,
             }) => {
                 if !tables.is_empty() {
                     plan_err!("DELETE <TABLE> not supported")?;
                 }
+                if on_cluster.is_some() {
+                    plan_err!("DELETE ON CLUSTER not supported")?;
+                }
+                if settings.is_some() {
+                    plan_err!("Delete with settings not supported")?;
+                }
 
                 if using.is_some() {
                     plan_err!("Using clause not supported")?;
@@ -1468,7 +1498,8 @@ impl<S: ContextProvider> SqlToRel<'_, S> {
                             ast::CreateFunctionBody::AsBeforeOptions{body: expr, link_symbol: _link_symbol} => expr,
                             ast::CreateFunctionBody::AsAfterOptions(expr) => expr,
                             ast::CreateFunctionBody::Return(expr) => expr,
-                            ast::CreateFunctionBody::AsBeginEnd(_) => {
+                            ast::CreateFunctionBody::AsBeginEnd(_)
+                            | ast::CreateFunctionBody::BeginAtomic(_) => {
                                 return not_impl_err!(
                                     "BEGIN/END enclosed function body syntax is not supported"
                                 )?;
@@ -2466,7 +2497,8 @@ impl<S: ContextProvider> SqlToRel<'_, S> {
                 partitions,
                 json_path,
                 sample,
-                index_hints,
+                index_directives,
+                inheritance,
             } => {
                 if alias
                     .as_ref()
@@ -2483,7 +2515,8 @@ impl<S: ContextProvider> SqlToRel<'_, S> {
                     || !partitions.is_empty()
                     || json_path.is_some()
                     || sample.is_some()
-                    || !index_hints.is_empty()
+                    || !index_directives.is_empty()
+                    || inheritance.is_some()
                 {
                     return not_impl_err!(
                         "MERGE target table modifiers are not supported"

datafusion/sql/src/unparser/ast.rs

@@ -590,7 +590,7 @@ pub struct TableRelationBuilder {
     with_hints: Vec<ast::Expr>,
     version: Option<ast::TableVersion>,
     partitions: Vec<ast::Ident>,
-    index_hints: Vec<ast::TableIndexHints>,
+    index_directives: Vec<ast::TableIndexDirective>,
 }
 
 impl TableRelationBuilder {
@@ -618,8 +618,11 @@ impl TableRelationBuilder {
         self.partitions = value;
         self
     }
-    pub fn index_hints(&mut self, value: Vec<ast::TableIndexHints>) -> &mut Self {
-        self.index_hints = value;
+    pub fn index_directives(
+        &mut self,
+        value: Vec<ast::TableIndexDirective>,
+    ) -> &mut Self {
+        self.index_directives = value;
         self
     }
     pub fn build(&self) -> Result<ast::TableFactor, BuilderError> {
@@ -639,7 +642,8 @@ impl TableRelationBuilder {
             with_ordinality: false,
             json_path: None,
             sample: None,
-            index_hints: self.index_hints.clone(),
+            index_directives: self.index_directives.clone(),
+            inheritance: None,
         })
     }
     fn create_empty() -> Self {
@@ -650,7 +654,7 @@ impl TableRelationBuilder {
             with_hints: Default::default(),
             version: Default::default(),
             partitions: Default::default(),
-            index_hints: Default::default(),
+            index_directives: Default::default(),
         }
     }
 }

datafusion/sql/src/unparser/expr.rs

@@ -158,10 +158,11 @@ impl Unparser<'_> {
                     .iter()
                     .map(|e| self.expr_to_sql_inner(e))
                     .collect::<Result<Vec<_>>>()?;
-                Ok(ast::Expr::InList {
+                Ok(ast::Expr::In {
                     expr: Box::new(self.expr_to_sql_inner(expr)?),
-                    list: list_expr,
+                    target: ast::InTarget::List(list_expr),
                     negated: *negated,
+                    global: false,
                 })
             }
             Expr::ScalarFunction(ScalarFunction { func, args }) => {
@@ -462,10 +463,11 @@ impl Unparser<'_> {
                         "Subquery must be a Query, but found {sub_statement:?}"
                     );
                 };
-                Ok(ast::Expr::InSubquery {
+                Ok(ast::Expr::In {
                     expr: inexpr,
-                    subquery: sub_query,
+                    target: ast::InTarget::Subquery(sub_query),
                     negated: insubq.negated,
+                    global: false,
                 })
             }
             Expr::SetComparison(set_cmp) => {
@@ -3926,7 +3928,7 @@ mod tests {
                 match tz {
                     Some(tz) => ast::DataType::Custom(
                         ObjectName::from(vec![Ident::new("TIMESTAMPTZ")]),
-                        vec![tz.to_string()],
+                        vec![ast::TypeModifier::Identifier(Ident::new(tz.to_string()))],
                     ),
                     None => ast::DataType::Timestamp(None, TimezoneInfo::None),
                 }

datafusion/sql/src/unparser/plan.rs

@@ -2763,6 +2763,7 @@ impl Unparser<'_> {
             JoinType::Inner => match &constraint {
                 ast::JoinConstraint::On(_)
                 | ast::JoinConstraint::Using(_)
+                | ast::JoinConstraint::UsingAlias(_, _)
                 | ast::JoinConstraint::Natural => ast::JoinOperator::Inner(constraint),
                 ast::JoinConstraint::None => {
                     // Inner joins with no conditions or filters are not valid SQL in most systems,